diff --git a/.gitattributes b/.gitattributes index 1aa7969e805..1b447a9189e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -31,6 +31,10 @@ # the reviewable change, and pin LF because they are compared byte-for-byte. # Not -diff: the shell diff is the review surface when a wrapper does change. /src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf +# Captured agent PTY transcripts. -text, not `text eol=lf` like the wrapper snapshots above: +# these carry real CR and CRLF bytes as the terminal emitted them, and line-ending +# normalisation on a Windows checkout would rewrite the evidence the fixture exists to be. +/src/main/runtime/__fixtures__/*.txt -text # Generated runtime English subset: compared byte-for-byte by # verify:localization-runtime-catalog, so a CRLF checkout would fail the gate. /src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf diff --git a/.github/workflows/cloud-deploy-relay-production-director.yml b/.github/workflows/cloud-deploy-relay-production-director.yml index 4489d67b845..07e1e74ec3c 100644 --- a/.github/workflows/cloud-deploy-relay-production-director.yml +++ b/.github/workflows/cloud-deploy-relay-production-director.yml @@ -13,6 +13,11 @@ on: default: preserve type: choice options: [preserve, enable, disable] + region-correction-cohort-percent: + description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled' + required: true + default: preserve + type: string prune-incompatible-revisions: description: Retain only the newly verified serving and rollback revisions required: true @@ -62,6 +67,7 @@ jobs: REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled IMAGE_DIGEST: ${{ inputs.image-digest }} REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }} + REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }} PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }} # Floor the served revision must keep, matching relay_min_instances in # environments/production.tfvars. This gate only fails a bad deploy; Terraform @@ -106,8 +112,11 @@ jobs: echo "image-digest must be an immutable lowercase sha256 digest" >&2 exit 1 fi + if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then + [[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]] + fi IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}" - SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')" + SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}" [[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]] [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] @@ -218,7 +227,8 @@ jobs: --max-instances "${DIRECTOR_MAX_INSTANCES}" \ --prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \ --release-id "${RELEASE_ID}" \ - --regional-placement-secret-version "${target_version}" + --regional-placement-secret-version "${target_version}" \ + --region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}" echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/pi-owner-runtime.yml b/.github/workflows/pi-owner-runtime.yml new file mode 100644 index 00000000000..373afb7a539 --- /dev/null +++ b/.github/workflows/pi-owner-runtime.yml @@ -0,0 +1,29 @@ +name: Pi owner runtime verification +on: + pull_request: + paths: + - 'src/main/pi/agent-status-handler-source.ts' + - 'tests/tools/pi-owner-runtime-smoke.mjs' + - '.github/workflows/pi-owner-runtime.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + runtime: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + env: + ORCA_BACKGROUND_LAUNCH: '1' + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: ./.github/actions/install-node-dependencies + - name: Install pinned extension loader + run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0 + - name: Verify real owner exit and hook delivery + run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent diff --git a/.github/workflows/pi-provider-runtime.yml b/.github/workflows/pi-provider-runtime.yml new file mode 100644 index 00000000000..837c38baf9a --- /dev/null +++ b/.github/workflows/pi-provider-runtime.yml @@ -0,0 +1,28 @@ +name: Pi extension provider verification +on: + pull_request: + paths: + - 'src/shared/commit-message-agent-specs-primary.ts' + - 'tests/tools/pi-provider-runtime-smoke.mjs' + - '.github/workflows/pi-provider-runtime.yml' +permissions: + contents: read +jobs: + runtime: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + env: + ORCA_BACKGROUND_LAUNCH: '1' + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: ./.github/actions/install-node-dependencies + - name: Install pinned Pi runtime + run: npm install --prefix .cache/pi-provider --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.84.2 + - name: Verify extension model generation before and after + run: node tests/tools/pi-provider-runtime-smoke.mjs .cache/pi-provider/node_modules/@earendil-works/pi-coding-agent/dist/cli.js diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index cdb334c64cd..0c215db94d3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -37,6 +37,13 @@ jobs: - name: Install Electron package binary for tests run: node config/scripts/install-electron-package-binary.mjs + # The real two-cell transport test imports cloud relay source and its contracts. + - name: Install relay integration dependencies + working-directory: cloud + run: | + npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts + npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build + - name: Test shard run: | pnpm exec vitest run --config config/vitest.config.ts \ diff --git a/.gitignore b/.gitignore index 913dfc4a045..e5207a25015 100644 --- a/.gitignore +++ b/.gitignore @@ -103,7 +103,9 @@ docs/** !docs/agent-skill-sharing-implementation-checklist.md !docs/mobile-terminal-shortcut-bar.md !docs/reference/ +!docs/reference/agent-pty-transcript-capture.md !docs/reference/agent-status-store.md +!docs/reference/antigravity-readiness-evidence.md !docs/reference/git-compatibility.md !docs/reference/headless-linux-server.md !docs/reference/ime-regression-checklist.md diff --git a/AGENTS.md b/AGENTS.md index 5ff66b95b0f..f1ce31e404b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,6 +72,10 @@ All changes must consider folder workspaces as well as git worktrees. Don't assu The execution host owns agent status in one store, the hook server's, and every reader (sidebar, `worktree ps`, mobile, dashboard) subscribes to it. Before adding a producer, a cache, or a reader-side precedence rule, read [`docs/reference/agent-status-store.md`](./docs/reference/agent-status-store.md): new producers write into that store, and readers keep only presentation policy. +## Agent Terminal Screens + +A rule that reads what an agent CLI paints on a terminal — readiness, blocked prompts, idle — must be written against a captured transcript, not a remembered screen. Record one with [`docs/reference/agent-pty-transcript-capture.md`](./docs/reference/agent-pty-transcript-capture.md), which keeps escapes and wrapping intact and scrubs account identifiers before they reach git. Antigravity readiness has no transcript yet and five failed attempts without one; before touching it, read [`docs/reference/antigravity-readiness-evidence.md`](./docs/reference/antigravity-readiness-evidence.md). + ## Remote Wire Compatibility Clients and remote Orca servers update independently, so mixed versions are the normal state. Before changing anything a paired client and host exchange — RPC params, stream frames, or the content either side publishes over them — follow [`docs/reference/remote-wire-compatibility.md`](./docs/reference/remote-wire-compatibility.md). A new optional field is safe; a new stream opcode must be capability-negotiated because decoders drop unknown opcodes silently; and changing what the host publishes reaches old clients even with no wire change. diff --git a/cloud/apps/push/src/fcm-client.test.ts b/cloud/apps/push/src/fcm-client.test.ts index 4a8d1fb41f7..8843e7aab95 100644 --- a/cloud/apps/push/src/fcm-client.test.ts +++ b/cloud/apps/push/src/fcm-client.test.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto' import { describe, expect, it } from 'vitest' -import { fcmCollapseKey, FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js' +import { FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js' import { buildPushDelivery } from './push-delivery-message.js' const NOW = 1_700_000_000_000 @@ -20,6 +20,7 @@ function delivery(agentState: 'needs-input' | null = 'needs-input') { agentState, title: 'Agent needs input', body: 'Waiting on your answer', + paneKey: 'tab-b:pane-1', worktreeId: 'wt-1' } }) @@ -59,27 +60,14 @@ describe('fcm client', () => { expect(JSON.parse(request.body)).toEqual({ message: { token: TOKEN, - notification: { title: 'Agent needs input', body: 'Waiting on your answer' }, - android: { - priority: 'HIGH', - ttl: '300s', - collapse_key: createHash('sha256') - .update( - createHash('sha256') - .update(JSON.stringify([HOST, 'note-1'])) - .digest('hex') - ) - .digest('hex') - .slice(0, 32), - notification: { - channel_id: 'orca-desktop', - tag: createHash('sha256') - .update(JSON.stringify([HOST, 'note-1'])) - .digest('hex') - } - }, + android: { priority: 'HIGH', ttl: '300s' }, data: { + title: 'Agent needs input', + message: 'Waiting on your answer', + tag: delivery().collapseId, + channelId: 'orca-desktop', hostFingerprint: HOST, + paneKey: 'tab-b:pane-1', worktreeId: 'wt-1', notificationId: 'note-1', notificationSeq: '7', @@ -96,7 +84,7 @@ describe('fcm client', () => { await fcm.send(delivery(null), { token: TOKEN }) const message = JSON.parse(fake.requests[0]!.body) as { message: { - android: { collapse_key: string; notification: { tag: string } } + android: Record data: Record } } @@ -108,9 +96,10 @@ describe('fcm client', () => { .update(JSON.stringify([HOST, 'note-1'])) .digest('hex') expect(message.message.data.coalescedCount).toBeUndefined() - expect(message.message.android.notification.tag).toBe(tag) - expect(message.message.android.collapse_key).toBe(fcmCollapseKey(tag)) - expect(message.message.android.collapse_key).toHaveLength(32) + expect(message.message.data.tag).toBe(tag) + expect(message.message.android).not.toHaveProperty('collapse_key') + expect(message.message).not.toHaveProperty('notification') + expect(message.message.data).not.toHaveProperty('body') }) it('marks an unregistered token dead from the status or the error detail', async () => { diff --git a/cloud/apps/push/src/fcm-client.ts b/cloud/apps/push/src/fcm-client.ts index c22bd3309cd..0b58aae80f5 100644 --- a/cloud/apps/push/src/fcm-client.ts +++ b/cloud/apps/push/src/fcm-client.ts @@ -1,5 +1,4 @@ import { providerRetryAfter } from './provider-retry-delay.js' -import { createHash } from 'node:crypto' import { PUSH_DEFAULTS } from '@orca-cloud/push-contract' import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js' import type { PushProviderOutcome } from './push-provider-outcome.js' @@ -22,12 +21,6 @@ type FcmErrorBody = { error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] } } -// FCM collapse_key is a short opaque string, so the collapse id is hashed -// rather than truncated: truncation would merge unrelated notifications. -export function fcmCollapseKey(collapseId: string): string { - return createHash('sha256').update(collapseId).digest('hex').slice(0, 32) -} - export function fcmMessageBody(input: { delivery: PushDelivery token: string @@ -39,24 +32,23 @@ export function fcmMessageBody(input: { return JSON.stringify({ message: { token: input.token, - ...(delivery.orca.kind === 'dismiss' - ? {} - : { notification: { title: delivery.title, body: delivery.body } }), android: { priority: 'HIGH', - ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`, - collapse_key: fcmCollapseKey(delivery.collapseId), + ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s` + }, + // Notification payloads collapse offline; Expo renders these data messages natively. + data: { + ...orcaDataStrings(delivery.orca), ...(delivery.orca.kind === 'dismiss' ? {} : { - notification: { - channel_id: - delivery.sound === false ? `${input.channelId}-silent` : input.channelId, - tag: delivery.collapseId - } + title: delivery.title, + message: delivery.body, + tag: delivery.collapseId, + channelId: delivery.sound === false ? `${input.channelId}-silent` : input.channelId, + ...(delivery.sound === false ? { sound: '' } : {}) }) - }, - data: orcaDataStrings(delivery.orca) + } } }) } diff --git a/cloud/apps/push/src/push-delivery-message.ts b/cloud/apps/push/src/push-delivery-message.ts index bc2c1a5d9b2..3bd2dae6e7c 100644 --- a/cloud/apps/push/src/push-delivery-message.ts +++ b/cloud/apps/push/src/push-delivery-message.ts @@ -5,6 +5,7 @@ export type PushOrcaData = { kind?: 'alert' | 'dismiss' hostFingerprint: string worktreeId?: string + paneKey?: string notificationId?: string notificationSeq: number notificationEpoch: string @@ -51,6 +52,7 @@ export function buildPushDelivery(input: { orca: { ...(notification.kind ? { kind: notification.kind } : {}), hostFingerprint, + ...(notification.paneKey === undefined ? {} : { paneKey: notification.paneKey }), ...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }), ...(notification.notificationId === undefined ? {} diff --git a/cloud/apps/push/src/push-dismissal-provider.test.ts b/cloud/apps/push/src/push-dismissal-provider.test.ts index 15362105777..65572e8401c 100644 --- a/cloud/apps/push/src/push-dismissal-provider.test.ts +++ b/cloud/apps/push/src/push-dismissal-provider.test.ts @@ -23,5 +23,9 @@ it('dismissal provider payloads cannot display a new alert or play a sound', () const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message expect(android).not.toHaveProperty('notification') expect(android.android).not.toHaveProperty('notification') + expect(android.android).not.toHaveProperty('collapse_key') + expect(android.data).not.toHaveProperty('title') + expect(android.data).not.toHaveProperty('message') + expect(android.data).not.toHaveProperty('sound') expect(android.data.kind).toBe('dismiss') }) diff --git a/cloud/apps/push/src/push-notification-sound.test.ts b/cloud/apps/push/src/push-notification-sound.test.ts index 4dd1b85504f..ede4dcd3291 100644 --- a/cloud/apps/push/src/push-notification-sound.test.ts +++ b/cloud/apps/push/src/push-notification-sound.test.ts @@ -23,7 +23,11 @@ it('carries a silent preference through validation to APNs and Android payloads' expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound') expect( JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message - .android.notification.channel_id + .data.channelId ).toBe('orca-desktop-silent') + expect( + JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message + .data.sound + ).toBe('') expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default') }) diff --git a/cloud/apps/push/src/push-pane-routing.test.ts b/cloud/apps/push/src/push-pane-routing.test.ts new file mode 100644 index 00000000000..66ce0b3a38c --- /dev/null +++ b/cloud/apps/push/src/push-pane-routing.test.ts @@ -0,0 +1,27 @@ +import { expect, it } from 'vitest' +import { PushNotificationSchema } from '@orca-cloud/push-contract' +import { buildPushDelivery, orcaDataStrings } from './push-delivery-message.js' + +it('preserves pane identity for both APNs and FCM, and accepts older workspace-only messages', () => { + const base = { + notificationSeq: 1, + notificationEpoch: 'epoch', + source: 'agent-task-complete', + agentState: 'finished', + title: 'Done', + body: '', + worktreeId: 'folder:/work' + } + const paneKey = 'tab-b:11111111-1111-4111-8111-111111111111' + for (const extra of [{}, { paneKey }]) { + const notification = PushNotificationSchema.parse({ ...base, ...extra }) + const delivery = buildPushDelivery({ + notification, + hostFingerprint: 'host', + registrationId: 'phone', + expiresAt: Date.now() + 300000 + }) + expect(delivery.orca.paneKey).toBe('paneKey' in extra ? paneKey : undefined) + expect(orcaDataStrings(delivery.orca).paneKey).toBe('paneKey' in extra ? paneKey : undefined) + } +}) diff --git a/cloud/apps/push/src/push-server-send.test.ts b/cloud/apps/push/src/push-server-send.test.ts index 2a93020ed1f..4b14b77eaec 100644 --- a/cloud/apps/push/src/push-server-send.test.ts +++ b/cloud/apps/push/src/push-server-send.test.ts @@ -56,7 +56,7 @@ describe('push gateway send route', () => { await harness.flushDeliveries() expect(harness.fcmRequests).toHaveLength(1) expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({ - message: { token: FCM_TOKEN, notification: { title: 'Agent needs input' } } + message: { token: FCM_TOKEN, data: { title: 'Agent needs input' } } }) const afterDeath = await harness.post( @@ -179,7 +179,7 @@ describe('push gateway send route', () => { const message = JSON.parse(harness.fcmRequests[0]!.body) as { message: { android: { notification: { tag: string } }; data: Record } } - expect(message.message.android.notification.tag).toMatch(/^[a-f0-9]{64}$/) + expect(message.message.data.tag).toMatch(/^[a-f0-9]{64}$/) expect(message.message.data.coalescedCount).toBeUndefined() }) diff --git a/cloud/apps/relay/src/admin-token-verifier.ts b/cloud/apps/relay/src/admin-token-verifier.ts index 4b8ad26e695..8b236d58473 100644 --- a/cloud/apps/relay/src/admin-token-verifier.ts +++ b/cloud/apps/relay/src/admin-token-verifier.ts @@ -6,6 +6,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [ '/v1/admin/cell-status', '/v1/admin/evacuation-status', '/v1/admin/regional-rehome-control', + '/v1/admin/regional-rehome-preview', '/v1/admin/runtime-status' ] as const diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts index c45e31c4a01..44f6a6fc293 100644 --- a/cloud/apps/relay/src/app.ts +++ b/cloud/apps/relay/src/app.ts @@ -1,5 +1,9 @@ import { AssignmentRequestSchema, + IdleRegionalRehomeRequestSchema, + type IdleRegionalRehomeRequest, + type IdleRegionalRehomeOutcome, + type RegionCorrectionResponse, isRelayCellConnectionHardCap, RELAY_ADMISSION_BUDGETS, RELAY_DEFAULT_REGION, @@ -39,7 +43,7 @@ import { type AssignmentAdmissionRejection } from './public-assignment-admission.js' import { relayHostLogDigest } from './relay-host-log-digest.js' -import type { RelayRuntimeCounts } from './relay-observability.js' +import type { RegionalRehomeSafetySnapshot, RelayRuntimeCounts } from './relay-observability.js' import { isRegionalRehomeTrustProbe, probeRegionalRehomeTrust @@ -68,21 +72,28 @@ export function createRelayApp( store: RelayCredentialStore assignments: RelayAssignmentStore drain: (graceMs: number) => void + idleRehome?: (input: IdleRegionalRehomeRequest & { + cohortPercent: number + directorSafety: RegionalRehomeSafetySnapshot + }) => Promise<{ outcome: IdleRegionalRehomeOutcome }> drainHost?: (input: { attemptId: string userId: string relayHostId: string sourceAssignmentEpoch: number + sourceCellIncarnation: string graceMs: number - }) => 'accepted' | 'already-accepted' | 'host-not-connected' + }) => + | 'accepted' + | 'already-accepted' + | 'host-not-connected' + | Promise<'accepted' | 'already-accepted' | 'host-not-connected'> regionalRehomeIdentityToken?: (audience: string) => Promise regionalRehomeFetch?: typeof fetch - regionalRehomeTrustProbeHostExists?: (input: { - userId: string - relayHostId: string - }) => boolean + regionalRehomeTrustProbeHostExists?: (input: { userId: string; relayHostId: string }) => boolean cellIncarnation?: string isDraining?: () => boolean + regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot runtimeCounts?: () => RelayRuntimeCounts ready: () => Promise recordAssignmentAdmission?: ( @@ -226,7 +237,8 @@ export function createRelayApp( return context.json({ error: 'host_identity_mismatch' }, 403) } const identity = { userId: claims.sub, relayHostId: claims.relayHostId } - const requestedRegion = body.data.preferredRegion + const requestedRegion = + body.data.regionCorrection?.action === 'report' ? undefined : body.data.preferredRegion const targetRegion = config.regionalPlacementEnabled !== false && requestedRegion ? requestedRegion @@ -295,10 +307,30 @@ export function createRelayApp( } } let assignment: RelayAssignment + let regionCorrection: RegionCorrectionResponse | undefined try { - assignment = requestedRegion - ? await operations.assignments.assign(identity, requestedRegion, targetRegion) - : await operations.assignments.assign(identity) + if (body.data.regionCorrection?.action === 'report') { + const current = await operations.assignments.resolve(identity) + if (!current) return context.json({ error: 'assignment_not_found' }, 409) + assignment = current + } else { + assignment = requestedRegion + ? await operations.assignments.assign(identity, requestedRegion, targetRegion) + : await operations.assignments.assign(identity) + } + if (body.data.regionCorrection) { + try { + regionCorrection = await operations.assignments.exchangeRegionCorrection( + identity, + body.data.regionCorrection, + assignment.assignmentEpoch + ) + } catch (error) { + if (body.data.regionCorrection.action === 'report') throw error + // Optional measurement setup must not discard an otherwise valid placement. + console.warn(JSON.stringify({ event: 'orca_relay_region_window_unavailable' })) + } + } } catch (error) { if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) { logAssignmentRejection({ @@ -353,13 +385,16 @@ export function createRelayApp( v: 1, cellUrl: assignment.cellUrl, assignmentEpoch: assignment.assignmentEpoch, - lease + lease, + ...(regionCorrection ? { regionCorrection } : {}) }) }) app.post('/v1/resolve', async (context) => { if (config.role === 'cell') return context.json({ error: 'director_only' }, 404) if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context) - if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) { + if ( + Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes + ) { return context.json({ error: 'request_too_large' }, 413) } const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null)) @@ -433,6 +468,34 @@ export function createRelayApp( operations.drain(body.data.graceMs) return context.json({ ok: true }) }) + app.post('/v1/admin/host-idle-rehome', async (context) => { + if (config.role !== 'cell' || !operations.idleRehome) { + return context.json({ error: 'cell_only' }, 404) + } + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyRegionalRehomeToken(bearer))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = IdleRegionalRehomeCommandSchema.safeParse( + await context.req.json().catch(() => null) + ) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + if ( + body.data.sourceCellId !== config.cellId || + !operations.cellIncarnation || + body.data.sourceCellIncarnation !== operations.cellIncarnation + ) { + return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409) + } + try { + return context.json({ v: 1, ...(await operations.idleRehome(body.data)) }) + } catch (error) { + return context.json({ error: operationError(error) }, 409) + } + }) app.post('/v1/admin/host-drain', async (context) => { if (config.role !== 'cell' || !operations.drainHost) { return context.json({ error: 'cell_only' }, 404) @@ -474,7 +537,7 @@ export function createRelayApp( } sharedRuntimeIdentityRejected = true } - const outcome = operations.drainHost(body.data) + const outcome = await operations.drainHost(body.data) return context.json({ v: 1, outcome, @@ -502,8 +565,7 @@ export function createRelayApp( region: config.region ?? RELAY_DEFAULT_REGION, imageDigest: config.imageDigest ?? null, draining: operations.isDraining?.() ?? false, - regionalRehomeProtocol: - config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0, + regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0, connectionCapacity: config.connectionHardCap === undefined ? null @@ -559,6 +621,18 @@ export function createRelayApp( return context.json({ error: operationError(error) }, 409) } }) + app.get('/v1/admin/regional-rehome-preview', async (context) => { + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + const bearer = readBearer(context.req.header('authorization')) + if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) { + return context.json({ error: 'invalid_token' }, 401) + } + const preview = await operations.assignments.previewRegionalRehomeEligibility( + operations.regionalRehomeSafetySnapshot?.() + ) + const outcomes = await operations.assignments.regionCorrectionOutcomes() + return context.json({ v: 1, preview, outcomes }) + }) app.post('/v1/admin/regional-rehome-control', async (context) => { if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) const bearer = readBearer(context.req.header('authorization')) @@ -1295,6 +1369,11 @@ const RegionalRehomeSafetySchema = z }) .strict() +const IdleRegionalRehomeCommandSchema = IdleRegionalRehomeRequestSchema.extend({ + cohortPercent: z.number().int().min(0).max(100), + directorSafety: RegionalRehomeSafetySchema +}) + const CellHeartbeatSchema = z .object({ v: z.literal(1), @@ -1394,45 +1473,48 @@ const CellRegionalRehomeStatusSchema = z v: z.literal(1), cellId: z.string().min(1).max(128), cellIncarnation: z.string().uuid(), - regionalRehomeProtocol: z.number().int().min(0).max(1), + regionalRehomeProtocol: z.number().int().min(0).max(3), safety: RegionalRehomeSafetySchema }) .strict() -const RegionalRehomeControlSchema = z.discriminatedUnion('action', [ - z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(), - z.object({ - v: z.literal(1), - action: z.literal('apply'), - expectedGeneration: z.number().int().nonnegative(), - enabled: z.boolean(), - notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), - ratePerMinute: z.number().int().min(1).max(120), - preferenceMaxAgeMs: z - .number() - .int() - .min(60_000) - .max(30 * 24 * 60 * 60_000), - hostCooldownMs: z - .number() - .int() - .min(60_000) - .max(30 * 24 * 60 * 60_000), - drainGraceMs: z.number().int().min(60_000).max(60 * 60_000), - confirmation: z.enum([ - 'ENABLE_REGIONAL_REHOMING', - 'DISABLE_REGIONAL_REHOMING' - ]) - }).strict() -]).superRefine((value, context) => { - if (value.action !== 'apply') return - const expected = value.enabled - ? 'ENABLE_REGIONAL_REHOMING' - : 'DISABLE_REGIONAL_REHOMING' - if (value.confirmation !== expected) { - context.addIssue({ code: 'custom', message: 'confirmation does not match state' }) - } -}) +const RegionalRehomeControlSchema = z + .discriminatedUnion('action', [ + z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(), + z + .object({ + v: z.literal(1), + action: z.literal('apply'), + expectedGeneration: z.number().int().nonnegative(), + enabled: z.boolean(), + notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + ratePerMinute: z.number().int().min(1).max(120), + preferenceMaxAgeMs: z + .number() + .int() + .min(60_000) + .max(30 * 24 * 60 * 60_000), + hostCooldownMs: z + .number() + .int() + .min(60_000) + .max(30 * 24 * 60 * 60_000), + drainGraceMs: z + .number() + .int() + .min(60_000) + .max(60 * 60_000), + confirmation: z.enum(['ENABLE_REGIONAL_REHOMING', 'DISABLE_REGIONAL_REHOMING']) + }) + .strict() + ]) + .superRefine((value, context) => { + if (value.action !== 'apply') return + const expected = value.enabled ? 'ENABLE_REGIONAL_REHOMING' : 'DISABLE_REGIONAL_REHOMING' + if (value.confirmation !== expected) { + context.addIssue({ code: 'custom', message: 'confirmation does not match state' }) + } + }) const RegionalRehomeTrustProbeSchema = z .object({ @@ -1776,7 +1858,11 @@ const RegionalHostDrainSchema = z sourceCellId: z.string().min(1).max(128), sourceCellIncarnation: z.string().uuid(), sourceAssignmentEpoch: z.number().int().positive(), - graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) + graceMs: z + .number() + .int() + .nonnegative() + .max(60 * 60 * 1000) }) .strict() diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts index 824cb1e0b2f..680caac715f 100644 --- a/cloud/apps/relay/src/assignment-store.ts +++ b/cloud/apps/relay/src/assignment-store.ts @@ -1,3 +1,14 @@ +import { IDLE_REHOME_PAGE_SIZE, selectIdleRegionalRehomes } from './idle-regional-rehome-selection.js' +import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js' +import { + previewRegionalRehomeEligibility, + type RegionCorrectionPreview +} from './region-correction-preview.js' +import { + exchangeRegionCorrection, + previewRegionCorrection, + REGIONAL_REHOME_CONCURRENT_LIMIT +} from './region-correction-state.js' import { randomUUID } from 'node:crypto' import { performance } from 'node:perf_hooks' import { @@ -7,7 +18,10 @@ import { RELAY_DEFAULT_REGION, RELAY_REGIONS, RELAY_PROTOCOL_LIMITS, - type RelayRegion + type RelayRegion, + type RegionCorrectionRequest, + type RegionCorrectionResponse, + type IdleRegionalRehomeRequest, } from '@orca-cloud/relay-contract' import { cellAdmissionState, @@ -80,6 +94,7 @@ type CellRegionalRehomeStatus = { } type RelayAssignmentStoreOptions = { + regionalRehomeCohortPercent?: number requireLiveCells?: boolean heartbeatTtlMs?: number recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void @@ -136,10 +151,7 @@ export type RegionalRehomeAttempt = AssignmentIdentity & { sendAttempts: number } -export type RegionalHostDrainOutcome = - | 'accepted' - | 'already-accepted' - | 'host-not-connected' +export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected' export type RegionalRehomeFleetSafety = RegionalRehomeSafetySnapshot & { requiredCells: number @@ -415,6 +427,7 @@ const ABORTABLE_EXPIRED_MIGRATION = `( )` export class RelayAssignmentStore { + private readonly regionalRehomeCohortPercent: number private readonly requireLiveCells: boolean private readonly heartbeatTtlMs: number // Poisoned attempts never complete or abort and stay the oldest rows, so @@ -430,13 +443,19 @@ export class RelayAssignmentStore { private readonly migrationCellRegistrar: RelayMigrationCellRegistrar private readonly activityQueue = new AssignmentIdentityQueue() private assignmentTail: Promise = Promise.resolve() - private pendingRegionalRehomeDisableLog: Record | null = null constructor( private readonly database: RelayDatabase, private readonly now: () => number = Date.now, options: RelayAssignmentStoreOptions = {} ) { + this.regionalRehomeCohortPercent = options.regionalRehomeCohortPercent ?? 0 + if ( + !Number.isInteger(this.regionalRehomeCohortPercent) || + this.regionalRehomeCohortPercent < 0 || + this.regionalRehomeCohortPercent > 100 + ) + throw new Error('invalid_regional_rehome_cohort') this.requireLiveCells = options.requireLiveCells ?? false this.heartbeatTtlMs = options.heartbeatTtlMs ?? 45_000 this.recordControlRenewal = options.recordControlRenewal @@ -3309,6 +3328,163 @@ export class RelayAssignmentStore { }) } + async exchangeRegionCorrection( + identity: AssignmentIdentity, + request: RegionCorrectionRequest, + assignmentEpoch: number + ): Promise { + return exchangeRegionCorrection(this.database, identity, request, assignmentEpoch, this.now()) + } + + async regionCorrectionOutcomes() { + return readRegionCorrectionOutcomes(this.database, this.now()) + } + + async previewRegionCorrection(): Promise> { + return previewRegionCorrection(this.database, this.now()) + } + + private idleRegionalCandidateOffset = 0 + + async selectIdleRegionalRehomeCandidates( + processSafety?: RegionalRehomeSafetySnapshot + ): Promise> { + const now = this.now() + if (!processSafety || this.regionalRehomeCohortPercent === 0) return [] + const fleetSafety = await this.readRegionalRehomeFleetSafety(this.database, now) + if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return [] + const candidates = await selectIdleRegionalRehomes({ + database: this.database, now, heartbeatTtlMs: this.heartbeatTtlMs, + cohortPercent: this.regionalRehomeCohortPercent, offset: this.idleRegionalCandidateOffset, + connectionHeadroom: await this.connectionHeadroomByCell(this.database), + cellIsClean: regionalRehomeCellSafetyIsClean + }) + this.idleRegionalCandidateOffset = candidates.length < IDLE_REHOME_PAGE_SIZE + ? 0 : this.idleRegionalCandidateOffset + candidates.length + return candidates + } + + async commitIdleRegionalRehome( + request: IdleRegionalRehomeRequest, + processSafety?: RegionalRehomeSafetySnapshot, + cohortPercent = this.regionalRehomeCohortPercent + ): Promise<{ outcome: 'committed' | 'deferred' | 'stale' }> { + const prior = await this.reconcileIdleRegionalRehome(request) + if (prior !== 'not-committed') return { outcome: prior } + if (!processSafety || !Number.isInteger(cohortPercent) || cohortPercent <= 0 || cohortPercent > 100) { + return { outcome: 'deferred' } + } + let safetyDisable: Record | null = null + const result = await this.database.transaction(async (transaction): Promise<{ outcome: 'committed' | 'deferred' | 'stale' }> => { + safetyDisable = null + const now = this.now() + const control = (await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'` + ))[0] + if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) { + return { outcome: 'deferred' } + } + await transaction.query( + `INSERT INTO relay_region_rehome_worker_state + (worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at) + VALUES ('global', 0, 0, 0, ?) ON CONFLICT (worker_id) DO NOTHING`, [now] + ) + const worker = (await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` + ))[0]! + if (Number(worker.paused_until) > now || Number(worker.next_dispatch_at) > now) { + return { outcome: 'deferred' } + } + const open = (await transaction.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations + WHERE completed_at IS NULL AND aborted_at IS NULL` + ))[0] + if (Number(open?.count ?? 0) >= REGIONAL_REHOME_CONCURRENT_LIMIT) return { outcome: 'deferred' } + const attempt = await this.startRegionalRehomeCandidate(transaction, { + identity: request, + sourceCellId: request.sourceCellId, + assignmentEpoch: request.sourceAssignmentEpoch, + preferenceCutoff: now - Number(control.preference_max_age_ms), + cooldownCutoff: now - Number(control.host_cooldown_ms), + drainGraceMs: 0, + processSafety, + worker, + now, + skips: [], + idleRequest: request, + cohortPercent, + onSafetyDisabled: (event) => { safetyDisable = event } + }) + if (!attempt) return { outcome: 'deferred' } + await this.markRegionalRehomeDispatchClaimed( + transaction, request.attemptId, now, Math.ceil(60_000 / Number(control.rate_per_minute)) + ) + await transaction.query( + `UPDATE relay_region_rehome_attempts SET drain_receipt_at = ?, drain_outcome = 'accepted' + WHERE attempt_id = ?`, [now, request.attemptId] + ) + return { outcome: 'committed' } + }) + if (safetyDisable) console.warn(JSON.stringify(safetyDisable)) + return result + } + + async reconcileIdleRegionalRehome(request: IdleRegionalRehomeRequest): Promise<'committed' | 'not-committed' | 'stale'> { + return this.database.transaction(async (transaction) => { + // Absence is definitive only after the same assignment lock as commit/activation. + const assignment = await this.assignmentRow(transaction, request) + const attempt = (await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [request.attemptId] + ))[0] + if (attempt) { + return attempt.user_id === request.userId && + attempt.relay_host_id === request.relayHostId && + attempt.source_cell_id === request.sourceCellId && + attempt.source_cell_incarnation === request.sourceCellIncarnation && + Number(attempt.previous_epoch) === request.sourceAssignmentEpoch && + Number(attempt.source_generation) === request.sourceGeneration && + attempt.target_cell_id === request.targetCellId && + attempt.aborted_at == null + ? 'committed' : 'stale' + } + if (!assignment || assignment.cell_id !== request.sourceCellId || + Number(assignment.assignment_epoch) !== request.sourceAssignmentEpoch) return 'stale' + const control = (await transaction.query( + `SELECT capability.generation, capability.cell_incarnation + FROM relay_control_capabilities capability + JOIN relay_assignment_activity_leases lease + ON lease.user_id = capability.user_id AND lease.relay_host_id = capability.relay_host_id + AND lease.activity_id = capability.activity_id + WHERE capability.user_id = ? AND capability.relay_host_id = ? + AND capability.cell_id = ? AND capability.assignment_epoch = ? + AND lease.activity_kind = 'control' AND lease.expires_at > ? + ORDER BY capability.generation DESC LIMIT 1`, + [request.userId, request.relayHostId, request.sourceCellId, request.sourceAssignmentEpoch, this.now()] + ))[0] + return control && Number(control.generation) === request.sourceGeneration && + control.cell_incarnation === request.sourceCellIncarnation ? 'not-committed' : 'stale' + }) + } + + async previewRegionalRehomeEligibility( + processSafety?: RegionalRehomeSafetySnapshot + ): Promise { + const now = this.now() + const fleetSafety = await this.readRegionalRehomeFleetSafety(this.database, now) + return previewRegionalRehomeEligibility({ + database: this.database, + now, + heartbeatTtlMs: this.heartbeatTtlMs, + cohortPercent: this.regionalRehomeCohortPercent, + globalSafetyFailure: processSafety + ? regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now) + : 'process-safety-unavailable', + connectionHeadroom: await this.connectionHeadroomByCell(this.database), + cellIsClean: regionalRehomeCellSafetyIsClean + }) + } + async renewControlActivity( identity: AssignmentIdentity, input: { activityId: string; cellId: string; expiresAt: number } @@ -3545,6 +3721,8 @@ export class RelayAssignmentStore { cellId: string assignmentEpoch: number generation: number + idleRegionalRehome?: boolean + cellIncarnation?: string connectionInclusionWatermark?: number } ): Promise { @@ -3626,6 +3804,33 @@ export class RelayAssignmentStore { input.connectionInclusionWatermark, now ) + await transaction.query( + `DELETE FROM relay_control_capabilities WHERE user_id = ? AND relay_host_id = ? + AND NOT EXISTS (SELECT 1 FROM relay_assignment_activity_leases lease + WHERE lease.user_id = relay_control_capabilities.user_id + AND lease.relay_host_id = relay_control_capabilities.relay_host_id + AND lease.activity_id = relay_control_capabilities.activity_id)`, + [identity.userId, identity.relayHostId] + ) + await transaction.query( + `INSERT INTO relay_control_capabilities + (user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing, idle_regional_rehome) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (user_id, relay_host_id, activity_id) DO UPDATE SET + cell_incarnation = excluded.cell_incarnation, assignment_epoch = excluded.assignment_epoch, + generation = excluded.generation, finish_existing = excluded.finish_existing, idle_regional_rehome = excluded.idle_regional_rehome`, + [ + identity.userId, + identity.relayHostId, + activityId, + input.cellId, + input.cellIncarnation ?? '', + input.assignmentEpoch, + input.generation, + 0, + input.idleRegionalRehome && input.cellIncarnation ? 1 : 0 + ] + ) return activityId }) }) @@ -5116,327 +5321,6 @@ export class RelayAssignmentStore { } } - async claimRegionalRehome( - processSafety?: RegionalRehomeSafetySnapshot - ): Promise { - const now = this.now() - // Directors poll every second; avoid taking the global worker-row lock while disabled. - const control = ( - await this.database.query( - `SELECT enabled, not_before - FROM relay_region_rehome_control - WHERE control_id = 'global'` - ) - )[0] - if (!control) { - await this.initializeRegionalRehomeControl(this.database, now) - return null - } - if (integer(control, 'enabled') !== 1 || integer(control, 'not_before') > now) { - return null - } - this.pendingRegionalRehomeDisableLog = null - const candidateSkips: RegionalRehomeCandidateSkip[] = [] - // A Postgres transaction is unusable after a NOWAIT abort, so a contended - // tick abandons the candidate it stopped on plus every one behind it. - let candidatesTotal = 0 - let candidatesFinished = 0 - const claimResult = await this.database.transaction(async (transaction) => { - candidatesTotal = 0 - candidatesFinished = 0 - candidateSkips.length = 0 - await this.initializeRegionalRehomeControl(transaction, now) - const control = ( - await transaction.queryLocked( - `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'` - ) - )[0]! - if (integer(control, 'enabled') !== 1 || integer(control, 'not_before') > now) { - return null - } - const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute')) - const preferenceCutoff = now - integer(control, 'preference_max_age_ms') - // A host that was rehomed recently is left alone whichever way its - // preference now points: a flapping region probe must not walk one host - // back and forth across an ocean. - const cooldownCutoff = now - integer(control, 'host_cooldown_ms') - await transaction.query( - `INSERT INTO relay_region_rehome_worker_state - (worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at) - VALUES ('global', 0, 0, 0, ?) - ON CONFLICT (worker_id) DO NOTHING`, - [now] - ) - const worker = ( - await transaction.queryLocked( - `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` - ) - )[0]! - if ( - integer(worker, 'paused_until') > now || - integer(worker, 'next_dispatch_at') > now - ) { - return null - } - const effectiveProcessSafety = processSafety ?? cleanRegionalRehomeSafety(now) - const fleetSafety = await this.readRegionalRehomeFleetSafety(transaction, now) - if ( - !(await this.regionalRehomeSafetyAllowsClaim( - transaction, - worker, - effectiveProcessSafety, - fleetSafety, - now - )) - ) { - return null - } - const retry = ( - await transaction.queryLocked( - `SELECT attempt.*, source.cell_url AS source_cell_url - FROM relay_region_rehome_attempts attempt - JOIN relay_cells source ON source.cell_id = attempt.source_cell_id - JOIN relay_cell_runtime runtime ON runtime.cell_id = attempt.source_cell_id - JOIN relay_cell_capabilities capability - ON capability.cell_id = runtime.cell_id - AND capability.cell_incarnation = runtime.cell_incarnation - JOIN relay_assignment_migrations migration - ON migration.user_id = attempt.user_id - AND migration.relay_host_id = attempt.relay_host_id - AND migration.assignment_epoch = attempt.assignment_epoch - WHERE attempt.drain_receipt_at IS NULL - AND attempt.completed_at IS NULL AND attempt.aborted_at IS NULL - AND attempt.send_attempts < 10 - AND (attempt.last_send_attempt_at IS NULL OR attempt.last_send_attempt_at <= ?) - AND runtime.cell_incarnation = attempt.source_cell_incarnation - AND runtime.ready = 1 AND runtime.last_heartbeat_at > ? - AND capability.regional_rehome_protocol >= 1 - AND migration.completed_at IS NULL AND migration.aborted_at IS NULL - ORDER BY attempt.created_at, attempt.attempt_id - LIMIT 1`, - [now - 30_000, now - this.heartbeatTtlMs] - ) - )[0] - if (retry) { - candidatesTotal = 1 - const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now) - if ( - !(await this.regionalRehomeSafetyAllowsClaim( - transaction, - worker, - effectiveProcessSafety, - fleetSafety, - now - )) - ) { - return null - } - await this.markRegionalRehomeDispatchClaimed( - transaction, - text(retry, 'attempt_id'), - now, - intervalMs - ) - retry.send_attempts = integer(retry, 'send_attempts') + 1 - return regionalRehomeAttempt(retry) - } - - // A drain receipt is not convergence: grace enforcement lives only in - // source-cell session state, and attempts have been observed stalled - // dual-homed well past grace with source leases still renewing. Such - // attempts are re-dispatched with the remaining (zero) grace so the - // source force-closes and the host re-resolves onto its registered - // target. - const redrain = ( - await transaction.queryLocked( - `SELECT attempt.*, source.cell_url AS source_cell_url - FROM relay_region_rehome_attempts attempt - JOIN relay_cells source ON source.cell_id = attempt.source_cell_id - JOIN relay_cell_runtime runtime ON runtime.cell_id = attempt.source_cell_id - JOIN relay_cell_capabilities capability - ON capability.cell_id = runtime.cell_id - AND capability.cell_incarnation = runtime.cell_incarnation - JOIN relay_assignment_migrations migration - ON migration.user_id = attempt.user_id - AND migration.relay_host_id = attempt.relay_host_id - AND migration.assignment_epoch = attempt.assignment_epoch - WHERE attempt.drain_receipt_at IS NOT NULL - AND attempt.completed_at IS NULL AND attempt.aborted_at IS NULL - AND attempt.created_at + attempt.drain_grace_ms <= ? - AND attempt.send_attempts < ? - AND (attempt.last_send_attempt_at IS NULL OR attempt.last_send_attempt_at <= ?) - AND runtime.cell_incarnation = attempt.source_cell_incarnation - AND runtime.ready = 1 AND runtime.last_heartbeat_at > ? - AND capability.regional_rehome_protocol >= 1 - AND migration.completed_at IS NULL AND migration.aborted_at IS NULL - AND migration.target_registered_at IS NOT NULL - AND EXISTS ( - SELECT 1 FROM relay_assignment_activity_leases source_lease - WHERE source_lease.user_id = attempt.user_id - AND source_lease.relay_host_id = attempt.relay_host_id - AND source_lease.cell_id = attempt.source_cell_id - ) - ORDER BY attempt.created_at, attempt.attempt_id - LIMIT 1`, - [ - now, - REGIONAL_REHOME_REDRAIN_SEND_LIMIT, - now - REGIONAL_REHOME_REDRAIN_INTERVAL_MS, - now - this.heartbeatTtlMs - ] - ) - )[0] - if (redrain) { - candidatesTotal = 1 - const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now) - if ( - !(await this.regionalRehomeSafetyAllowsClaim( - transaction, - worker, - effectiveProcessSafety, - fleetSafety, - now - )) - ) { - return null - } - await this.markRegionalRehomeDispatchClaimed( - transaction, - text(redrain, 'attempt_id'), - now, - intervalMs - ) - redrain.send_attempts = integer(redrain, 'send_attempts') + 1 - redrain.drain_grace_ms = 0 - return regionalRehomeAttempt(redrain) - } - - const candidates = await transaction.query( - `SELECT preference.user_id, preference.relay_host_id, - preference.observed_at, assignment.cell_id AS source_cell_id, - assignment.assignment_epoch - FROM relay_assignment_region_preferences preference - JOIN relay_assignments assignment - ON assignment.user_id = preference.user_id - AND assignment.relay_host_id = preference.relay_host_id - JOIN relay_cell_regions region ON region.cell_id = assignment.cell_id - JOIN relay_cell_admission admission ON admission.cell_id = assignment.cell_id - JOIN relay_cell_runtime runtime ON runtime.cell_id = assignment.cell_id - JOIN relay_cell_capabilities capability - ON capability.cell_id = runtime.cell_id - AND capability.cell_incarnation = runtime.cell_incarnation - WHERE preference.preferred_region <> region.region - AND preference.observed_at >= ? - AND admission.admission_state = 'general' - AND runtime.ready = 1 AND runtime.last_heartbeat_at > ? - AND capability.regional_rehome_protocol >= 1 - AND EXISTS ( - SELECT 1 FROM relay_assignment_activity_leases control - WHERE control.user_id = assignment.user_id - AND control.relay_host_id = assignment.relay_host_id - AND control.cell_id = assignment.cell_id - AND control.activity_kind = 'control' - AND control.activity_id NOT LIKE 'control-pending:%' - AND control.expires_at > ? - AND control.updated_at >= runtime.started_at - ) - AND NOT EXISTS ( - SELECT 1 FROM relay_assignment_migrations migration - WHERE migration.user_id = assignment.user_id - AND migration.relay_host_id = assignment.relay_host_id - AND migration.completed_at IS NULL AND migration.aborted_at IS NULL - ) - AND NOT EXISTS ( - SELECT 1 FROM relay_region_rehome_attempts recent - WHERE recent.user_id = preference.user_id - AND recent.relay_host_id = preference.relay_host_id - AND recent.created_at > ? - ) - AND EXISTS ( - SELECT 1 FROM relay_cell_regions target_region - JOIN relay_cells target_cell ON target_cell.cell_id = target_region.cell_id - JOIN relay_cell_admission target_admission - ON target_admission.cell_id = target_region.cell_id - JOIN relay_cell_runtime target_runtime - ON target_runtime.cell_id = target_region.cell_id - JOIN relay_cell_capabilities target_capability - ON target_capability.cell_id = target_runtime.cell_id - AND target_capability.cell_incarnation = target_runtime.cell_incarnation - WHERE target_region.region = preference.preferred_region - AND target_cell.enabled = 1 - AND target_admission.admission_state = 'general' - AND target_runtime.ready = 1 - AND target_runtime.last_heartbeat_at > ? - AND target_capability.regional_rehome_protocol >= 1 - ) - ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id - LIMIT 10`, - [ - preferenceCutoff, - now - this.heartbeatTtlMs, - now, - cooldownCutoff, - now - this.heartbeatTtlMs - ] - ) - candidatesTotal = candidates.length - for (const candidate of candidates) { - const claimed = await this.startRegionalRehomeCandidate(transaction, { - identity: { - userId: text(candidate, 'user_id'), - relayHostId: text(candidate, 'relay_host_id') - }, - sourceCellId: text(candidate, 'source_cell_id'), - assignmentEpoch: integer(candidate, 'assignment_epoch'), - preferenceCutoff, - cooldownCutoff, - drainGraceMs: integer(control, 'drain_grace_ms'), - processSafety: effectiveProcessSafety, - worker, - now, - skips: candidateSkips - }) - candidatesFinished++ - if (!claimed) continue - await this.markRegionalRehomeDispatchClaimed( - transaction, - claimed.attemptId, - now, - intervalMs - ) - return { ...claimed, sendAttempts: 1 } - } - if (candidates.length > 0) { - // Skipped candidates still cost all-rows FOR UPDATE inventory scans; - // charge the dispatch interval so skips are rate-limited like claims. - await this.markRegionalRehomeTickSkipped(transaction, now, intervalMs) - } - return null - }).catch((error: unknown): RegionalRehomeAttempt | null => { - // Only inventory contention is swallowed here; every other failure keeps - // its existing propagation and its dispatch-failure accounting. - if (!isDatabaseLockUnavailable(error)) throw error - // The dispatch tick runs every second; losing one to inventory contention - // costs a second of latency and never loses durable rehome state. The - // rolled-back transaction never disabled anything, so its pending disable - // log would describe a decision that did not happen. - candidateSkips.length = 0 - this.pendingRegionalRehomeDisableLog = null - warnSweepCellInventoryBusy( - 'claim-regional-rehome', - Math.max(1, candidatesTotal - candidatesFinished) - ) - return null - }) - const pendingDisableLog = this.pendingRegionalRehomeDisableLog - this.pendingRegionalRehomeDisableLog = null - if (pendingDisableLog) console.warn(JSON.stringify(pendingDisableLog)) - if (claimResult === null && candidateSkips.length > 0) { - console.warn(JSON.stringify(aggregateRegionalRehomeCandidateSkips(candidateSkips))) - } - return claimResult - } - private async startRegionalRehomeCandidate( transaction: RelayDatabase, input: { @@ -5450,6 +5334,9 @@ export class RelayAssignmentStore { worker: SqlRow now: number skips: RegionalRehomeCandidateSkip[] + idleRequest: IdleRegionalRehomeRequest + cohortPercent: number + onSafetyDisabled: (event: Record | null) => void } ): Promise | null> { const assignment = await this.assignmentRow(transaction, input.identity) @@ -5463,12 +5350,21 @@ export class RelayAssignmentStore { } const preference = ( await transaction.queryLocked( - `SELECT * FROM relay_assignment_region_preferences + `SELECT * FROM relay_region_decisions WHERE user_id = ? AND relay_host_id = ?`, [input.identity.userId, input.identity.relayHostId] ) )[0] - if (!preference || integer(preference, 'observed_at') < input.preferenceCutoff) { + if ( + !preference || + integer(preference, 'observed_at') < input.preferenceCutoff || + Number(preference.expires_at) <= input.now || + preference.outcome !== 'conclusive' || + Number(preference.policy_version) !== 1 || + Number(preference.assignment_epoch) !== input.assignmentEpoch || + !preference.preferred_region || + Number(preference.cohort_bucket) >= input.cohortPercent + ) { input.skips.push({ reason: 'candidate_stale' }) return null } @@ -5540,13 +5436,13 @@ export class RelayAssignmentStore { input.now ) if (safetyFailure) { - await this.pauseRegionalRehomeForSafety( + input.onSafetyDisabled(await this.pauseRegionalRehomeForSafety( transaction, input.worker, input.now, safetyFailure, fleetSafety - ) + )) return null } // The preference read under lock can now agree with the cell the host is @@ -5564,9 +5460,9 @@ export class RelayAssignmentStore { integer(sourceRuntime, 'ready') !== 1 || integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs || !sourceCapability || - text(sourceCapability, 'cell_incarnation') !== - text(sourceRuntime, 'cell_incarnation') || - integer(sourceCapability, 'regional_rehome_protocol') < 1 + text(sourceCapability, 'cell_incarnation') !== text(sourceRuntime, 'cell_incarnation') || + integer(sourceCapability, 'regional_rehome_protocol') < 3 || + sourceRuntime.cell_incarnation !== input.idleRequest.sourceCellIncarnation ) { input.skips.push({ reason: 'source_ineligible', cellId: input.sourceCellId }) return null @@ -5575,6 +5471,32 @@ export class RelayAssignmentStore { input.skips.push(cellUncleanSkip('source_unclean', input.sourceCellId, sourceSafety)) return null } + const hostCapability = ( + await transaction.query( + `SELECT capability.* FROM relay_control_capabilities capability + JOIN relay_assignment_activity_leases lease + ON lease.user_id = capability.user_id AND lease.relay_host_id = capability.relay_host_id + AND lease.activity_id = capability.activity_id + WHERE capability.user_id = ? AND capability.relay_host_id = ? + AND capability.cell_id = ? AND capability.assignment_epoch = ? + AND capability.cell_incarnation = ? AND capability.idle_regional_rehome = 1 + AND lease.expires_at > ? AND lease.activity_kind = 'control' + ORDER BY capability.generation DESC LIMIT 1`, + [ + input.identity.userId, + input.identity.relayHostId, + input.sourceCellId, + input.assignmentEpoch, + sourceRuntime.cell_incarnation, + input.now + ] + ) + )[0] + if (!hostCapability || preference.incumbent_region !== regions.get(input.sourceCellId) || + Number(hostCapability.generation) !== input.idleRequest.sourceGeneration) { + input.skips.push({ reason: 'candidate_stale' }) + return null + } const sourceControlActive = activityLeases.some( (lease) => text(lease, 'cell_id') === input.sourceCellId && @@ -5606,7 +5528,8 @@ export class RelayAssignmentStore { integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs && capability !== undefined && text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') && - integer(capability, 'regional_rehome_protocol') >= 1 + integer(capability, 'regional_rehome_protocol') >= 3 && + cellId === input.idleRequest.targetCellId ) }) const targetIsClean = (row: SqlRow): boolean => { @@ -5753,7 +5676,7 @@ export class RelayAssignmentStore { text(targetRuntime, 'cell_incarnation') ] ) - const attemptId = randomUUID() + const attemptId = input.idleRequest.attemptId await transaction.query( `INSERT INTO relay_region_rehome_attempts (attempt_id, user_id, relay_host_id, preferred_region, @@ -5780,6 +5703,10 @@ export class RelayAssignmentStore { input.now ] ) + await transaction.query( + `UPDATE relay_region_rehome_attempts SET source_generation = ? WHERE attempt_id = ?`, + [input.idleRequest.sourceGeneration, attemptId] + ) return { ...input.identity, attemptId, @@ -5795,61 +5722,13 @@ export class RelayAssignmentStore { } } - private async lockedRegionalRehomeFleetSafety( - transaction: RelayDatabase, - now: number - ): Promise { - const cells = await this.lockCellInventory(transaction, 'nowait') - const admission = await cellAdmissionStates(transaction) - const regions = new Map( - (await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [ - text(row, 'cell_id'), - relayRegion(row, 'region') - ]) - ) - const runtimes = await transaction.queryLocked( - `SELECT * FROM relay_cell_runtime ORDER BY cell_id` - ) - const capabilities = await transaction.queryLocked( - `SELECT * FROM relay_cell_capabilities ORDER BY cell_id` - ) - const safetyRows = await transaction.queryLocked( - `SELECT * FROM relay_cell_rehome_safety ORDER BY cell_id` - ) - return regionalRehomeFleetSafetyFromInventory({ - cells, - admission, - regions, - runtimes, - capabilities, - safetyRows, - now, - heartbeatTtlMs: this.heartbeatTtlMs - }) - } - - private async regionalRehomeSafetyAllowsClaim( - transaction: RelayDatabase, - worker: SqlRow, - processSafety: RegionalRehomeSafetySnapshot, - fleetSafety: RegionalRehomeFleetSafety, - now: number - ): Promise { - const failure = regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now) - if (!failure) { - return true - } - await this.pauseRegionalRehomeForSafety(transaction, worker, now, failure, fleetSafety) - return false - } - private async pauseRegionalRehomeForSafety( transaction: RelayDatabase, worker: SqlRow, now: number, reason: string, fleetSafety: RegionalRehomeFleetSafety - ): Promise { + ): Promise | null> { const disabled = await transaction.query( `UPDATE relay_region_rehome_control SET generation = generation + 1, enabled = 0, updated_at = ? @@ -5860,8 +5739,9 @@ export class RelayAssignmentStore { // The durable disable is otherwise invisible: nothing else records why // claims stopped and inspection only shows enabled=false. Logged after // the transaction commits so a rollback cannot fabricate the record. + let event: Record | null = null if (disabled.length > 0) { - this.pendingRegionalRehomeDisableLog = { + event = { event: 'orca_relay_regional_rehome_safety_disabled', reason, controlGeneration: integer(disabled[0]!, 'generation'), @@ -5879,19 +5759,9 @@ export class RelayAssignmentStore { } } await this.incrementRegionalRehomeWorkerFailure(transaction, worker, now) + return event } - private async markRegionalRehomeTickSkipped( - transaction: RelayDatabase, - now: number, - intervalMs: number - ): Promise { - await transaction.query( - `UPDATE relay_region_rehome_worker_state - SET next_dispatch_at = ?, updated_at = ? WHERE worker_id = 'global'`, - [now + intervalMs, now] - ) - } private async markRegionalRehomeDispatchClaimed( transaction: RelayDatabase, @@ -5912,74 +5782,6 @@ export class RelayAssignmentStore { ) } - async recordRegionalRehomeDrainReceipt( - attemptId: string, - outcome: RegionalHostDrainOutcome - ): Promise { - const now = this.now() - return await this.database.transaction(async (transaction) => { - const worker = ( - await transaction.queryLocked( - `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` - ) - )[0] - const attempt = ( - await transaction.queryLocked( - `SELECT * FROM relay_region_rehome_attempts WHERE attempt_id = ?`, - [attemptId] - ) - )[0] - if (!attempt) throw new Error('regional_rehome_attempt_not_found') - // Any receipt proves the source cell answered: reset the failure budget - // even when a redrain repeats the stored outcome; otherwise a - // redrain-dominated stream lets scattered transient failures reach the - // durable three-failure disable. - if (worker) { - await transaction.query( - `UPDATE relay_region_rehome_worker_state - SET consecutive_failures = 0, paused_until = 0, updated_at = ? - WHERE worker_id = 'global'`, - [now] - ) - } - const existingOutcome = optionalText(attempt, 'drain_outcome') - if (existingOutcome === outcome) return false - // Redrains produce one receipt per dispatch; the latest outcome wins. - await transaction.query( - `UPDATE relay_region_rehome_attempts - SET drain_receipt_at = ?, drain_outcome = ?, updated_at = ? - WHERE attempt_id = ?`, - [now, outcome, now, attemptId] - ) - return true - }) - } - - async recordRegionalRehomeDispatchFailure(attemptId: string): Promise { - const now = this.now() - const disableLog = await this.database.transaction(async (transaction) => { - // Match claim and enable ordering before a spent budget updates the control. - await transaction.queryLocked( - `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'` - ) - const worker = ( - await transaction.queryLocked( - `SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` - ) - )[0] - const attempt = ( - await transaction.queryLocked( - `SELECT attempt_id FROM relay_region_rehome_attempts WHERE attempt_id = ?`, - [attemptId] - ) - )[0] - if (!worker || !attempt) return null - return await this.incrementRegionalRehomeWorkerFailure(transaction, worker, now) - }) - // Logged after the commit so a rollback cannot fabricate the record. - if (disableLog) console.warn(JSON.stringify(disableLog)) - } - // Returns the durable disable this failure caused, for the caller to log once // its transaction commits; null when the budget survives or was already spent. private async incrementRegionalRehomeWorkerFailure( @@ -6074,7 +5876,7 @@ export class RelayAssignmentStore { // LIMIT pages: poisoned rows are permanent and always the oldest, so // without exclusion they eventually starve every healthy candidate. private recordRegionalRehomeCandidateFailure( - operation: 'complete' | 'abort', + operation: 'complete' | 'abort' | 'refresh', attemptId: string, now: number, error: unknown @@ -6116,12 +5918,16 @@ export class RelayAssignmentStore { async refreshRegionalRehomeLeases(limit = 100): Promise { const now = this.now() + const quarantined = this.quarantinedRegionalRehomeAttemptIds(now) + const exclusion = quarantined.length + ? ` AND attempt_id NOT IN (${quarantined.map(() => '?').join(', ')})` + : '' const candidates = await this.database.query( - `SELECT user_id, relay_host_id, assignment_epoch + `SELECT attempt_id, user_id, relay_host_id, assignment_epoch FROM relay_region_rehome_attempts - WHERE completed_at IS NULL AND aborted_at IS NULL - ORDER BY created_at, attempt_id LIMIT ?`, - [limit] + WHERE completed_at IS NULL AND aborted_at IS NULL${exclusion} + ORDER BY updated_at, attempt_id LIMIT ?`, + [...quarantined, limit] ) let refreshed = 0 for (const candidate of candidates) { @@ -6130,50 +5936,91 @@ export class RelayAssignmentStore { relayHostId: text(candidate, 'relay_host_id') } const assignmentEpoch = integer(candidate, 'assignment_epoch') - const changed = await this.database.transaction(async (transaction) => { - const assignment = await this.assignmentRow(transaction, identity) - const attempt = ( - await transaction.queryLocked( - `SELECT * FROM relay_region_rehome_attempts + const attemptId = text(candidate, 'attempt_id') + try { + const changed = await this.database.transaction(async (transaction) => { + const assignment = await this.assignmentRow(transaction, identity) + const attempt = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_rehome_attempts WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [identity.userId, identity.relayHostId, assignmentEpoch] - ) - )[0] - const migration = ( - await transaction.queryLocked( - `SELECT * FROM relay_assignment_migrations + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + const migration = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignment_migrations WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [identity.userId, identity.relayHostId, assignmentEpoch] - ) - )[0] - if ( - !assignment || - !attempt || - !migration || - optionalInteger(attempt, 'completed_at') !== undefined || - optionalInteger(attempt, 'aborted_at') !== undefined || - optionalInteger(migration, 'completed_at') !== undefined || - optionalInteger(migration, 'aborted_at') !== undefined - ) { - return false - } - const attemptAgeMs = now - integer(attempt, 'created_at') - if (attemptAgeMs >= REGIONAL_REHOME_MAX_REFRESH_MS) { - return false - } - if ( - optionalInteger(migration, 'target_registered_at') === undefined && - attemptAgeMs >= REGIONAL_REHOME_UNREGISTERED_REFRESH_MS - ) { - await transaction.query( - `UPDATE relay_assignment_activity_leases + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + if (attempt && attempt.completed_at == null && attempt.aborted_at == null) { + await transaction.query( + `UPDATE relay_region_rehome_attempts SET updated_at = ? WHERE attempt_id = ?`, + [now, attempt.attempt_id] + ) + } + if ( + !assignment || + !attempt || + !migration || + optionalInteger(attempt, 'completed_at') !== undefined || + optionalInteger(attempt, 'aborted_at') !== undefined || + optionalInteger(migration, 'completed_at') !== undefined || + optionalInteger(migration, 'aborted_at') !== undefined + ) { + return false + } + const attemptAgeMs = now - integer(attempt, 'created_at') + if (attemptAgeMs >= REGIONAL_REHOME_MAX_REFRESH_MS) { + return false + } + if ( + optionalInteger(migration, 'target_registered_at') === undefined && + attemptAgeMs >= REGIONAL_REHOME_UNREGISTERED_REFRESH_MS + ) { + await transaction.query( + `UPDATE relay_assignment_activity_leases SET expires_at = CASE WHEN expires_at < ? THEN expires_at ELSE ? END, updated_at = ? WHERE user_id = ? AND relay_host_id = ? AND activity_id IN (?, ?)`, + [ + now, + now, + now, + identity.userId, + identity.relayHostId, + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ] + ) + await transaction.query( + `UPDATE relay_assignment_migrations + SET expires_at = CASE WHEN expires_at < ? THEN expires_at ELSE ? END, + updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return false + } + const leases = await this.lockAssignmentActivities(transaction, identity) + const protectedIds = new Set([ + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ]) + const protectedLeases = leases.filter((lease) => + protectedIds.has(text(lease, 'activity_id')) + ) + if (protectedLeases.length === 0) return false + const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs + await transaction.query( + `UPDATE relay_assignment_activity_leases + SET expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? + AND activity_id IN (?, ?)`, [ - now, - now, + expiresAt, now, identity.userId, identity.relayHostId, @@ -6182,53 +6029,25 @@ export class RelayAssignmentStore { ] ) await transaction.query( - `UPDATE relay_assignment_migrations - SET expires_at = CASE WHEN expires_at < ? THEN expires_at ELSE ? END, - updated_at = ? - WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [now, now, now, identity.userId, identity.relayHostId, assignmentEpoch] - ) - return false - } - const leases = await this.lockAssignmentActivities(transaction, identity) - const protectedIds = new Set([ - pendingControlActivityId(assignmentEpoch), - migrationActivityId(assignmentEpoch) - ]) - const protectedLeases = leases.filter((lease) => - protectedIds.has(text(lease, 'activity_id')) - ) - if (protectedLeases.length === 0) return false - const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs - await transaction.query( - `UPDATE relay_assignment_activity_leases - SET expires_at = ?, updated_at = ? - WHERE user_id = ? AND relay_host_id = ? - AND activity_id IN (?, ?)`, - [ - expiresAt, - now, - identity.userId, - identity.relayHostId, - pendingControlActivityId(assignmentEpoch), - migrationActivityId(assignmentEpoch) - ] - ) - await transaction.query( - `UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ? + `UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ? WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch] - ) - await transaction.query( - `UPDATE relay_assignments SET lease_expires_at = + [expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + await transaction.query( + `UPDATE relay_assignments SET lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?`, - [expiresAt, expiresAt, now, identity.userId, identity.relayHostId] - ) - return true - }) - if (changed) refreshed++ + [expiresAt, expiresAt, now, identity.userId, identity.relayHostId] + ) + return true + }) + if (changed) refreshed++ + this.regionalRehomeCandidateQuarantine.delete(attemptId) + } catch (error) { + if (!isDatabaseLockUnavailable(error)) + this.recordRegionalRehomeCandidateFailure('refresh', attemptId, now, error) + } } return refreshed } @@ -6575,92 +6394,169 @@ export class RelayAssignmentStore { let aborted = 0 let inventoryBusy = 0 for (const candidate of candidates) { - const didAbort = await this.database.transaction(async (transaction) => { - const identity = { - userId: text(candidate, 'user_id'), - relayHostId: text(candidate, 'relay_host_id') - } - // Migration cleanup follows the same assignment-first order as evacuation. - const assignment = await this.assignmentRow(transaction, identity) - const assignmentEpoch = integer(candidate, 'assignment_epoch') - const regionalAttempt = ( - await transaction.queryLocked( - `SELECT attempt_id FROM relay_region_rehome_attempts + const didAbort = await this.database + .transaction(async (transaction) => { + const identity = { + userId: text(candidate, 'user_id'), + relayHostId: text(candidate, 'relay_host_id') + } + // Migration cleanup follows the same assignment-first order as evacuation. + const assignment = await this.assignmentRow(transaction, identity) + const assignmentEpoch = integer(candidate, 'assignment_epoch') + const regionalAttempt = ( + await transaction.queryLocked( + `SELECT attempt_id FROM relay_region_rehome_attempts WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? AND completed_at IS NULL AND aborted_at IS NULL`, - [identity.userId, identity.relayHostId, assignmentEpoch] - ) - )[0] - const row = ( - await transaction.queryLocked( - `SELECT migration.* FROM relay_assignment_migrations migration + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + )[0] + const row = ( + await transaction.queryLocked( + `SELECT migration.* FROM relay_assignment_migrations migration WHERE migration.user_id = ? AND migration.relay_host_id = ? AND migration.assignment_epoch = ? AND migration.expires_at <= ? AND migration.completed_at IS NULL AND migration.aborted_at IS NULL AND ${ABORTABLE_EXPIRED_MIGRATION}`, - [ - identity.userId, - identity.relayHostId, - assignmentEpoch, - now, - now, - abandonedBefore, - abandonedBefore - ] + [ + identity.userId, + identity.relayHostId, + assignmentEpoch, + now, + now, + abandonedBefore, + abandonedBefore + ] + ) + )[0] + if (!row) return false + const targetCellId = text(row, 'target_cell_id') + const activityLeases = await this.lockAssignmentActivities(transaction, identity) + if (!assignment) throw new Error('migration_assignment_missing') + const currentAssignmentEpoch = integer(assignment, 'assignment_epoch') + const assignmentEpochMatches = + text(assignment, 'cell_id') === targetCellId && + currentAssignmentEpoch === assignmentEpoch + const pendingTargetControl = activityLeaseById( + activityLeases, + pendingControlActivityId(assignmentEpoch) ) - )[0] - if (!row) return false - const targetCellId = text(row, 'target_cell_id') - const activityLeases = await this.lockAssignmentActivities(transaction, identity) - if (!assignment) throw new Error('migration_assignment_missing') - const currentAssignmentEpoch = integer(assignment, 'assignment_epoch') - const assignmentEpochMatches = - text(assignment, 'cell_id') === targetCellId && - currentAssignmentEpoch === assignmentEpoch - const pendingTargetControl = activityLeaseById( - activityLeases, - pendingControlActivityId(assignmentEpoch) - ) - const targetGrantIsFresh = - assignmentEpochMatches && - pendingTargetControl !== undefined && - text(pendingTargetControl, 'cell_id') === targetCellId && - text(pendingTargetControl, 'activity_kind') === 'control' && - integer(pendingTargetControl, 'expires_at') > now - const targetIsActive = activityLeases.some( - (lease) => - text(lease, 'cell_id') === targetCellId && - text(lease, 'activity_kind') === 'control' && - text(lease, 'activity_id') !== pendingControlActivityId(assignmentEpoch) - ) - if (targetGrantIsFresh) return false - if (targetIsActive && assignmentEpochMatches) { - // A committed target control is stronger evidence than a failed follow-up - // write; repair the marker instead of rolling a live desktop backward. - await transaction.query( - `UPDATE relay_assignment_migrations + const targetGrantIsFresh = + assignmentEpochMatches && + pendingTargetControl !== undefined && + text(pendingTargetControl, 'cell_id') === targetCellId && + text(pendingTargetControl, 'activity_kind') === 'control' && + integer(pendingTargetControl, 'expires_at') > now + const targetIsActive = activityLeases.some( + (lease) => + text(lease, 'cell_id') === targetCellId && + text(lease, 'activity_kind') === 'control' && + text(lease, 'activity_id') !== pendingControlActivityId(assignmentEpoch) + ) + if (targetGrantIsFresh) return false + if (targetIsActive && assignmentEpochMatches) { + // A committed target control is stronger evidence than a failed follow-up + // write; repair the marker instead of rolling a live desktop backward. + await transaction.query( + `UPDATE relay_assignment_migrations SET target_registered_at = COALESCE(target_registered_at, ?), updated_at = ? WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [now, now, identity.userId, identity.relayHostId, assignmentEpoch] - ) - return false - } - if (!assignmentEpochMatches) { - if (currentAssignmentEpoch <= assignmentEpoch) { - throw new Error('migration_assignment_mismatch') + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return false } - // A newer assignment is authoritative regardless of where it landed. - // Retire only this obsolete migration; never rewrite the newer epoch. - const obsoleteLeases = [ + if (!assignmentEpochMatches) { + if (currentAssignmentEpoch <= assignmentEpoch) { + throw new Error('migration_assignment_mismatch') + } + // A newer assignment is authoritative regardless of where it landed. + // Retire only this obsolete migration; never rewrite the newer epoch. + const obsoleteLeases = [ + pendingControlActivityId(assignmentEpoch), + migrationActivityId(assignmentEpoch) + ] + .map((activityId) => activityLeaseById(activityLeases, activityId)) + .filter((lease): lease is SqlRow => lease !== undefined) + if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'nowait') + for (const lease of obsoleteLeases) { + await this.removeActivityLease(transaction, identity, lease, now) + } + await this.releaseSupersededControlConnectionReservations( + transaction, + identity, + targetCellId, + assignmentEpoch, + now + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return true + } + const cells = await this.lockCellInventory(transaction, 'nowait') + const sourceCellId = text(row, 'source_cell_id') + const admissionRows = await transaction.query( + `SELECT cell_id, admission_state, updated_at FROM relay_cell_admission + WHERE cell_id IN (?, ?)`, + [sourceCellId, targetCellId] + ) + const sourceCell = cells.find((cell) => text(cell, 'cell_id') === sourceCellId) + const targetCell = cells.find((cell) => text(cell, 'cell_id') === targetCellId) + const sourceAdmission = admissionRows.find( + (admission) => text(admission, 'cell_id') === sourceCellId + ) + const targetAdmission = admissionRows.find( + (admission) => text(admission, 'cell_id') === targetCellId + ) + const registered = optionalInteger(row, 'target_registered_at') !== undefined + const sourceIsDurablyFenced = + registered && + ( + await transaction.query( + `SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = ? AND migration.relay_host_id = ? + AND migration.assignment_epoch = ? + AND ${DURABLY_FENCED_MIGRATION_SOURCE}`, + [identity.userId, identity.relayHostId, assignmentEpoch] + ) + ).length === 1 + const retireOnTarget = + registered && + activityUnitsForCell(activityLeases, sourceCellId) === 0 && + sourceCell !== undefined && + integer(sourceCell, 'enabled') === 0 && + sourceAdmission !== undefined && + text(sourceAdmission, 'admission_state') === 'existing-only' && + (integer(sourceAdmission, 'updated_at') <= abandonedBefore || sourceIsDurablyFenced) && + targetCell !== undefined && + integer(targetCell, 'enabled') === 1 && + targetAdmission !== undefined && + ['migration-only', 'general'].includes(text(targetAdmission, 'admission_state')) + const rollbackReason = + !registered || + (targetCell !== undefined && + integer(targetCell, 'enabled') === 0 && + targetAdmission !== undefined && + text(targetAdmission, 'admission_state') === 'existing-only' && + integer(targetAdmission, 'updated_at') <= abandonedBefore) + const regionalRollbackSourceAvailable = + !regionalAttempt || + (sourceCell !== undefined && + integer(sourceCell, 'enabled') === 1 && + sourceAdmission !== undefined && + text(sourceAdmission, 'admission_state') === 'general' && + (await this.cellIsLive(transaction, sourceCellId, now))) + const rollbackToSource = rollbackReason && regionalRollbackSourceAvailable + if (!retireOnTarget && !rollbackToSource) return false + for (const activityId of [ pendingControlActivityId(assignmentEpoch), migrationActivityId(assignmentEpoch) - ] - .map((activityId) => activityLeaseById(activityLeases, activityId)) - .filter((lease): lease is SqlRow => lease !== undefined) - if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'nowait') - for (const lease of obsoleteLeases) { - await this.removeActivityLease(transaction, identity, lease, now) + ]) { + const lease = activityLeaseById(activityLeases, activityId) + if (lease) await this.removeActivityLease(transaction, identity, lease, now) } await this.releaseSupersededControlConnectionReservations( transaction, @@ -6669,116 +6565,47 @@ export class RelayAssignmentStore { assignmentEpoch, now ) - await transaction.query( - `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? - WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [now, now, identity.userId, identity.relayHostId, assignmentEpoch] - ) - return true - } - const cells = await this.lockCellInventory(transaction, 'nowait') - const sourceCellId = text(row, 'source_cell_id') - const admissionRows = await transaction.query( - `SELECT cell_id, admission_state, updated_at FROM relay_cell_admission - WHERE cell_id IN (?, ?)`, - [sourceCellId, targetCellId] - ) - const sourceCell = cells.find((cell) => text(cell, 'cell_id') === sourceCellId) - const targetCell = cells.find((cell) => text(cell, 'cell_id') === targetCellId) - const sourceAdmission = admissionRows.find( - (admission) => text(admission, 'cell_id') === sourceCellId - ) - const targetAdmission = admissionRows.find( - (admission) => text(admission, 'cell_id') === targetCellId - ) - const registered = optionalInteger(row, 'target_registered_at') !== undefined - const sourceIsDurablyFenced = - registered && - ( + if (retireOnTarget) { await transaction.query( - `SELECT 1 FROM relay_assignment_migrations migration - WHERE migration.user_id = ? AND migration.relay_host_id = ? - AND migration.assignment_epoch = ? - AND ${DURABLY_FENCED_MIGRATION_SOURCE}`, - [identity.userId, identity.relayHostId, assignmentEpoch] - ) - ).length === 1 - const retireOnTarget = - registered && - activityUnitsForCell(activityLeases, sourceCellId) === 0 && - sourceCell !== undefined && - integer(sourceCell, 'enabled') === 0 && - sourceAdmission !== undefined && - text(sourceAdmission, 'admission_state') === 'existing-only' && - (integer(sourceAdmission, 'updated_at') <= abandonedBefore || - sourceIsDurablyFenced) && - targetCell !== undefined && - integer(targetCell, 'enabled') === 1 && - targetAdmission !== undefined && - ['migration-only', 'general'].includes(text(targetAdmission, 'admission_state')) - const rollbackReason = - !registered || - (targetCell !== undefined && - integer(targetCell, 'enabled') === 0 && - targetAdmission !== undefined && - text(targetAdmission, 'admission_state') === 'existing-only' && - integer(targetAdmission, 'updated_at') <= abandonedBefore) - const regionalRollbackSourceAvailable = - !regionalAttempt || - (sourceCell !== undefined && - integer(sourceCell, 'enabled') === 1 && - sourceAdmission !== undefined && - text(sourceAdmission, 'admission_state') === 'general' && - (await this.cellIsLive(transaction, sourceCellId, now))) - const rollbackToSource = rollbackReason && regionalRollbackSourceAvailable - if (!retireOnTarget && !rollbackToSource) return false - for (const activityId of [ - pendingControlActivityId(assignmentEpoch), - migrationActivityId(assignmentEpoch) - ]) { - const lease = activityLeaseById(activityLeases, activityId) - if (lease) await this.removeActivityLease(transaction, identity, lease, now) - } - await this.releaseSupersededControlConnectionReservations( - transaction, - identity, - targetCellId, - assignmentEpoch, - now - ) - if (retireOnTarget) { - await transaction.query( - `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ? + `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ? WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [now, now, identity.userId, identity.relayHostId, assignmentEpoch] - ) - return true - } - await transaction.query( - `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + return true + } + await transaction.query( + `UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?, lease_expires_at = ?, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?`, - [ - sourceCellId, - assignmentEpoch + 1, - now + ASSIGNMENT_LIMITS.activityLeaseMs, - now, - identity.userId, - identity.relayHostId - ] - ) - await transaction.query( - `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? + [ + sourceCellId, + assignmentEpoch + 1, + now + ASSIGNMENT_LIMITS.activityLeaseMs, + now, + identity.userId, + identity.relayHostId + ] + ) + await transaction.query( + `UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ? WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`, - [now, now, identity.userId, identity.relayHostId, assignmentEpoch] - ) - return true - }).catch((error: unknown): boolean => { - // Expiry is durable; another director settling this row is not a failure. - if (!isDatabaseLockUnavailable(error)) throw error - inventoryBusy++ - return false - }) + [now, now, identity.userId, identity.relayHostId, assignmentEpoch] + ) + if (regionalAttempt) { + await transaction.query( + `UPDATE relay_region_rehome_attempts SET aborted_at = ?, updated_at = ? WHERE attempt_id = ?`, + [now, now, regionalAttempt.attempt_id] + ) + } + return true + }) + .catch((error: unknown): boolean => { + // Expiry is durable; another director settling this row is not a failure. + // Invariant failures remain fatal so operators see corrupt migration state. + if (!isDatabaseLockUnavailable(error)) throw error + inventoryBusy++ + return false + }) if (didAbort) aborted++ } warnSweepCellInventoryBusy('abort-expired-evacuations', inventoryBusy) @@ -8165,7 +7992,7 @@ function migration(identity: AssignmentIdentity, row: SqlRow): RelayAssignmentMi // Attempt ids are server-minted UUIDs and this codebase's invariant messages // are snake_case slugs; anything else could carry secrets and logs redacted. function warnRegionalRehomeCandidateFailure( - operation: 'complete' | 'abort', + operation: 'complete' | 'abort' | 'refresh', attemptId: string, error: unknown ): void { @@ -8190,24 +8017,6 @@ function noteRegionalRehomeActivityCountsRepaired(attemptId: string): void { ) } -function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt { - return { - attemptId: text(row, 'attempt_id'), - userId: text(row, 'user_id'), - relayHostId: text(row, 'relay_host_id'), - preferredRegion: relayRegion(row, 'preferred_region'), - sourceCellId: text(row, 'source_cell_id'), - sourceCellUrl: text(row, 'source_cell_url'), - sourceCellIncarnation: text(row, 'source_cell_incarnation'), - targetCellId: text(row, 'target_cell_id'), - targetCellIncarnation: text(row, 'target_cell_incarnation'), - previousEpoch: integer(row, 'previous_epoch'), - assignmentEpoch: integer(row, 'assignment_epoch'), - drainGraceMs: integer(row, 'drain_grace_ms'), - sendAttempts: integer(row, 'send_attempts') - } -} - function regionalRehomeControl(row: SqlRow): RegionalRehomeControl { return { generation: integer(row, 'generation'), @@ -8221,17 +8030,6 @@ function regionalRehomeControl(row: SqlRow): RegionalRehomeControl { } } -function cleanRegionalRehomeSafety(now: number): RegionalRehomeSafetySnapshot { - return { - observedAt: now, - sqlFailures: 0, - reconnects: 0, - controlActivityRecoveryFailures: 0, - databasePoolWaiting: 0, - databasePoolWaitersMax: 0, - databasePoolWaitMsMax: 0 - } -} function regionalRehomeFleetSafetyFromInventory(input: { cells: SqlRow[] @@ -8353,23 +8151,6 @@ function cellUncleanSkip( // Candidate skips are otherwise invisible: they neither latch the control off // nor produce attempts, so an operator cannot tell "skipping" from "idle". // Cell ids and counters only — never free-form error text. -function aggregateRegionalRehomeCandidateSkips( - skips: readonly RegionalRehomeCandidateSkip[] -): Record { - // `candidates` counts skipped candidate iterations, not distinct cells: one - // unclean cell blocking six candidates reports candidates=6 on one cellId. - const aggregated = new Map() - for (const skip of skips) { - const key = `${skip.reason}:${skip.cellId ?? ''}` - const entry = aggregated.get(key) - if (entry) entry.candidates += 1 - else aggregated.set(key, { ...skip, candidates: 1 }) - } - return { - event: 'orca_relay_regional_rehome_candidates_skipped', - skips: [...aggregated.values()] - } -} function regionalRehomeCellSafetyIsClean( safety: SqlRow | undefined, diff --git a/cloud/apps/relay/src/cell-heartbeat-client.test.ts b/cloud/apps/relay/src/cell-heartbeat-client.test.ts index 2aa708bed1b..6c36829e768 100644 --- a/cloud/apps/relay/src/cell-heartbeat-client.test.ts +++ b/cloud/apps/relay/src/cell-heartbeat-client.test.ts @@ -92,7 +92,7 @@ describe('cell heartbeat client', () => { client.stop() expect(JSON.parse(String(requests[1]!.body))).toMatchObject({ - regionalRehomeProtocol: 1, + regionalRehomeProtocol: 3, safety: { observedAt: 120, sqlFailures: 0, @@ -149,28 +149,34 @@ describe('cell heartbeat client', () => { it('does not start outside an explicitly configured cell role', () => { expect( - startCellHeartbeat({ ...CONFIG, role: 'director' }, { - ready: async () => true, - observedRequests: () => 0, - connectionCounts: () => ({ - totalConnections: 0, - inFlightConnections: 0, - reservedConnectionUnits: 0, - enforcedConnectionUnits: 0 - }) - }) + startCellHeartbeat( + { ...CONFIG, role: 'director' }, + { + ready: async () => true, + observedRequests: () => 0, + connectionCounts: () => ({ + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }) + } + ) ).toBeNull() expect( - startCellHeartbeat({ ...CONFIG, directorUrl: undefined }, { - ready: async () => true, - observedRequests: () => 0, - connectionCounts: () => ({ - totalConnections: 0, - inFlightConnections: 0, - reservedConnectionUnits: 0, - enforcedConnectionUnits: 0 - }) - }) + startCellHeartbeat( + { ...CONFIG, directorUrl: undefined }, + { + ready: async () => true, + observedRequests: () => 0, + connectionCounts: () => ({ + totalConnections: 0, + inFlightConnections: 0, + reservedConnectionUnits: 0, + enforcedConnectionUnits: 0 + }) + } + ) ).toBeNull() }) }) diff --git a/cloud/apps/relay/src/cell-heartbeat-client.ts b/cloud/apps/relay/src/cell-heartbeat-client.ts index 5c990310413..54f97a17af8 100644 --- a/cloud/apps/relay/src/cell-heartbeat-client.ts +++ b/cloud/apps/relay/src/cell-heartbeat-client.ts @@ -70,8 +70,7 @@ export function startCellHeartbeat( inFlightConnections: connectionCounts!.inFlightConnections, reservedConnectionUnits: connectionCounts!.reservedConnectionUnits, enforcedConnectionUnits: connectionCounts!.enforcedConnectionUnits, - connectionInclusionWatermark: - connectionCounts!.inclusionWatermark, + connectionInclusionWatermark: connectionCounts!.inclusionWatermark, connectionHardCap: config.connectionHardCap, connectionUnobservedBound: config.connectionUnobservedBound }) @@ -94,7 +93,7 @@ export function startCellHeartbeat( cellId: config.cellId, cellIncarnation, regionalRehomeProtocol: - config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0, + config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0, safety: options.regionalRehomeSafety() }), signal: AbortSignal.timeout(10_000) @@ -106,7 +105,10 @@ export function startCellHeartbeat( } } catch (error) { // A heartbeat must fail closed without ever logging its bearer token. - console.warn('[orca-relay] cell heartbeat failed', error instanceof Error ? error.message : '') + console.warn( + '[orca-relay] cell heartbeat failed', + error instanceof Error ? error.message : '' + ) } finally { inFlight = false } diff --git a/cloud/apps/relay/src/cell-inventory-lock-census.test.ts b/cloud/apps/relay/src/cell-inventory-lock-census.test.ts index 0ac4c8225e3..929173eb9da 100644 --- a/cloud/apps/relay/src/cell-inventory-lock-census.test.ts +++ b/cloud/apps/relay/src/cell-inventory-lock-census.test.ts @@ -43,14 +43,13 @@ const CENSUS: CensusEntry[] = [ { method: 'completeEvacuation', mode: 'nowait', reach: 'both' }, { method: 'completeEvacuation', mode: 'pool-default', reach: 'both' }, { method: 'rebalanceDormant', mode: 'request', reach: 'request' }, - { method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' }, - { method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' }, + { method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'request' }, { method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' }, { method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' }, { method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' }, { method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' }, { method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' }, - { method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' }, + { method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' } // reconcileReservationAccounting and leastLoadedCell are gone too: the first // repairs exactly two cells' counters and now holds only those rows, and the // second selects from the inventory its single caller has already locked. @@ -119,9 +118,10 @@ function storeCallGraph(lines: string[]): Map> { bounds.forEach((method, index) => { const end = bounds[index + 1]?.start ?? lines.length const names = callees.get(method.name) ?? new Set() - for (const call of lines.slice(method.start, end).join('\n').matchAll( - /this\.([A-Za-z_][\w]*)\s*\(/g - )) { + for (const call of lines + .slice(method.start, end) + .join('\n') + .matchAll(/this\.([A-Za-z_][\w]*)\s*\(/g)) { names.add(call[1]!) } callees.set(method.name, names) @@ -174,9 +174,7 @@ function readCallSites(): { method: string; mode: CensusMode }[] { describe('cell inventory lock call-site census', () => { it('classifies every call site exactly as recorded', () => { - expect(readCallSites()).toEqual( - CENSUS.map(({ method, mode }) => ({ method, mode })) - ) + expect(readCallSites()).toEqual(CENSUS.map(({ method, mode }) => ({ method, mode }))) }) // Why: the census only sees lockCellInventory calls, so a hand-written diff --git a/cloud/apps/relay/src/config.test.ts b/cloud/apps/relay/src/config.test.ts index bb0522dcbd3..01661a61826 100644 --- a/cloud/apps/relay/src/config.test.ts +++ b/cloud/apps/relay/src/config.test.ts @@ -25,6 +25,17 @@ function cellEnvironment(capacity: number): NodeJS.ProcessEnv { } describe('GCE relay capacity configuration', () => { + it('defaults optional region correction off and bounds the cohort', () => { + const env = cellEnvironment(4_000) + expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(0) + env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = '5' + expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(5) + for (const invalid of ['-1', '101', '1.5', 'not-a-number']) { + env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = invalid + expect(() => loadRelayConfig(env)).toThrow() + } + }) + it('requires distinct dedicated admin identities and accepts omitted values', () => { const env = cellEnvironment(4_000) expect(loadRelayConfig(env)).toMatchObject({ diff --git a/cloud/apps/relay/src/config.ts b/cloud/apps/relay/src/config.ts index 2bf23444a71..83dc9708f74 100644 --- a/cloud/apps/relay/src/config.ts +++ b/cloud/apps/relay/src/config.ts @@ -75,11 +75,15 @@ const EnvSchema = z.object({ ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: z.string().email().optional(), ORCA_RELAY_DIRECTOR_URL: z.string().url().optional(), ORCA_RELAY_HEARTBEAT_AUDIENCE: z.string().url().optional(), - ORCA_RELAY_IMAGE_DIGEST: z.string().regex(/^sha256:[a-f0-9]{64}$/).optional(), + ORCA_RELAY_IMAGE_DIGEST: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'), ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(), ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema, ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema, + ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT: z.coerce.number().int().min(0).max(100).default(0), ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: z.coerce.number().int().positive().max(100).default(2), ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY: z.coerce.number().int().positive().max(100).default(1), ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX: z.coerce.number().int().positive().max(4_096).default(64), @@ -185,6 +189,7 @@ export type RelayConfig = { databasePoolMax: number publicAssignmentsEnabled: boolean regionalPlacementEnabled?: boolean + regionCorrectionCohortPercent?: number publicAssignmentConcurrency: number publicAssignmentQueueMax: number publicAssignmentWaitMs: number @@ -332,6 +337,7 @@ export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConf databasePoolMax, publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED, regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED, + regionCorrectionCohortPercent: parsed.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT, publicAssignmentConcurrency: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY, publicAssignmentQueueMax: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX, publicAssignmentWaitMs: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS, diff --git a/cloud/apps/relay/src/database.test.ts b/cloud/apps/relay/src/database.test.ts index 56122def4be..0c987f95d40 100644 --- a/cloud/apps/relay/src/database.test.ts +++ b/cloud/apps/relay/src/database.test.ts @@ -18,6 +18,34 @@ afterEach(() => { }) describe('relay database', () => { + it('upgrades an existing SQLite relay without treating legacy controls as idle-capable', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'orca-idle-schema-')) + temporaryDirectories.push(dataDir) + const legacy = await openRelayDatabase({ dataDir }) + await legacy.query('ALTER TABLE relay_control_capabilities DROP COLUMN idle_regional_rehome') + await legacy.query('ALTER TABLE relay_region_rehome_attempts DROP COLUMN source_generation') + await legacy.query( + `INSERT INTO relay_control_capabilities + (user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing) + VALUES ('legacy-user', 'abcdefghijklmnop', 'control:source:1', 'source', 'legacy-incarnation', 1, 1, 1)` + ) + await legacy.close() + const upgraded = await openRelayDatabase({ dataDir }) + try { + expect( + await upgraded.query('SELECT idle_regional_rehome FROM relay_control_capabilities') + ).toEqual([{ idle_regional_rehome: 0 }]) + const columns = await upgraded.query( + "SELECT * FROM pragma_table_info('relay_region_rehome_attempts')" + ) + expect(columns.find((column) => column.name === 'source_generation')).toMatchObject({ + dflt_value: '0' + }) + } finally { + await upgraded.close() + } + }) + it('creates every durable relay state table', async () => { const database = await openInMemoryRelayDatabase() const rows = await database.query( @@ -54,6 +82,7 @@ describe('relay database', () => { 'relay_confirm_results', 'relay_confirmable_splices', 'relay_connection_bases', + 'relay_control_capabilities', 'relay_control_connection_reservations', 'relay_devices', 'relay_direct_authorizations', @@ -62,6 +91,7 @@ describe('relay database', () => { 'relay_migration_leases', 'relay_post_drain_migration_pins', 'relay_rate_windows', + 'relay_region_decisions', 'relay_region_rehome_attempts', 'relay_region_rehome_control', 'relay_region_rehome_worker_state' @@ -140,9 +170,7 @@ describe('relay database', () => { const second = await openRelayDatabase({ dataDir }) expect( - await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [ - 'legacy-cell' - ]) + await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, ['legacy-cell']) ).toEqual([{ region: 'us-central1' }]) await second.close() }) @@ -165,9 +193,7 @@ describe('relay database', () => { 'relay_region_rehome_attempts' ]) expect(checked.every((row) => String(row.sql).includes(list))).toBe(true) - expect( - POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list)) - ).toBe(true) + expect(POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))).toBe(true) await database.close() }) diff --git a/cloud/apps/relay/src/database.ts b/cloud/apps/relay/src/database.ts index d51f4e7a423..41ead67ea60 100644 --- a/cloud/apps/relay/src/database.ts +++ b/cloud/apps/relay/src/database.ts @@ -197,6 +197,24 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences ( CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed ON relay_assignment_region_preferences(observed_at); +CREATE TABLE IF NOT EXISTS relay_region_decisions ( + user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, + generation BIGINT NOT NULL, expires_at BIGINT NOT NULL, + assignment_epoch BIGINT NOT NULL, incumbent_region TEXT NOT NULL, + policy_version BIGINT NOT NULL, outcome TEXT NOT NULL, + cohort_bucket BIGINT NOT NULL DEFAULT 0, + last_considered_at BIGINT NOT NULL DEFAULT 0, + preferred_region TEXT, observed_at BIGINT NOT NULL, report_json TEXT, + PRIMARY KEY (user_id, relay_host_id) +); +CREATE TABLE IF NOT EXISTS relay_control_capabilities ( + user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, activity_id TEXT NOT NULL, + cell_id TEXT NOT NULL, cell_incarnation TEXT NOT NULL, + assignment_epoch BIGINT NOT NULL, generation BIGINT NOT NULL, + finish_existing BIGINT NOT NULL, + idle_regional_rehome BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (user_id, relay_host_id, activity_id) +); CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state ( worker_id TEXT PRIMARY KEY, next_dispatch_at BIGINT NOT NULL, @@ -228,6 +246,7 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts ( CHECK (preferred_region IN (${REGION_LIST})), source_cell_id TEXT NOT NULL, source_cell_incarnation TEXT NOT NULL, + source_generation BIGINT NOT NULL DEFAULT 0, target_cell_id TEXT NOT NULL, target_cell_incarnation TEXT NOT NULL, previous_epoch BIGINT NOT NULL, @@ -600,6 +619,8 @@ CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at); // auto-named; the replacement is named, so both statements are no-ops on a // database the current schema created and neither can drop the other. export const POSTGRES_SCHEMA_MIGRATIONS = [ + `ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`, + `ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS cohort_bucket BIGINT NOT NULL DEFAULT 0`, `ALTER TABLE relay_region_rehome_attempts DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`, `ALTER TABLE relay_region_rehome_attempts @@ -607,7 +628,9 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [ CHECK (preferred_region IN (${REGION_LIST}))`, `ALTER TABLE relay_region_rehome_control ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL - DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}` + DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`, + `ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`, + `ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0` ] function postgresSql(sql: string): string { @@ -1013,6 +1036,15 @@ async function applySchema(database: RelayDatabase): Promise { for (const statement of SCHEMA.split(';')) { if (statement.trim()) await database.query(statement) } + for (const [table, column] of [ + ['relay_control_capabilities', 'idle_regional_rehome'], + ['relay_region_rehome_attempts', 'source_generation'] + ]) { + const columns = await database.query('SELECT name FROM pragma_table_info(?)', [table]) + if (!columns.some((existing) => existing.name === column)) { + await database.query(`ALTER TABLE ${table} ADD COLUMN ${column} BIGINT NOT NULL DEFAULT 0`) + } + } } // Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs diff --git a/cloud/apps/relay/src/host-session-client-accept.test.ts b/cloud/apps/relay/src/host-session-client-accept.test.ts index 0cec6531e3f..04f86c533e4 100644 --- a/cloud/apps/relay/src/host-session-client-accept.test.ts +++ b/cloud/apps/relay/src/host-session-client-accept.test.ts @@ -155,6 +155,66 @@ describe('client accept abandoned mid-DB-phase', () => { vi.useRealTimers() }) + it('does not admit new source work after a drain crosses activity acquisition', async () => { + const h = harness() + const control = await activeHost(h) + const slow = deferred() + h.acquireActivity.mockReturnValueOnce(slow.promise) + const client = new FakeSocket() + const capacity = { bind: vi.fn(), release: vi.fn() } + const accepting = h.registry.acceptClient( + client as unknown as WebSocket, + identity.relayHostId, + 'credential', + capacity + ) + await vi.advanceTimersByTimeAsync(0) + h.registry.drainHost({ + attemptId: 'attempt', + userId: identity.sub, + relayHostId: identity.relayHostId, + sourceAssignmentEpoch: 1, + graceMs: 60_000 + }) + slow.resolve() + await accepting + expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open')) + expect(capacity.bind).not.toHaveBeenCalled() + expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String)) + expect(h.releaseActivity).toHaveBeenCalled() + }) + + it('does not splice an attachment whose generation retired during basis persistence', async () => { + const h = harness() + await activeHost(h) + const client = new FakeSocket() + await h.registry.acceptClient( + client as unknown as WebSocket, + identity.relayHostId, + 'credential' + ) + const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })! + const pending = [...session.pendingConns.values()][0]! + const slow = deferred() + h.store.recordConnectionBasis.mockReturnValueOnce(slow.promise) + const host = new FakeSocket() + const attaching = h.registry.acceptHostData( + host as unknown as WebSocket, + pending.connId, + pending.connTicket, + 1 + ) + await vi.advanceTimersByTimeAsync(0) + h.registry.drain(0) + await vi.advanceTimersByTimeAsync(0) + slow.resolve() + expect(await attaching).toBe(false) + expect(session.activeSplices.size).toBe(0) + expect(h.store.deactivateBasis).toHaveBeenCalledWith(pending.connId) + expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('\"ok\":true')) + expect(host.close).toHaveBeenCalled() + }) + it('stops after a slow activity acquire when the phone already hung up', async () => { const h = harness() const control = await activeHost(h) @@ -390,6 +450,7 @@ describe('successful client accept timing', () => { relayHostIdDigest: string } expect(event.credentialKind).toBe('resume') + expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' }) // Joins the line back to the emitting process, like the runtime metrics event. expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' }) expect(Object.keys(event.stageMs).sort()).toEqual([ @@ -460,6 +521,9 @@ describe('control round-trip sampling', () => { cellId: config.cellId, region: 'us-central1', rttMsMedian: 40, + assignmentEpoch: 1, + controlGeneration: 1, + drainMode: 'none', sampleCount: 4 }) expect(rttLines()[0]).not.toContain(identity.relayHostId) diff --git a/cloud/apps/relay/src/host-session-registry.test.ts b/cloud/apps/relay/src/host-session-registry.test.ts index 920faa6f4b8..dcfcd3f36c5 100644 --- a/cloud/apps/relay/src/host-session-registry.test.ts +++ b/cloud/apps/relay/src/host-session-registry.test.ts @@ -4,6 +4,7 @@ import { CONTROL_CONTINUITY_LIMITS, RELAY_CLOSE_CODE, RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS, + RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -140,7 +141,10 @@ function createRegistry( store as RelayCredentialStore, assignments, new ProcessQueuedByteBudget(), - observer + observer, + Date.now, + Math.random, + 'incarnation-1' ) // Mirrors the production signature exactly so a future positional shift fails to compile. const bound = ( @@ -166,7 +170,14 @@ function createRegistry( assignmentEpoch, appVersion = '1.4.173' ) => bound(socket, identity, existing, generation, rebind, assignmentEpoch, appVersion) - return { registry, activate, acquireActivity, renewControlActivity, releaseActivity, observer } + return { + registry, + activate, + acquireActivity, + renewControlActivity, + releaseActivity, + observer + } } describe('host session cleanup races', () => { @@ -398,26 +409,20 @@ describe('host session cleanup races', () => { attemptId: '22222222-2222-4222-8222-222222222222' }) ).toThrow('regional_rehome_attempt_conflict') - expect(() => - registry.drainHost({ ...request, sourceAssignmentEpoch: 8 }) - ).toThrow('regional_rehome_assignment_epoch_mismatch') + expect(() => registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })).toThrow( + 'regional_rehome_assignment_epoch_mismatch' + ) const rebound = new FakeSocket() - await activate( - rebound as unknown as WebSocket, - identity, - registry.get(request), - 1, - true, - 7 - ) + await activate(rebound as unknown as WebSocket, identity, registry.get(request), 1, true, 7) expect(registry.get(request)?.state).toBe('drain-only') expect(rebound.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"')) await vi.advanceTimersByTimeAsync(30_000) expect(registry.get(request)).toBeNull() - expect(registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId })) - .not.toBeNull() + expect( + registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId }) + ).not.toBeNull() expect(secondSocket.close).not.toHaveBeenCalled() }) @@ -513,14 +518,7 @@ describe('host session cleanup races', () => { expect(original).not.toBeNull() const rebindSocket = new FakeSocket() - const rebinding = activate( - rebindSocket as unknown as WebSocket, - identity, - original, - 1, - true, - 1 - ) + const rebinding = activate(rebindSocket as unknown as WebSocket, identity, original, 1, true, 1) rebindSocket.close() blocked.resolve('control:production-gce-c3:1') await rebinding @@ -659,14 +657,7 @@ describe('host session cleanup races', () => { originalSocket.close() const replacementSocket = new FakeSocket() - await activate( - replacementSocket as unknown as WebSocket, - identity, - original, - 2, - false, - 1 - ) + await activate(replacementSocket as unknown as WebSocket, identity, original, 2, false, 1) const replacement = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId @@ -696,14 +687,7 @@ describe('host session cleanup races', () => { }) expect(original).not.toBeNull() - await activate( - new FakeSocket() as unknown as WebSocket, - identity, - original, - 2, - false, - 1 - ) + await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1) vi.advanceTimersByTime(15_000) expect(renewControlActivity).toHaveBeenCalledOnce() @@ -718,6 +702,53 @@ describe('host session cleanup races', () => { vi.advanceTimersByTime(0) }) + it('ignores a denial belonging to the socket before a same-generation rebind', async () => { + const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1')) + const oldSocket = new FakeSocket() + await h.activate(oldSocket as unknown as WebSocket, identity, null, 1, false, 1) + const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })! + let reject!: (error: Error) => void + h.renewControlActivity.mockReturnValueOnce( + new Promise((_, fail) => { + reject = fail + }) + ) + await vi.advanceTimersByTimeAsync(15_000) + const replacement = new FakeSocket() + await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1) + reject(new Error('activity_cell_not_authoritative')) + await vi.advanceTimersByTimeAsync(0) + expect(replacement.close).not.toHaveBeenCalled() + expect(session.socket).toBe(replacement) + expect(session.generation).toBe(1) + }) + + it('ignores missing-activity recovery denial after an authority transition', async () => { + const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1')) + const socket = new FakeSocket() + await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })! + h.renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found')) + let reject!: (error: Error) => void + h.acquireActivity.mockReturnValueOnce( + new Promise((_, fail) => { + reject = fail + }) + ) + await vi.advanceTimersByTimeAsync(15_000) + h.registry.drainHost({ + attemptId: 'attempt', + userId: identity.sub, + relayHostId: identity.relayHostId, + sourceAssignmentEpoch: 1, + graceMs: 60_000 + }) + reject(new Error('activity_cell_not_authoritative')) + await vi.advanceTimersByTimeAsync(0) + expect(socket.close).not.toHaveBeenCalled() + expect(session.state).toBe('drain-only') + }) + it('keeps 15s pings while halving steady-state control renewals', async () => { const activateControl = vi .fn() @@ -732,9 +763,7 @@ describe('host session cleanup races', () => { socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false) } - const pings = socket.send.mock.calls.filter((call) => - String(call[0]).includes('"ping"') - ) + const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"')) expect(pings).toHaveLength(4) expect(renewControlActivity).toHaveBeenCalledTimes(2) const firstExpiry = Number(renewControlActivity.mock.calls[0]![1].expiresAt) @@ -1118,3 +1147,277 @@ describe('host hello ack pending connections', () => { expect(rebound.pendingConns).toEqual([DETAILED_ENTRY]) }) }) + +describe('source-owned idle cutover', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + const request = { + attemptId: 'idle-1', + userId: identity.sub, + relayHostId: identity.relayHostId, + sourceAssignmentEpoch: 1, + sourceGeneration: 1, + sourceCellIncarnation: 'incarnation-1', + targetCellId: 'target' + } + async function source(store: Partial = {}) { + const h = createRegistry(vi.fn().mockResolvedValue('control:1'), store) + const socket = new FakeSocket() + h.registry.acceptControl( + socket as unknown as WebSocket, + identity, + undefined, + new Set([RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME]) + ) + socket.removeAllListeners('message') + await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + return { ...h, socket, session: h.registry.get(request)! } + } + it('keeps either established client busy until both actually leave', async () => { + const h = await source() + h.session.activeConnIds.add('phone') + h.session.activeConnIds.add('ipad') + const commit = vi.fn().mockResolvedValue({ outcome: 'committed' }) + h.session.activeConnIds.delete('ipad') + expect( + await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed')) + ).toEqual({ outcome: 'busy' }) + expect(commit).not.toHaveBeenCalled() + h.session.activeConnIds.delete('phone') + expect( + await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed')) + ).toEqual({ outcome: 'committed' }) + expect(h.socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String)) + expect(h.releaseActivity).toHaveBeenCalled() + }) + it.each([ + { userId: 'other-user' }, + { sourceAssignmentEpoch: 2 }, + { sourceGeneration: 2 }, + { sourceCellIncarnation: 'other-incarnation' }, + { targetCellId: 'other-target' } + ])('rejects a reused operation ID with changed authority %j', async (change) => { + const h = await source() + const result = deferred<{ outcome: 'deferred' }>() + const commit = vi.fn().mockReturnValue(result.promise) + const reconcile = vi.fn().mockResolvedValue('not-committed') + const moving = h.registry.idleRehome(request, commit, reconcile) + const conflicting = h.registry.idleRehome({ ...request, ...change }, commit, reconcile) + result.resolve({ outcome: 'deferred' }) + expect(await conflicting).toEqual({ outcome: 'stale' }) + expect(await moving).toEqual({ outcome: 'deferred' }) + expect(commit).toHaveBeenCalledOnce() + expect(h.socket.close).not.toHaveBeenCalled() + }) + it('accounts for accepts before credential identity resolves', async () => { + const lookup = deferred() + const h = await source({ + resolveResume: vi.fn().mockReturnValue(lookup.promise), + resolveInviteForMove: vi.fn().mockResolvedValue(null) + }) + const client = new FakeSocket() + const accept = h.registry.acceptClient( + client as unknown as WebSocket, + identity.relayHostId, + 'credential' + ) + expect( + await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed')) + ).toEqual({ outcome: 'busy' }) + lookup.resolve(null) + await accept + expect(h.socket.close).not.toHaveBeenCalled() + }) + it('rejects new accepts and replacements synchronously while a commit awaits', async () => { + const h = await source() + const result = deferred<{ outcome: 'deferred' }>() + const commit = vi.fn().mockReturnValue(result.promise) + const moving = h.registry.idleRehome( + request, + commit, + vi.fn().mockResolvedValue('not-committed') + ) + const duplicate = h.registry.idleRehome( + request, + commit, + vi.fn().mockResolvedValue('not-committed') + ) + const client = new FakeSocket() + const release = vi.fn() + await h.registry.acceptClient( + client as unknown as WebSocket, + identity.relayHostId, + 'credential', + { release } as never + ) + expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String)) + expect(release).toHaveBeenCalledOnce() + const replacement = new FakeSocket() + await h.activate(replacement as unknown as WebSocket, identity, h.session, 2, false, 1) + expect(replacement.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String)) + result.resolve({ outcome: 'deferred' }) + await moving + await duplicate + expect(commit).toHaveBeenCalledOnce() + expect(h.socket.close).not.toHaveBeenCalled() + expect( + await h.registry.idleRehome( + { ...request, attemptId: 'next' }, + vi.fn().mockResolvedValue({ outcome: 'committed' }), + vi.fn().mockResolvedValue('not-committed') + ) + ).toEqual({ outcome: 'committed' }) + }) + it.each(['ambiguous', 'deferred'])( + 'keeps %s outcomes fenced until locked reconciliation succeeds', + async (claim) => { + const h = await source() + const reconcile = vi + .fn() + .mockRejectedValueOnce(new Error('database unavailable')) + .mockRejectedValueOnce(new Error('database unavailable')) + .mockResolvedValue('not-committed') + const moving = h.registry.idleRehome( + request, + claim === 'ambiguous' + ? vi.fn().mockRejectedValue(new Error('lost commit reply')) + : vi.fn().mockResolvedValue({ outcome: 'deferred' }), + reconcile + ) + await vi.advanceTimersByTimeAsync(50) + expect( + await h.registry.idleRehome( + { ...request, attemptId: 'other' }, + vi.fn(), + vi.fn().mockResolvedValue('not-committed') + ) + ).toEqual({ outcome: 'busy' }) + expect(h.socket.close).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(300) + expect(await moving).toEqual({ outcome: 'deferred' }) + expect(reconcile).toHaveBeenCalledTimes(3) + expect(h.socket.close).not.toHaveBeenCalled() + } + ) + it('owns accepted control mutations before the handler first awaits', async () => { + const mutation = deferred() + const h = await source() + ;(h.registry as unknown as { verifyRelayToken: unknown }).verifyRelayToken = vi + .fn() + .mockReturnValue(mutation.promise) + h.socket.emit( + 'message', + Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'token' })), + false + ) + const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' }) + expect( + await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed')) + ).toEqual({ outcome: 'busy' }) + mutation.resolve(identity) + await vi.advanceTimersByTimeAsync(0) + expect( + await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed')) + ).toEqual({ outcome: 'deferred' }) + }) + it('owns queued replacement activation before its first persistence await', async () => { + const h = await source() + const activation = deferred() + const assignments = (h.registry as unknown as { assignments: { activateControl: unknown } }) + .assignments + assignments.activateControl = vi.fn().mockReturnValue(activation.promise) + const replacement = new FakeSocket() + const activating = h.activate( + replacement as unknown as WebSocket, + identity, + h.session, + 2, + false, + 1 + ) + expect( + await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed')) + ).toEqual({ outcome: 'busy' }) + activation.resolve('control:2') + await activating + }) + it('retires changed authority even when the claim definitively deferred', async () => { + const h = await source() + expect( + await h.registry.idleRehome( + request, + vi.fn().mockResolvedValue({ outcome: 'deferred' }), + vi.fn().mockResolvedValue('stale') + ) + ).toEqual({ outcome: 'stale' }) + expect(h.session.state).toBe('closed') + expect(h.releaseActivity).toHaveBeenCalled() + }) + it('holds attach ownership through basis failure reservation cleanup', async () => { + const basis = deferred() + const cleanup = deferred() + const h = await source({ + recordConnectionBasis: vi.fn().mockImplementation(async () => { + await basis.promise + throw new Error('basis failed') + }), + failReservation: vi.fn().mockReturnValue(cleanup.promise) + }) + const client = new FakeSocket() + h.session.pendingConns.set('conn', { + connId: 'conn', + connTicket: 'ticket', + client: client as unknown as WebSocket, + reservation: { + userId: identity.sub, + relayHostId: identity.relayHostId, + credentialKind: 'invite', + leaseExpiresAt: Date.now() + 1000 + }, + attachTimer: setTimeout(() => {}, 1000), + credentialActivityId: null + } as never) + const attached = h.registry.acceptHostData( + new FakeSocket() as unknown as WebSocket, + 'conn', + 'ticket', + 1 + ) + const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' }) + expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' }) + basis.resolve() + await vi.advanceTimersByTimeAsync(0) + expect(h.session.activeConnIds.size).toBe(0) + expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' }) + expect(commit).not.toHaveBeenCalled() + cleanup.resolve() + await attached + }) + it('returns the durable operation outcome after source retirement', async () => { + const h = await source() + const commit = vi.fn().mockResolvedValue({ outcome: 'committed' }) + await h.registry.idleRehome(request, commit, vi.fn()) + expect( + await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('committed')) + ).toEqual({ outcome: 'committed' }) + expect(commit).toHaveBeenCalledOnce() + }) + it('does not reopen a source overtaken by emergency drain', async () => { + const h = await source() + const result = deferred<{ outcome: 'deferred' }>() + const moving = h.registry.idleRehome( + request, + () => result.promise, + vi.fn().mockResolvedValue('not-committed') + ) + h.registry.drain(0) + await vi.advanceTimersByTimeAsync(0) + result.resolve({ outcome: 'deferred' }) + await moving + expect(h.session.state).toBe('closed') + expect(h.registry.get(request)).toBeNull() + }) +}) diff --git a/cloud/apps/relay/src/host-session-registry.ts b/cloud/apps/relay/src/host-session-registry.ts index 3b4e616a692..61a1b706fa9 100644 --- a/cloud/apps/relay/src/host-session-registry.ts +++ b/cloud/apps/relay/src/host-session-registry.ts @@ -15,6 +15,7 @@ import { HostHelloSchema, InviteCreateSchema, RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS, + RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME, RELAY_PROTOCOL_LIMITS, RELAY_CLOSE_CODE, type RelayHostCloseReason, @@ -25,10 +26,7 @@ import type WebSocket from 'ws' import type { RawData } from 'ws' import type { RelayConfig } from './config.js' import type { RelayAssignmentStore } from './assignment-store.js' -import { - RelayCredentialStore, - type CredentialReservation -} from './credential-store.js' +import { RelayCredentialStore, type CredentialReservation } from './credential-store.js' import { HostCloseReasonMemory } from './host-close-reason-memory.js' import { relayHostLogDigest } from './relay-host-log-digest.js' import type { RelayTokenClaims } from './relay-token-verifier.js' @@ -78,7 +76,7 @@ export type HostSession = { identity: RelayTokenClaims readonly relayHostId: string readonly generation: number - readonly assignmentEpoch: number + assignmentEpoch: number readonly controlActivityId: string | null readonly controlResumeSecret: string // Why: reconnect churn is only actionable once it can be pinned to a client build. @@ -94,6 +92,7 @@ export type HostSession = { pendingPingAt: number | null controlRttSamplesMs: number[] controlRttLoggedAt: number | null + authorityRevision: number activityRenewalDueAt: number activityRenewalAttempt: number activityRenewalCompletedAttempt: number @@ -108,10 +107,7 @@ export type HostSession = { regionalDrainExpiresAt: number | null } -export type RegionalHostDrainOutcome = - | 'accepted' - | 'already-accepted' - | 'host-not-connected' +export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected' type PendingConnection = { connId: string @@ -184,6 +180,113 @@ export class HostSessionRegistry { private readonly hostCapabilities = new WeakMap>() private draining = false + private readonly idleWork = new Map() + private readonly idleAttempts = new Map< + string, + { + attemptId: string + authorityKey: string + promise: Promise<{ outcome: 'committed' | 'deferred' | 'stale' }> + } + >() + + async idleRehome( + input: { + attemptId: string + userId: string + relayHostId: string + sourceAssignmentEpoch: number + sourceGeneration: number + sourceCellIncarnation: string + targetCellId: string + }, + commit: () => Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>, + reconcile: () => Promise<'committed' | 'not-committed' | 'stale'> + ): Promise<{ outcome: 'busy' | 'committed' | 'deferred' | 'stale' }> { + const authorityKey = JSON.stringify([ + input.userId, + input.sourceAssignmentEpoch, + input.sourceGeneration, + input.sourceCellIncarnation, + input.targetCellId + ]) + const prior = this.idleAttempts.get(input.relayHostId) + if (prior) { + if (prior.attemptId !== input.attemptId) return { outcome: 'busy' } + return prior.authorityKey === authorityKey ? prior.promise : { outcome: 'stale' } + } + const session = this.get(input) + if ( + this.draining || + !session || + session.state !== 'active' || + session.generation !== input.sourceGeneration || + session.assignmentEpoch !== input.sourceAssignmentEpoch || + this.cellIncarnation !== input.sourceCellIncarnation + ) { + const durable = await reconcile() + return { outcome: durable === 'committed' ? 'committed' : 'stale' } + } + if ( + !session.socket || + !this.hostCapabilities.get(session.socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME) + ) + return { outcome: 'deferred' } + if ( + (this.idleWork.get(input.relayHostId) ?? 0) !== 0 || + session.activeConnIds.size !== 0 || + session.activeSplices.size !== 0 || + session.pendingConns.size !== 0 + ) + return { outcome: 'busy' } + const revision = session.authorityRevision + const promise = Promise.resolve().then(async () => { + let outcome: 'committed' | 'deferred' | 'stale' + try { + outcome = (await commit()).outcome + if (outcome === 'deferred') { + const durable = await reconcile() + outcome = durable === 'not-committed' ? 'deferred' : durable + } + } catch { + let delay = 100 + for (;;) { + try { + const durable = await reconcile() + outcome = durable === 'not-committed' ? 'deferred' : durable + break + } catch { + await new Promise((resolve) => { + const timer = setTimeout(resolve, delay) + timer.unref?.() + }) + delay = Math.min(delay * 2, 5000) + } + } + } + if (this.get(input) === session) { + if (outcome !== 'deferred' || this.draining || session.authorityRevision !== revision) { + this.closeDrainedSession(session) + } + } + if (this.idleAttempts.get(input.relayHostId)?.promise === promise) + this.idleAttempts.delete(input.relayHostId) + return { outcome } + }) + this.idleAttempts.set(input.relayHostId, { attemptId: input.attemptId, authorityKey, promise }) + return promise + } + + private beginIdleWork(hostId: string): (() => void) | null { + if (this.idleAttempts.has(hostId)) return null + this.idleWork.set(hostId, (this.idleWork.get(hostId) ?? 0) + 1) + return () => { + const remaining = (this.idleWork.get(hostId) ?? 1) - 1 + if (remaining === 0) this.idleWork.delete(hostId) + else this.idleWork.set(hostId, remaining) + } + } + constructor( private readonly config: RelayConfig, private readonly verifyRelayToken: VerifyRelayToken, @@ -192,7 +295,8 @@ export class HostSessionRegistry { private readonly queuedByteBudget: ProcessQueuedByteBudget, private readonly observer: RelayRuntimeObserver, private readonly now: () => number = Date.now, - private readonly random: () => number = Math.random + private readonly random: () => number = Math.random, + private readonly cellIncarnation?: string ) {} // Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter). @@ -206,6 +310,25 @@ export class HostSessionRegistry { hostId: string, credential: string, capacityReservation?: PendingHostDataReservation + ): Promise { + const release = this.beginIdleWork(hostId) + if (!release) { + capacityReservation?.release() + this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL) + return + } + try { + await this.acceptClientUnfenced(socket, hostId, credential, capacityReservation) + } finally { + release() + } + } + + private async acceptClientUnfenced( + socket: WebSocket, + hostId: string, + credential: string, + capacityReservation?: PendingHostDataReservation ): Promise { if (this.draining) { capacityReservation?.release() @@ -295,6 +418,7 @@ export class HostSessionRegistry { this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED) return } + const admittingSocket = session.socket const connId = randomUUID() const connTicket = randomBytes(32).toString('base64url') const identity = { userId: reservation.userId, relayHostId: hostId } @@ -324,6 +448,20 @@ export class HostSessionRegistry { ) { return } + // Admission may have crossed a drain or control replacement while persisting activity. + if ( + this.draining || + this.sessions.get(sessionKey) !== session || + session.state !== 'active' || + session.socket !== admittingSocket || + admittingSocket.readyState !== admittingSocket.OPEN + ) { + capacityReservation?.release() + this.failReservationBestEffort(reservation) + if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId) + this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL) + return + } markStage('activity') const attachTimer = setTimeout(() => { session.pendingConns.delete(connId) @@ -372,6 +510,27 @@ export class HostSessionRegistry { connId: string, connTicket: string, generation: number + ): Promise { + const owner = [...this.sessions.values()].find((candidate) => + candidate.pendingConns.has(connId) + ) + const release = owner ? this.beginIdleWork(owner.relayHostId) : () => {} + if (!release) { + socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress') + return false + } + try { + return await this.acceptHostDataUnfenced(socket, connId, connTicket, generation) + } finally { + release() + } + } + + private async acceptHostDataUnfenced( + socket: WebSocket, + connId: string, + connTicket: string, + generation: number ): Promise { const session = [...this.sessions.values()].find((candidate) => candidate.pendingConns.has(connId) @@ -427,6 +586,27 @@ export class HostSessionRegistry { socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'basis persistence failed') return false } + // Already admitted attachments may finish a regional drain, but never a retired generation. + if ( + this.draining || + this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session || + this.get(identity)?.state === 'closed' || + !session.activeConnIds.has(connId) || + socket.readyState !== socket.OPEN || + pending.client.readyState !== pending.client.OPEN + ) { + session.activeConnIds.delete(connId) + pending.capacityReservation?.release() + this.deactivateBasisBestEffort(connId) + this.failReservationBestEffort(pending.reservation) + if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId) + if (pending.credentialActivityId) { + this.releaseActivityBestEffort(identity, pending.credentialActivityId) + } + this.rejectClient(pending.client, RELAY_CLOSE_CODE.DRAINING) + socket.close(RELAY_CLOSE_CODE.DRAINING, 'host retired during attachment') + return false + } const close = wireSplice({ client: pending.client, host: socket, @@ -505,6 +685,7 @@ export class HostSessionRegistry { JSON.stringify({ event: 'orca_relay_client_accept_completed', ...this.logIdentity(), + ...this.sessionPlacementLogFields(session), credentialKind: pending.reservation.credentialKind, stageMs, totalMs, @@ -513,6 +694,14 @@ export class HostSessionRegistry { ) } + private sessionPlacementLogFields(session: HostSession) { + return { + assignmentEpoch: session.assignmentEpoch, + controlGeneration: session.generation, + drainMode: session.regionalDrainAttemptId ? 'deadline' : 'none' + } + } + // Matches the runtime metrics event so a log line and a metric point can be // joined back to the process that emitted them. private logIdentity(): { role: string; cellId: string; region: RelayRegion } { @@ -549,6 +738,7 @@ export class HostSessionRegistry { JSON.stringify({ event: 'orca_relay_host_control_rtt', ...this.logIdentity(), + ...this.sessionPlacementLogFields(session), relayHostIdDigest: relayHostLogDigest(session.relayHostId), rttMsMedian: percentile(samples, 0.5), sampleCount: samples.length @@ -562,6 +752,10 @@ export class HostSessionRegistry { connectionInclusionWatermark?: number, hostCapabilities?: ReadonlySet ): void { + if (this.idleAttempts.has(identity.relayHostId)) { + socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress') + return + } // Keyed by socket, not session: a rebind swaps the session's socket, and the // successor's own advertisement is the only one that describes its decoder. if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities) @@ -602,8 +796,7 @@ export class HostSessionRegistry { socket: WebSocket | null, context: string ): void { - void Promise.resolve() - .then(task) + void (async () => task())() .catch((error: unknown) => { const message = (error instanceof Error ? error.message : 'unknown') // Untruncated, unlike peer-supplied close reasons: this is the @@ -653,6 +846,7 @@ export class HostSessionRegistry { this.draining = true for (const session of this.sessions.values()) { if (session.state === 'closed') continue + session.authorityRevision += 1 session.state = 'drain-only' if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' }) setTimeout(() => this.closeDrainedSession(session), graceMs) @@ -665,7 +859,8 @@ export class HostSessionRegistry { relayHostId: string sourceAssignmentEpoch: number graceMs: number - }): RegionalHostDrainOutcome { + sourceCellIncarnation?: string + }): RegionalHostDrainOutcome | Promise { const session = this.get(input) if (!session || session.state === 'closed') return 'host-not-connected' if (session.assignmentEpoch !== input.sourceAssignmentEpoch) { @@ -678,13 +873,11 @@ export class HostSessionRegistry { this.reassertRegionalDrain(session) return 'already-accepted' } + session.authorityRevision += 1 session.regionalDrainAttemptId = input.attemptId session.regionalDrainExpiresAt = this.now() + input.graceMs this.reassertRegionalDrain(session) - session.regionalDrainTimer = setTimeout( - () => this.closeDrainedSession(session), - input.graceMs - ) + session.regionalDrainTimer = setTimeout(() => this.closeDrainedSession(session), input.graceMs) return 'accepted' } @@ -740,9 +933,9 @@ export class HostSessionRegistry { const existing = this.sessions.get(key) const rebind = Boolean( existing && - hello.data.controlResumeSecret && - hello.data.controlResumeSecret === existing.controlResumeSecret && - (existing.state === 'orphaned' || existing.state === 'active') + hello.data.controlResumeSecret && + hello.data.controlResumeSecret === existing.controlResumeSecret && + (existing.state === 'orphaned' || existing.state === 'active') ) const generation = rebind ? existing!.generation : (existing?.generation ?? 0) + 1 const ephemeral = nacl.box.keyPair() @@ -784,7 +977,9 @@ export class HostSessionRegistry { }, 10_000) socket.once('message', (raw, isBinary) => { clearTimeout(proofTimer) - const ack = isBinary ? null : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack')) + const ack = isBinary + ? null + : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack')) const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null if ( !ack?.success || @@ -826,6 +1021,11 @@ export class HostSessionRegistry { appVersion: string, connectionInclusionWatermark?: number ): Promise { + const release = this.beginIdleWork(identity.relayHostId) + if (!release) { + socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress') + return Promise.resolve() + } const key = this.key(identity.sub, identity.relayHostId) const previous = this.activationQueues.get(key) ?? Promise.resolve() // The timeout only fails this waiting socket; the queue entry still chains @@ -836,26 +1036,29 @@ export class HostSessionRegistry { socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled') }, ACTIVATION_QUEUE_WAIT_MS) queueWaitTimer.unref?.() - const activation = previous.catch(() => undefined).then(async () => { - clearTimeout(queueWaitTimer) - if (queueWaitExpired) return - if ((this.sessions.get(key) ?? null) !== existing) { - socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded') - return - } - await this.activateCurrent( - socket, - identity, - existing, - generation, - rebind, - assignmentEpoch, - appVersion, - connectionInclusionWatermark - ) - }) + const activation = previous + .catch(() => undefined) + .then(async () => { + clearTimeout(queueWaitTimer) + if (queueWaitExpired) return + if ((this.sessions.get(key) ?? null) !== existing) { + socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded') + return + } + await this.activateCurrent( + socket, + identity, + existing, + generation, + rebind, + assignmentEpoch, + appVersion, + connectionInclusionWatermark + ) + }) this.activationQueues.set(key, activation) const cleanup = (): void => { + release() if (this.activationQueues.get(key) === activation) this.activationQueues.delete(key) } void activation.then(cleanup, cleanup) @@ -881,7 +1084,11 @@ export class HostSessionRegistry { cellId: this.config.cellId, assignmentEpoch, generation, - connectionInclusionWatermark + connectionInclusionWatermark, + idleRegionalRehome: + this.hostCapabilities.get(socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME) ?? + false, + cellIncarnation: this.cellIncarnation } ) await this.assignments.markMigrationTargetRegistered( @@ -918,14 +1125,15 @@ export class HostSessionRegistry { const previousSocket = existing.socket if (existing.orphanTimer) clearTimeout(existing.orphanTimer) existing.orphanTimer = null + existing.authorityRevision += 1 + existing.assignmentEpoch = assignmentEpoch existing.socket = socket existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active' existing.appVersion = appVersion existing.leaseExpiresAt = this.controlLeaseExpiresAt() existing.lastPongAt = this.now() existing.pendingPingAt = null - existing.activityRenewalDueAt = - this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + existing.activityRenewalDueAt = this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs this.wireActiveControl(existing) this.sendHelloAck(existing) if (existing.regionalDrainAttemptId) this.reassertRegionalDrain(existing) @@ -980,6 +1188,7 @@ export class HostSessionRegistry { pendingPingAt: null, controlRttSamplesMs: [], controlRttLoggedAt: null, + authorityRevision: 0, activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs, activityRenewalAttempt: 0, activityRenewalCompletedAttempt: 0, @@ -1028,7 +1237,9 @@ export class HostSessionRegistry { ` splices=${session.closingCounts?.splices ?? session.activeSplices.size}` + ` pending=${session.closingCounts?.pending ?? session.pendingConns.size}` + ` code=${code} reason=${JSON.stringify(printableCloseReason(reason))}` + - (socketError === null ? '' : ` error=${JSON.stringify(printableCloseReason(socketError))}`) + (socketError === null + ? '' + : ` error=${JSON.stringify(printableCloseReason(socketError))}`) ) }) socket.on('message', (raw, isBinary) => { @@ -1077,6 +1288,19 @@ export class HostSessionRegistry { } private async acceptRefresh(session: HostSession, raw: RawData): Promise { + const release = this.beginIdleWork(session.relayHostId) + if (!release) { + session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director') + return + } + try { + await this.acceptRefreshUnfenced(session, raw) + } finally { + release() + } + } + + private async acceptRefreshUnfenced(session: HostSession, raw: RawData): Promise { const parsed = AuthRefreshSchema.safeParse(payload(raw, 'auth-refresh')) if (!parsed.success) return const refreshed = await this.verifyRelayToken(parsed.data.relayJwt) @@ -1107,6 +1331,16 @@ export class HostSessionRegistry { if (controlActivityId && now >= session.activityRenewalDueAt) { const attempt = ++session.activityRenewalAttempt const startedAt = now + const socket = session.socket + const authorityRevision = session.authorityRevision + const current = (): boolean => + this.sessions.get(key) === session && + session.state !== 'closed' && + session.socket === socket && + socket.readyState === socket.OPEN && + session.controlActivityId === controlActivityId && + session.authorityRevision === authorityRevision && + attempt > session.activityRenewalCompletedAttempt void this.assignments .renewControlActivity( { userId: session.identity.sub, relayHostId: session.relayHostId }, @@ -1117,11 +1351,16 @@ export class HostSessionRegistry { } ) .then(() => { - if (attempt <= session.activityRenewalCompletedAttempt) return + if (!current()) return session.activityRenewalCompletedAttempt = attempt session.activityRenewalDueAt = startedAt + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS }) .catch(async (error: unknown) => { + if (!current()) return + if (error instanceof Error && error.message === 'assignment_not_found') { + socket.close(RELAY_CLOSE_CODE.DRAINING, 'control assignment missing') + return + } if (error instanceof Error && error.message === 'activity_cell_not_authoritative') { // Completion fences a late drain-only heartbeat after all source work is gone. session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed') @@ -1144,8 +1383,22 @@ export class HostSessionRegistry { cellId: this.config.cellId } ) + if (!current()) { + // A replaced activity must not remain leased after its owner disappears. + if ( + !this.sessions.get(key) || + this.sessions.get(key)?.controlActivityId !== controlActivityId + ) { + this.releaseActivityBestEffort( + { userId: session.identity.sub, relayHostId: session.relayHostId }, + controlActivityId + ) + } + return + } this.observer.recordControlActivityRecovery?.(true) } catch (acquireError: unknown) { + if (!current()) return this.observer.recordControlActivityRecovery?.(false) if ( acquireError instanceof Error && @@ -1218,6 +1471,21 @@ export class HostSessionRegistry { private closeDrainedSession(session: HostSession): void { if (session.state === 'closed') return + const forcedConnections = session.activeConnIds.size + session.pendingConns.size + if (forcedConnections > 0) { + console.warn( + JSON.stringify({ + event: 'orca_relay_host_drain_forced_close', + ...this.logIdentity(), + ...this.sessionPlacementLogFields(session), + relayHostIdDigest: relayHostLogDigest(session.relayHostId), + reason: this.draining ? 'emergency' : 'regional-deadline', + forcedConnections, + splices: session.activeSplices.size, + pending: session.pendingConns.size + }) + ) + } if (session.heartbeatTimer) clearInterval(session.heartbeatTimer) if (session.orphanTimer) clearTimeout(session.orphanTimer) if (session.regionalDrainTimer) clearTimeout(session.regionalDrainTimer) @@ -1250,11 +1518,7 @@ export class HostSessionRegistry { session.pendingConns.clear() session.state = 'closed' if (session.socket) { - closeRelayWebSocket( - session.socket, - RELAY_CLOSE_CODE.DRAINING, - 'resolve configured director' - ) + closeRelayWebSocket(session.socket, RELAY_CLOSE_CODE.DRAINING, 'resolve configured director') } const key = this.key(session.identity.sub, session.relayHostId) if (this.sessions.get(key) === session) this.sessions.delete(key) @@ -1278,6 +1542,23 @@ export class HostSessionRegistry { session: HostSession, type: unknown, raw: RawData + ): Promise { + const release = this.beginIdleWork(session.relayHostId) + if (!release) { + session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director') + return + } + try { + await this.acceptControlCommandUnfenced(session, type, raw) + } finally { + release() + } + } + + private async acceptControlCommandUnfenced( + session: HostSession, + type: unknown, + raw: RawData ): Promise { if (typeof type !== 'string' || !session.socket) return try { @@ -1315,10 +1596,7 @@ export class HostSessionRegistry { } if (type === 'device-credential-install') { const request = DeviceCredentialInstallSchema.parse(payload(raw, type)) - if ( - session.state !== 'active' && - request.authorization.mode === 'authenticated-direct' - ) { + if (session.state !== 'active' && request.authorization.mode === 'authenticated-direct') { throw new Error('authorization_expired') } const installActivityId = `install:${request.reqId}` diff --git a/cloud/apps/relay/src/idle-regional-rehome-reconciliation.test.ts b/cloud/apps/relay/src/idle-regional-rehome-reconciliation.test.ts new file mode 100644 index 00000000000..6fc2017ab48 --- /dev/null +++ b/cloud/apps/relay/src/idle-regional-rehome-reconciliation.test.ts @@ -0,0 +1,103 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayDatabase } from './database.js' +import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js' + +const request = { + v: 1 as const, + attemptId: '33333333-3333-4333-8333-333333333333', + userId: 'idle-reconciliation-test', + relayHostId: 'abcdefghijklmnop', + sourceCellId: 'source', + sourceCellIncarnation: '11111111-1111-4111-8111-111111111111', + sourceAssignmentEpoch: 1, + sourceGeneration: 7, + targetCellId: 'target' +} +const databases: RelayDatabase[] = [] +afterEach(async () => { + vi.restoreAllMocks() + for (const database of databases.splice(0)) await database.close() +}) + +async function setup() { + const database = await openIdleRehomeTestDatabase() + databases.push(database) + const store = new RelayAssignmentStore(database, () => 100_000_000) + await store.reconcileCells([ + { id: 'source', url: 'https://source.example.test', capacityRequests: 100 }, + { id: 'target', url: 'https://target.example.test', capacityRequests: 100 } + ]) + await store.assign(request) + await store.activateControl(request, { + cellId: request.sourceCellId, + assignmentEpoch: request.sourceAssignmentEpoch, + generation: request.sourceGeneration, + cellIncarnation: request.sourceCellIncarnation + }) + return { database, store } +} + +describe('idle cutover durable reconciliation', () => { + it('only permits reopening when the exact source still owns the assignment', async () => { + const { store } = await setup() + expect(await store.reconcileIdleRegionalRehome(request)).toBe('not-committed') + await store.activateControl(request, { + cellId: request.sourceCellId, + assignmentEpoch: request.sourceAssignmentEpoch, + generation: request.sourceGeneration + 1, + cellIncarnation: request.sourceCellIncarnation + }) + expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale') + }) + + it('does not reopen an obsolete source after an assignment change', async () => { + const { store } = await setup() + await store.startEvacuation(request, request.targetCellId) + expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale') + }) + + it('propagates unavailable durable state instead of declaring rollback', async () => { + const { database, store } = await setup() + vi.spyOn(database, 'transaction').mockRejectedValue(new Error('database_unavailable')) + await expect(store.reconcileIdleRegionalRehome(request)).rejects.toThrow('database_unavailable') + }) + + it('waits for an outstanding assignment transaction before deciding authority', async () => { + const { database, store } = await setup() + let release!: () => void + let entered!: () => void + const locked = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + release = resolve + }) + const commit = database.transaction(async (transaction) => { + await transaction.queryLocked( + 'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?', + [request.userId, request.relayHostId] + ) + entered() + await gate + await transaction.query( + 'UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 1 WHERE user_id = ? AND relay_host_id = ?', + [request.userId, request.relayHostId] + ) + }) + await locked + let settled = false + const reconciliation = store.reconcileIdleRegionalRehome(request).finally(() => { + settled = true + }) + try { + await new Promise((resolve) => setImmediate(resolve)) + expect(settled).toBe(false) + } finally { + release() + await commit + await reconciliation + } + expect(await reconciliation).toBe('stale') + }) +}) diff --git a/cloud/apps/relay/src/idle-regional-rehome-selection.ts b/cloud/apps/relay/src/idle-regional-rehome-selection.ts new file mode 100644 index 00000000000..f8790459c0b --- /dev/null +++ b/cloud/apps/relay/src/idle-regional-rehome-selection.ts @@ -0,0 +1,117 @@ +import { createHash } from 'node:crypto' +import type { IdleRegionalRehomeRequest } from '@orca-cloud/relay-contract' +import type { RelayDatabase, SqlRow } from './database.js' + +export const IDLE_REHOME_PAGE_SIZE = 100 + +export async function selectIdleRegionalRehomes(input: { + database: RelayDatabase + now: number + heartbeatTtlMs: number + cohortPercent: number + offset: number + connectionHeadroom: Map + cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean +}): Promise> { + const [runtimes, safetyRows] = await Promise.all([ + input.database.query('SELECT * FROM relay_cell_runtime'), + input.database.query('SELECT * FROM relay_cell_rehome_safety') + ]) + const cleanCells = runtimes + .filter((runtime) => + input.cellIsClean( + safetyRows.find((safety) => safety.cell_id === runtime.cell_id), + runtime, + input.now + ) + ) + .map((runtime) => String(runtime.cell_id)) + const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false) + if (!cleanCells.length || !targetCells.length) return [] + const rows = await input.database.query( + `SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id, + a.assignment_epoch, host.generation, r.cell_incarnation, + s.cell_url, target.cell_id AS target_cell_id + FROM relay_region_rehome_control policy + JOIN relay_region_decisions d ON d.outcome = 'conclusive' + JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id + JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1 + JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id + JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general' + JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1 + JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation + JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id + AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch + AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1 + JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id + AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id + AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control' + JOIN relay_cell_regions tr ON tr.region = d.preferred_region + JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1 + JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general' + JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1 + JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation + WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ? + AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region + AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1 + AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms + AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at + AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ? + AND s.cell_id IN (${cleanCells.map(() => '?').join(',')}) + AND target.cell_id IN (${targetCells.map(() => '?').join(',')}) + -- Reserve the moving host's source activity plus its assignment on the target. + AND target.reserved_requests + 1 + ( + SELECT COALESCE(SUM(activity.request_units), 0) + FROM relay_assignment_activity_leases activity + WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id + AND activity.cell_id = a.cell_id + ) <= target.capacity_requests + AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3 + AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration + WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt + WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id + AND attempt.created_at > ? - policy.host_cooldown_ms) + ORDER BY a.user_id, a.relay_host_id, host.generation DESC, + (target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests, + target.cell_id + LIMIT ? OFFSET ?`, + [ + input.now, + input.now, + input.now, + input.cohortPercent, + input.now, + input.now - input.heartbeatTtlMs, + input.now - input.heartbeatTtlMs, + ...cleanCells, + ...targetCells, + input.now, + IDLE_REHOME_PAGE_SIZE, + input.offset + ] + ) + return rows.map((row) => { + const request = { + v: 1 as const, + userId: String(row.user_id), + relayHostId: String(row.relay_host_id), + sourceCellId: String(row.source_cell_id), + sourceCellIncarnation: String(row.cell_incarnation), + sourceAssignmentEpoch: Number(row.assignment_epoch), + sourceGeneration: Number(row.generation), + targetCellId: String(row.target_cell_id) + } + // UUIDv5 keeps retries on every director bound to the same source authority and target. + const digest = createHash('sha1') + .update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex')) + .update(JSON.stringify(request)) + .digest() + digest[6] = (digest[6]! & 0x0f) | 0x50 + digest[8] = (digest[8]! & 0x3f) | 0x80 + const hex = digest.subarray(0, 16).toString('hex') + const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` + return { ...request, attemptId, sourceCellUrl: String(row.cell_url) } + }) +} diff --git a/cloud/apps/relay/src/idle-regional-rehome-store.test.ts b/cloud/apps/relay/src/idle-regional-rehome-store.test.ts new file mode 100644 index 00000000000..5d0d3cff343 --- /dev/null +++ b/cloud/apps/relay/src/idle-regional-rehome-store.test.ts @@ -0,0 +1,350 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayDatabase, RelayLockOptions } from './database.js' +import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js' + +const identity = { userId: 'idle-store-test', relayHostId: 'abcdefghijklmnop' } +const incarnations = [ + '11111111-1111-4111-8111-111111111111', + '22222222-2222-4222-8222-222222222222' +] +const cells = [ + { + id: 'source', + url: 'https://source.example.test', + region: 'us-central1' as const, + capacityRequests: 100 + }, + { + id: 'target', + url: 'https://target.example.test', + region: 'asia-east2' as const, + capacityRequests: 100 + } +] +const databases: RelayDatabase[] = [] +afterEach(async () => { + vi.restoreAllMocks() + for (const database of databases.splice(0)) await database.close() +}) + +async function setup() { + const database = await openIdleRehomeTestDatabase() + databases.push(database) + let now = 100_000_000 + const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 }) + await store.inspectRegionalRehomeControl() + now += 86_400_000 + await store.applyRegionalRehomeControl({ + expectedGeneration: 0, + enabled: true, + notBefore: now, + ratePerMinute: 10, + preferenceMaxAgeMs: 86_400_000, + hostCooldownMs: 604_800_000, + drainGraceMs: 60_000 + }) + await store.reconcileCells(cells) + const safety = { + observedAt: now, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + for (const [index, cell] of cells.entries()) { + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + region: cell.region, + cellIncarnation: incarnations[index]!, + startedAt: now - 1_000, + ready: true, + observedRequests: 0 + }) + await store.recordCellRegionalRehomeStatus({ + cellId: cell.id, + cellIncarnation: incarnations[index]!, + regionalRehomeProtocol: 3, + safety + }) + } + const assignment = await store.assign(identity, undefined, 'us-central1') + await store.activateControl(identity, { + cellId: cells[0]!.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 7, + cellIncarnation: incarnations[0], + idleRegionalRehome: true + }) + const issued = await store.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + assignment.assignmentEpoch + ) + await store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.window!.generation, + assignmentEpoch: assignment.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 180, 'asia-east2': 40 } + }, + assignment.assignmentEpoch + ) + const request = { + v: 1 as const, + ...identity, + attemptId: '33333333-3333-4333-8333-333333333333', + sourceCellId: cells[0]!.id, + sourceCellIncarnation: incarnations[0]!, + sourceAssignmentEpoch: assignment.assignmentEpoch, + sourceGeneration: 7, + targetCellId: cells[1]!.id + } + return { store, database, safety, request } +} + +describe('constrained idle regional assignment transaction', () => { + it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => { + const { store, database, safety, request } = await setup() + // Model three source activity units and seven units already reserved at the target. + await database.query( + 'UPDATE relay_assignment_activity_leases SET request_units = 3 WHERE user_id = ? AND relay_host_id = ?', + [identity.userId, identity.relayHostId] + ) + await database.query("UPDATE relay_cells SET reserved_requests = 4 WHERE cell_id = 'source'") + await database.query( + "UPDATE relay_cells SET reserved_requests = 7, capacity_requests = ? WHERE cell_id = 'target'", + [capacity] + ) + const candidates = await store.selectIdleRegionalRehomeCandidates(safety) + expect(candidates).toHaveLength(capacity === 11 ? 1 : 0) + expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ + outcome: capacity === 11 ? 'committed' : 'deferred' + }) + const [target] = await database.query("SELECT reserved_requests FROM relay_cells WHERE cell_id = 'target'") + expect(Number(target!.reserved_requests)).toBe(capacity === 11 ? 11 : 7) + expect(await store.resolve(identity)).toMatchObject({ + cellId: capacity === 11 ? 'target' : 'source', + assignmentEpoch: capacity === 11 ? 2 : 1 + }) + }) + + it('progresses past a full page of busy candidates without writing eligibility state', async () => { + const { store, database, safety } = await setup() + for (const table of [ + 'relay_assignments', + 'relay_assignment_activity_leases', + 'relay_control_capabilities', + 'relay_region_decisions' + ]) { + const template = ( + await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [ + identity.userId, + identity.relayHostId + ]) + )[0]! + const columns = Object.keys(template) + for (let index = 0; index < 100; index++) { + const values = columns.map((column) => + column === 'user_id' || column === 'relay_host_id' ? '?' : column + ) + await database.query( + `INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table} + WHERE user_id = ? AND relay_host_id = ?`, + [ + `idle-store-test-${String(index).padStart(3, '0')}`, + `pagehost${String(index).padStart(8, '0')}`, + identity.userId, + identity.relayHostId + ] + ) + } + } + const first = await store.selectIdleRegionalRehomeCandidates(safety) + const next = await store.selectIdleRegionalRehomeCandidates(safety) + expect(first).toHaveLength(100) + expect(next).toHaveLength(1) + expect(next[0]!.relayHostId).toBe('pagehost00000099') + const restarted = new RelayAssignmentStore(database, () => safety.observedAt, { + regionalRehomeCohortPercent: 100 + }) + expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first) + expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([]) + const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions') + expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true) + }) + + it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))( + 'rechecks generation when replacement wins after the initial authority lookup', + async () => { + const { store, safety, request, database } = await setup() + const held = holdStatement(database, 'SELECT * FROM relay_region_rehome_control') + const commit = store.commitIdleRegionalRehome(request, safety) + await held.entered + try { + await store.activateControl(identity, { + cellId: 'source', + assignmentEpoch: 1, + generation: 8, + cellIncarnation: incarnations[0], + idleRegionalRehome: true + }) + } finally { + held.release() + } + expect(await commit).toEqual({ outcome: 'deferred' }) + expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale') + expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([]) + } + ) + + it('rejects source replacement when the cutover already holds assignment authority', async () => { + const { store, safety, request, database } = await setup() + const held = holdStatement(database, 'UPDATE relay_assignments SET cell_id') + const commit = store.commitIdleRegionalRehome(request, safety) + await held.entered + const replacement = store.activateControl(identity, { + cellId: 'source', + assignmentEpoch: 1, + generation: 8, + cellIncarnation: incarnations[0], + idleRegionalRehome: true + }) + const rejected = expect(replacement).rejects.toThrow('wrong_assignment') + held.release() + expect(await commit).toEqual({ outcome: 'committed' }) + await rejected + expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 }) + }) + + it('finds the committed attempt after its database reply is lost', async () => { + const { store, safety, request, database } = await setup() + const transaction = database.transaction.bind(database) + const intercepted = vi + .spyOn(database, 'transaction') + .mockImplementation(async (operation, options) => { + let changed = false + const result = await transaction( + async (tx) => + operation( + new Proxy(tx, { + get(target, key) { + if (key === 'query') + return async (sql: string, params?: unknown[]) => { + if (sql.includes('INSERT INTO relay_region_rehome_attempts')) changed = true + return target.query(sql, params) + } + const value = Reflect.get(target, key) + return typeof value === 'function' ? value.bind(target) : value + } + }) + ), + options + ) + if (changed) throw new Error('simulated_commit_reply_lost') + return result + }) + await expect(store.commitIdleRegionalRehome(request, safety)).rejects.toThrow( + 'simulated_commit_reply_lost' + ) + intercepted.mockRestore() + expect(await store.reconcileIdleRegionalRehome(request)).toBe('committed') + expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' }) + expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toHaveLength(1) + }) + + it('commits the requested move once and records its outcome without source retention', async () => { + const { store, database, safety, request } = await setup() + expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' }) + expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' }) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 }) + const attempts = await database.query('SELECT * FROM relay_region_rehome_attempts') + expect(attempts).toHaveLength(1) + expect(attempts[0]!.attempt_id).toBe(request.attemptId) + expect(Number(attempts[0]!.source_generation)).toBe(7) + }) + + it('rejects a replaced control and never substitutes a different target', async () => { + const { store, safety, request } = await setup() + expect( + await store.commitIdleRegionalRehome({ ...request, targetCellId: 'missing' }, safety) + ).toEqual({ outcome: 'deferred' }) + await store.activateControl(identity, { + cellId: 'source', + assignmentEpoch: 1, + generation: 8, + cellIncarnation: incarnations[0], + idleRegionalRehome: true + }) + expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'stale' }) + expect(await store.resolve(identity)).toMatchObject({ cellId: 'source', assignmentEpoch: 1 }) + }) + + it('does not commit without process safety or cohort authorization', async () => { + const { store, safety, request, database } = await setup() + expect(await store.commitIdleRegionalRehome(request)).toEqual({ outcome: 'deferred' }) + expect(await store.commitIdleRegionalRehome(request, safety, 0)).toEqual({ + outcome: 'deferred' + }) + expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([]) + }) + + it('selects read-only with stable identity and the control generation, not probe generation', async () => { + const { store, safety, database } = await setup() + const before = await database.query('SELECT * FROM relay_assignments') + const candidates = await store.selectIdleRegionalRehomeCandidates(safety) + expect(candidates).toHaveLength(1) + expect(candidates[0]).toMatchObject({ + sourceGeneration: 7, + sourceCellId: 'source', + targetCellId: 'target' + }) + expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates) + expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before) + expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([]) + }) +}) + +function holdStatement(database: RelayDatabase, fragment: string) { + let entered!: () => void + let release!: () => void + const arrival = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + release = resolve + }) + let held = false + const transaction = database.transaction.bind(database) + vi.spyOn(database, 'transaction').mockImplementation((operation, options) => + transaction(async (tx) => { + return operation( + new Proxy(tx, { + get(target, key) { + if (key === 'query' || key === 'queryLocked') + return async (sql: string, params?: unknown[], lockOptions?: RelayLockOptions) => { + if (!held && sql.includes(fragment)) { + held = true + entered() + await gate + } + return key === 'queryLocked' + ? target.queryLocked(sql, params, lockOptions) + : target.query(sql, params) + } + const value = Reflect.get(target, key) + return typeof value === 'function' ? value.bind(target) : value + } + }) + ) + }, options) + ) + return { entered: arrival, release } +} diff --git a/cloud/apps/relay/src/idle-regional-rehome-test-database.ts b/cloud/apps/relay/src/idle-regional-rehome-test-database.ts new file mode 100644 index 00000000000..a541e7d1126 --- /dev/null +++ b/cloud/apps/relay/src/idle-regional-rehome-test-database.ts @@ -0,0 +1,40 @@ +import { randomUUID } from 'node:crypto' +import pg from 'pg' +import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js' + +export async function openIdleRehomeTestDatabase(): Promise { + const configured = process.env.ORCA_IDLE_REHOME_POSTGRES_URL + if (!configured) return openInMemoryRelayDatabase() + const url = new URL(configured) + if (url.port !== '55440' || !['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) { + throw new Error('idle_rehome_tests_require_local_postgres_55440') + } + const schema = `idle_rehome_${randomUUID().replaceAll('-', '')}` + const admin = new pg.Client({ connectionString: configured }) + await admin.connect() + try { + await admin.query(`CREATE SCHEMA ${schema}`) + url.searchParams.set('options', `-c search_path=${schema}`) + const database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' }) + const close = database.close.bind(database) + database.close = async () => { + try { + await close() + } finally { + try { + await admin.query(`DROP SCHEMA ${schema} CASCADE`) + } finally { + await admin.end() + } + } + } + return database + } catch (error) { + try { + await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) + } finally { + await admin.end() + } + throw error + } +} diff --git a/cloud/apps/relay/src/idle-regional-rehome-worker.test.ts b/cloud/apps/relay/src/idle-regional-rehome-worker.test.ts new file mode 100644 index 00000000000..9b3f6d7fa07 --- /dev/null +++ b/cloud/apps/relay/src/idle-regional-rehome-worker.test.ts @@ -0,0 +1,105 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import { startRegionalRehomeWorker } from './regional-rehome-worker.js' + +const candidate = { + v: 1, + attemptId: '11111111-1111-4111-8111-111111111111', + userId: 'private-user', + relayHostId: 'abcdefghijklmnop', + sourceCellId: 'source', + sourceCellUrl: 'https://source.example.test', + sourceCellIncarnation: '22222222-2222-4222-8222-222222222222', + sourceAssignmentEpoch: 7, + sourceGeneration: 3, + targetCellId: 'target' +} +const config = { + role: 'director', + regionCorrectionCohortPercent: 100, + rehomeAudience: 'https://relay.example.test/v1/admin/host-drain', + rehomeDirectorServiceAccount: 'director@example.test' +} as RelayConfig + +function setup(fetch: typeof globalThis.fetch) { + const selectIdleRegionalRehomeCandidates = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValue([candidate]) + const claimRegionalRehome = vi.fn() + const recordRegionalRehomeDispatchFailure = vi.fn() + const worker = startRegionalRehomeWorker( + config, + { + selectIdleRegionalRehomeCandidates, + claimRegionalRehome, + recordRegionalRehomeDispatchFailure + } as unknown as RelayAssignmentStore, + { + safetySnapshot: () => ({ observedAt: 100 }) as never, + intervalMs: 60_000, + identityToken: async () => 'private-token', + fetch + } + )! + return { + worker, + selectIdleRegionalRehomeCandidates, + claimRegionalRehome, + recordRegionalRehomeDispatchFailure + } +} + +describe('idle regional worker dispatch', () => { + afterEach(() => vi.restoreAllMocks()) + it('sends an idle request without claiming an assignment first', async () => { + const fetch = vi.fn(async () => + Response.json({ v: 1, outcome: 'committed' }) + ) + const c = setup(fetch) + await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()) + await c.worker.run() + c.worker.stop() + expect(c.claimRegionalRehome).not.toHaveBeenCalled() + expect(fetch).toHaveBeenCalledOnce() + const [url, init] = fetch.mock.calls[0]! + expect(String(url)).toBe('https://source.example.test/v1/admin/host-idle-rehome') + const { sourceCellUrl: _, ...request } = candidate + expect(JSON.parse(String(init?.body))).toEqual({ + ...request, + cohortPercent: 100, + directorSafety: { observedAt: 100 } + }) + }) + it('progresses past busy hosts without charging a dispatch failure', async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(Response.json({ v: 1, outcome: 'busy' })) + .mockResolvedValueOnce(Response.json({ v: 1, outcome: 'committed' })) + const c = setup(fetch) + await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()) + c.selectIdleRegionalRehomeCandidates.mockResolvedValue([ + candidate, + { + ...candidate, + relayHostId: 'ponmlkjihgfedcba', + attemptId: '33333333-3333-4333-8333-333333333333' + } + ]) + await c.worker.run() + c.worker.stop() + expect(fetch).toHaveBeenCalledTimes(2) + expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled() + }) + it('does not charge a lost response as a claimed migration failure', async () => { + const fetch = vi.fn(async () => { + throw new Error('response lost') + }) + const c = setup(fetch) + await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()) + await c.worker.run() + c.worker.stop() + expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled() + }) +}) diff --git a/cloud/apps/relay/src/index.ts b/cloud/apps/relay/src/index.ts index 541884362c2..8e7b6a56941 100644 --- a/cloud/apps/relay/src/index.ts +++ b/cloud/apps/relay/src/index.ts @@ -2,6 +2,7 @@ import { formatAssignmentInventorySnapshot, readAssignmentInventorySnapshot } from './assignment-inventory-snapshot.js' +import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js' import { RelayAssignmentStore } from './assignment-store.js' import { loadRelayConfig } from './config.js' import { startCellHeartbeat } from './cell-heartbeat-client.js' @@ -71,6 +72,13 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role) void runRelayBackgroundOperation(async () => { const inventory = await readRegisteredMigrationInventory(database, Date.now()) for (const line of formatRegisteredMigrationInventory(inventory)) console.warn(line) + console.log( + JSON.stringify({ + event: 'orca_relay_region_correction_outcomes', + observedAt: Date.now(), + outcomes: await readRegionCorrectionOutcomes(database, Date.now()) + }) + ) }, '[orca-relay] migration inventory failed') }, 5 * 60_000) : null diff --git a/cloud/apps/relay/src/region-correction-outcomes.ts b/cloud/apps/relay/src/region-correction-outcomes.ts new file mode 100644 index 00000000000..d3a6f8d3be3 --- /dev/null +++ b/cloud/apps/relay/src/region-correction-outcomes.ts @@ -0,0 +1,29 @@ +import type { RelayDatabase } from './database.js' + +export async function readRegionCorrectionOutcomes(database: RelayDatabase, now: number) { + const rows = await database.query( + `SELECT attempt.source_cell_id, attempt.target_cell_id, + CASE WHEN attempt.aborted_at IS NOT NULL THEN 'aborted' + WHEN attempt.completed_at IS NOT NULL THEN 'completed' + WHEN migration.target_registered_at IS NOT NULL THEN 'registered' ELSE 'registering' END AS state, + COUNT(*) AS count, + COALESCE(MAX(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + THEN ? - attempt.created_at ELSE 0 END), 0) AS oldest_open_ms, + COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL + THEN migration.target_reserved_units ELSE 0 END), 0) AS target_reserved_units + FROM relay_region_rehome_attempts attempt + JOIN relay_assignment_migrations migration ON migration.user_id = attempt.user_id + AND migration.relay_host_id = attempt.relay_host_id AND migration.assignment_epoch = attempt.assignment_epoch + GROUP BY attempt.source_cell_id, attempt.target_cell_id, state + ORDER BY attempt.source_cell_id, attempt.target_cell_id, state`, + [now] + ) + return rows.map((row) => ({ + sourceCellId: String(row.source_cell_id), + targetCellId: String(row.target_cell_id), + state: String(row.state), + count: Number(row.count), + oldestOpenMs: Number(row.oldest_open_ms), + targetReservedUnits: Number(row.target_reserved_units) + })) +} diff --git a/cloud/apps/relay/src/region-correction-preview.ts b/cloud/apps/relay/src/region-correction-preview.ts new file mode 100644 index 00000000000..c120dbb2272 --- /dev/null +++ b/cloud/apps/relay/src/region-correction-preview.ts @@ -0,0 +1,157 @@ +import type { RelayDatabase, SqlRow } from './database.js' +import { REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS } from './database.js' +import { + REGIONAL_REHOME_CONCURRENT_LIMIT, + REGION_DECISION_TTL_MS +} from './region-correction-state.js' + +export type RegionCorrectionPreview = { + observedAt: number + newClaimsEnabled: boolean + cohortPercent: number + openMigrations: number + availableMigrationSlots: number + globalSafetyFailure: string | null + counts: Record +} + +export async function previewRegionalRehomeEligibility(input: { + database: RelayDatabase + now: number + heartbeatTtlMs: number + cohortPercent: number + globalSafetyFailure: string | null + connectionHeadroom: ReadonlyMap + cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean +}): Promise { + const { database, now } = input + const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations] = + await Promise.all([ + database.query( + `SELECT assignment.cell_id, assignment.assignment_epoch, + decision.generation, decision.assignment_epoch AS decision_epoch, decision.expires_at, + decision.incumbent_region, decision.preferred_region, decision.outcome, decision.policy_version, + decision.observed_at, decision.cohort_bucket, + (SELECT MAX(attempt.created_at) FROM relay_region_rehome_attempts attempt + WHERE attempt.user_id = assignment.user_id AND attempt.relay_host_id = assignment.relay_host_id) AS last_attempt_at, + (SELECT COUNT(*) FROM relay_assignment_migrations migration + WHERE migration.user_id = assignment.user_id AND migration.relay_host_id = assignment.relay_host_id + AND migration.completed_at IS NULL AND migration.aborted_at IS NULL) AS open_migrations, + (SELECT COALESCE(SUM(lease.request_units),0) FROM relay_assignment_activity_leases lease + WHERE lease.user_id = assignment.user_id AND lease.relay_host_id = assignment.relay_host_id + AND lease.cell_id = assignment.cell_id) AS source_units, + (SELECT COUNT(*) FROM relay_control_capabilities host_capability + JOIN relay_assignment_activity_leases lease ON lease.user_id = host_capability.user_id + AND lease.relay_host_id = host_capability.relay_host_id AND lease.activity_id = host_capability.activity_id + JOIN relay_cell_runtime runtime ON runtime.cell_id = host_capability.cell_id + AND runtime.cell_incarnation = host_capability.cell_incarnation + WHERE host_capability.user_id = assignment.user_id AND host_capability.relay_host_id = assignment.relay_host_id + AND host_capability.cell_id = assignment.cell_id AND host_capability.assignment_epoch = assignment.assignment_epoch + AND host_capability.idle_regional_rehome = 1 AND lease.activity_kind = 'control' + AND lease.activity_id NOT LIKE 'control-pending:%' AND lease.expires_at > ? + AND lease.updated_at >= runtime.started_at) AS capable_controls + FROM relay_assignments assignment LEFT JOIN relay_region_decisions decision + ON decision.user_id = assignment.user_id AND decision.relay_host_id = assignment.relay_host_id`, + [now] + ), + database.query(`SELECT cell.*, region.region, admission.admission_state FROM relay_cells cell + LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id + LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id`), + database.query(`SELECT * FROM relay_cell_runtime`), + database.query(`SELECT * FROM relay_cell_capabilities`), + database.query(`SELECT * FROM relay_cell_rehome_safety`), + database.query(`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`), + database.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE completed_at IS NULL AND aborted_at IS NULL` + ) + ]) + const byCell = (rows: SqlRow[]) => new Map(rows.map((row) => [String(row.cell_id), row])) + const runtimes = byCell(runtimeRows) + const capabilities = byCell(capabilityRows) + const safety = byCell(safetyRows) + const inventory = byCell(cells) + const control = controls[0] + const cooldown = Number(control?.host_cooldown_ms ?? REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS) + const maxAge = Number(control?.preference_max_age_ms ?? REGION_DECISION_TTL_MS) + const openMigrations = Number(migrations[0]?.count ?? 0) + const counts: Record = {} + const count = (reason: string) => { + counts[reason] = (counts[reason] ?? 0) + 1 + } + const available = (cell: SqlRow): boolean => { + const id = String(cell.cell_id) + const runtime = runtimes.get(id) + const capability = capabilities.get(id) + return ( + Number(cell.enabled) === 1 && + cell.admission_state === 'general' && + cell.region != null && + runtime !== undefined && + Number(runtime.ready) === 1 && + Number(runtime.last_heartbeat_at) > now - input.heartbeatTtlMs && + capability !== undefined && + capability.cell_incarnation === runtime.cell_incarnation && + Number(capability.regional_rehome_protocol) >= 3 + ) + } + for (const host of hosts) { + let reason: string | null = null + const source = inventory.get(String(host.cell_id)) + if (host.generation == null) reason = 'no-verified-decision' + else if (Number(host.expires_at) <= now || Number(host.observed_at) < now - maxAge) + reason = 'expired' + else if ( + Number(host.decision_epoch) !== Number(host.assignment_epoch) || + host.incumbent_region !== source?.region + ) + reason = 'basis-changed' + else if ( + host.outcome !== 'conclusive' || + Number(host.policy_version) !== 1 || + host.preferred_region == null + ) + reason = 'inconclusive-or-insufficient-improvement' + else if (Number(host.cohort_bucket) >= input.cohortPercent) reason = 'outside-cohort' + else if (Number(host.open_migrations) > 0) reason = 'migration-open' + else if (host.last_attempt_at != null && Number(host.last_attempt_at) > now - cooldown) + reason = 'host-cooldown' + else if (!source || !available(source)) reason = 'source-ineligible' + else if (Number(host.capable_controls) === 0) reason = 'source-control-unsupported-or-inactive' + else if ( + !input.cellIsClean(safety.get(String(host.cell_id)), runtimes.get(String(host.cell_id))!, now) + ) + reason = 'source-unclean' + if (reason) { + count(reason) + continue + } + const targets = cells.filter( + (cell) => + cell.cell_id !== host.cell_id && cell.region === host.preferred_region && available(cell) + ) + const clean = targets.filter((cell) => + input.cellIsClean(safety.get(String(cell.cell_id)), runtimes.get(String(cell.cell_id))!, now) + ) + const capacity = clean.filter( + (cell) => + input.connectionHeadroom.get(String(cell.cell_id)) !== false && + Number(cell.reserved_requests) + Number(host.source_units) + 1 <= + Number(cell.capacity_requests) + ) + if (targets.length === 0) count('no-eligible-target') + else if (clean.length === 0) count('target-unclean') + else if (capacity.length === 0) count('no-target-headroom') + else if (input.globalSafetyFailure) count('global-safety-blocked') + else if (openMigrations >= REGIONAL_REHOME_CONCURRENT_LIMIT) count('concurrent-migration-cap') + else count(`eligible:${host.incumbent_region}-to-${host.preferred_region}`) + } + return { + observedAt: now, + newClaimsEnabled: Number(control?.enabled ?? 0) === 1, + cohortPercent: input.cohortPercent, + openMigrations, + availableMigrationSlots: Math.max(0, REGIONAL_REHOME_CONCURRENT_LIMIT - openMigrations), + globalSafetyFailure: input.globalSafetyFailure, + counts + } +} diff --git a/cloud/apps/relay/src/region-correction-restart.test.ts b/cloud/apps/relay/src/region-correction-restart.test.ts new file mode 100644 index 00000000000..02f315ab218 --- /dev/null +++ b/cloud/apps/relay/src/region-correction-restart.test.ts @@ -0,0 +1,119 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openRelayDatabase, type RelayDatabase } from './database.js' + +const identity = { userId: 'restart-test-user', relayHostId: 'abcdefghijklmnop' } +const paths: string[] = [] +const databases = new Set() +afterEach(async () => { + for (const database of databases) await database.close() + databases.clear() + for (const path of paths.splice(0)) await rm(path, { recursive: true, force: true }) +}) + +async function setup() { + const dataDir = await mkdtemp(join(tmpdir(), 'relay-region-restart-')) + paths.push(dataDir) + let now = 1_000_000_000 + const open = async () => { + const database = await openRelayDatabase({ dataDir }) + databases.add(database) + return { database, store: new RelayAssignmentStore(database, () => now) } + } + const first = await open() + const cell = { + id: 'restart-us', + url: 'https://restart-us.example.test', + region: 'us-central1' as const, + capacityRequests: 100 + } + await first.store.reconcileCells([cell]) + await first.store.setCellEnabled(cell.id, true) + await first.store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: '11111111-1111-4111-8111-111111111111', + region: cell.region, + startedAt: now - 1_000, + ready: true, + observedRequests: 0 + }) + const assignment = await first.store.assign(identity) + const issue = (store: RelayAssignmentStore) => + store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch) + const window = (await issue(first.store)).window! + const report = { + v: 1 as const, + action: 'report' as const, + generation: window.generation, + assignmentEpoch: window.assignmentEpoch, + policyVersion: 1 as const, + outcome: 'conclusive' as const, + measurements: { 'us-central1': 200, 'asia-east2': 40 } + } + const restart = async () => { + await first.database.close() + databases.delete(first.database) + return open() + } + return { + ...first, window, report, issue, restart, + setNow: (value: number) => { now = value } + } +} + +describe('persisted region decisions across director restart', () => { + it('keeps tombstones and fixed expiry, then invalidates the prior generation after restart', async () => { + const context = await setup() + const epoch = context.window.assignmentEpoch + await context.store.exchangeRegionCorrection(identity, { + v: 1, action: 'report', generation: context.window.generation, + assignmentEpoch: epoch, policyVersion: 1, outcome: 'inconclusive', reason: 'jitter' + }, epoch) + const restarted = await context.restart() + expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch)) + .toMatchObject({ reportStatus: 'duplicate' }) + const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]! + expect(row.outcome).toBe('inconclusive') + expect(Number(row.expires_at)).toBe(context.window.expiresAt) + const successor = (await context.issue(restarted.store)).window! + expect(successor.generation).toBe(context.window.generation + 1) + expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch)) + .toMatchObject({ reportStatus: 'stale' }) + }) + + it('uses server expiry after a restart regardless of an old client report', async () => { + const context = await setup() + context.setNow(context.window.expiresAt) + const restarted = await context.restart() + expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch)) + .toMatchObject({ reportStatus: 'expired' }) + expect(await restarted.store.previewRegionCorrection()).toEqual({ expired: 1 }) + }) + + it('does not interpret a persisted future-policy window using the old policy after rollback', async () => { + const context = await setup() + await context.database.query('UPDATE relay_region_decisions SET policy_version = 2') + const restarted = await context.restart() + expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch)) + .toMatchObject({ reportStatus: 'stale' }) + const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]! + expect(row.outcome).toBe('pending') + expect(row.preferred_region).toBeNull() + expect(row.report_json).toBeNull() + }) + + it('keeps generation ordering when the server clock moves backwards across restart', async () => { + const context = await setup() + context.setNow(1_000_000_000 - 60_000) + const restarted = await context.restart() + const successor = (await context.issue(restarted.store)).window! + expect(successor.generation).toBe(context.window.generation + 1) + expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000) + expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch)) + .toMatchObject({ reportStatus: 'stale' }) + }) +}) diff --git a/cloud/apps/relay/src/region-correction-state.ts b/cloud/apps/relay/src/region-correction-state.ts new file mode 100644 index 00000000000..a1b15520b1c --- /dev/null +++ b/cloud/apps/relay/src/region-correction-state.ts @@ -0,0 +1,158 @@ +import { relayHostLogDigest } from './relay-host-log-digest.js' +import { createHash } from 'node:crypto' +import type { + RegionCorrectionRequest, + RegionCorrectionResponse, + RelayRegion +} from '@orca-cloud/relay-contract' +import type { RelayDatabase } from './database.js' + +type Identity = { userId: string; relayHostId: string } +export const REGION_DECISION_TTL_MS = 24 * 60 * 60_000 +export const REGIONAL_REHOME_CONCURRENT_LIMIT = 8 + +export async function exchangeRegionCorrection( + database: RelayDatabase, + identity: Identity, + request: RegionCorrectionRequest, + assignmentEpoch: number, + now: number +): Promise { + const result: RegionCorrectionResponse = await database.transaction(async (transaction) => { + const assignment = ( + await transaction.queryLocked( + `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0] + const region = + assignment && + ( + await transaction.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [ + assignment.cell_id + ]) + )[0] + if (!assignment || !region || Number(assignment.assignment_epoch) !== assignmentEpoch) { + return { v: 1, reportStatus: 'basis-changed' } + } + const prior = ( + await transaction.queryLocked( + `SELECT * FROM relay_region_decisions WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0] + if (request.action === 'issue-window') { + const generation = Number(prior?.generation ?? 0) + 1 + if (!Number.isSafeInteger(generation)) throw new Error('region_generation_exhausted') + const expiresAt = now + REGION_DECISION_TTL_MS + const cohortBucket = + createHash('sha256') + .update(JSON.stringify([identity.userId, identity.relayHostId])) + .digest() + .readUInt32BE(0) % 100 + await transaction.query( + `INSERT INTO relay_region_decisions + (user_id, relay_host_id, generation, expires_at, assignment_epoch, incumbent_region, + policy_version, outcome, preferred_region, observed_at, report_json, cohort_bucket) + VALUES (?, ?, ?, ?, ?, ?, 1, 'pending', NULL, ?, NULL, ?) + ON CONFLICT (user_id, relay_host_id) DO UPDATE SET + generation = excluded.generation, expires_at = excluded.expires_at, + assignment_epoch = excluded.assignment_epoch, incumbent_region = excluded.incumbent_region, + policy_version = 1, outcome = 'pending', preferred_region = NULL, + observed_at = excluded.observed_at, report_json = NULL, cohort_bucket = excluded.cohort_bucket`, + [ + identity.userId, + identity.relayHostId, + generation, + expiresAt, + assignmentEpoch, + region.region, + now, + cohortBucket + ] + ) + return { + v: 1, + window: { + generation, + expiresAt, + assignmentEpoch, + incumbentRegion: region.region as RelayRegion, + policyVersion: 1 + } + } + } + if (!prior || Number(prior.generation) !== request.generation) + return { v: 1, reportStatus: 'stale' } + if (Number(prior.policy_version) !== request.policyVersion) + return { v: 1, reportStatus: 'stale' } + if (Number(prior.expires_at) <= now) return { v: 1, reportStatus: 'expired' } + if ( + request.assignmentEpoch !== assignmentEpoch || + Number(prior.assignment_epoch) !== assignmentEpoch || + prior.incumbent_region !== region.region + ) { + return { v: 1, reportStatus: 'basis-changed' } + } + // The first report wins, including an inconclusive tombstone. + if (prior.outcome !== 'pending') return { v: 1, reportStatus: 'duplicate' } + let preferredRegion: RelayRegion | null = null + if (request.outcome === 'conclusive') { + const incumbent = request.measurements[region.region as RelayRegion] + const target: RelayRegion = region.region === 'us-central1' ? 'asia-east2' : 'us-central1' + const targetRtt = request.measurements[target] + if (incumbent - targetRtt >= 25 && targetRtt <= incumbent * 0.8) preferredRegion = target + } + await transaction.query( + `UPDATE relay_region_decisions SET outcome = ?, preferred_region = ?, report_json = ? + WHERE user_id = ? AND relay_host_id = ? AND generation = ?`, + [ + request.outcome, + preferredRegion, + JSON.stringify(request), + identity.userId, + identity.relayHostId, + request.generation + ] + ) + return { v: 1, reportStatus: 'accepted' } + }) + if (request.action === 'report' && result.reportStatus === 'accepted') { + const digest = relayHostLogDigest(identity.relayHostId) + // Stable sampling includes unchanged hosts for before/after comparisons. + if (Number.parseInt(digest.slice(0, 8), 16) % 10 === 0) { + console.log( + JSON.stringify({ + event: 'orca_relay_region_comparison', + relayHostIdDigest: digest, + assignmentEpoch, + generation: request.generation, + policyVersion: request.policyVersion, + outcome: request.outcome, + ...(request.outcome === 'conclusive' ? { measurements: request.measurements } : {}) + }) + ) + } + } + return result +} + +export async function previewRegionCorrection( + database: RelayDatabase, + now: number +): Promise> { + const rows = await database.query( + `SELECT CASE WHEN decision.expires_at <= ? THEN 'expired' + WHEN decision.assignment_epoch <> assignment.assignment_epoch THEN 'basis-changed' + WHEN decision.outcome = 'pending' THEN 'pending' + WHEN decision.preferred_region IS NULL THEN 'ineligible' + ELSE decision.incumbent_region || '-to-' || decision.preferred_region END AS reason, + COUNT(*) AS count + FROM relay_region_decisions decision + JOIN relay_assignments assignment ON assignment.user_id = decision.user_id + AND assignment.relay_host_id = decision.relay_host_id + GROUP BY reason`, + [now] + ) + return Object.fromEntries(rows.map((row) => [String(row.reason), Number(row.count)])) +} diff --git a/cloud/apps/relay/src/region-correction-store.test.ts b/cloud/apps/relay/src/region-correction-store.test.ts new file mode 100644 index 00000000000..11af7b3c5bd --- /dev/null +++ b/cloud/apps/relay/src/region-correction-store.test.ts @@ -0,0 +1,359 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js' + +const identity = { userId: 'region-correction-test-user', relayHostId: 'abcdefghijklmnop' } +const cells = [ + { + id: 'decision-us', + url: 'https://decision-us.example.test', + region: 'us-central1' as const, + capacityRequests: 100 + }, + { + id: 'decision-asia', + url: 'https://decision-asia.example.test', + region: 'asia-east2' as const, + capacityRequests: 100 + } +] +const incarnations = [ + '11111111-1111-4111-8111-111111111111', + '22222222-2222-4222-8222-222222222222' +] +const opened: RelayDatabase[] = [] +afterEach(async () => { + for (const database of opened.splice(0)) { + if (database.dialect === 'postgres') await cleanupPostgres(database) + await database.close() + } +}) + +async function cleanupPostgres(database: RelayDatabase) { + for (const table of [ + 'relay_control_connection_reservations', + 'relay_region_decisions', + 'relay_control_capabilities', + 'relay_assignment_activity_leases', + 'relay_assignment_migrations', + 'relay_assignment_migration_incarnations', + 'relay_assignment_region_preferences', + 'relay_region_rehome_attempts', + 'relay_assignments' + ]) { + await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId]) + } + for (const table of [ + 'relay_cell_rehome_safety', + 'relay_cell_capabilities', + 'relay_cell_connection_snapshots', + 'relay_cell_connection_runtime', + 'relay_cell_runtime', + 'relay_cell_connection_limits', + 'relay_cell_admission', + 'relay_cell_regions', + 'relay_cells' + ]) { + await database.query( + `DELETE FROM ${table} WHERE cell_id IN (?, ?)`, + cells.map((cell) => cell.id) + ) + } +} + +async function setup() { + const database = + process.env.ORCA_REGION_CORRECTION_POSTGRES === '1' + ? await openRelayDatabase({ + databaseUrl: requiredPostgresUrl(), + dataDir: '/tmp/orca-region-correction-unused' + }) + : await openInMemoryRelayDatabase() + opened.push(database) + if (database.dialect === 'postgres') await cleanupPostgres(database) + let clock = 100_000_000 + const store = new RelayAssignmentStore(database, () => clock, { + regionalRehomeCohortPercent: 100 + }) + await store.reconcileCells(cells) + for (const cell of cells) await store.setCellEnabled(cell.id, true) + for (const [index, cell] of cells.entries()) { + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + region: cell.region, + cellIncarnation: incarnations[index]!, + startedAt: clock - 1_000, + ready: true, + observedRequests: 0 + }) + } + const assignment = await store.assign(identity, undefined, 'us-central1') + const activityId = await store.activateControl(identity, { + cellId: cells[0]!.id, + assignmentEpoch: assignment.assignmentEpoch, + generation: 7, + cellIncarnation: incarnations[0], + idleRegionalRehome: true + }) + return { + database, + store, + assignment, + activityId, + now: () => clock, + advance: (ms: number) => { + clock += ms + } + } +} + +function requiredPostgresUrl(): string { + const url = process.env.ORCA_RELAY_TEST_POSTGRES_URL + if (!url || new URL(url).port !== '55440') + throw new Error('PostgreSQL tests require configured port 55440') + return url +} + +async function window(context: Awaited>) { + const result = await context.store.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + context.assignment.assignmentEpoch + ) + return result.window! +} + +async function regionalMigration(context: Awaited>) { + const migration = await context.store.startEvacuation(identity, cells[1]!.id) + const attemptId = '33333333-3333-4333-8333-333333333333' + await context.database.query( + `INSERT INTO relay_region_rehome_attempts + (attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation, + target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at) + VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`, + [ + attemptId, + identity.userId, + identity.relayHostId, + cells[0]!.id, + incarnations[0], + cells[1]!.id, + incarnations[1], + migration.previousEpoch, + migration.assignmentEpoch, + context.now(), + context.now() + ] + ) + return { migration } +} + +describe('ordered region decisions and migration outcomes', () => { + it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => { + const context = await setup() + const { migration } = await regionalMigration(context) + context.advance(1_000) + const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts') + const outcomes = await context.store.regionCorrectionOutcomes() + expect(outcomes).toEqual([ + expect.objectContaining({ + sourceCellId: cells[0]!.id, + targetCellId: cells[1]!.id, + state: 'registering', + count: 1, + oldestOpenMs: 1_000 + }) + ]) + expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0) + expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId) + expect(JSON.stringify(outcomes)).not.toContain(identity.userId) + expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual( + before + ) + await context.store.activateControl(identity, { + cellId: cells[1]!.id, + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: cells[1]!.id, + assignmentEpoch: migration.assignmentEpoch + }) + expect(await context.store.regionCorrectionOutcomes()).toEqual([ + expect.objectContaining({ state: 'registered' }) + ]) + await context.store.releaseActivity(identity, context.activityId) + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + expect(await context.store.regionCorrectionOutcomes()).toEqual([ + expect.objectContaining({ + state: 'completed', + targetReservedUnits: 0, + oldestOpenMs: 0 + }) + ]) + }) + + it('supersedes prior windows and keeps an inconclusive tombstone immutable', async () => { + const context = await setup() + const first = await window(context) + const second = await window(context) + expect(second.generation).toBe(first.generation + 1) + const report = { + v: 1 as const, + action: 'report' as const, + assignmentEpoch: first.assignmentEpoch, + policyVersion: 1 as const, + outcome: 'conclusive' as const, + measurements: { 'us-central1': 200, 'asia-east2': 40 } + } + expect( + await context.store.exchangeRegionCorrection( + identity, + { ...report, generation: first.generation }, + first.assignmentEpoch + ) + ).toMatchObject({ reportStatus: 'stale' }) + expect( + await context.store.exchangeRegionCorrection( + identity, + { ...report, generation: second.generation, outcome: 'inconclusive', reason: 'jitter' }, + second.assignmentEpoch + ) + ).toMatchObject({ reportStatus: 'accepted' }) + expect( + await context.store.exchangeRegionCorrection( + identity, + { ...report, generation: second.generation }, + second.assignmentEpoch + ) + ).toMatchObject({ reportStatus: 'duplicate' }) + expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 }) + }) + + it('previews the uncapped fleet without writes, claims, or locked reads', async () => { + const context = await setup() + const query = context.database.query.bind(context.database) + const transaction = context.database.transaction.bind(context.database) + const queryLocked = context.database.queryLocked.bind(context.database) + context.database.query = async (sql, params) => { + expect(sql.trim()).toMatch(/^(SELECT|WITH)/i) + return query(sql, params) + } + context.database.transaction = async () => { + throw new Error('preview_must_not_open_mutating_transaction') + } + context.database.queryLocked = async () => { + throw new Error('preview_must_not_lock') + } + try { + const preview = await context.store.previewRegionalRehomeEligibility() + expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1) + expect(preview.globalSafetyFailure).toBe('process-safety-unavailable') + expect(JSON.stringify(preview)).not.toContain(identity.relayHostId) + expect(JSON.stringify(preview)).not.toContain(identity.userId) + } finally { + context.database.query = query + context.database.transaction = transaction + context.database.queryLocked = queryLocked + } + }) + + it('allocates distinct ordered generations for concurrent window issuers', async () => { + const context = await setup() + const replies = await Promise.all([window(context), window(context), window(context)]) + expect(replies.map((reply) => reply.generation).sort((a, b) => a - b)).toEqual([1, 2, 3]) + const older = replies.find((reply) => reply.generation === 2)! + expect( + await context.store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: older.generation, + assignmentEpoch: older.assignmentEpoch, + policyVersion: 1, + outcome: 'inconclusive', + reason: 'delayed' + }, + older.assignmentEpoch + ) + ).toMatchObject({ reportStatus: 'stale' }) + }) + + it('compares with assigned region, preserves hints, and never extends a window on report', async () => { + const context = await setup() + await context.store.assign(identity, 'asia-east2') + const issued = await window(context) + expect(issued.incumbentRegion).toBe('us-central1') + context.advance(50) + await context.store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.generation, + assignmentEpoch: issued.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 110, 'asia-east2': 90 } + }, + issued.assignmentEpoch + ) + expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 }) + const row = (await context.database.query(`SELECT * FROM relay_region_decisions`))[0]! + expect(Number(row.expires_at)).toBe(issued.expiresAt) + const hint = ( + await context.database.query( + `SELECT preferred_region FROM relay_assignment_region_preferences WHERE user_id = ?`, + [identity.userId] + ) + )[0] + expect(hint?.preferred_region).toBe('asia-east2') + context.advance(24 * 60 * 60_000) + expect( + await context.store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.generation, + assignmentEpoch: issued.assignmentEpoch, + policyVersion: 1, + outcome: 'inconclusive', + reason: 'late' + }, + issued.assignmentEpoch + ) + ).toMatchObject({ reportStatus: 'expired' }) + }) + + it('rejects stale assignment basis and requires both thresholds', async () => { + const context = await setup() + const issued = await window(context) + await context.store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.generation, + assignmentEpoch: issued.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 150, 'asia-east2': 100 } + }, + issued.assignmentEpoch + ) + expect(await context.store.previewRegionCorrection()).toEqual({ + 'us-central1-to-asia-east2': 1 + }) + await context.store.startEvacuation(identity, cells[1]!.id) + expect( + await context.store.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + issued.assignmentEpoch + ) + ).toMatchObject({ reportStatus: 'basis-changed' }) + }) +}) diff --git a/cloud/apps/relay/src/regional-host-drain-app.test.ts b/cloud/apps/relay/src/regional-host-drain-app.test.ts index e2a33a07bb0..cf7e3798155 100644 --- a/cloud/apps/relay/src/regional-host-drain-app.test.ts +++ b/cloud/apps/relay/src/regional-host-drain-app.test.ts @@ -5,7 +5,9 @@ vi.mock('./admin-token-verifier.js', () => ({ createAdminTokenVerifier: () => async (token: string, route?: string) => token === 'deploy-token' || (token === 'monitor-token' && - (!route || route === '/v1/admin/regional-rehome-control')), + (!route || + route === '/v1/admin/regional-rehome-control' || + route === '/v1/admin/regional-rehome-preview')), createReadOnlyAdminTokenVerifier: () => async () => false, createRegionalRehomeControlApplyTokenVerifier: () => async (token: string) => token === 'deploy-token', @@ -41,7 +43,83 @@ const request = { graceMs: 60_000 } +describe('idle regional cutover endpoint', () => { + it('authenticates and fences the source before invoking a cutover', async () => { + const idleRehome = vi.fn(async () => ({ outcome: 'busy' })) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + idleRehome, + cellIncarnation, + ready: vi.fn(async () => true) + } as Parameters[1]) + const input = { + v: 1, + attemptId: request.attemptId, + userId: request.userId, + relayHostId: request.relayHostId, + sourceCellId: request.sourceCellId, + sourceCellIncarnation: cellIncarnation, + sourceAssignmentEpoch: 7, + sourceGeneration: 1, + targetCellId: 'target-cell', + cohortPercent: 100, + directorSafety: { + observedAt: 100, sqlFailures: 0, reconnects: 0, controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0 + } + } + const path = '/v1/admin/host-idle-rehome' + expect((await postPath(app, path, 'runtime-token', input)).status).toBe(401) + expect( + ( + await postPath(app, path, 'rehome-token', { + ...input, + sourceCellIncarnation: '33333333-3333-4333-8333-333333333333' + }) + ).status + ).toBe(409) + expect(idleRehome).not.toHaveBeenCalled() + const response = await postPath(app, path, 'rehome-token', input) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ v: 1, outcome: 'busy' }) + expect(idleRehome).toHaveBeenCalledExactlyOnceWith(input) + }) +}) + describe('regional host drain endpoint', () => { + it('exposes aggregate preview to monitors without a mutation path', async () => { + const preview = { counts: { 'eligible:asia-east2-to-us-central1': 2 } } + const safety = { observedAt: 100 } + const previewRegionalRehomeEligibility = vi.fn(async () => preview) + const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { + store: {} as never, + assignments: { + previewRegionalRehomeEligibility, + regionCorrectionOutcomes: async () => [] + } as never, + regionalRehomeSafetySnapshot: () => safety as never, + drain: vi.fn(), + ready: vi.fn(async () => true) + }) + const path = '/v1/admin/regional-rehome-preview' + expect((await app.request(path)).status).toBe(401) + expect(previewRegionalRehomeEligibility).not.toHaveBeenCalled() + const response = await app.request(path, { headers: { authorization: 'Bearer monitor-token' } }) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ v: 1, preview, outcomes: [] }) + expect(previewRegionalRehomeEligibility).toHaveBeenCalledExactlyOnceWith(safety) + expect( + ( + await app.request(path, { + method: 'POST', + headers: { authorization: 'Bearer deploy-token' } + }) + ).status + ).toBe(404) + }) + it('accepts only the dedicated identity and exact cell generation', async () => { const drainHost = vi.fn(() => 'accepted' as const) const app = createRelayApp(config(), { @@ -60,14 +138,66 @@ describe('regional host drain endpoint', () => { expect((await post(app, 'deploy-token', request)).status).toBe(401) expect( - (await post(app, 'rehome-token', { - ...request, - sourceCellIncarnation: '33333333-3333-4333-8333-333333333333' - })).status + ( + await post(app, 'rehome-token', { + ...request, + sourceCellIncarnation: '33333333-3333-4333-8333-333333333333' + }) + ).status ).toBe(409) expect(drainHost).toHaveBeenCalledOnce() }) + it('waits for an asynchronous drain operation before acknowledging', async () => { + let grant!: (value: 'accepted') => void + let entered!: () => void + const started = new Promise((resolve) => { + entered = resolve + }) + const drainHost = vi.fn(() => { + entered() + return new Promise<'accepted'>((resolve) => { + grant = resolve + }) + }) + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost, + cellIncarnation, + ready: vi.fn(async () => true) + }) + const pending = post(app, 'rehome-token', request) + let acknowledged = false + void pending.then(() => { + acknowledged = true + }) + await started + await Promise.resolve() + expect(acknowledged).toBe(false) + grant('accepted') + const response = await pending + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ v: 1, outcome: 'accepted' }) + }) + + it('rejects a failed asynchronous drain instead of acknowledging it', async () => { + const app = createRelayApp(config(), { + store: {} as never, + assignments: {} as never, + drain: vi.fn(), + drainHost: async () => { + throw new Error('activity_cell_not_authoritative') + }, + cellIncarnation, + ready: vi.fn(async () => true) + }) + const response = await post(app, 'rehome-token', request) + expect(response.status).toBe(409) + expect(await response.json()).toEqual({ error: 'activity_cell_not_authoritative' }) + }) + it('rejects malformed identities before touching the session registry', async () => { const drainHost = vi.fn(() => 'accepted' as const) const app = createRelayApp(config(), { @@ -224,7 +354,7 @@ describe('regional rehome director controls', () => { v: 1, cellId: 'production-gce-c7', cellIncarnation, - regionalRehomeProtocol: 1, + regionalRehomeProtocol: 2, safety: { observedAt: 100, sqlFailures: 0, @@ -235,12 +365,7 @@ describe('regional rehome director controls', () => { databasePoolWaitMsMax: 0 } } - const response = await postPath( - app, - '/v1/admin/cell-rehome-status', - 'runtime-token', - body - ) + const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body) expect(response.status).toBe(200) expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body) @@ -266,18 +391,13 @@ describe('regional rehome director controls', () => { v: 1, cellId: 'production-gce-c7', cellIncarnation, - regionalRehomeProtocol: 1, + regionalRehomeProtocol: 2, safety: { ...observability.regionalRehomeRuntimeSafety(), ...emptyPostgresPoolPressureCounts() } } - const response = await postPath( - app, - '/v1/admin/cell-rehome-status', - 'runtime-token', - body - ) + const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body) expect(response.status).toBe(200) expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body) @@ -301,12 +421,14 @@ describe('regional rehome director controls', () => { drain: vi.fn(), ready: vi.fn(async () => true) }) - expect((await postPath( - app, - '/v1/admin/regional-rehome-control', - 'deploy-token', - { v: 1, action: 'inspect' } - )).status).toBe(200) + expect( + ( + await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', { + v: 1, + action: 'inspect' + }) + ).status + ).toBe(200) const apply = { v: 1, action: 'apply', @@ -319,39 +441,35 @@ describe('regional rehome director controls', () => { drainGraceMs: 60_000, confirmation: 'ENABLE_REGIONAL_REHOMING' } - expect((await postPath( - app, - '/v1/admin/regional-rehome-control', - 'deploy-token', - apply - )).status).toBe(200) + expect( + (await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', apply)).status + ).toBe(200) expect(applyRegionalRehomeControl).toHaveBeenCalledOnce() - expect((await postPath( - app, - '/v1/admin/regional-rehome-control', - 'monitor-token', - { v: 1, action: 'inspect' } - )).status).toBe(200) - expect((await postPath( - app, - '/v1/admin/regional-rehome-control', - 'monitor-token', - apply - )).status).toBe(403) - expect((await postPath( - app, - '/v1/admin/regional-rehome-control', - 'deploy-token', - { ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' } - )).status).toBe(400) + expect( + ( + await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', { + v: 1, + action: 'inspect' + }) + ).status + ).toBe(200) + expect( + (await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', apply)).status + ).toBe(403) + expect( + ( + await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', { + ...apply, + confirmation: 'DISABLE_REGIONAL_REHOMING' + }) + ).status + ).toBe(400) // The per-host cooldown is part of the durable shape an operator must state. const { hostCooldownMs: _omitted, ...withoutCooldown } = apply - expect((await postPath( - app, - '/v1/admin/regional-rehome-control', - 'deploy-token', - withoutCooldown - )).status).toBe(400) + expect( + (await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', withoutCooldown)) + .status + ).toBe(400) }) it('probes dedicated trust twice and returns only aggregate proof', async () => { @@ -382,12 +500,11 @@ describe('regional rehome director controls', () => { }) as typeof fetch, ready: vi.fn(async () => true) }) - const response = await postPath( - app, - '/v1/admin/regional-rehome-trust-probe', - 'deploy-token', - { v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation } - ) + const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', { + v: 1, + sourceCellId: 'production-gce-c7', + sourceCellIncarnation: cellIncarnation + }) expect(response.status).toBe(200) const responseBody = await response.json() @@ -448,12 +565,11 @@ describe('regional rehome director controls', () => { ready: vi.fn(async () => true) }) - const response = await postPath( - app, - '/v1/admin/regional-rehome-trust-probe', - 'deploy-token', - { v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation } - ) + const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', { + v: 1, + sourceCellId: 'production-gce-c27', + sourceCellIncarnation: cellIncarnation + }) expect(response.status).toBe(200) expect(await response.json()).toMatchObject({ proven: true }) @@ -481,12 +597,11 @@ describe('regional rehome director controls', () => { ready: vi.fn(async () => true) }) - const response = await postPath( - app, - '/v1/admin/regional-rehome-trust-probe', - 'deploy-token', - { v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation } - ) + const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', { + v: 1, + sourceCellId: 'production-gce-c27', + sourceCellIncarnation: cellIncarnation + }) expect(response.status).toBe(409) expect(sourceFetch).not.toHaveBeenCalled() @@ -504,18 +619,17 @@ describe('regional rehome director controls', () => { sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation } - expect((await postPath( - app, - '/v1/admin/regional-rehome-trust-probe', - 'monitor-token', - body - )).status).toBe(401) - expect((await postPath( - app, - '/v1/admin/regional-rehome-trust-probe', - 'deploy-token', - { ...body, unexpected: true } - )).status).toBe(400) + expect( + (await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'monitor-token', body)).status + ).toBe(401) + expect( + ( + await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', { + ...body, + unexpected: true + }) + ).status + ).toBe(400) }) it('fails closed when the source rejects the dedicated identity', async () => { @@ -529,9 +643,9 @@ describe('regional rehome director controls', () => { regionalRehomeProtocol: 1 } }) - const sourceFetch = vi.fn().mockResolvedValue( - Response.json({ error: 'invalid_token' }, { status: 401 }) - ) + const sourceFetch = vi + .fn() + .mockResolvedValue(Response.json({ error: 'invalid_token' }, { status: 401 })) const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { store: {} as never, assignments: { cellDeploymentStatus } as never, @@ -540,12 +654,11 @@ describe('regional rehome director controls', () => { regionalRehomeFetch: sourceFetch, ready: vi.fn(async () => true) }) - const response = await postPath( - app, - '/v1/admin/regional-rehome-trust-probe', - 'deploy-token', - { v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation } - ) + const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', { + v: 1, + sourceCellId: 'production-gce-c7', + sourceCellIncarnation: cellIncarnation + }) expect(response.status).toBe(409) expect(sourceFetch).toHaveBeenCalledOnce() diff --git a/cloud/apps/relay/src/regional-rehome-postgres.test.ts b/cloud/apps/relay/src/regional-rehome-postgres.test.ts index fdefda54401..07307707124 100644 --- a/cloud/apps/relay/src/regional-rehome-postgres.test.ts +++ b/cloud/apps/relay/src/regional-rehome-postgres.test.ts @@ -29,6 +29,9 @@ describePostgres('PostgreSQL regional rehoming', () => { }) async function cleanup(): Promise { + for (const table of ['relay_region_decisions', 'relay_control_capabilities']) { + await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`) + } await primary.query( `DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'` ) @@ -69,6 +72,81 @@ describePostgres('PostgreSQL regional rehoming', () => { } } + it('defaults to a closed correction cohort even with enabled durable control', async () => { + const context = await fixture() + const closed = new RelayAssignmentStore(primary, context.now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + expect(await cutover(closed, context.now())).toBeNull() + const preview = await closed.previewRegionalRehomeEligibility({ + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + expect(preview.cohortPercent).toBe(0) + expect(preview.counts['outside-cohort']).toBe(1) + expect(await attemptAndMigrationCounts(context.identity)).toEqual({ + attempts: 0, + migrations: 0 + }) + }) + + it('counts existing generic migrations against the optimization cap and preview', async () => { + const context = await fixture() + const safety = { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + const before = await context.store.previewRegionalRehomeEligibility(safety) + expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1) + for (let index = 0; index < 8; index++) { + const identity = { + userId: `pg-rehome-user-budget-${sequence}-${index}`, + relayHostId: `budgethost${String(index).padStart(6, '0')}` + } + await context.store.assign(identity, undefined, 'us-central1') + await context.store.startEvacuation(identity, context.target.id) + } + const preview = await context.store.previewRegionalRehomeEligibility(safety) + expect(preview.openMigrations).toBe(8) + expect(preview.availableMigrationSlots).toBe(0) + expect(preview.counts['concurrent-migration-cap']).toBe(1) + expect(await cutover(context.store, context.now())).toBeNull() + expect(await attemptAndMigrationCounts(context.identity)).toEqual({ + attempts: 0, + migrations: 0 + }) + }) + + it('preview excludes request capacity exhaustion before a claim', async () => { + const context = await fixture() + await primary.query( + `UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`, + [context.target.id] + ) + const preview = await context.store.previewRegionalRehomeEligibility({ + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + expect(preview.counts['no-target-headroom']).toBe(1) + expect(await cutover(context.store, context.now())).toBeNull() + }) + it('claims through ambient per-cell sql retry noise', async () => { const context = await fixture() await primary.query( @@ -78,27 +156,31 @@ describePostgres('PostgreSQL regional rehoming', () => { [context.source.id, context.target.id] ) - expect(await context.store.claimRegionalRehome()).not.toBeNull() + expect(await cutover(context.store, context.now())).not.toBeNull() }) it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => { const context = await fixture() - const attempt = await context.store.claimRegionalRehome() + const attempt = await cutover(context.store, context.now()) expect(attempt).toMatchObject({ preferredRegion: 'asia-east2', sourceCellId: context.source.id, targetCellId: context.target.id }) - expect(await primary.query( - `SELECT preferred_region, source_cell_id, target_cell_id + expect( + await primary.query( + `SELECT preferred_region, source_cell_id, target_cell_id FROM relay_region_rehome_attempts WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ - preferred_region: 'asia-east2', - source_cell_id: context.source.id, - target_cell_id: context.target.id - }]) + [context.identity.userId] + ) + ).toEqual([ + { + preferred_region: 'asia-east2', + source_cell_id: context.source.id, + target_cell_id: context.target.id + } + ]) }) it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => { @@ -107,32 +189,37 @@ describePostgres('PostgreSQL regional rehoming', () => { targetRegion: 'us-central1' }) - const attempt = await context.store.claimRegionalRehome() + const attempt = await cutover(context.store, context.now()) expect(attempt).toMatchObject({ preferredRegion: 'us-central1', sourceCellId: context.source.id, targetCellId: context.target.id }) // The durable attempt row must accept the reverse direction too. - expect(await primary.query( - `SELECT preferred_region, source_cell_id, target_cell_id + expect( + await primary.query( + `SELECT preferred_region, source_cell_id, target_cell_id FROM relay_region_rehome_attempts WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ - preferred_region: 'us-central1', - source_cell_id: context.source.id, - target_cell_id: context.target.id - }]) - expect(await primary.query( - `SELECT cell_id FROM relay_assignments WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ cell_id: context.target.id }]) + [context.identity.userId] + ) + ).toEqual([ + { + preferred_region: 'us-central1', + source_cell_id: context.source.id, + target_cell_id: context.target.id + } + ]) + expect( + await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [ + context.identity.userId + ]) + ).toEqual([{ cell_id: context.target.id }]) }) it('leaves a host whose preference already matches its own region', async () => { const context = await fixture({ preferredRegion: 'us-central1' }) - await expect(context.store.claimRegionalRehome()).resolves.toBeNull() + await expect(cutover(context.store, context.now())).resolves.toBeNull() await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({ generation: 1, enabled: true @@ -146,16 +233,12 @@ describePostgres('PostgreSQL regional rehoming', () => { it('leaves a host whose preference is older than the configured max age', async () => { const context = await fixture() await primary.query( - `UPDATE relay_assignment_region_preferences SET observed_at = ? + `UPDATE relay_region_decisions SET observed_at = ? WHERE user_id = ? AND relay_host_id = ?`, - [ - context.now() - 24 * 60 * 60_000 - 1, - context.identity.userId, - context.identity.relayHostId - ] + [context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId] ) - await expect(context.store.claimRegionalRehome()).resolves.toBeNull() + await expect(cutover(context.store, context.now())).resolves.toBeNull() await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({ generation: 1, enabled: true @@ -190,7 +273,7 @@ describePostgres('PostgreSQL regional rehoming', () => { ] ) - await expect(context.store.claimRegionalRehome()).resolves.toBeNull() + await expect(cutover(context.store, context.now())).resolves.toBeNull() await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({ generation: 1, enabled: true, @@ -206,7 +289,7 @@ describePostgres('PostgreSQL regional rehoming', () => { `UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`, [context.now() - 3 * 24 * 60 * 60_000, context.identity.userId] ) - await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({ + await expect(cutover(context.store, context.now())).resolves.toMatchObject({ sourceCellId: context.source.id, targetCellId: context.target.id }) @@ -217,7 +300,7 @@ describePostgres('PostgreSQL regional rehoming', () => { // where no later rehome could move it out again. const context = await fixture({ targetProtocol: 0 }) - await expect(context.store.claimRegionalRehome()).resolves.toBeNull() + await expect(cutover(context.store, context.now())).resolves.toBeNull() await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({ generation: 1, enabled: true @@ -237,119 +320,39 @@ describePostgres('PostgreSQL regional rehoming', () => { [context.target.id] ) - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await cutover(context.store, context.now())).toBeNull() expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ generation: 1, enabled: true }) - expect(await primary.query( - `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` - )).toEqual([{ next_dispatch_at: String(context.now() + 6_000) }]) + expect(await attemptAndMigrationCounts(context.identity)).toEqual({ + attempts: 0, + migrations: 0 + }) }) it('lets only one director claim a host', async () => { const context = await fixture() const claims = await Promise.all([ - context.store.claimRegionalRehome(), - context.competingStore.claimRegionalRehome() + cutover(context.store, context.now()), + cutover(context.competingStore, context.now()) ]) - expect(claims.filter(Boolean)).toHaveLength(1) - expect(await primary.query( - `SELECT COUNT(*) AS count FROM relay_region_rehome_attempts + expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1) + expect( + await primary.query( + `SELECT COUNT(*) AS count FROM relay_region_rehome_attempts WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ count: '1' }]) - expect(await primary.query( - `SELECT COUNT(*) AS count FROM relay_assignment_migrations + [context.identity.userId] + ) + ).toEqual([{ count: '1' }]) + expect( + await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`, - [context.identity.userId] - )).toEqual([{ count: '1' }]) - }) - - it('serializes an enable with a budget-exhausting failure without retries', async () => { - const context = await fixture() - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - const locked = Promise.withResolvers() - const release = Promise.withResolvers() - const primaryTransaction = primary.transaction.bind(primary) - const secondaryTransaction = secondary.transaction.bind(secondary) - let enableTransactions = 0 - let failureTransactions = 0 - let enablePid = 0 - let failurePid = 0 - const enableSpy = vi.spyOn(primary, 'transaction').mockImplementation((operation, options) => - primaryTransaction(async (transaction) => { - enableTransactions++ - enablePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid) - return await operation({ - dialect: 'postgres', - query: transaction.query.bind(transaction), - queryLocked: async (sql, params, lockOptions) => { - const rows = await transaction.queryLocked(sql, params, lockOptions) - if (sql.includes('FROM relay_region_rehome_control')) { - locked.resolve() - await release.promise - } - return rows - }, - transaction: transaction.transaction.bind(transaction), - close: transaction.close.bind(transaction) - }) - }, options) - ) - const failureSpy = vi.spyOn(secondary, 'transaction').mockImplementation((operation, options) => - secondaryTransaction(async (transaction) => { - failureTransactions++ - failurePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid) - return await operation(transaction) - }, options) - ) - const enable = context.store.applyRegionalRehomeControl({ - expectedGeneration: 1, - enabled: true, - notBefore: context.now(), - ratePerMinute: 10, - preferenceMaxAgeMs: 24 * 60 * 60_000, - hostCooldownMs: 7 * 24 * 60 * 60_000, - drainGraceMs: 60_000 - }) - let failure: Promise | undefined - let outcomes: PromiseSettledResult[] = [] - try { - await Promise.race([ - locked.promise, - enable.then(() => { - throw new Error('enable completed before the control lock') - }) - ]) - failure = context.competingStore.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - // Observe the actual PostgreSQL wait before letting enable acquire the worker row. - await vi.waitFor(async () => { - expect(failurePid).not.toBe(0) - const rows = await primary.query('SELECT pg_blocking_pids(?) AS blockers', [failurePid]) - expect(rows[0]!.blockers).toContain(enablePid) - }, { interval: 10, timeout: 800 }) - } finally { - release.resolve() - outcomes = await Promise.allSettled([enable, ...(failure ? [failure] : [])]) - enableSpy.mockRestore() - failureSpy.mockRestore() - } - expect(outcomes.map((outcome) => outcome.status)).toEqual(['fulfilled', 'fulfilled']) - expect({ enableTransactions, failureTransactions }).toEqual({ - enableTransactions: 1, - failureTransactions: 1 - }) - expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - generation: 2, - enabled: true - }) - expect(await primary.query( - `SELECT consecutive_failures, paused_until FROM relay_region_rehome_worker_state` - )).toEqual([{ consecutive_failures: '1', paused_until: '0' }]) + [context.identity.userId] + ) + ).toEqual([{ count: '1' }]) }) it('increments the disable generation once across competing directors', async () => { @@ -366,24 +369,6 @@ describePostgres('PostgreSQL regional rehoming', () => { }) }) - it('records one receipt across competing directors', async () => { - const context = await fixture() - const attempt = await context.store.claimRegionalRehome() - const receipts = await Promise.all([ - context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted'), - context.competingStore.recordRegionalRehomeDrainReceipt( - attempt!.attemptId, - 'accepted' - ) - ]) - - expect(receipts.sort()).toEqual([false, true]) - expect(await primary.query( - `SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?`, - [attempt!.attemptId] - )).toEqual([{ drain_outcome: 'accepted' }]) - }) - it('rechecks a preference changed while the assignment row is locked', async () => { const context = await fixture() let unlock!: () => void @@ -399,9 +384,9 @@ describePostgres('PostgreSQL regional rehoming', () => { await unlockPromise }) await lockedPromise - const claim = context.store.claimRegionalRehome() + const claim = cutover(context.store, context.now()) await primary.query( - `UPDATE relay_assignment_region_preferences SET preferred_region = 'us-central1', + `UPDATE relay_region_decisions SET preferred_region = 'us-central1', observed_at = ? WHERE user_id = ? AND relay_host_id = ?`, [context.now(), context.identity.userId, context.identity.relayHostId] ) @@ -409,23 +394,26 @@ describePostgres('PostgreSQL regional rehoming', () => { await held await expect(claim).resolves.toBeNull() - expect(await primary.query( - `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ count: '0' }]) + expect( + await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + ) + ).toEqual([{ count: '0' }]) }) it('rechecks fleet safety under locks before mutating a candidate', async () => { const context = await fixture() + const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now())) + expect(request).toBeDefined() let unlock!: () => void let locked!: () => void const lockedPromise = new Promise((resolve) => (locked = resolve)) const unlockPromise = new Promise((resolve) => (unlock = resolve)) const held = secondary.transaction(async (transaction) => { - await transaction.queryLocked( - `SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, - [context.target.id] - ) + await transaction.queryLocked(`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, [ + context.target.id + ]) await transaction.query( `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, [context.target.id] @@ -434,62 +422,50 @@ describePostgres('PostgreSQL regional rehoming', () => { await unlockPromise }) await lockedPromise - const claim = context.store.claimRegionalRehome() + const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now())) unlock() await held - await expect(claim).resolves.toBeNull() + await expect(claim).resolves.toEqual({ outcome: 'deferred' }) expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ generation: 2, enabled: false }) - expect(await primary.query( - `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ count: '0' }]) + expect( + await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + ) + ).toEqual([{ count: '0' }]) }) it('pauses when one required cell exceeds the reconnect limit', async () => { const context = await fixture() - await primary.query( - `UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, - [REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, context.source.id] - ) + const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now())) + expect(request).toBeDefined() + await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [ + REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, + context.source.id + ]) - await expect(context.store.claimRegionalRehome()).resolves.toBeNull() + await expect( + context.store.commitIdleRegionalRehome(request!, safety(context.now())) + ).resolves.toEqual({ outcome: 'deferred' }) await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({ generation: 2, enabled: false }) - expect(await primary.query( - `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ count: '0' }]) - }) - - it('does not retry a drain against a replacement source incarnation', async () => { - const context = await fixture() - const attempt = await context.store.claimRegionalRehome() - context.advance(31_000) - await heartbeat( - context.store, - context.source, - '33333333-3333-4333-8333-333333333333', - 1, - context.now() - ) - - await expect(context.competingStore.claimRegionalRehome()).resolves.toBeNull() - expect(await primary.query( - `SELECT send_attempts FROM relay_region_rehome_attempts WHERE attempt_id = ?`, - [attempt!.attemptId] - )).toEqual([{ send_attempts: '1' }]) + expect( + await primary.query( + `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + ) + ).toEqual([{ count: '0' }]) }) it('makes concurrent completion and expiry cleanup idempotent', async () => { const context = await fixture() - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await cutover(context.store, context.now()) const targetControl = await context.store.activateControl(context.identity, { cellId: context.target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -528,17 +504,18 @@ describePostgres('PostgreSQL regional rehoming', () => { context.competingStore.abortExpiredRegionalRehomes() ]) expect(outcomes).toEqual(expect.arrayContaining([0, 1])) - expect(await primary.query( - `SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted + expect( + await primary.query( + `SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted FROM relay_assignment_migrations WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ completed: true, aborted: false }]) + [context.identity.userId] + ) + ).toEqual([{ completed: true, aborted: false }]) }) it('will not complete against a replacement target incarnation', async () => { const context = await fixture() - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await cutover(context.store, context.now()) await context.store.activateControl(context.identity, { cellId: context.target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -559,15 +536,17 @@ describePostgres('PostgreSQL regional rehoming', () => { ) await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0) - expect(await primary.query( - `SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ completed_at: null, aborted_at: null }]) + expect( + await primary.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`, + [context.identity.userId] + ) + ).toEqual([{ completed_at: null, aborted_at: null }]) }) it('does not roll an unregistered target back to a stale regional source', async () => { const context = await fixture() - await context.store.claimRegionalRehome() + await cutover(context.store, context.now()) context.advance(6 * 60_000) await heartbeat( context.store, @@ -580,16 +559,17 @@ describePostgres('PostgreSQL regional rehoming', () => { await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0) await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0) - expect(await primary.query( - `SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }]) + expect( + await primary.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`, + [context.identity.userId] + ) + ).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }]) }) it('completes after the drained host re-resolves through the director', async () => { const context = await fixture() - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await cutover(context.store, context.now()) // The drain recovery lands while both controls are still live. await context.store.assign(context.identity, 'asia-east2') expect(await controlAccounting(context.identity)).toEqual({ @@ -609,11 +589,13 @@ describePostgres('PostgreSQL regional rehoming', () => { await context.store.releaseActivity(context.identity, context.sourceControl) await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1) - expect(await primary.query( - `SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations + expect( + await primary.query( + `SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations WHERE user_id = ?`, - [context.identity.userId] - )).toEqual([{ completed: true }]) + [context.identity.userId] + ) + ).toEqual([{ completed: true }]) expect(await controlAccounting(context.identity)).toEqual({ reservedControls: 1, controlLeases: 1 @@ -622,7 +604,7 @@ describePostgres('PostgreSQL regional rehoming', () => { it('repairs a skewed control counter before completing the rehome', async () => { const context = await fixture() - const attempt = await context.store.claimRegionalRehome() + const attempt = await cutover(context.store, context.now()) await context.store.activateControl(context.identity, { cellId: context.target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -634,10 +616,9 @@ describePostgres('PostgreSQL regional rehoming', () => { }) await context.store.releaseActivity(context.identity, context.sourceControl) // Damage already written by a pre-fix sticky grant. - await primary.query( - `UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, - [context.identity.userId] - ) + await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [ + context.identity.userId + ]) await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1) expect(await controlAccounting(context.identity)).toEqual({ @@ -646,6 +627,22 @@ describePostgres('PostgreSQL regional rehoming', () => { }) }) + async function cutover(store: RelayAssignmentStore, now: number) { + const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now)) + if (!request) return null + const result = await store.commitIdleRegionalRehome(request, safety(now)) + if (result.outcome !== 'committed') return null + const [attempt] = await primary.query( + `SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [request.attemptId] + ) + return { + ...request, + preferredRegion: String(attempt!.preferred_region), + assignmentEpoch: Number(attempt!.assignment_epoch) + } + } + async function attemptAndMigrationCounts(identity: { userId: string relayHostId: string @@ -711,18 +708,12 @@ describePostgres('PostgreSQL regional rehoming', () => { drainGraceMs: 60_000 }) await store.reconcileCells([source, target]) - await heartbeat( - store, - source, - '11111111-1111-4111-8111-111111111111', - 1, - 900_000 - ) + await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000) await heartbeat( store, target, '22222222-2222-4222-8222-222222222222', - options.targetProtocol ?? 1, + options.targetProtocol ?? 3, 900_000 ) const identity = { @@ -733,9 +724,32 @@ describePostgres('PostgreSQL regional rehoming', () => { const sourceControl = await store.activateControl(identity, { cellId: source.id, assignmentEpoch: assignment.assignmentEpoch, - generation: 1 + generation: 1, + idleRegionalRehome: true, + cellIncarnation: '11111111-1111-4111-8111-111111111111' }) await store.assign(identity, preferredRegion) + const issued = await store.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + assignment.assignmentEpoch + ) + await store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.window!.generation, + assignmentEpoch: assignment.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { + 'us-central1': preferredRegion === 'us-central1' ? 50 : 150, + 'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150 + } + }, + assignment.assignmentEpoch + ) return { preferredRegion, store, @@ -753,6 +767,7 @@ describePostgres('PostgreSQL regional rehoming', () => { }) const storeOptions = { + regionalRehomeCohortPercent: 100, requireLiveCells: true, heartbeatTtlMs: 45_000 } @@ -816,3 +831,15 @@ async function heartbeat( } }) } + +function safety(now: number) { + return { + observedAt: now, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } +} diff --git a/cloud/apps/relay/src/regional-rehome-store.test.ts b/cloud/apps/relay/src/regional-rehome-store.test.ts index 662876ef66e..e4a355699da 100644 --- a/cloud/apps/relay/src/regional-rehome-store.test.ts +++ b/cloud/apps/relay/src/regional-rehome-store.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { - RelayAssignmentStore, + RelayAssignmentStore as BaseRelayAssignmentStore, + type RegionalRehomeAttempt, REGIONAL_REHOME_QUARANTINE_FAILURES, REGIONAL_REHOME_QUARANTINE_MS, REGIONAL_REHOME_REDRAIN_SEND_LIMIT @@ -37,14 +38,115 @@ const sourceIncarnation = '11111111-1111-4111-8111-111111111111' const targetIncarnation = '22222222-2222-4222-8222-222222222222' describe('regional rehome assignment state', () => { + it('advances past a full candidate page whose destination lacks capacity', async () => { + const context = await setup() + for (let i = 0; i < 10; i++) { + await activatePreferredSource(context, { + userId: `blocked-${i}`, + relayHostId: 'abcdefghijklmnop' + }) + } + context.advance(1) + const reverse = { userId: 'healthy-reverse', relayHostId: 'abcdefghijklmnop' } + await activateReversePreferredSource(context, reverse) + await context.database.query( + 'UPDATE relay_cells SET capacity_requests = reserved_requests WHERE cell_id = ?', + [target.id] + ) + expect(await context.store.tryIdleRehome()).toMatchObject({ + userId: reverse.userId, + sourceCellId: target.id, + targetCellId: source.id + }) + await context.database.close() + }) + + it('defaults optional correction off even with enabled durable control', async () => { + const context = await setup() + await activatePreferredSource(context, { userId: 'cohort', relayHostId: 'abcdefghijklmnop' }) + const defaultStore = new IdleRehomeTestStore(context.database, context.now, { + requireLiveCells: true + }) + expect(await defaultStore.tryIdleRehome()).toBeNull() + expect( + await context.database.query('SELECT attempt_id FROM relay_region_rehome_attempts') + ).toEqual([]) + expect(await context.store.tryIdleRehome()).not.toBeNull() + await context.database.close() + }) + + it('counts pre-existing generic migrations against the eight-migration cap', async () => { + const context = await setup() + await activatePreferredSource(context, { userId: 'cap', relayHostId: 'abcdefghijklmnop' }) + for (let i = 0; i < 8; i++) { + await context.database.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch, assignment_epoch, + source_request_units, target_reserved_units, expires_at, created_at, updated_at) + VALUES (?, ?, ?, ?, 1, 2, 1, 1, ?, ?, ?)`, + [ + 'generic', + `synthetic-migration-${i}`, + source.id, + target.id, + context.now() + 60_000, + context.now(), + context.now() + ] + ) + } + expect(await context.store.tryIdleRehome()).toBeNull() + await context.database.query( + `UPDATE relay_assignment_migrations SET completed_at = ? + WHERE user_id = 'generic' AND relay_host_id = 'synthetic-migration-0'`, + [context.now()] + ) + expect(await context.store.tryIdleRehome()).not.toBeNull() + const open = await context.database + .query(`SELECT COUNT(*) AS count FROM relay_assignment_migrations + WHERE completed_at IS NULL AND aborted_at IS NULL`) + expect(Number(open[0]?.count)).toBe(8) + await context.database.close() + }) + + it('does not let legacy hints certify a move', async () => { + const context = await setup() + await activatePreferredSource(context, { userId: 'legacy', relayHostId: 'abcdefghijklmnop' }) + await context.database.query('DELETE FROM relay_region_decisions') + expect(await context.store.tryIdleRehome()).toBeNull() + await context.database.close() + }) + + it('refreshes later open attempts when an older attempt occupies the first page', async () => { + const context = await setup() + await activatePreferredSource(context, { userId: 'page-1', relayHostId: 'abcdefghijklmnop' }) + const first = await context.store.tryIdleRehome() + context.advance(10_000) + await freshHeartbeats(context) + await activatePreferredSource(context, { userId: 'page-2', relayHostId: 'abcdefghijklmnop' }) + const second = await context.store.tryIdleRehome() + expect(second).not.toBeNull() + context.advance(1_000) + expect(await context.store.refreshRegionalRehomeLeases(1)).toBe(1) + context.advance(1_000) + expect(await context.store.refreshRegionalRehomeLeases(1)).toBe(1) + const rows = await context.database.query( + `SELECT attempt_id, updated_at FROM relay_region_rehome_attempts + WHERE attempt_id = ?`, + [second!.attemptId] + ) + expect(Number(rows[0]?.updated_at)).toBe(context.now()) + await context.database.close() + }) + it('does not open a transaction while the worker is disabled', async () => { const delegate = await openInMemoryRelayDatabase() const database = new TransactionCountingDatabase(delegate) - const store = new RelayAssignmentStore(database, () => 1_000_000) + const store = new IdleRehomeTestStore(database, () => 1_000_000) await store.inspectRegionalRehomeControl() database.transactionCalls = 0 - await expect(store.claimRegionalRehome()).resolves.toBeNull() + await expect(store.tryIdleRehome()).resolves.toBeNull() expect(database.transactionCalls).toBe(0) await database.close() }) @@ -52,9 +154,9 @@ describe('regional rehome assignment state', () => { it('initializes a missing control row without opening a transaction', async () => { const delegate = await openInMemoryRelayDatabase() const database = new TransactionCountingDatabase(delegate) - const store = new RelayAssignmentStore(database, () => 1_000_000) + const store = new IdleRehomeTestStore(database, () => 1_000_000) - await expect(store.claimRegionalRehome()).resolves.toBeNull() + await expect(store.tryIdleRehome()).resolves.toBeNull() expect(database.transactionCalls).toBe(0) await expect(store.inspectRegionalRehomeControl()).resolves.toMatchObject({ generation: 0, @@ -75,24 +177,28 @@ describe('regional rehome assignment state', () => { generation: 2, enabled: false }) - await expect(context.store.applyRegionalRehomeControl({ - expectedGeneration: 1, - enabled: true, - notBefore: context.now(), - ratePerMinute: 10, - preferenceMaxAgeMs: 24 * 60 * 60_000, - hostCooldownMs: 7 * 24 * 60 * 60_000, - drainGraceMs: 60_000 - })).rejects.toThrow('regional_rehome_generation_mismatch') - await expect(context.store.applyRegionalRehomeControl({ - expectedGeneration: 2, - enabled: true, - notBefore: context.now(), - ratePerMinute: 10, - preferenceMaxAgeMs: 24 * 60 * 60_000, - hostCooldownMs: 7 * 24 * 60 * 60_000, - drainGraceMs: 60_000 - })).resolves.toMatchObject({ generation: 3, enabled: true }) + await expect( + context.store.applyRegionalRehomeControl({ + expectedGeneration: 1, + enabled: true, + notBefore: context.now(), + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + hostCooldownMs: 7 * 24 * 60 * 60_000, + drainGraceMs: 60_000 + }) + ).rejects.toThrow('regional_rehome_generation_mismatch') + await expect( + context.store.applyRegionalRehomeControl({ + expectedGeneration: 2, + enabled: true, + notBefore: context.now(), + ratePerMinute: 10, + preferenceMaxAgeMs: 24 * 60 * 60_000, + hostCooldownMs: 7 * 24 * 60 * 60_000, + drainGraceMs: 60_000 + }) + ).resolves.toMatchObject({ generation: 3, enabled: true }) await context.database.close() }) @@ -103,7 +209,7 @@ describe('regional rehome assignment state', () => { const sourceControl = await activatePreferredSource(context, identity) await activateSource(context, neighbor) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() expect(attempt).toMatchObject({ userId: identity.userId, relayHostId: identity.relayHostId, @@ -118,11 +224,11 @@ describe('regional rehome assignment state', () => { expect(await context.store.resolve(neighbor)).toMatchObject({ cellId: source.id }) expect(await context.store.completeReadyRegionalRehomes()).toBe(0) expect( - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - ).toBe(true) - expect( - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - ).toBe(false) + await context.database.query( + 'SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?', + [attempt!.attemptId] + ) + ).toEqual([{ drain_outcome: 'accepted' }]) const targetControl = await context.store.activateControl(identity, { cellId: target.id, @@ -142,23 +248,27 @@ describe('regional rehome assignment state', () => { assignmentEpoch: 2 }) expect(await context.store.resolve(neighbor)).toMatchObject({ cellId: source.id }) - expect(await context.database.query( - `SELECT completed_at, aborted_at FROM relay_assignment_migrations + expect( + await context.database.query( + `SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ? AND relay_host_id = ?`, - [identity.userId, identity.relayHostId] - )).toEqual([{ completed_at: context.now(), aborted_at: null }]) - expect(await context.database.query( - `SELECT completed_at, aborted_at FROM relay_region_rehome_attempts` - )).toEqual([{ completed_at: context.now(), aborted_at: null }]) + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ completed_at: context.now(), aborted_at: null }]) + expect( + await context.database.query( + `SELECT completed_at, aborted_at FROM relay_region_rehome_attempts` + ) + ).toEqual([{ completed_at: context.now(), aborted_at: null }]) expect(targetControl).toMatch(/^control:/) await context.database.close() }) - it('completes from durable activity when the drain response was lost', async () => { + it('completes from durable activity with the source-owned receipt', async () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -171,10 +281,12 @@ describe('regional rehome assignment state', () => { await context.store.releaseActivity(identity, sourceControl) expect(await context.store.completeReadyRegionalRehomes()).toBe(1) - expect(await context.database.query( - `SELECT drain_receipt_at, completed_at, aborted_at + expect( + await context.database.query( + `SELECT drain_receipt_at, completed_at, aborted_at FROM relay_region_rehome_attempts` - )).toEqual([{ drain_receipt_at: null, completed_at: context.now(), aborted_at: null }]) + ) + ).toEqual([{ drain_receipt_at: context.now(), completed_at: context.now(), aborted_at: null }]) await context.database.close() }) @@ -184,23 +296,22 @@ describe('regional rehome assignment state', () => { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.tryIdleRehome()).toBeNull() expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) await context.database.close() }) it('fails fleet safety closed until source and target telemetry is fresh', async () => { const context = await setup() - await context.database.query( - `DELETE FROM relay_cell_rehome_safety WHERE cell_id = ?`, - [target.id] - ) + await context.database.query(`DELETE FROM relay_cell_rehome_safety WHERE cell_id = ?`, [ + target.id + ]) expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({ requiredCells: 2, missingCells: 1, observedAt: 0 }) - await heartbeat(context.store, source, sourceIncarnation, 1, 2, { + await heartbeat(context.store, source, sourceIncarnation, 3, 2, { observedAt: context.now(), sqlFailures: 0, reconnects: 2, @@ -209,7 +320,7 @@ describe('regional rehome assignment state', () => { databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0 }) - await heartbeat(context.store, target, targetIncarnation, 1, 2, { + await heartbeat(context.store, target, targetIncarnation, 3, 2, { observedAt: context.now(), sqlFailures: 1, reconnects: 3, @@ -237,7 +348,7 @@ describe('regional rehome assignment state', () => { requiredCells: 1, missingCells: 0 }) - await heartbeat(context.store, target, targetIncarnation, 1, 2) + await heartbeat(context.store, target, targetIncarnation, 3, 2) expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({ requiredCells: 2, missingCells: 0 @@ -256,14 +367,14 @@ describe('regional rehome assignment state', () => { databasePoolWaitersMax: 2, databasePoolWaitMsMax: 1 } - await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline) - await heartbeat(context.store, target, targetIncarnation, 1, 2, baseline) + await heartbeat(context.store, source, sourceIncarnation, 3, 2, baseline) + await heartbeat(context.store, target, targetIncarnation, 3, 2, baseline) await activatePreferredSource(context, { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) - expect(await context.store.claimRegionalRehome()).toMatchObject({ + expect(await context.store.tryIdleRehome()).toMatchObject({ sourceCellId: source.id, targetCellId: target.id }) @@ -279,6 +390,17 @@ describe('regional rehome assignment state', () => { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) + const safety: RegionalRehomeSafetySnapshot = { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + const [candidate] = await context.store.selectIdleRegionalRehomeCandidates(safety) + expect(candidate).toBeDefined() await context.database.query( `UPDATE relay_cell_rehome_safety SET reconnects = 251 WHERE cell_id = ?`, [source.id] @@ -286,7 +408,9 @@ describe('regional rehome assignment state', () => { const warnings = collectDisableWarnings() try { - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({ + outcome: 'deferred' + }) } finally { warnings.restore() } @@ -306,6 +430,17 @@ describe('regional rehome assignment state', () => { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) + const safety: RegionalRehomeSafetySnapshot = { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + const [candidate] = await context.store.selectIdleRegionalRehomeCandidates(safety) + expect(candidate).toBeDefined() await context.database.query( `UPDATE relay_cell_rehome_safety SET database_pool_waiters_max = 17 WHERE cell_id = ?`, [target.id] @@ -313,9 +448,11 @@ describe('regional rehome assignment state', () => { const warnings = collectDisableWarnings() try { - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({ + outcome: 'deferred' + }) // Already disabled: the next tick returns before the gate and stays silent. - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.tryIdleRehome()).toBeNull() } finally { warnings.restore() } @@ -339,7 +476,7 @@ describe('regional rehome assignment state', () => { `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT}` ) - expect(await context.store.claimRegionalRehome()).not.toBeNull() + expect(await context.store.tryIdleRehome()).not.toBeNull() await context.database.close() }) @@ -348,7 +485,7 @@ describe('regional rehome assignment state', () => { const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } await activateReversePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() expect(attempt).toMatchObject({ userId: identity.userId, relayHostId: identity.relayHostId, @@ -366,11 +503,13 @@ describe('regional rehome assignment state', () => { `SELECT preferred_region, source_cell_id, target_cell_id FROM relay_region_rehome_attempts` ) - ).toEqual([{ - preferred_region: 'us-central1', - source_cell_id: target.id, - target_cell_id: source.id - }]) + ).toEqual([ + { + preferred_region: 'us-central1', + source_cell_id: target.id, + target_cell_id: source.id + } + ]) expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id }) await context.database.close() }) @@ -389,21 +528,19 @@ describe('regional rehome assignment state', () => { const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') try { - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.tryIdleRehome()).toBeNull() } finally { warnings.restore() } expect(warnings.entries).toEqual([]) expect( - await context.database.query( - `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` - ) + await context.database.query(`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`) ).toEqual([{ next_dispatch_at: 0 }]) expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) await context.database.close() }) - it('names the skip when the last target is lost between scan and claim', async () => { + it('does not migrate when the last target is lost between selection and commit', async () => { const database = await openInMemoryRelayDatabase() const context = await setup({ database, @@ -419,15 +556,8 @@ describe('regional rehome assignment state', () => { relayHostId: 'abcdefghijklmnop' }) - const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') - try { - expect(await context.store.claimRegionalRehome()).toBeNull() - } finally { - warnings.restore() - } - expect(warnings.entries).toMatchObject([ - { skips: [{ reason: 'no_eligible_target', candidates: 1 }] } - ]) + expect(await context.store.tryIdleRehome()).toBeNull() + expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ generation: 1, enabled: true @@ -444,11 +574,11 @@ describe('regional rehome assignment state', () => { // really does scan and the cooldown is the only thing holding this host. context.advance(10_000) // The desktop's region probe now says us-central1 again. - await context.store.assign(identity, 'us-central1') + await activateReversePreferredSource(context, identity) const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') try { - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.tryIdleRehome()).toBeNull() } finally { warnings.restore() } @@ -462,9 +592,9 @@ describe('regional rehome assignment state', () => { cellId: target.id, expiresAt: context.now() + 90_000 }) - await context.store.assign(identity, 'us-central1') + await activateReversePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() expect(attempt).toMatchObject({ preferredRegion: 'us-central1', sourceCellId: target.id, @@ -502,15 +632,8 @@ describe('regional rehome assignment state', () => { }) await activatePreferredSource(context, identity) - const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') - try { - expect(await context.store.claimRegionalRehome()).toBeNull() - } finally { - warnings.restore() - } - expect(warnings.entries).toMatchObject([ - { skips: [{ reason: 'host_cooldown', candidates: 1 }] } - ]) + expect(await context.store.tryIdleRehome()).toBeNull() + expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) await database.close() }) @@ -527,15 +650,13 @@ describe('regional rehome assignment state', () => { const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') try { - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.tryIdleRehome()).toBeNull() } finally { warnings.restore() } expect(warnings.entries).toEqual([]) expect( - await context.database.query( - `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` - ) + await context.database.query(`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`) ).toEqual([{ next_dispatch_at: 0 }]) expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) await context.database.close() @@ -558,37 +679,16 @@ describe('regional rehome assignment state', () => { [target.id] ) - const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') - try { - expect(await context.store.claimRegionalRehome()).toBeNull() - } finally { - warnings.restore() - } + expect(await context.store.tryIdleRehome()).toBeNull() expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ generation: 1, enabled: true }) - // The skip is visible and named, and both candidates blocked by the one - // unclean cell accumulate into a single entry. - expect(warnings.entries).toMatchObject([ - { - skips: [ - { - reason: 'target_unclean', - cellId: target.id, - sqlFailures: REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1, - candidates: 2 - } - ] - } - ]) - // A skipped tick is charged the dispatch interval: candidate scans stay - // rate-limited even when nothing claims. + + // Read-only selection does not spend the commit rate budget. expect( - await context.database.query( - `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` - ) - ).toEqual([{ next_dispatch_at: context.now() + 6_000 }]) + await context.database.query(`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`) + ).toEqual([{ next_dispatch_at: 0 }]) expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([]) await context.database.close() }) @@ -598,15 +698,13 @@ describe('regional rehome assignment state', () => { const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') try { - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.tryIdleRehome()).toBeNull() } finally { warnings.restore() } expect(warnings.entries).toEqual([]) expect( - await context.database.query( - `SELECT next_dispatch_at FROM relay_region_rehome_worker_state` - ) + await context.database.query(`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`) ).toEqual([{ next_dispatch_at: 0 }]) await context.database.close() }) @@ -617,12 +715,25 @@ describe('regional rehome assignment state', () => { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) + const safety: RegionalRehomeSafetySnapshot = { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + const [candidate] = await context.store.selectIdleRegionalRehomeCandidates(safety) + expect(candidate).toBeDefined() await context.database.query( `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, [target.id] ) - expect(await context.store.claimRegionalRehome()).toBeNull() + expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({ + outcome: 'deferred' + }) expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ generation: 2, enabled: false @@ -631,80 +742,6 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - it('rechecks locked fleet safety before retrying a drain dispatch', async () => { - const context = await setup() - await activatePreferredSource(context, { - userId: 'user-1', - relayHostId: 'abcdefghijklmnop' - }) - expect(await context.store.claimRegionalRehome()).not.toBeNull() - context.advance(31_000) - await context.database.query( - `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, - [target.id] - ) - - expect(await context.store.claimRegionalRehome()).toBeNull() - expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - generation: 2, - enabled: false - }) - await context.database.close() - }) - - it('latches off after three dispatch failures and resumes only through CAS', async () => { - const context = await setup() - await activatePreferredSource(context, { - userId: 'user-1', - relayHostId: 'abcdefghijklmnop' - }) - const first = await context.store.claimRegionalRehome() - for (let index = 0; index < 3; index++) { - await context.store.recordRegionalRehomeDispatchFailure(first!.attemptId) - } - context.advance(5 * 60_000 - 1) - expect(await context.store.claimRegionalRehome()).toBeNull() - context.advance(1) - await heartbeat(context.store, source, sourceIncarnation, 1, 2, { - observedAt: context.now(), - sqlFailures: 0, - reconnects: 0, - controlActivityRecoveryFailures: 0, - databasePoolWaiting: 0, - databasePoolWaitersMax: 0, - databasePoolWaitMsMax: 0 - }) - await heartbeat(context.store, target, targetIncarnation, 1, 2, { - observedAt: context.now(), - sqlFailures: 0, - reconnects: 0, - controlActivityRecoveryFailures: 0, - databasePoolWaiting: 0, - databasePoolWaitersMax: 0, - databasePoolWaitMsMax: 0 - }) - expect(await context.store.claimRegionalRehome()).toBeNull() - expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - generation: 2, - enabled: false - }) - await context.store.applyRegionalRehomeControl({ - expectedGeneration: 2, - enabled: true, - notBefore: context.now(), - ratePerMinute: 10, - preferenceMaxAgeMs: 24 * 60 * 60_000, - hostCooldownMs: 7 * 24 * 60 * 60_000, - drainGraceMs: 60_000 - }) - const retry = await context.store.claimRegionalRehome() - expect(retry).toMatchObject({ attemptId: first!.attemptId, sendAttempts: 2 }) - expect(await context.database.query( - `SELECT COUNT(*) AS count FROM relay_assignment_migrations` - )).toEqual([{ count: 1 }]) - await context.database.close() - }) - it('refreshes only the migration leases while source splices drain', async () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } @@ -714,7 +751,7 @@ describe('regional rehome assignment state', () => { kind: 'splice', cellId: source.id }) - await context.store.claimRegionalRehome() + await context.store.tryIdleRehome() const before = await context.database.query( `SELECT activity_id, expires_at FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`, @@ -743,19 +780,21 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } await activatePreferredSource(context, identity) - await context.store.claimRegionalRehome() + await context.store.tryIdleRehome() context.advance(6 * 60_000) expect(await context.store.refreshRegionalRehomeLeases()).toBe(0) - await heartbeat(context.store, source, sourceIncarnation, 1, 2) + await heartbeat(context.store, source, sourceIncarnation, 3, 2) expect(await context.store.abortExpiredEvacuations()).toBe(1) - expect(await context.store.reapRegionalRehomeAttempts()).toBe(1) + expect(await context.store.reapRegionalRehomeAttempts()).toBe(0) expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id, assignmentEpoch: 3 }) - expect(await context.database.query( - `SELECT completed_at, aborted_at FROM relay_region_rehome_attempts` - )).toEqual([{ completed_at: null, aborted_at: context.now() }]) + expect( + await context.database.query( + `SELECT completed_at, aborted_at FROM relay_region_rehome_attempts` + ) + ).toEqual([{ completed_at: null, aborted_at: context.now() }]) await context.database.close() }) @@ -763,9 +802,9 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } await activatePreferredSource(context, identity) - await context.store.claimRegionalRehome() + await context.store.tryIdleRehome() context.advance(6 * 60_000) - await heartbeat(context.store, target, targetIncarnation, 1, 2) + await heartbeat(context.store, target, targetIncarnation, 3, 2) expect(await context.store.refreshRegionalRehomeLeases()).toBe(0) expect(await context.store.abortExpiredEvacuations()).toBe(0) @@ -779,41 +818,6 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - it('skips a rehome dispatch tick on a contended cell inventory', async () => { - const probe = new CellInventoryLockProbe() - const context = await setup({ wrap: (database) => probe.wrap(database) }) - const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } - await activatePreferredSource(context, identity) - probe.reset() - probe.failNoWait = true - const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') - - let attempt: unknown - try { - attempt = await context.store.claimRegionalRehome() - } finally { - busy.restore() - } - - expect(attempt).toBeNull() - expect(probe.locks).not.toEqual([]) - expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true) - expect(busy.entries).toEqual([ - { - event: 'orca_relay_sweep_cell_inventory_busy', - sweep: 'claim-regional-rehome', - skipped: 1 - } - ]) - - probe.failNoWait = false - expect(await context.store.claimRegionalRehome()).toMatchObject({ - sourceCellId: source.id, - targetCellId: target.id - }) - await context.database.close() - }) - // Why: the redrain lane reaches the inventory through the fleet-safety read // rather than through candidate selection, so it needs its own coverage. // Why: one contended candidate must cost its own tick, not the whole page. The @@ -830,8 +834,7 @@ describe('regional rehome assignment state', () => { context.advance(60_000) await freshHeartbeats(context) const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: 2, @@ -881,8 +884,7 @@ describe('regional rehome assignment state', () => { context.advance(60_000) await freshHeartbeats(context) const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: 2, @@ -927,9 +929,7 @@ describe('regional rehome assignment state', () => { const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') try { - await expect(context.store.claimRegionalRehome()).rejects.toThrow( - 'relay_capacity_exhausted' - ) + await expect(context.store.tryIdleRehome()).rejects.toThrow('relay_capacity_exhausted') } finally { busy.restore() } @@ -940,87 +940,13 @@ describe('regional rehome assignment state', () => { // Why: the transaction dies at the first contended candidate, so every // candidate behind it is abandoned too. Reporting one would understate the tick. - it('reports every candidate the contended tick abandoned', async () => { - const probe = new CellInventoryLockProbe() - const context = await setup({ wrap: (database) => probe.wrap(database) }) - await activatePreferredSource(context, { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) - await activatePreferredSource(context, { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' }) - await activatePreferredSource(context, { userId: 'user-3', relayHostId: 'aaaabbbbccccdddd' }) - probe.reset() - probe.failNoWait = true - const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') - - try { - expect(await context.store.claimRegionalRehome()).toBeNull() - } finally { - busy.restore() - } - - expect(busy.entries).toEqual([ - { - event: 'orca_relay_sweep_cell_inventory_busy', - sweep: 'claim-regional-rehome', - skipped: 3 - } - ]) - await context.database.close() - }) - - it('skips a redrain tick on a contended cell inventory', async () => { - const probe = new CellInventoryLockProbe() - const context = await setup({ wrap: (database) => probe.wrap(database) }) - const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } - await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - await context.store.activateControl(identity, { - cellId: target.id, - assignmentEpoch: 2, - generation: 1 - }) - await context.store.markMigrationTargetRegistered(identity, { - cellId: target.id, - assignmentEpoch: 2 - }) - context.advance(60 * 60_000 + 1) - await freshHeartbeats(context) - probe.reset() - probe.failNoWait = true - const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') - - let redrain: unknown - try { - redrain = await context.store.claimRegionalRehome() - } finally { - busy.restore() - } - - expect(redrain).toBeNull() - expect(probe.locks).not.toEqual([]) - expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true) - expect(busy.entries).toEqual([ - { - event: 'orca_relay_sweep_cell_inventory_busy', - sweep: 'claim-regional-rehome', - skipped: 1 - } - ]) - - probe.failNoWait = false - expect(await context.store.claimRegionalRehome()).toMatchObject({ - attemptId: attempt!.attemptId, - sendAttempts: 2 - }) - await context.database.close() - }) it('skips a completion tick on a contended cell inventory without quarantining it', async () => { const probe = new CellInventoryLockProbe() const context = await setup({ wrap: (database) => probe.wrap(database) }) const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: 2, @@ -1069,8 +995,7 @@ describe('regional rehome assignment state', () => { const context = await setup({ wrap: (database) => probe.wrap(database) }) const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await context.store.tryIdleRehome() const targetControl = await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: 2, @@ -1083,7 +1008,7 @@ describe('regional rehome assignment state', () => { await context.store.releaseActivity(identity, sourceControl) await context.store.releaseActivity(identity, targetControl) context.advance(24 * 60 * 60_000) - await heartbeat(context.store, source, sourceIncarnation, 1, 2) + await heartbeat(context.store, source, sourceIncarnation, 3, 2) probe.reset() probe.failNoWait = true const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') @@ -1118,11 +1043,7 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt( - attempt!.attemptId, - 'accepted' - ) + const attempt = await context.store.tryIdleRehome() const targetControl = await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: 2, @@ -1135,7 +1056,7 @@ describe('regional rehome assignment state', () => { await context.store.releaseActivity(identity, sourceControl) await context.store.releaseActivity(identity, targetControl) context.advance(24 * 60 * 60_000) - await heartbeat(context.store, source, sourceIncarnation, 1, 2) + await heartbeat(context.store, source, sourceIncarnation, 3, 2) expect(await context.store.abortExpiredRegionalRehomes()).toBe(1) expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id, @@ -1144,151 +1065,21 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - it('redrains a receipted dual-homed attempt once its grace elapses', async () => { - const context = await setup() - const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } - const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - await context.store.activateControl(identity, { - cellId: target.id, - assignmentEpoch: 2, - generation: 1 - }) - await context.store.markMigrationTargetRegistered(identity, { - cellId: target.id, - assignmentEpoch: 2 - }) - - // Before grace elapses a receipted attempt is not re-dispatched. - context.advance(30 * 60_000) - await freshHeartbeats(context) - expect(await context.store.claimRegionalRehome()).toBeNull() - - context.advance(30 * 60_000 + 1) - await freshHeartbeats(context) - const redrain = await context.store.claimRegionalRehome() - expect(redrain).toMatchObject({ - attemptId: attempt!.attemptId, - drainGraceMs: 0, - sendAttempts: 2 - }) - // The per-dispatch receipt replaces the original without a mismatch. - await expect( - context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'host-not-connected') - ).resolves.toBe(true) - - // Redrains are spaced: nothing new inside the redrain interval. - context.advance(30_000) - await freshHeartbeats(context) - expect(await context.store.claimRegionalRehome()).toBeNull() - context.advance(30_001) - await freshHeartbeats(context) - expect(await context.store.claimRegionalRehome()).toMatchObject({ - attemptId: attempt!.attemptId, - drainGraceMs: 0, - sendAttempts: 3 - }) - - // Once the host actually leaves the source, completion wins over redrain. - await context.store.releaseActivity(identity, sourceControl) - context.advance(60_001) - await freshHeartbeats(context) - await context.store.activateControl(identity, { - cellId: target.id, - assignmentEpoch: 2, - generation: 2 - }) - expect(await context.store.claimRegionalRehome()).toBeNull() - expect(await context.store.completeReadyRegionalRehomes()).toBe(1) - await context.database.close() - }) - - it('resets the failure budget on a repeated redrain receipt outcome', async () => { - const context = await setup() - const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } - await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - await context.store.activateControl(identity, { - cellId: target.id, - assignmentEpoch: 2, - generation: 1 - }) - await context.store.markMigrationTargetRegistered(identity, { - cellId: target.id, - assignmentEpoch: 2 - }) - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - - context.advance(60 * 60_000 + 1) - await freshHeartbeats(context) - expect(await context.store.claimRegionalRehome()).toMatchObject({ - attemptId: attempt!.attemptId, - drainGraceMs: 0 - }) - // The repeated outcome still proves the source answered. - expect( - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - ).toBe(false) - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - generation: 1, - enabled: true - }) - await context.database.close() - }) - - it('does not redrain before the target registers or when the fleet is unsafe', async () => { - const context = await setup() - const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } - await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - await context.store.activateControl(identity, { - cellId: target.id, - assignmentEpoch: 2, - generation: 1 - }) - - // Past grace but the target never registered: force-closing the source - // would disconnect the host with nowhere proven to land. - context.advance(60 * 60_000 + 1) - await freshHeartbeats(context) - expect(await context.store.claimRegionalRehome()).toBeNull() - - await context.store.markMigrationTargetRegistered(identity, { - cellId: target.id, - assignmentEpoch: 2 - }) - await context.database.query( - `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`, - [target.id] - ) - expect(await context.store.claimRegionalRehome()).toBeNull() - expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - enabled: false - }) - await context.database.close() - }) - it('completes healthy candidates past a poisoned attempt and logs it', async () => { const context = await setup() const poisoned = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const healthy = { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } const poisonedSource = await activatePreferredSource(context, poisoned) const healthySource = await activatePreferredSource(context, healthy) - const first = await context.store.claimRegionalRehome() + const first = await context.store.tryIdleRehome() context.advance(6_000) - const second = await context.store.claimRegionalRehome() + const second = await context.store.tryIdleRehome() expect(first!.userId).toBe(poisoned.userId) expect(second!.userId).toBe(healthy.userId) for (const [identity, attempt, sourceControl] of [ [poisoned, first, poisonedSource], [healthy, second, healthySource] ] as const) { - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -1320,10 +1111,12 @@ describe('regional rehome assignment state', () => { reason: 'regional_rehome_assignment_mismatch' } ]) - expect(await context.database.query( - `SELECT completed_at FROM relay_region_rehome_attempts WHERE attempt_id = ?`, - [second!.attemptId] - )).toEqual([{ completed_at: context.now() }]) + expect( + await context.database.query( + `SELECT completed_at FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [second!.attemptId] + ) + ).toEqual([{ completed_at: context.now() }]) await context.database.close() }) @@ -1331,8 +1124,7 @@ describe('regional rehome assignment state', () => { const context = await setup() const poisoned = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const source1 = await activatePreferredSource(context, poisoned) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(poisoned, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -1363,9 +1155,7 @@ describe('regional rehome assignment state', () => { expect(warnings.entries).toHaveLength(REGIONAL_REHOME_QUARANTINE_FAILURES + 1) // A free-form error (never a slug) reaches the log only as 'redacted'. expect( - warnings.entries.every( - (entry) => entry.reason === 'regional_rehome_assignment_mismatch' - ) + warnings.entries.every((entry) => entry.reason === 'regional_rehome_assignment_mismatch') ).toBe(true) context.advance(REGIONAL_REHOME_QUARANTINE_MS + 1) const database = context.database @@ -1393,14 +1183,13 @@ describe('regional rehome assignment state', () => { const healthy = { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } const poisonedSource = await activatePreferredSource(context, poisoned) const healthySource = await activatePreferredSource(context, healthy) - const first = await context.store.claimRegionalRehome() + const first = await context.store.tryIdleRehome() context.advance(6_000) - const second = await context.store.claimRegionalRehome() + const second = await context.store.tryIdleRehome() for (const [identity, attempt, sourceControl] of [ [poisoned, first, poisonedSource], [healthy, second, healthySource] ] as const) { - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') const targetControl = await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -1434,10 +1223,12 @@ describe('regional rehome assignment state', () => { reason: 'regional_rehome_assignment_mismatch' } ]) - expect(await context.database.query( - `SELECT aborted_at FROM relay_region_rehome_attempts WHERE attempt_id = ?`, - [second!.attemptId] - )).toEqual([{ aborted_at: context.now() }]) + expect( + await context.database.query( + `SELECT aborted_at FROM relay_region_rehome_attempts WHERE attempt_id = ?`, + [second!.attemptId] + ) + ).toEqual([{ aborted_at: context.now() }]) await context.database.close() }) @@ -1445,7 +1236,7 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() expect(await controlAccounting(context, identity)).toEqual({ reservedControls: 2, controlLeases: 2 @@ -1475,11 +1266,13 @@ describe('regional rehome assignment state', () => { }) expect(await context.store.completeReadyRegionalRehomes()).toBe(1) - expect(await context.database.query( - `SELECT completed_at FROM relay_assignment_migrations + expect( + await context.database.query( + `SELECT completed_at FROM relay_assignment_migrations WHERE user_id = ? AND relay_host_id = ?`, - [identity.userId, identity.relayHostId] - )).toEqual([{ completed_at: context.now() }]) + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ completed_at: context.now() }]) expect(await cellReservations(context)).toEqual({ [source.id]: 0, [target.id]: 1 }) await context.database.close() }) @@ -1488,7 +1281,7 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -1503,11 +1296,13 @@ describe('regional rehome assignment state', () => { ) await context.store.assign(identity, 'asia-east2') - expect(await context.database.query( - `SELECT activity_id FROM relay_assignment_activity_leases + expect( + await context.database.query( + `SELECT activity_id FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`, - [identity.userId, identity.relayHostId] - )).toEqual([{ activity_id: `control:${target.id}:1` }]) + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ activity_id: `control:${target.id}:1` }]) expect(await cellReservations(context)).toEqual({ [source.id]: 0, [target.id]: 2 }) await context.database.close() }) @@ -1516,7 +1311,7 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -1539,11 +1334,13 @@ describe('regional rehome assignment state', () => { reservedControls: 1, controlLeases: 1 }) - expect(await context.database.query( - `SELECT migration_leases FROM relay_assignments + expect( + await context.database.query( + `SELECT migration_leases FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, - [identity.userId, identity.relayHostId] - )).toEqual([{ migration_leases: 0 }]) + [identity.userId, identity.relayHostId] + ) + ).toEqual([{ migration_leases: 0 }]) await context.database.close() }) @@ -1553,9 +1350,9 @@ describe('regional rehome assignment state', () => { const clean = { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } const skewedSource = await activatePreferredSource(context, skewed) const cleanSource = await activatePreferredSource(context, clean) - const first = await context.store.claimRegionalRehome() + const first = await context.store.tryIdleRehome() context.advance(6_000) - const second = await context.store.claimRegionalRehome() + const second = await context.store.tryIdleRehome() for (const [identity, attempt, sourceControl] of [ [skewed, first, skewedSource], [clean, second, cleanSource] @@ -1599,7 +1396,7 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -1646,7 +1443,7 @@ describe('regional rehome assignment state', () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -1684,102 +1481,6 @@ describe('regional rehome assignment state', () => { ]) await context.database.close() }) - - it('caps redrain dispatches at the send limit', async () => { - const context = await setup() - const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } - await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() - await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') - await context.store.activateControl(identity, { - cellId: target.id, - assignmentEpoch: 2, - generation: 1 - }) - await context.store.markMigrationTargetRegistered(identity, { - cellId: target.id, - assignmentEpoch: 2 - }) - await context.database.query( - `UPDATE relay_region_rehome_attempts SET send_attempts = ? WHERE attempt_id = ?`, - [REGIONAL_REHOME_REDRAIN_SEND_LIMIT, attempt!.attemptId] - ) - context.advance(60 * 60_000 + 1) - await freshHeartbeats(context) - expect(await context.store.claimRegionalRehome()).toBeNull() - await context.database.close() - }) - - it('clears a stale failure budget when the control is enabled again', async () => { - const context = await setup() - await activatePreferredSource(context, { - userId: 'user-1', - relayHostId: 'abcdefghijklmnop' - }) - const attempt = await context.store.claimRegionalRehome() - for (let index = 0; index < 3; index++) { - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - } - expect(await workerState(context)).toMatchObject({ consecutiveFailures: 3 }) - const latched = await context.store.inspectRegionalRehomeControl() - expect(latched).toMatchObject({ generation: 2, enabled: false }) - - await context.store.applyRegionalRehomeControl({ - expectedGeneration: latched.generation, - enabled: true, - notBefore: context.now(), - ratePerMinute: 10, - preferenceMaxAgeMs: 24 * 60 * 60_000, - hostCooldownMs: 7 * 24 * 60 * 60_000, - drainGraceMs: 60 * 60_000 - }) - - // A budget spent under the previous enable is not evidence about this one. - expect(await workerState(context)).toMatchObject({ - consecutiveFailures: 0, - pausedUntil: 0 - }) - // One transient failure must not latch the fresh enable straight back off. - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - generation: 3, - enabled: true - }) - await context.database.close() - }) - - it('reports the durable disable when the failure budget latches the control off', async () => { - const context = await setup() - await activatePreferredSource(context, { - userId: 'user-1', - relayHostId: 'abcdefghijklmnop' - }) - const attempt = await context.store.claimRegionalRehome() - const warnings = collectEventWarnings( - 'orca_relay_regional_rehome_failure_budget_disabled' - ) - try { - for (let index = 0; index < 5; index++) { - await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId) - } - } finally { - warnings.restore() - } - - // Only the transition is reported; later failures find the control already off. - expect(warnings.entries).toEqual([ - expect.objectContaining({ - event: 'orca_relay_regional_rehome_failure_budget_disabled', - controlGeneration: 2, - consecutiveFailures: 3 - }) - ]) - expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - generation: 2, - enabled: false - }) - await context.database.close() - }) }) class TransactionCountingDatabase implements RelayDatabase { @@ -1848,7 +1549,8 @@ async function setup( ) { let clock = 1_000_000 const database = options.database ?? (await openInMemoryRelayDatabase()) - const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, { + const store = new IdleRehomeTestStore(options.wrap?.(database) ?? database, () => clock, { + regionalRehomeCohortPercent: 100, requireLiveCells: true, heartbeatTtlMs: 45_000 }) @@ -1864,8 +1566,8 @@ async function setup( drainGraceMs: 60 * 60_000 }) await store.reconcileCells([source, target]) - await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1) - await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 1) + await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 3) + await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 3) return { database, store, @@ -1950,9 +1652,7 @@ async function cellReservations(context: Context): Promise [String(row.cell_id), Number(row.reserved_requests)]) - ) + return Object.fromEntries(rows.map((row) => [String(row.cell_id), Number(row.reserved_requests)])) } async function freshHeartbeats(context: Context): Promise { @@ -1966,8 +1666,8 @@ async function freshHeartbeats(context: Context): Promise { databasePoolWaitMsMax: 0 } // The clock doubles as a strictly-increasing connection inclusion watermark. - await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety) - await heartbeat(context.store, target, targetIncarnation, 1, context.now(), safety) + await heartbeat(context.store, source, sourceIncarnation, 3, context.now(), safety) + await heartbeat(context.store, target, targetIncarnation, 3, context.now(), safety) } async function activatePreferredSource( @@ -1978,9 +1678,29 @@ async function activatePreferredSource( const control = await context.store.activateControl(identity, { cellId: source.id, assignmentEpoch: assignment.assignmentEpoch, - generation: 1 + generation: 1, + idleRegionalRehome: true, + cellIncarnation: sourceIncarnation }) await context.store.assign(identity, 'asia-east2') + const issued = await context.store.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + assignment.assignmentEpoch + ) + await context.store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.window!.generation, + assignmentEpoch: assignment.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 150, 'asia-east2': 50 } + }, + assignment.assignmentEpoch + ) return control } @@ -1994,7 +1714,7 @@ function hookAfterCandidateScan( const decorate = (delegate: RelayDatabase): RelayDatabase => ({ query: async (sql, params) => { const rows = await delegate.query(sql, params) - if (!fired && sql.includes('FROM relay_assignment_region_preferences preference')) { + if (!fired && sql.includes('SELECT a.user_id, a.relay_host_id')) { fired = true await hook(delegate) } @@ -2017,7 +1737,7 @@ async function completeRehomeToTarget( identity: { userId: string; relayHostId: string } ): Promise { const sourceControl = await activatePreferredSource(context, identity) - const attempt = await context.store.claimRegionalRehome() + const attempt = await context.store.tryIdleRehome() const targetControl = await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: attempt!.assignmentEpoch, @@ -2040,9 +1760,29 @@ async function activateReversePreferredSource( const control = await context.store.activateControl(identity, { cellId: target.id, assignmentEpoch: assignment.assignmentEpoch, - generation: 1 + generation: 1, + idleRegionalRehome: true, + cellIncarnation: targetIncarnation }) await context.store.assign(identity, 'us-central1') + const issued = await context.store.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + assignment.assignmentEpoch + ) + await context.store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.window!.generation, + assignmentEpoch: assignment.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 50, 'asia-east2': 150 } + }, + assignment.assignmentEpoch + ) return control } @@ -2059,7 +1799,7 @@ async function activateSource( } async function heartbeat( - store: RelayAssignmentStore, + store: IdleRehomeTestStore, cell: typeof source | typeof target, cellIncarnation: string, regionalRehomeProtocol: number, @@ -2152,3 +1892,46 @@ async function workerState( pausedUntil: Number(row.paused_until) } } + +class IdleRehomeTestStore extends BaseRelayAssignmentStore { + private readonly fixtureDatabase: RelayDatabase + private readonly fixtureNow: () => number + constructor(...args: ConstructorParameters) { + super(...args) + this.fixtureDatabase = args[0] + this.fixtureNow = args[1] ?? Date.now + } + async tryIdleRehome( + processSafety?: RegionalRehomeSafetySnapshot + ): Promise { + const safety = processSafety ?? { + observedAt: this.fixtureNow(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + for (const candidate of await this.selectIdleRegionalRehomeCandidates(safety)) { + const result = await this.commitIdleRegionalRehome(candidate, safety) + if (result.outcome !== 'committed') continue + const row = ( + await this.fixtureDatabase.query( + 'SELECT * FROM relay_region_rehome_attempts WHERE attempt_id = ?', + [candidate.attemptId] + ) + )[0]! + return { + ...candidate, + preferredRegion: row.preferred_region as RegionalRehomeAttempt['preferredRegion'], + targetCellIncarnation: String(row.target_cell_incarnation), + previousEpoch: Number(row.previous_epoch), + assignmentEpoch: Number(row.assignment_epoch), + drainGraceMs: Number(row.drain_grace_ms), + sendAttempts: Number(row.send_attempts) + } + } + return null + } +} diff --git a/cloud/apps/relay/src/regional-rehome-target-selection.test.ts b/cloud/apps/relay/src/regional-rehome-target-selection.test.ts index 493eaa50a61..89ae6f88544 100644 --- a/cloud/apps/relay/src/regional-rehome-target-selection.test.ts +++ b/cloud/apps/relay/src/regional-rehome-target-selection.test.ts @@ -25,6 +25,7 @@ async function setup() { let clock = 1_000_000 const database = await openInMemoryRelayDatabase() const store = new RelayAssignmentStore(database, () => clock, { + regionalRehomeCohortPercent: 100, requireLiveCells: true, heartbeatTtlMs: 45_000 }) @@ -83,11 +84,40 @@ async function setup() { await store.activateControl(identity, { cellId: source.id, assignmentEpoch: assignment.assignmentEpoch, - generation: 1 + generation: 1, + idleRegionalRehome: true, + cellIncarnation: incarnation(1) }) - await store.assign(identity, 'asia-east2') + const { window } = await store.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + assignment.assignmentEpoch + ) + expect(window).toBeDefined() + await store.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: window!.generation, + assignmentEpoch: assignment.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 180, 'asia-east2': 40 } + }, + assignment.assignmentEpoch + ) } - return { database, store, beat, activatePreferredSource } + const safety = () => ({ + observedAt: clock, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + return { database, store, beat, activatePreferredSource, safety } } const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1 @@ -95,70 +125,78 @@ const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1 describe('regional rehome target selection', () => { it('never selects a target without connection headroom, even at lowest load', async () => { const context = await setup() - await context.beat(source, 1, 1, { + await context.beat(source, 1, 3, { observedRequests: 0, enforcedConnections: 0, sqlFailures: 0 }) // Lowest load but the connection hard cap is exhausted. - await context.beat(noHeadroom, 2, 1, { + await context.beat(noHeadroom, 2, 3, { observedRequests: 0, enforcedConnections: 999, sqlFailures: 0 }) - await context.beat(unclean, 3, 1, { + await context.beat(unclean, 3, 3, { observedRequests: 0, enforcedConnections: 0, sqlFailures: UNCLEAN }) - await context.beat(highLoad, 4, 1, { + await context.beat(highLoad, 4, 3, { observedRequests: 50, enforcedConnections: 0, sqlFailures: 0 }) - await context.beat(lowLoad, 5, 1, { + await context.beat(lowLoad, 5, 3, { observedRequests: 10, enforcedConnections: 0, sqlFailures: 0 }) await context.activatePreferredSource() - const attempt = await context.store.claimRegionalRehome() + const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety()) + const attempt = candidates[0] expect(attempt?.targetCellId).toBe(lowLoad.id) + expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({ + outcome: 'committed' + }) await context.database.close() }) it('falls to the next clean target when the load winner goes unclean', async () => { const context = await setup() - await context.beat(source, 1, 1, { + await context.beat(source, 1, 3, { observedRequests: 0, enforcedConnections: 0, sqlFailures: 0 }) - await context.beat(noHeadroom, 2, 1, { + await context.beat(noHeadroom, 2, 3, { observedRequests: 0, enforcedConnections: 999, sqlFailures: 0 }) - await context.beat(unclean, 3, 1, { + await context.beat(unclean, 3, 3, { observedRequests: 0, enforcedConnections: 0, sqlFailures: UNCLEAN }) - await context.beat(highLoad, 4, 1, { + await context.beat(highLoad, 4, 3, { observedRequests: 50, enforcedConnections: 0, sqlFailures: 0 }) - await context.beat(lowLoad, 5, 1, { + await context.beat(lowLoad, 5, 3, { observedRequests: 10, enforcedConnections: 0, sqlFailures: UNCLEAN }) await context.activatePreferredSource() - const attempt = await context.store.claimRegionalRehome() + const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety()) + const attempt = candidates[0] expect(attempt?.targetCellId).toBe(highLoad.id) + expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({ + outcome: 'committed' + }) await context.database.close() }) }) diff --git a/cloud/apps/relay/src/regional-rehome-worker.test.ts b/cloud/apps/relay/src/regional-rehome-worker.test.ts index 33e7f01f737..bd47923bdf1 100644 --- a/cloud/apps/relay/src/regional-rehome-worker.test.ts +++ b/cloud/apps/relay/src/regional-rehome-worker.test.ts @@ -11,148 +11,12 @@ import { startRegionalRehomeWorker } from './regional-rehome-worker.js' describe('regional rehome worker', () => { afterEach(() => vi.restoreAllMocks()) - it('sends an incarnation- and source-epoch-bound drain without exposing identity', async () => { - let now = 0 - const attempt = { - attemptId: '11111111-1111-4111-8111-111111111111', - userId: 'private-user', - relayHostId: 'abcdefghijklmnop', - preferredRegion: 'asia-east2', - sourceCellId: 'production-gce-c7', - sourceCellUrl: 'https://c7.relay.example.test', - sourceCellIncarnation: '22222222-2222-4222-8222-222222222222', - targetCellId: 'production-gce-c27', - targetCellIncarnation: '33333333-3333-4333-8333-333333333333', - previousEpoch: 7, - assignmentEpoch: 8, - drainGraceMs: 60_000, - sendAttempts: 1 - } - const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt) - const recordRegionalRehomeDrainReceipt = vi.fn().mockResolvedValue(true) - const assignments = { - claimRegionalRehome, - recordRegionalRehomeDrainReceipt - } as unknown as RelayAssignmentStore - const requests: Array<{ url: string; init?: RequestInit }> = [] - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const worker = startRegionalRehomeWorker(config(), assignments, { - now: () => now, - safetySnapshot: () => safety(now), - intervalMs: 60_000, - identityToken: async (audience) => { - expect(audience).toBe('https://relay.example.test/v1/admin/host-drain') - return 'secret-token' - }, - fetch: (async (url, init) => { - requests.push({ url: String(url), init }) - return Response.json({ v: 1, outcome: 'accepted' }) - }) as typeof fetch - })! - await settleWorker() - now = 1_000 - await worker.run() - worker.stop() - - expect(requests).toHaveLength(1) - expect(requests[0]!.url).toBe('https://c7.relay.example.test/v1/admin/host-drain') - expect(requests[0]!.url).not.toContain('secret-token') - expect(requests[0]!.init?.headers).toMatchObject({ - authorization: 'Bearer secret-token' - }) - expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({ - v: 1, - attemptId: '11111111-1111-4111-8111-111111111111', - userId: 'private-user', - relayHostId: 'abcdefghijklmnop', - sourceCellId: 'production-gce-c7', - sourceCellIncarnation: '22222222-2222-4222-8222-222222222222', - sourceAssignmentEpoch: 7, - graceMs: 60_000 - }) - expect(recordRegionalRehomeDrainReceipt).toHaveBeenCalledWith( - '11111111-1111-4111-8111-111111111111', - 'accepted' - ) - const logs = warn.mock.calls.map((call) => String(call[0])).join('\n') - expect(logs).not.toContain('private-user') - expect(logs).not.toContain('abcdefghijklmnop') - }) - - it('fails closed before the observation gate and records bounded dispatch failures', async () => { - let now = 0 - const attempt = { - attemptId: '11111111-1111-4111-8111-111111111111', - userId: 'private-user', - relayHostId: 'abcdefghijklmnop', - sourceCellId: 'source', - sourceCellUrl: 'https://source.example.test', - sourceCellIncarnation: '22222222-2222-4222-8222-222222222222', - targetCellId: 'target', - previousEpoch: 1, - assignmentEpoch: 2, - drainGraceMs: 60_000, - sendAttempts: 1 - } - const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt) - const assignments = { - claimRegionalRehome, - recordRegionalRehomeDispatchFailure: vi.fn().mockResolvedValue(undefined) - } as unknown as RelayAssignmentStore - const worker = startRegionalRehomeWorker(config(), assignments, { - now: () => now, - safetySnapshot: () => safety(now), - intervalMs: 60_000, - identityToken: async () => { - throw new Error('token unavailable') - } - })! - await settleWorker() - claimRegionalRehome.mockClear() - now = 100 - await worker.run() - worker.stop() - expect(assignments.recordRegionalRehomeDispatchFailure).toHaveBeenCalledWith( - '11111111-1111-4111-8111-111111111111' - ) - }) - - it('keeps a failed poll out of the durable dispatch-failure budget', async () => { - let now = 0 - const claimRegionalRehome = vi - .fn() - .mockResolvedValueOnce(null) - .mockRejectedValue(new Error('Connection terminated due to connection timeout')) - const recordRegionalRehomeDispatchFailure = vi.fn().mockResolvedValue(undefined) - const assignments = { - claimRegionalRehome, - recordRegionalRehomeDispatchFailure - } as unknown as RelayAssignmentStore - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const worker = startRegionalRehomeWorker(config(), assignments, { - now: () => now, - safetySnapshot: () => safety(now), - intervalMs: 60_000 - })! - await settleWorker() - now = 1_000 - await expect(worker.run()).resolves.toBeUndefined() - worker.stop() - - // The poll never claimed an attempt, so nothing was drained and nothing may - // be charged to the budget that latches the durable control off. - expect(recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled() - expect(warn.mock.calls.map((call) => JSON.parse(String(call[0])).event)).toEqual([ - 'orca_relay_regional_rehome_poll_failed' - ]) - }) - it('passes unsafe process telemetry to the durable claim gate', async () => { let now = 0 let sqlFailures = 0 - const claimRegionalRehome = vi.fn().mockResolvedValue(null) + const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([]) const assignments = { - claimRegionalRehome + selectIdleRegionalRehomeCandidates } as unknown as RelayAssignmentStore const worker = startRegionalRehomeWorker(config(), assignments, { now: () => now, @@ -160,46 +24,40 @@ describe('regional rehome worker', () => { intervalMs: 60_000 })! await settleWorker() - claimRegionalRehome.mockClear() + selectIdleRegionalRehomeCandidates.mockClear() now = 100 sqlFailures = 1 await worker.run() worker.stop() - expect(claimRegionalRehome).toHaveBeenCalledWith( + expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledWith( expect.objectContaining({ observedAt: 100, sqlFailures: 1 }) ) }) it('starts inert on directors so durable control can enable without a restart', async () => { let now = 0 - const claimRegionalRehome = vi.fn().mockResolvedValue(null) + const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([]) const assignments = { - claimRegionalRehome + selectIdleRegionalRehomeCandidates } as unknown as RelayAssignmentStore - const worker = startRegionalRehomeWorker( - config(), - assignments, - { - now: () => now, - safetySnapshot: () => safety(now), - intervalMs: 60_000 - } - ) + const worker = startRegionalRehomeWorker(config(), assignments, { + now: () => now, + safetySnapshot: () => safety(now), + intervalMs: 60_000 + }) expect(worker).not.toBeNull() await settleWorker() - claimRegionalRehome.mockClear() + selectIdleRegionalRehomeCandidates.mockClear() now = 100 await worker!.run() worker!.stop() - expect(claimRegionalRehome).toHaveBeenCalledOnce() + expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce() expect( - startRegionalRehomeWorker( - config({ role: 'cell' }), - {} as RelayAssignmentStore, - { safetySnapshot: () => safety(1) } - ) + startRegionalRehomeWorker(config({ role: 'cell' }), {} as RelayAssignmentStore, { + safetySnapshot: () => safety(1) + }) ).toBeNull() }) @@ -208,16 +66,20 @@ describe('regional rehome worker', () => { const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT const processSafety = { ...safety(100), reconnects: limit * 10 } const fleetSafety = { ...safety(100), reconnects: limit } - expect(regionalRehomeSafetyFailure( - combineRegionalRehomeSafety(processSafety, fleetSafety), - 100, - cells - )).toBeNull() - expect(regionalRehomeSafetyFailure( - combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }), - 100, - cells - )).toBe('elevated_reconnects') + expect( + regionalRehomeSafetyFailure( + combineRegionalRehomeSafety(processSafety, fleetSafety), + 100, + cells + ) + ).toBeNull() + expect( + regionalRehomeSafetyFailure( + combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }), + 100, + cells + ) + ).toBe('elevated_reconnects') }) }) diff --git a/cloud/apps/relay/src/regional-rehome-worker.ts b/cloud/apps/relay/src/regional-rehome-worker.ts index 4d8fa694afd..5f3827f5188 100644 --- a/cloud/apps/relay/src/regional-rehome-worker.ts +++ b/cloud/apps/relay/src/regional-rehome-worker.ts @@ -1,4 +1,4 @@ -import { z } from 'zod' +import { IdleRegionalRehomeResponseSchema } from '@orca-cloud/relay-contract' import type { RelayAssignmentStore } from './assignment-store.js' import type { RelayConfig } from './config.js' import { googleMetadataIdentityToken } from './google-metadata-identity-token.js' @@ -20,13 +20,6 @@ export type RegionalRehomeWorker = { stop: () => void } -const RegionalHostDrainResponseSchema = z - .object({ - v: z.literal(1), - outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected']) - }) - .strict() - export function startRegionalRehomeWorker( config: RelayConfig, assignments: RelayAssignmentStore, @@ -42,7 +35,6 @@ export function startRegionalRehomeWorker( } const audience = config.rehomeAudience const safetySnapshot = options.safetySnapshot - const now = options.now ?? Date.now const fetchImpl = options.fetch ?? fetch const tokenProvider = options.identityToken ?? @@ -52,64 +44,50 @@ export function startRegionalRehomeWorker( const run = async (): Promise => { if (stopped || inFlight) return inFlight = true - let attemptId: string | null = null try { - const processSafety = safetySnapshot() - const attempt = await assignments.claimRegionalRehome(processSafety) - if (!attempt) return - attemptId = attempt.attemptId + const candidates = await assignments.selectIdleRegionalRehomeCandidates(safetySnapshot()) + if (candidates.length === 0) return const token = await tokenProvider(audience) - const response = await fetchImpl( - new URL('/v1/admin/host-drain', attempt.sourceCellUrl), - { - method: 'POST', - headers: { - authorization: `Bearer ${token}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ - v: 1, - attemptId: attempt.attemptId, - userId: attempt.userId, - relayHostId: attempt.relayHostId, - sourceCellId: attempt.sourceCellId, - sourceCellIncarnation: attempt.sourceCellIncarnation, - sourceAssignmentEpoch: attempt.previousEpoch, - graceMs: attempt.drainGraceMs - }), - signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000) + for (const candidate of candidates) { + if (stopped) return + const { sourceCellUrl, ...request } = candidate + try { + const response = await fetchImpl(new URL('/v1/admin/host-idle-rehome', sourceCellUrl), { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + ...request, + cohortPercent: config.regionCorrectionCohortPercent ?? 0, + directorSafety: safetySnapshot() + }), + signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000) + }) + if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`) + const body = IdleRegionalRehomeResponseSchema.parse(await response.json()) + if (body.outcome === 'committed') { + console.warn( + JSON.stringify({ + event: 'orca_relay_idle_rehome_committed', + sourceCellId: candidate.sourceCellId, + targetCellId: candidate.targetCellId + }) + ) + return + } + } catch (error) { + // The source may have committed; its durable outcome owns recovery. + console.warn( + JSON.stringify({ + event: 'orca_relay_idle_rehome_request_failed', + reason: error instanceof Error ? error.message : 'unknown' + }) + ) } - ) - if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`) - const body = RegionalHostDrainResponseSchema.safeParse(await response.json()) - if (!body.success) throw new Error('regional_rehome_source_invalid_response') - await assignments.recordRegionalRehomeDrainReceipt( - attempt.attemptId, - body.data.outcome - ) - console.warn( - JSON.stringify({ - event: 'orca_relay_regional_rehome_dispatched', - sourceCellId: attempt.sourceCellId, - targetCellId: attempt.targetCellId, - outcome: body.data.outcome, - sendAttempts: attempt.sendAttempts - }) - ) - } catch (error) { - // Only a claimed attempt was drained. A poll that failed before the claim - // - a pool timeout on the once-a-second control read - dispatched nothing, - // so it must not spend the budget that latches the durable control off. - if (attemptId) { - await assignments - .recordRegionalRehomeDispatchFailure(attemptId) - .catch(() => undefined) } + } catch (error) { console.warn( JSON.stringify({ - event: attemptId - ? 'orca_relay_regional_rehome_dispatch_failed' - : 'orca_relay_regional_rehome_poll_failed', + event: 'orca_relay_regional_rehome_poll_failed', reason: error instanceof Error ? error.message : 'unknown' }) ) diff --git a/cloud/apps/relay/src/relay-region-app.test.ts b/cloud/apps/relay/src/relay-region-app.test.ts index 30cf3bf3e26..54e8da670b8 100644 --- a/cloud/apps/relay/src/relay-region-app.test.ts +++ b/cloud/apps/relay/src/relay-region-app.test.ts @@ -40,6 +40,7 @@ describe('Relay region API', () => { ) expect(response.status).toBe(200) + expect(await response.clone().json()).not.toHaveProperty('regionCorrection') expect(assign).toHaveBeenCalledWith( { userId: 'user-1', relayHostId: 'asiahost00000001' }, 'asia-east2', @@ -83,6 +84,160 @@ describe('Relay region API', () => { ) }) + it('preserves the cold-start hint and binds a negotiated window after placement', async () => { + const assignment = { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop', + cellId: 'asia-c1', + cellUrl: 'https://asia-c1.relay.example.test', + region: 'asia-east2', + assignmentEpoch: 7, + leaseExpiresAt: Date.now() + 300_000 + } + const assign = vi.fn(async () => assignment) + const window = { + generation: 2, + expiresAt: Date.now() + 86_400_000, + assignmentEpoch: 7, + incumbentRegion: 'asia-east2', + policyVersion: 1 + } + const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, window })) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, exchangeRegionCorrection } as never, + drain: vi.fn(), + ready: async () => true + }) + const regionCorrection = { v: 1, action: 'issue-window' } + const response = await app.request( + '/v1/assign', + assignmentRequest('abcdefghijklmnop', { + preferredRegion: 'asia-east2', + regionCorrection + }) + ) + expect(response.status).toBe(200) + expect(assign).toHaveBeenCalledWith( + { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }, + 'asia-east2', + 'asia-east2' + ) + expect(exchangeRegionCorrection).toHaveBeenCalledWith( + { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }, + regionCorrection, + 7 + ) + expect(((await response.json()) as { regionCorrection: unknown }).regionCorrection).toEqual({ + v: 1, + window + }) + }) + + it('returns successful placement when optional window storage is unavailable', async () => { + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign: async () => ({ + cellId: 'asia-c1', + region: 'asia-east2', + cellUrl: 'https://asia-c1.relay.example.test', + assignmentEpoch: 7 + }), + exchangeRegionCorrection: async () => { + throw new Error('database unavailable') + } + } as never, + drain: vi.fn(), + ready: async () => true + }) + const response = await app.request( + '/v1/assign', + assignmentRequest('abcdefghijklmnop', { + preferredRegion: 'asia-east2', + regionCorrection: { v: 1, action: 'issue-window' } + }) + ) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ + cellUrl: 'https://asia-c1.relay.example.test', + assignmentEpoch: 7 + }) + }) + + it('does not place or write a legacy hint when reporting migration evidence', async () => { + const current = { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop', + cellId: 'asia-c1', + cellUrl: 'https://asia-c1.relay.example.test', + region: 'asia-east2', + assignmentEpoch: 7, + leaseExpiresAt: Date.now() + 300_000 + } + const assign = vi.fn() + const resolve = vi.fn(async () => current) + const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, reportStatus: 'accepted' })) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, resolve, exchangeRegionCorrection } as never, + drain: vi.fn(), + ready: async () => true + }) + const regionCorrection = { + v: 1, + action: 'report', + generation: 2, + assignmentEpoch: 7, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 40, 'asia-east2': 180 } + } + const response = await app.request( + '/v1/assign', + assignmentRequest('abcdefghijklmnop', { + preferredRegion: 'us-central1', + regionCorrection + }) + ) + expect(response.status).toBe(200) + expect(assign).not.toHaveBeenCalled() + expect(exchangeRegionCorrection).toHaveBeenCalledWith( + { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }, + regionCorrection, + 7 + ) + expect(((await response.json()) as { assignmentEpoch: number }).assignmentEpoch).toBe(7) + }) + + it('does not manufacture an assignment for a report whose assignment disappeared', async () => { + const assign = vi.fn() + const exchangeRegionCorrection = vi.fn() + const app = createRelayApp(config(), { + store: {} as never, + assignments: { assign, resolve: async () => null, exchangeRegionCorrection } as never, + drain: vi.fn(), + ready: async () => true + }) + const response = await app.request( + '/v1/assign', + assignmentRequest('abcdefghijklmnop', { + regionCorrection: { + v: 1, + action: 'report', + generation: 2, + assignmentEpoch: 7, + policyVersion: 1, + outcome: 'inconclusive', + reason: 'timeout' + } + }) + ) + expect(response.status).toBe(409) + expect(assign).not.toHaveBeenCalled() + expect(exchangeRegionCorrection).not.toHaveBeenCalled() + }) + it('exposes only the store-provided healthy catalog from directors', async () => { const regionCatalog = vi.fn(async () => [ { region: 'us-central1' as const, probeOrigins: ['https://us.relay.example.test'] } diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts index 77a15a1d259..7cee77e52de 100644 --- a/cloud/apps/relay/src/relay-server.ts +++ b/cloud/apps/relay/src/relay-server.ts @@ -20,14 +20,12 @@ import { createRelayApp } from './app.js' import { RelayAssignmentStore } from './assignment-store.js' import type { RelayConfig } from './config.js' import { RelayCredentialStore } from './credential-store.js' -import type { RelayDatabase } from './database.js' +import { readRelayDatabasePoolPressure, type RelayDatabase } from './database.js' import { HostSessionRegistry } from './host-session-registry.js' import { observeRelayDatabase } from './observed-relay-database.js' import { RelayObservability } from './relay-observability.js' -import { - RelayConnectionLedger, - type RelayConnectionUpgrade -} from './relay-connection-ledger.js' +import { combineRegionalRehomeSafety } from './regional-rehome-safety.js' +import { RelayConnectionLedger, type RelayConnectionUpgrade } from './relay-connection-ledger.js' import { createRelayReadiness } from './relay-readiness.js' import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js' import { closeRelayWebSocket } from './relay-websocket-close.js' @@ -65,7 +63,7 @@ function guardSocketErrors(socket: WebSocket, kind: string): void { function admissionSource(request: IncomingMessage): string { const forwarded = request.headers['x-forwarded-for'] - const chain = (Array.isArray(forwarded) ? forwarded.join(',') : forwarded ?? '') + const chain = (Array.isArray(forwarded) ? forwarded.join(',') : (forwarded ?? '')) .split(',') .map((entry) => entry.trim()) .filter(Boolean) @@ -112,6 +110,7 @@ export function createRelayServer( const store = new RelayCredentialStore(observedDatabase, options.now) const assignments = new RelayAssignmentStore(observedDatabase, options.now, { requireLiveCells: config.role === 'director', + regionalRehomeCohortPercent: config.regionCorrectionCohortPercent ?? 0, recordControlRenewal: (durationMs, outcome) => observability.recordControlRenewal?.(durationMs, outcome) }) @@ -127,17 +126,35 @@ export function createRelayServer( queuedBytes, observability, options.now, - options.random + options.random, + cellIncarnation ) const app = createRelayApp(config, { store, assignments, drain: (graceMs) => sessions.drain(graceMs), drainHost: (input) => sessions.drainHost(input), + idleRehome: (input) => { + const now = (options.now ?? Date.now)() + if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) { + return Promise.resolve({ outcome: 'deferred' }) + } + return sessions.idleRehome(input, + () => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety( + input.directorSafety, + { ...observability.regionalRehomeRuntimeSafety(), ...readRelayDatabasePoolPressure(database) } + ), input.cohortPercent), + () => assignments.reconcileIdleRegionalRehome(input) + ) + }, regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null, cellIncarnation, isDraining: () => sessions.isDraining(), runtimeCounts: () => runtimeCounts(), + regionalRehomeSafetySnapshot: () => ({ + ...observability.regionalRehomeRuntimeSafety(), + ...readRelayDatabasePoolPressure(database) + }), ready, recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome), recordAssignmentRejectionReason: (lane, reason) => @@ -339,7 +356,7 @@ export function createRelayServer( const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null // Released combined-service invites gain their first durable cell assignment here. const assignment = identity - ? (await assignments.resolve(identity)) ?? (await assignments.assign(identity)) + ? ((await assignments.resolve(identity)) ?? (await assignments.assign(identity))) : null if (!invite || !assignment) { phoneAdmission?.hostData.release() diff --git a/cloud/apps/relay/src/relay-sweep-schedule.test.ts b/cloud/apps/relay/src/relay-sweep-schedule.test.ts index d5ef450cc43..55469cab91c 100644 --- a/cloud/apps/relay/src/relay-sweep-schedule.test.ts +++ b/cloud/apps/relay/src/relay-sweep-schedule.test.ts @@ -35,7 +35,7 @@ describe('sweep schedule jitter', () => { rehomeAudience: 'https://rehome.example.test', rehomeDirectorServiceAccount: 'rehome@example.test' } as never, - { claimRegionalRehome: async () => null } as never, + { selectIdleRegionalRehomeCandidates: async () => [] } as never, { random: () => 0.5, safetySnapshot: () => ({}) as never } ) } finally { diff --git a/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/README.md b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/README.md new file mode 100644 index 00000000000..2ac4ad78391 --- /dev/null +++ b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/README.md @@ -0,0 +1,8 @@ +Exact relay contract snapshots from `027acb4efa2e6b226d40df266b86367423946d62`, `cloud/packages/relay-contract/src/`. Used to exercise the pre-correction strict wire parsers. Do not format or edit these baseline sources. + +```text +aba94e108a5cd0f1af8b38875429ad8636d24c43a728273e3df60d9a1a1d1b6d director-messages.ts +bd13b5a694a5d683a5b680c14e46ab33f4ef4a5bfedf040d046b09d540cb4c17 wire-scalars.ts +bc89116f884a2f20a6588f9b91219aa596bc2410d28b499a93a78350def109d5 relay-regions.ts +8fcae470a5fc72f2fcdde9d2f09cd20289c256356dd490484ac1cfa53839fbe4 control-messages.ts +``` diff --git a/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/control-messages.ts b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/control-messages.ts new file mode 100644 index 00000000000..0daf21e7c28 --- /dev/null +++ b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/control-messages.ts @@ -0,0 +1,146 @@ +import { z } from 'zod' +import { + Base6432ByteSchema, + Base64Raw24ByteSchema, + Base64Url32ByteSchema, + EpochMsSchema, + GenerationSchema, + OpaqueIdSchema, + PositiveDurationMsSchema, + RelayHostIdSchema +} from './wire-scalars.js' + +const AppVersionSchema = z.string().min(1).max(128) +const BoundedCiphertextSchema = z + .string() + .min(1) + .max(16 * 1024) + .regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/) +const ConnectionKindSchema = z.enum(['invite', 'resume']) + +export const HostHelloSchema = z + .object({ + v: z.literal(1), + relayHostId: RelayHostIdSchema, + assignmentEpoch: GenerationSchema, + hostPublicKeyB64: Base6432ByteSchema, + appVersion: AppVersionSchema, + previousGeneration: GenerationSchema.optional(), + controlResumeSecret: Base64Url32ByteSchema.optional() + }) + .strict() + +export const HostChallengeSchema = z + .object({ + challengeId: OpaqueIdSchema, + relayEphemeralPublicKeyB64: Base6432ByteSchema, + nonceB64: Base64Raw24ByteSchema, + ciphertextB64: BoundedCiphertextSchema, + expiresAt: EpochMsSchema + }) + .strict() + +export const HostChallengeAckSchema = z + .object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema }) + .strict() + +// Advertised on the control upgrade rather than in host-hello: HostHelloSchema +// is strict, so a new hello key is refused by every already-deployed cell. +export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities' +// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does +// not advertise this parses those entries strictly and would drop the whole ack. +export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details' + +export function parseRelayHostCapabilities( + header: string | string[] | undefined +): ReadonlySet { + const raw = Array.isArray(header) ? header.join(',') : (header ?? '') + return new Set( + raw + .split(',') + .map((token) => token.trim()) + .filter((token) => token.length > 0 && token.length <= 64) + .slice(0, 16) + ) +} + +// kind/relayDeviceId are optional so an entry stays readable by a host that +// predates them; the cell only emits them to a host that advertised support. +const PendingConnectionSchema = z + .object({ + connId: OpaqueIdSchema, + connTicket: Base64Url32ByteSchema, + kind: ConnectionKindSchema.optional(), + relayDeviceId: OpaqueIdSchema.optional() + }) + .strict() + +export const HostHelloAckSchema = z + .object({ + v: z.literal(1), + generation: GenerationSchema, + controlResumeSecret: Base64Url32ByteSchema, + leaseExpiresAt: EpochMsSchema, + activeConnIds: z.array(OpaqueIdSchema).max(8), + pendingConns: z.array(PendingConnectionSchema).max(8) + }) + .strict() + +export const ConnectionOpenSchema = z + .object({ + connId: OpaqueIdSchema, + connTicket: Base64Url32ByteSchema, + kind: ConnectionKindSchema, + relayDeviceId: OpaqueIdSchema, + attachDeadlineMs: PositiveDurationMsSchema + }) + .strict() + +export const HostDataAuthSchema = z + .object({ + v: z.literal(1), + connTicket: Base64Url32ByteSchema, + generation: GenerationSchema + }) + .strict() + +export const InviteCreateSchema = z + .object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema }) + .strict() + +export const InviteCreatedSchema = z + .object({ + reqId: OpaqueIdSchema, + inviteToken: Base64Url32ByteSchema, + expiresAt: EpochMsSchema, + maxAttempts: z.number().int().positive().max(16) + }) + .strict() + +export const DeviceRevokeSchema = z + .object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema }) + .strict() + +export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict() + +export const DrainSchema = z + .object({ + graceMs: z.number().int().nonnegative().max(60 * 60 * 1000), + recovery: z.literal('resolve-director') + }) + .strict() + +export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict() + +export type HostHello = z.infer +export type HostChallenge = z.infer +export type HostChallengeAck = z.infer +export type HostHelloAck = z.infer +export type ConnectionOpen = z.infer +export type HostDataAuth = z.infer +export type InviteCreate = z.infer +export type InviteCreated = z.infer +export type DeviceRevoke = z.infer +export type AuthRefresh = z.infer +export type Drain = z.infer +export type Heartbeat = z.infer diff --git a/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/director-messages.ts b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/director-messages.ts new file mode 100644 index 00000000000..e697135b68e --- /dev/null +++ b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/director-messages.ts @@ -0,0 +1,75 @@ +import { z } from 'zod' +import { + Base64Url32ByteSchema, + CanonicalHttpsOriginSchema, + EpochMsSchema, + GenerationSchema, + RelayHostIdSchema +} from './wire-scalars.js' +import { RelayRegionSchema } from './relay-regions.js' + +const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024) + +export const AssignmentRequestSchema = z + .object({ + v: z.literal(1), + relayHostId: RelayHostIdSchema, + // Client-declared reconnection; the director verifies it against the + // durable assignment before granting fast-lane admission. + reconnect: z.boolean().optional(), + preferredRegion: RelayRegionSchema.optional() + }) + .strict() + +export const AssignmentResponseSchema = z + .object({ + v: z.literal(1), + cellUrl: CanonicalHttpsOriginSchema, + assignmentEpoch: GenerationSchema, + lease: SignedAssignmentLeaseSchema + }) + .strict() + +export const ResolveRequestSchema = z + .object({ + v: z.literal(1), + relayHostId: RelayHostIdSchema, + resumeToken: Base64Url32ByteSchema + }) + .strict() + +export const ResolveResponseSchema = z + .object({ + v: z.literal(1), + cellUrl: CanonicalHttpsOriginSchema, + assignmentEpoch: GenerationSchema, + leaseExpiresAt: EpochMsSchema + }) + .strict() + +export const RelayMovedSchema = z + .object({ + v: z.literal(1), + cellUrl: CanonicalHttpsOriginSchema, + assignmentEpoch: GenerationSchema + }) + .strict() + +export function isTrustedNewerMove(input: { + sourceOrigin: string + configuredDirectorOrigin: string + currentAssignmentEpoch: number + move: z.infer +}): boolean { + // Why: cells and stale director responses must never redirect a credential-bearing client. + return ( + input.sourceOrigin === input.configuredDirectorOrigin && + input.move.assignmentEpoch > input.currentAssignmentEpoch + ) +} + +export type AssignmentRequest = z.infer +export type AssignmentResponse = z.infer +export type ResolveRequest = z.infer +export type ResolveResponse = z.infer +export type RelayMoved = z.infer diff --git a/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/relay-regions.ts b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/relay-regions.ts new file mode 100644 index 00000000000..6b8837829df --- /dev/null +++ b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/relay-regions.ts @@ -0,0 +1,71 @@ +import { z } from 'zod' + +export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const + +export const RelayRegionSchema = z.enum(RELAY_REGIONS) + +export type RelayRegion = z.infer + +export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1' + +// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so +// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a +// new region a compile error here, which is the point: a region with no segment would silently +// drop out of the region-skew alert's denominators. +export const RELAY_REGION_METRIC_SEGMENTS = { + 'us-central1': 'UsCentral1', + 'asia-east2': 'AsiaEast2' +} as const satisfies Record + +const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin) + +export const RelayRegionCatalogResponseSchema = z + .object({ + v: z.literal(1), + regions: z + .array( + z + .object({ + region: RelayRegionSchema, + probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2) + }) + .strict() + ) + .max(RELAY_REGIONS.length) + }) + .strict() + .superRefine((catalog, context) => { + const regions = new Set() + const origins = new Set() + for (const [regionIndex, entry] of catalog.regions.entries()) { + if (regions.has(entry.region)) { + context.addIssue({ + code: 'custom', + message: 'duplicate relay region', + path: ['regions', regionIndex, 'region'] + }) + } + regions.add(entry.region) + for (const [originIndex, origin] of entry.probeOrigins.entries()) { + if (origins.has(origin)) { + context.addIssue({ + code: 'custom', + message: 'duplicate relay probe origin', + path: ['regions', regionIndex, 'probeOrigins', originIndex] + }) + } + origins.add(origin) + } + } + }) + +export type RelayRegionCatalogResponse = z.infer + +function isCanonicalHttpsOrigin(value: string): boolean { + try { + const url = new URL(value) + return url.protocol === 'https:' && url.origin === value + } catch { + return false + } +} diff --git a/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/wire-scalars.ts b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/wire-scalars.ts new file mode 100644 index 00000000000..27dd3a8b30f --- /dev/null +++ b/cloud/apps/relay/src/test-fixtures/relay-contract-baseline/wire-scalars.ts @@ -0,0 +1,20 @@ +import { z } from 'zod' + +export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/) +export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/) +export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/) +export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/) +export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/) +export const OpaqueIdSchema = z.string().min(1).max(128) +export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000) + +export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => { + try { + const url = new URL(value) + return url.protocol === 'https:' && url.origin === value && url.pathname === '/' + } catch { + return false + } +}, 'must be a canonical HTTPS origin') diff --git a/cloud/apps/relay/tsconfig.build.json b/cloud/apps/relay/tsconfig.build.json index 489ddfd34d6..38eb0396cf2 100644 --- a/cloud/apps/relay/tsconfig.build.json +++ b/cloud/apps/relay/tsconfig.build.json @@ -6,5 +6,5 @@ "outDir": "dist", "rootDir": "src" }, - "exclude": ["src/**/*.test.ts"] + "exclude": ["src/**/*.test.ts", "src/test-fixtures/**"] } diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs index 88e4f8ccc60..ddfe2bce8fc 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs @@ -10,6 +10,7 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET = 'orca-cloud-relay-regional-placement-enabled' export const DIRECTOR_REGIONAL_PLACEMENT_ENV = 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' +export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT' export const DIRECTOR_REHOME_IDENTITY_ENV = 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT' export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE' @@ -228,6 +229,13 @@ export function directorCellSetAddition(currentValue, desiredValue) { return { changed: additions.length > 0, value: JSON.stringify(desired) } } +export function correctionCohortPercent(value) { + if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) { + throw new Error('region correction cohort must be an integer from 0 to 100') + } + return String(value) +} + export function directorDeploymentEnvironment(config) { const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1] if (config.image !== undefined && imageDigest === undefined) { @@ -238,6 +246,10 @@ export function directorDeploymentEnvironment(config) { ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER, ...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest }) } + if (config['region-correction-cohort-percent'] !== undefined && + config['region-correction-cohort-percent'] !== 'preserve') { + environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent']) + } const serviceAccount = projectServiceAccount(config, 'capacity-service-account') const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account') const rehomeDirectorServiceAccount = projectServiceAccount( @@ -302,7 +314,8 @@ export function parseArguments(argv) { values['rehome-director-service-account'] !== undefined || values['rehome-audience'] !== undefined || values['expected-rehome-generation'] !== undefined || - values['rehome-control-origin'] !== undefined + values['rehome-control-origin'] !== undefined || + values['region-correction-cohort-percent'] !== undefined ) { throw new Error('director configuration arguments require --role director') } @@ -785,6 +798,14 @@ export async function deployDirector(config, tag, overrides = {}) { config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined const currentEnvironment = revisionEnvironment(servingRevision) const deploymentEnvironment = directorDeploymentEnvironment(config) + deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent( + currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0' + ) + if (config['region-correction-cohort-percent'] !== undefined && + config['region-correction-cohort-percent'] !== 'preserve' && + config['expected-rehome-generation'] === undefined) { + throw new Error('cohort changes require an exact disabled regional-rehome generation') + } const mutableEnvironment = { ...deploymentEnvironment, [DIRECTOR_REGIONAL_PLACEMENT_ENV]: '' diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs index 6e56676098b..a68ce50e912 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs @@ -4,6 +4,8 @@ import { test } from 'node:test' import { fileURLToPath } from 'node:url' import { activeRevision, + correctionCohortPercent, + DIRECTOR_CORRECTION_COHORT_ENV, cloudRunTrafficTag, DIRECTOR_ADMISSION_ENVIRONMENT, DIRECTOR_REGIONAL_PLACEMENT_ENV, @@ -812,3 +814,43 @@ test('waits for authenticated target readiness without hiding other capacity err /forbidden/ ) }) + + +test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => { + for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value) + for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) { + assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/) + } + assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined) + assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined) + assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1') +}) + +test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => { + const harness = directorHarness() + harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3' + await deployDirector({}, 'candidate-new', harness.operations) + for (const revision of ['relay-00002-new', 'relay-00003-new']) { + assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3') + } +}) + +test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => { + const harness = directorHarness() + await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' }, + 'candidate-new', harness.operations), /exact disabled regional-rehome generation/) + assert.equal(harness.state.activeRevision, 'relay-00001-old') + assert.equal(harness.state.nextRevision, 2) + await deployDirector({}, 'candidate-new', harness.operations) + assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0') +}) + +test('sets a reviewed cohort only behind repeated disabled-control verification', async () => { + const harness = directorHarness() + let verified = 0 + const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' } + await deployDirector(config, 'candidate-new', { ...harness.operations, + assertRegionalRehomeDisabled: async () => { verified++ } }) + assert.ok(verified >= 2) + assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1') +}) diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs index 80d40277e5a..796d0e9d91a 100644 --- a/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs +++ b/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs @@ -53,7 +53,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = { try { service = run(gcloudArguments('services', input)) } catch (error) { - if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version } + if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' } throw error } const serving = (service.status?.traffic ?? []).filter( @@ -67,12 +67,22 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = { throw new Error('Relay director must have exactly one revision serving 100% traffic') } const revision = run(gcloudArguments('revisions', input, serving[0].revisionName)) + const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) => + (container.env ?? []).filter((environment) => + environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT') + ) + if (cohortSettings.length > 1 || (cohortSettings.length === 1 && + (typeof cohortSettings[0].value !== 'string' || + !/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) { + throw new Error('serving region correction cohort is invalid') + } + const cohort_percent = cohortSettings[0]?.value ?? '0' const references = (revision.spec?.containers ?? []).flatMap((container) => (container.env ?? []).filter( (environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' ) ) - if (references.length === 0) return { version: input.bootstrap_version } + if (references.length === 0) return { version: input.bootstrap_version, cohort_percent } const reference = normalizeSecretReference(references[0]) if ( references.length !== 1 || @@ -81,7 +91,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = { ) { throw new Error('serving regional placement secret reference is invalid') } - return { version: reference.version } + return { version: reference.version, cohort_percent } } // Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs index 8043fc23e94..9c49d4127a5 100644 --- a/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs +++ b/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs @@ -67,7 +67,7 @@ test('reads the exact version from the sole traffic-serving revision', () => { } }) - assert.deepEqual(result, { version: '11' }) + assert.deepEqual(result, { version: '11', cohort_percent: '0' }) assert.equal(calls[1][3], 'relay-serving') }) @@ -78,7 +78,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource ]) { assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1') - }), { version: '1' }) + }), { version: '1', cohort_percent: '0' }) } }) @@ -100,12 +100,12 @@ test('falls back only when the service or setting is absent', () => { notFound.code = 'NOT_FOUND' assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { run: () => { throw notFound } - }), { version: '7' }) + }), { version: '7', cohort_percent: '0' }) assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { run: (args) => args[1] === 'services' ? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } : { spec: { containers: [{ env: [] }] } } - }), { version: '7' }) + }), { version: '7', cohort_percent: '0' }) }) test('classifies real absent-service stderr without weakening revision failures', () => { @@ -136,3 +136,31 @@ test('rejects ambiguous traffic, malformed references, and read failures', () => run: () => { throw denied } }), denied) }) + + +test('preserves the serving cohort including explicit disable across later Terraform plans', () => { + for (const value of ['0', '1', '17', '100']) { + const servingRevision = revision() + servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value }) + assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' ? serving() : servingRevision + }), { version: '11', cohort_percent: value }) + } +}) + +test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => { + const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT' + const cases = [ + [{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }], + [{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }], + [{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }], + [{ name, value: '1' }, { name, value: '2' }] + ] + for (const settings of cases) { + const servingRevision = revision() + servingRevision.spec.containers[0].env.push(...settings) + assert.throws(() => readRelayServingRegionalPlacementVersion(input, { + run: (args) => args[1] === 'services' ? serving() : servingRevision + }), /cohort is invalid/) + } +}) diff --git a/cloud/docs/orca-relay-operations.md b/cloud/docs/orca-relay-operations.md index cd989b58e94..f27b437fff9 100644 --- a/cloud/docs/orca-relay-operations.md +++ b/cloud/docs/orca-relay-operations.md @@ -491,3 +491,64 @@ Run and record each scenario in staging before launch: - return a dormant host, overload a cell, kill a cell, evacuate active work, and exercise pre-registration rollback. The served black-box relay suite validates the protocol/state transitions used by these procedures. The physical-device and real-GFE canaries remain separate launch gates; unit/black-box success cannot replace them. + +## Optional measured region correction (deployment gated) + +New optimization claims require both the durable regional-rehome control and +`ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT` (integer 0–100, default **0**). +Turning either gate off stops new optional moves; ordinary migration cleanup and +recovery continue. Legacy preferred-region hints do not certify a correction. Both +cells must advertise regional protocol3 and the authenticated desktop control must +advertise idle-regional-rehome-v1. The source must have no actual client sockets or +pending admission/control work; a live control socket alone does not prevent a move. + +The monitor/deploy identity can read **GET `/v1/admin/regional-rehome-preview`**. +It returns full-population eligibility/exclusion counts, open-migration capacity, +process-safety gating and aggregate migration outcomes; it never claims a +host or changes the failure budget. This is advisory, with separately read state: +concurrent assignments, capacity changes, rate pauses and control changes can make +the next claim differ. Inspect the durable control separately before enabling. +Do not treat an unavailable/failed preview as zero eligible hosts. + +`orca_relay_region_correction_outcomes` reports attempts by source/target, +registration/completion/abort state, oldest open age and target +reservation units every five minutes. `orca_relay_region_comparison` samples a +stable 10% of accepted reports (including unchanged hosts), keyed by host digest, +assignment epoch and decision generation. Existing control RTT and client-accept +logs include assignment epoch, control generation and drain mode; join those for +matched before/after and unchanged-cohort comparisons. Client accept latency is +connection setup, not application command round trip. No application-latency +improvement has been demonstrated by probe differences alone. + +Quiet live connections count as work and defer optional correction indefinitely. +A returning client may race with the short admission gate and retry normally. No +optimization timer may close an established client. Investigate failed registration, +ambiguous authority, stuck reservations and reconnect/failure rates against agreed +limits. A database outage can keep the source fenced until locked reconciliation +establishes its authority; timeout alone is not permission to reopen admissions. + +All directors must run the reviewed idle worker before enabling. Record the tested +immutable source and rollback revisions, then verify the ordinary migration recovery +path before rollout. There is no retained-source table or renewal protocol. Deploying +supporting cells/desktops and enabling a cohort require separate rollout authorization +and explicit numerical stop criteria; this change enables neither. + +### Setting the correction cohort during a reviewed director rollout + +The existing **Deploy Relay Production Director** workflow accepts +`region-correction-cohort-percent`: `preserve` (default) or an integer0–100. +It carries the cohort onto both candidate and compatible rollback revisions and +verifies the environment before promotion. If the predecessor has no setting, +`preserve` stamps zero. An explicit change requires the exact disabled durable +rehome generation; configuring a nonzero cohort does not itself enable the sweep. +The usual image, identity, health and traffic checks remain in force. No workflow +was dispatched as part of implementation. + +Terraform reads the cohort from the same traffic-serving revision used to preserve +regional placement. A later apply therefore preserves a workflow-set cohort, +including explicit zero; only an absent service/setting bootstraps to0. Malformed +or ambiguous live settings fail the plan instead of silently resetting the cohort. +The audited director workflow owns subsequent changes. +Before the first nonzero cohort, verify compatible protocol2 cells, updated +cleanup workers, preview eligibility, both serving/rollback images and the +explicitly approved observation/stop criteria. diff --git a/cloud/infra/terraform/relay.tf b/cloud/infra/terraform/relay.tf index 7a5124a00b1..5df7372ff07 100644 --- a/cloud/infra/terraform/relay.tf +++ b/cloud/infra/terraform/relay.tf @@ -169,6 +169,11 @@ resource "google_cloud_run_v2_service" "relay" { } } + env { + name = "ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT" + value = data.external.relay_serving_regional_placement_version.result.cohort_percent + } + ports { container_port = 8080 } diff --git a/cloud/packages/push-contract/src/send-messages.ts b/cloud/packages/push-contract/src/send-messages.ts index 57d1c2e2745..a8959c18b05 100644 --- a/cloud/packages/push-contract/src/send-messages.ts +++ b/cloud/packages/push-contract/src/send-messages.ts @@ -26,7 +26,8 @@ export const PushNotificationSchema = z agentState: PushAgentStateSchema.nullable(), title: z.string().min(1).max(PUSH_LIMITS.titleMaxChars), body: z.string().max(PUSH_LIMITS.bodyMaxChars), - worktreeId: z.string().min(1).max(2048).optional() + worktreeId: z.string().min(1).max(2048).optional(), + paneKey: z.string().min(1).max(2048).optional() }) .strict() .refine( diff --git a/cloud/packages/relay-contract/src/control-messages.ts b/cloud/packages/relay-contract/src/control-messages.ts index 0daf21e7c28..ebe9586407e 100644 --- a/cloud/packages/relay-contract/src/control-messages.ts +++ b/cloud/packages/relay-contract/src/control-messages.ts @@ -50,6 +50,7 @@ export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities' // The host accepts kind/relayDeviceId on a pendingConns entry. A host that does // not advertise this parses those entries strictly and would drop the whole ack. export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details' +export const RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME = 'idle-regional-rehome-v1' export function parseRelayHostCapabilities( header: string | string[] | undefined @@ -121,11 +122,22 @@ export const DeviceRevokeSchema = z .object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema }) .strict() -export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict() +export const AuthRefreshSchema = z + .object({ + relayJwt: z + .string() + .min(1) + .max(8 * 1024) + }) + .strict() export const DrainSchema = z .object({ - graceMs: z.number().int().nonnegative().max(60 * 60 * 1000), + graceMs: z + .number() + .int() + .nonnegative() + .max(60 * 60 * 1000), recovery: z.literal('resolve-director') }) .strict() diff --git a/cloud/packages/relay-contract/src/director-messages.ts b/cloud/packages/relay-contract/src/director-messages.ts index e697135b68e..0f57081014a 100644 --- a/cloud/packages/relay-contract/src/director-messages.ts +++ b/cloud/packages/relay-contract/src/director-messages.ts @@ -7,8 +7,15 @@ import { RelayHostIdSchema } from './wire-scalars.js' import { RelayRegionSchema } from './relay-regions.js' +import { + RegionCorrectionRequestSchema, + RegionCorrectionResponseSchema +} from './region-correction.js' -const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024) +const SignedAssignmentLeaseSchema = z + .string() + .min(1) + .max(8 * 1024) export const AssignmentRequestSchema = z .object({ @@ -17,7 +24,8 @@ export const AssignmentRequestSchema = z // Client-declared reconnection; the director verifies it against the // durable assignment before granting fast-lane admission. reconnect: z.boolean().optional(), - preferredRegion: RelayRegionSchema.optional() + preferredRegion: RelayRegionSchema.optional(), + regionCorrection: RegionCorrectionRequestSchema.optional() }) .strict() @@ -26,7 +34,8 @@ export const AssignmentResponseSchema = z v: z.literal(1), cellUrl: CanonicalHttpsOriginSchema, assignmentEpoch: GenerationSchema, - lease: SignedAssignmentLeaseSchema + lease: SignedAssignmentLeaseSchema, + regionCorrection: RegionCorrectionResponseSchema.optional() }) .strict() diff --git a/cloud/packages/relay-contract/src/idle-regional-rehome.ts b/cloud/packages/relay-contract/src/idle-regional-rehome.ts new file mode 100644 index 00000000000..9f9eff36593 --- /dev/null +++ b/cloud/packages/relay-contract/src/idle-regional-rehome.ts @@ -0,0 +1,26 @@ +import { z } from 'zod' +import { GenerationSchema, RelayHostIdSchema } from './wire-scalars.js' + +export const IdleRegionalRehomeRequestSchema = z + .object({ + v: z.literal(1), + attemptId: z.string().uuid(), + userId: z.string().min(1).max(256), + relayHostId: RelayHostIdSchema, + sourceCellId: z.string().min(1).max(128), + sourceCellIncarnation: z.string().uuid(), + sourceAssignmentEpoch: GenerationSchema.refine((value) => value > 0), + sourceGeneration: GenerationSchema.refine((value) => value > 0), + targetCellId: z.string().min(1).max(128) + }) + .strict() + +export const IdleRegionalRehomeResponseSchema = z + .object({ + v: z.literal(1), + outcome: z.enum(['busy', 'committed', 'deferred', 'stale']) + }) + .strict() + +export type IdleRegionalRehomeRequest = z.infer +export type IdleRegionalRehomeOutcome = z.infer['outcome'] diff --git a/cloud/packages/relay-contract/src/index.ts b/cloud/packages/relay-contract/src/index.ts index aab3b53b5f3..3b52ec503a1 100644 --- a/cloud/packages/relay-contract/src/index.ts +++ b/cloud/packages/relay-contract/src/index.ts @@ -13,3 +13,5 @@ export * from './resume-confirmation-contract.js' export * from './relay-regions.js' export * from './splice-state-machine.js' export * from './wire-scalars.js' +export * from './region-correction.js' +export * from './idle-regional-rehome.js' diff --git a/cloud/packages/relay-contract/src/region-correction.test.ts b/cloud/packages/relay-contract/src/region-correction.test.ts new file mode 100644 index 00000000000..8c0122341d4 --- /dev/null +++ b/cloud/packages/relay-contract/src/region-correction.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' +import { AssignmentRequestSchema, AssignmentResponseSchema } from './director-messages.js' +import { DrainSchema } from './control-messages.js' +import { RegionCorrectionRequestSchema } from './region-correction.js' + +const report = { + v: 1, + action: 'report', + generation: 3, + assignmentEpoch: 7, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 40, 'asia-east2': 180 } +} +const retention = { + mode: 'finish-existing', + attemptId: '11111111-1111-4111-8111-111111111111', + sourceGeneration: 3, + sourceAssignmentEpoch: 7 +} + +describe('region correction wire boundaries', () => { + it('keeps legacy assignment shapes readable without negotiated fields', () => { + expect( + AssignmentRequestSchema.parse({ v: 1, relayHostId: 'abcdefghijklmnop' }) + ).not.toHaveProperty('regionCorrection') + expect( + AssignmentResponseSchema.parse({ + v: 1, + cellUrl: 'https://cell.example', + assignmentEpoch: 1, + lease: 'synthetic-lease' + }) + ).not.toHaveProperty('regionCorrection') + }) + + it('accepts complete comparison evidence and explicit inconclusive reports', () => { + expect(RegionCorrectionRequestSchema.safeParse(report).success).toBe(true) + const { measurements: _measurements, ...basis } = report + expect( + RegionCorrectionRequestSchema.safeParse({ + ...basis, + outcome: 'inconclusive', + reason: 'probe-unavailable' + }).success + ).toBe(true) + }) + + it.each([ + { measurements: { 'us-central1': 40 } }, + { measurements: { 'us-central1': -1, 'asia-east2': 10 } }, + { measurements: { 'us-central1': Infinity, 'asia-east2': 10 } }, + { measurements: { 'us-central1': 120_001, 'asia-east2': 10 } }, + { generation: Number.MAX_SAFE_INTEGER + 1 }, + { assignmentEpoch: 1.2 }, + { policyVersion: 2 }, + { outcome: 'inconclusive', reason: 'timeout' } + ])('rejects ambiguous or unbounded evidence: %j', (override) => { + expect(RegionCorrectionRequestSchema.safeParse({ ...report, ...override }).success).toBe(false) + }) + + it('rejects reporting and issuing a window in the same request', () => { + expect( + RegionCorrectionRequestSchema.safeParse({ + ...report, + action: 'issue-window' + }).success + ).toBe(false) + }) + + it('uses ordinary drain and rejects the superseded retention extension', () => { + const ordinary = { graceMs: 0, recovery: 'resolve-director' } + expect(DrainSchema.parse(ordinary)).toEqual(ordinary) + expect(DrainSchema.safeParse({ ...ordinary, retention }).success).toBe(false) + }) +}) diff --git a/cloud/packages/relay-contract/src/region-correction.ts b/cloud/packages/relay-contract/src/region-correction.ts new file mode 100644 index 00000000000..5fffca0ad8c --- /dev/null +++ b/cloud/packages/relay-contract/src/region-correction.ts @@ -0,0 +1,60 @@ +import { z } from 'zod' +import { EpochMsSchema, GenerationSchema } from './wire-scalars.js' +import { RelayRegionSchema } from './relay-regions.js' + +const RttSchema = z.number().finite().nonnegative().max(120_000) +export const RegionMeasurementsSchema = z + .object({ + 'us-central1': RttSchema, + 'asia-east2': RttSchema + }) + .strict() + +export const RegionMeasurementWindowSchema = z + .object({ + generation: GenerationSchema, + expiresAt: EpochMsSchema, + assignmentEpoch: GenerationSchema, + incumbentRegion: RelayRegionSchema, + policyVersion: z.literal(1) + }) + .strict() + +const ReportBasis = { + v: z.literal(1), + action: z.literal('report'), + generation: GenerationSchema, + assignmentEpoch: GenerationSchema, + policyVersion: z.literal(1) +} + +export const RegionCorrectionRequestSchema = z.union([ + z.object({ v: z.literal(1), action: z.literal('issue-window') }).strict(), + z + .object({ + ...ReportBasis, + outcome: z.literal('conclusive'), + measurements: RegionMeasurementsSchema + }) + .strict(), + z + .object({ + ...ReportBasis, + outcome: z.literal('inconclusive'), + reason: z.string().min(1).max(64) + }) + .strict() +]) + +export const RegionCorrectionResponseSchema = z + .object({ + v: z.literal(1), + window: RegionMeasurementWindowSchema.optional(), + reportStatus: z.enum(['accepted', 'duplicate', 'stale', 'expired', 'basis-changed']).optional() + }) + .strict() + +export type RegionMeasurements = z.infer +export type RegionMeasurementWindow = z.infer +export type RegionCorrectionRequest = z.infer +export type RegionCorrectionResponse = z.infer diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 0ba70ddcf1e..5b21fa9b0ad 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10,6 +10,81 @@ } }, "gates": [ + { + "id": "runtime.connection-owned-host-status", + "title": "Host status recovers with its owning connection", + "maturity": "experimental", + "protection": "partial", + "owner": "runtime", + "layer": "service-integration-and-e2e", + "surfaces": [ + "sidebar host status", + "desktop runtime connection", + "browser primary connection" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["remote-runtime"], + "coverageNotes": "Real authenticated sockets plus isolated desktop and headless hosts with desktop and browser viewers; deterministic lifecycle tests cover stale results and reader deadlines.", + "motivatingLinks": ["https://github.com/stablyai/orca/pull/19163"], + "invariant": "Failed bootstrap and authenticated reconnect converge without UI triggers; one connection owner publishes verified status, with no independent healthy status polling.", + "oracle": "Observe automatic recovery, retained runtime identity on failure, ordered publications, exact request counts, isolated viewer outages, and retirement on disconnect.", + "commands": [ + "ORCA_BACKGROUND_LAUNCH=1 pnpm test src/shared/runtime-host-status-owner.test.ts src/main/ipc/runtime-environment-status-recovery.test.ts src/main/ipc/runtime-environment-status-connection.test.ts src/renderer/src/store/slices/runtime-status-snapshot.test.ts src/renderer/src/web/web-runtime-status-owner.test.ts", + "ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" + ], + "testFiles": [ + "src/shared/runtime-host-status-owner.test.ts", + "src/main/ipc/runtime-environment-status-recovery.test.ts", + "src/main/ipc/runtime-environment-status-connection.test.ts", + "src/renderer/src/store/slices/runtime-status-snapshot.test.ts", + "src/renderer/src/web/web-runtime-status-owner.test.ts", + "tests/e2e/runtime-host-status-recovery.spec.ts" + ], + "assertionRefs": [ + { + "file": "src/main/ipc/runtime-environment-status-recovery.test.ts", + "assertions": [ + "recovers a saved host after its first status check fails, without another UI request" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-09-10", + "runner": "local", + "platform": "macos", + "command": "ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "summary": "Desktop-host and headless-host journeys passed with desktop and browser viewers.", + "durationSeconds": 31.3 + } + ], + "runtimeBudget": { + "p95Seconds": 180, + "scope": "Target excluding builds; measured p95 not established." + }, + "flakeHistory": { + "status": "soaking", + "evidence": "Local candidate runs passed; no sustained CI history yet." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "First-status-failure oracle failed on main 58ff95becb40 (one request instead of two) and passes on the candidate. E2E verifies candidate recovery, not a baseline comparison." + }, + "performanceBudget": { + "required": false, + "evidence": "Deterministic tests assert one shared request and no healthy owner polling." + }, + "promotionCriteria": ["Collect repeated CI runs without unexplained flakes."], + "knownGaps": [ + "No live Linux, Windows, SSH, or mixed-version pair validation.", + "TCP interruption exercises reconnect, not a full real host process restart.", + "The outage begins on the first saved-host check, not by relaunching a preseeded desktop profile." + ], + "demotionRule": "Keep experimental until repeated runs establish reliability; preserve request-count and lifecycle assertions." + }, { "id": "mobile-push.headless-startup-and-policy", "title": "Headless push lifecycle and mobile delivery policy", @@ -2852,6 +2927,78 @@ ], "demotionRule": "Keep experimental or demote if assignment calls overlap, duplicate drain events bypass backoff, Retry-After is ignored, close resurrects work, or mixed-version request rate exceeds the reviewed director budget." }, + { + "id": "desktop-relay.region-correction-idle-cutover", + "title": "Regional correction moves only an idle relay source", + "maturity": "experimental", + "protection": "partial", + "owner": "desktop-runtime", + "layer": "cell-desktop-real-websocket", + "surfaces": ["regional correction", "idle source cutover", "desktop relay reconnect"], + "platforms": ["macos", "linux", "windows"], + "providers": ["cloud-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["cloud-relay"], + "coverageNotes": "Real local WebSocket/control/proof/splice traffic and production SQLite store run with synthetic clock and synthetic token verification. Separate PostgreSQL16 suites validate SQL concurrency. This does not measure production network latency, physical phones, UI, or the production token issuer.", + "motivatingLinks": [ + "docs/relay-region-correction/RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md" + ], + "invariant": "Regional optimization never closes an established relay client. The source gates admissions only after actual work is idle, commits the exact assignment atomically, and releases its empty control. A failed target uses ordinary migration recovery; previously sent mutations are not replayed.", + "oracle": "Two real TCP WebSocket cells, the actual desktop origin pool, SQLite and an independent execution child verify busy phone/iPad deferral, idle movement, racing arrival rejection, definite-abort admission recovery and observed target-registration failure with ordinary rollback.", + "commands": [ + "ORCA_BACKGROUND_LAUNCH=1 pnpm test tests/e2e/relay-region-correction.unit.test.ts" + ], + "testFiles": ["tests/e2e/relay-region-correction.unit.test.ts"], + "assertionRefs": [ + { + "file": "tests/e2e/relay-region-correction.unit.test.ts", + "assertions": [ + "releases the empty source and recovers normally when the target never registers", + "rejects an arrival during cutover and restores admissions after a definite failed commit", + "defers for either connected device, then moves after both disconnect without replaying work" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-09-11", + "runner": "local", + "platform": "macos", + "command": "ORCA_BACKGROUND_LAUNCH=1 pnpm test tests/e2e/relay-region-correction.unit.test.ts", + "result": "passed", + "durationSeconds": 6.44, + "summary": "Three real TCP WebSocket cases passed after removing store retention. Log: .tmp/idle-cutover-review/transport-without-retention.log. Does not validate packaged or physical clients." + } + ], + "runtimeBudget": { + "p95Seconds": 180, + "scope": "three local real-WebSocket scenarios with synthetic elapsed time for ordinary recovery" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "Local implementation validation; no CI soak history yet." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "Registry admission accounting negative control and five conflicting operation-identity regressions fail before their fixes and pass afterward. Locked database reconciliation has separate red/green evidence. Full cross-layer counterfactual remains unverified." + }, + "performanceBudget": { + "required": true, + "evidence": "No retained source lease or new mobile timer. Candidate selection is read-only; busy sources immediately defer. Idle cutover reuses normal desktop reconnect and existing migration recovery." + }, + "promotionCriteria": [ + "Collect 100 consecutive CI passes or 14 days of soak.", + "Complete mixed-version and packaged-client validation.", + "Validate bounded rollout latency and reliability against reviewed numerical limits." + ], + "knownGaps": [ + "Production authentication verifier is mocked.", + "Synthetic elapsed time is not a wall-clock soak.", + "Physical phone lifecycle, packaged mixed versions, SSH execution and production network behavior require separate validation." + ], + "demotionRule": "Keep experimental or demote if optimization closes an established client, a gate reopens on ambiguous authority, a mutation is replayed, cleanup is lost, or eligible idle hosts starve." + }, + { "id": "git-worktree.refresh-event-semantics", "title": "Index-only Git metadata cannot trigger structural worktree refresh fanout", diff --git a/config/scripts/capture-agent-pty-transcript.mjs b/config/scripts/capture-agent-pty-transcript.mjs new file mode 100644 index 00000000000..a60d0adbdd5 --- /dev/null +++ b/config/scripts/capture-agent-pty-transcript.mjs @@ -0,0 +1,283 @@ +/** + * Records a live agent CLI session through a real PTY into a test fixture, bytes intact. + * + * Why a PTY and not `agy | tee`: a pipe is not a terminal, so the CLI renders its + * non-interactive path — no alternate screen, no caret, no dialogs. The detector under + * test only ever sees the PTY shape, so that is the only shape worth capturing. + * + * Nothing here strips escapes, folds CRs, or rewraps lines: the transcript is written + * exactly as the terminal received it. See docs/reference/agent-pty-transcript-capture.md. + */ +import { createWriteStream, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { + formatFindings, + redactTranscript, + scanTranscriptForSecrets +} from './pty-transcript-secret-scan.mjs' + +const REPO_ROOT = resolve(import.meta.dirname, '..', '..') +const FIXTURE_DIR = join(REPO_ROOT, 'src', 'main', 'runtime', '__fixtures__') +const STOP_KEY = 0x1d // Ctrl-], consumed by the recorder and never forwarded to the agent. +const NAME_RE = /^[a-z0-9][a-z0-9-]*$/ + +const USAGE = `Capture a raw agent PTY transcript into src/main/runtime/__fixtures__/. + + node config/scripts/capture-agent-pty-transcript.mjs --name [options] -- [args...] + node config/scripts/capture-agent-pty-transcript.mjs --scan [--redact] + +Options + --name Output fixture name, e.g. antigravity-ready-personal-non-gemini + --out Write somewhere other than the fixture directory + --cols --rows Pin the PTY size (default: this terminal's size, else 120x40) + --duration Stop unattended after N seconds + --send ":" Type into the PTY at (repeatable; \\r \\n \\t \\e escapes) + --note "" Recorded in the .meta.json sidecar + --scan Scan existing transcripts for identifiers/credentials and exit + --redact With --scan: rewrite each finding as a same-length placeholder + +Press Ctrl-] to end a capture. That key is consumed here, so the agent keeps whatever +dialog it is showing — which is the only way to capture a dialog that owns the screen.` + +function parseArgs(argv) { + const options = { cols: null, rows: null, duration: null, scan: [], sends: [], redact: false } + const command = [] + let cursor = 0 + let afterSeparator = false + while (cursor < argv.length) { + const arg = argv[cursor] + if (afterSeparator) { + command.push(arg) + cursor += 1 + continue + } + if (arg === '--') { + afterSeparator = true + } else if (arg === '--redact') { + options.redact = true + } else if (arg === '--help' || arg === '-h') { + options.help = true + } else if (arg === '--scan') { + while (cursor + 1 < argv.length && !argv[cursor + 1].startsWith('--')) { + cursor += 1 + options.scan.push(argv[cursor]) + } + } else if (arg === '--send') { + cursor += 1 + options.sends.push(parseSend(argv[cursor])) + } else if (arg.startsWith('--')) { + const key = arg.slice(2) + cursor += 1 + options[key] = argv[cursor] + } + cursor += 1 + } + for (const key of ['cols', 'rows', 'duration']) { + options[key] = options[key] == null ? null : Number(options[key]) + } + return { options, command } +} + +// String.fromCharCode, not a literal: the formatter rewrites an escape sequence into a raw +// control byte in source, which is unreadable and survives badly in diffs. +const ESC = String.fromCharCode(27) +const SEND_ESCAPES = { r: '\r', n: '\n', t: '\t', e: ESC, '\\': '\\' } + +/** `":"` — a keystroke to deliver at a fixed offset, for an unattended dialog capture. */ +function parseSend(value) { + const separator = String(value ?? '').indexOf(':') + if (separator === -1) { + throw new Error(`--send expects ":", got ${String(value)}`) + } + const atMs = Number(value.slice(0, separator)) + if (!Number.isFinite(atMs)) { + throw new Error( + `--send delay must be a number of milliseconds, got ${value.slice(0, separator)}` + ) + } + const text = value + .slice(separator + 1) + .replace(/\\(.)/g, (whole, code) => SEND_ESCAPES[code] ?? whole) + return { atMs, text } +} + +function runScan(files, redact) { + let failed = false + for (const file of files) { + const path = resolve(file) + const text = readFileSync(path, 'utf8') + if (redact) { + const { text: redacted, redacted: count } = redactTranscript(text) + writeFileSync(path, redacted) + console.log(`${file}: redacted ${count} span(s) in place, same length each.`) + continue + } + const findings = scanTranscriptForSecrets(text) + console.log(formatFindings(file, findings)) + failed ||= findings.length > 0 + } + return failed ? 1 : 0 +} + +function resolveSpawn(command) { + // node-pty cannot run a .cmd/.bat shim directly on Windows; those need cmd.exe. + if (process.platform === 'win32' && /\.(cmd|bat)$/i.test(command[0])) { + return { file: 'cmd.exe', args: ['/c', `"${command[0]}"`, ...command.slice(1)] } + } + return { file: command[0], args: command.slice(1) } +} + +async function runCapture(options, command) { + const name = options.name + if (typeof name === 'string' && !NAME_RE.test(name)) { + console.error(`--name must be lowercase kebab-case; got ${name}`) + return 2 + } + const outPath = options.out ? resolve(options.out) : join(FIXTURE_DIR, `${name}.txt`) + mkdirSync(dirname(outPath), { recursive: true }) + + const pty = await import('node-pty').catch((error) => { + console.error( + `node-pty failed to load. Build it for plain node first: + node config/scripts/ensure-native-runtime.mjs --runtime=node +${String(error)}` + ) + return null + }) + if (pty === null) { + return 2 + } + + const cols = options.cols ?? process.stdout.columns ?? 120 + const rows = options.rows ?? process.stdout.rows ?? 40 + const { file, args } = resolveSpawn(command) + const term = pty.spawn(file, args, { + name: 'xterm-256color', + cols, + rows, + cwd: process.cwd(), + env: { ...process.env, TERM: 'xterm-256color' }, + encoding: null + }) + + const sink = createWriteStream(outPath) + let recording = true + term.onData((chunk) => { + const bytes = typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : chunk + // Why recording stops before the kill: an agent repaints an idle frame on its way out, so + // a transcript that keeps writing through shutdown ends on that frame instead of on the + // state you stopped to capture. A mid-turn or dialog capture cannot survive that. + if (recording) { + sink.write(bytes) + } + process.stdout.write(bytes) + }) + + const wasRaw = process.stdin.isTTY === true && process.stdin.isRaw === true + if (process.stdin.isTTY) { + process.stdin.setRawMode(true) + } + process.stdin.resume() + let stopping = false + const stop = () => { + if (stopping) { + return + } + stopping = true + recording = false + try { + term.kill() + } catch { + // The agent may have exited on its own; the transcript is already on disk. + } + } + process.stdin.on('data', (chunk) => { + if (chunk.includes(STOP_KEY)) { + stop() + return + } + term.write(chunk.toString('binary')) + }) + // Why scripted input: a dialog capture has to be driven, and CI (or an agent) has no TTY to + // type into. The keystrokes ride the same PTY a human's would, so the capture is unchanged. + const sendTimers = options.sends.map((send) => setTimeout(() => term.write(send.text), send.atMs)) + const durationTimer = options.duration === null ? null : setTimeout(stop, options.duration * 1000) + + const exitCode = await new Promise((resolveExit) => { + term.onExit(({ exitCode: code }) => resolveExit(code ?? 0)) + }) + for (const timer of sendTimers) { + clearTimeout(timer) + } + if (durationTimer !== null) { + clearTimeout(durationTimer) + } + if (process.stdin.isTTY) { + process.stdin.setRawMode(wasRaw) + } + process.stdin.pause() + await new Promise((done) => sink.end(done)) + + writeMeta(outPath, { command, cols, rows, note: options.note ?? null, exitCode }) + const findings = scanTranscriptForSecrets(readFileSync(outPath, 'utf8')) + console.log(`\nTranscript: ${outPath}`) + console.log(formatFindings('scrub check', findings)) + if (findings.length > 0) { + console.log( + `Scrub with: + node config/scripts/capture-agent-pty-transcript.mjs --scan ${outPath} --redact` + ) + } + return 0 +} + +function writeMeta(outPath, details) { + const metaPath = outPath.replace(/\.txt$/, '.meta.json') + writeFileSync( + metaPath, + `${JSON.stringify( + { + capturedAt: new Date().toISOString(), + platform: process.platform, + command: details.command, + cols: details.cols, + rows: details.rows, + note: details.note, + exitCode: details.exitCode + }, + null, + 2 + )}\n` + ) +} + +async function main() { + const { options, command } = parseArgs(process.argv.slice(2)) + if (options.help === true) { + console.log(USAGE) + return 0 + } + if (options.scan.length > 0) { + return runScan(options.scan, options.redact) + } + if (command.length === 0 || (options.name === undefined && options.out === undefined)) { + console.error(USAGE) + return 2 + } + return runCapture(options, command) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().then( + (code) => { + process.exitCode = code + }, + (error) => { + console.error(error) + process.exitCode = 1 + } + ) +} + +export { parseArgs, resolveSpawn } diff --git a/config/scripts/pty-transcript-secret-scan.mjs b/config/scripts/pty-transcript-secret-scan.mjs new file mode 100644 index 00000000000..1d93204ccda --- /dev/null +++ b/config/scripts/pty-transcript-secret-scan.mjs @@ -0,0 +1,135 @@ +// Finds account identifiers and credentials in a captured PTY transcript before it is committed. +import os from 'node:os' + +// Why same-length replacements: a transcript's value is its exact wrapping and column +// alignment. Shortening a redacted span reflows the screen and destroys the evidence. +const EMAIL_DOMAIN = '@example.com' +const PLACEHOLDER_UUID = '00000000-0000-4000-8000-000000000000' + +/** Ordered most-specific first; the first pattern to claim a span owns it. */ +function buildPatterns() { + const username = os.userInfo().username + const hostname = os.hostname() + const patterns = [ + { kind: 'jwt', re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}/g }, + { kind: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{20,}/g }, + { kind: 'google-refresh-token', re: /\b1\/\/[0-9A-Za-z_-]{20,}/g }, + { kind: 'vendor-key', re: /\b(?:sk-|ghp_|gho_|github_pat_|xoxb-|xoxp-)[A-Za-z0-9_-]{16,}/g }, + { kind: 'bearer-token', re: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/gi }, + { kind: 'email', re: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g }, + // Why a UUID counts: agy prints a resumable conversation id on exit, and installation and + // project ids look the same. They identify the operator's session, not just its shape. + { kind: 'uuid', re: /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi }, + { kind: 'opaque-token', re: /\b[A-Za-z0-9_-]{40,}\b/g } + ] + if (username.length >= 3) { + patterns.splice(5, 0, { kind: 'local-username', re: literalPattern(username) }) + } + if (hostname.length >= 3) { + patterns.splice(5, 0, { kind: 'local-hostname', re: literalPattern(hostname) }) + } + return patterns +} + +function literalPattern(value) { + return new RegExp(value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g') +} + +/** + * @param {string} text raw transcript, escapes intact + * @returns {{kind: string, line: number, column: number, index: number, match: string}[]} + */ +export function scanTranscriptForSecrets(text) { + const claimed = [] + const findings = [] + for (const { kind, re } of buildPatterns()) { + re.lastIndex = 0 + let match = re.exec(text) + while (match !== null) { + const start = match.index + const end = start + match[0].length + if (!claimed.some(([from, to]) => start < to && end > from)) { + claimed.push([start, end]) + if (!isAlreadyScrubbed(kind, match[0])) { + findings.push({ kind, index: start, match: match[0], ...locate(text, start) }) + } + } + match = re.exec(text) + } + } + return findings.sort((left, right) => left.index - right.index) +} + +// Why: a scrubbed fixture must verify clean, so this scanner has to recognise its own +// placeholders — otherwise "prove it's gone" can never pass and the check gets ignored. +const PLACEHOLDER_DOMAIN_RE = /@(?:example\.(?:com|org|net)|localhost)$/i + +function isAlreadyScrubbed(kind, match) { + if (kind === 'email') { + return PLACEHOLDER_DOMAIN_RE.test(match) + } + if (kind === 'uuid') { + return match.toLowerCase() === PLACEHOLDER_UUID + } + return /^(.)\1*$/.test(match) +} + +function locate(text, index) { + let line = 1 + let lineStart = 0 + for (let cursor = 0; cursor < index; cursor += 1) { + if (text.charCodeAt(cursor) === 10) { + line += 1 + lineStart = cursor + 1 + } + } + return { line, column: index - lineStart + 1 } +} + +/** Same-length stand-in so redaction cannot reflow the captured screen. */ +export function placeholderFor(kind, length) { + if (kind === 'uuid' && length === PLACEHOLDER_UUID.length) { + return PLACEHOLDER_UUID + } + if (kind === 'email' && length > EMAIL_DOMAIN.length) { + return 'u'.repeat(length - EMAIL_DOMAIN.length) + EMAIL_DOMAIN + } + return kind === 'local-username' || kind === 'local-hostname' + ? 'x'.repeat(length) + : 'X'.repeat(length) +} + +/** @returns {{text: string, redacted: number}} */ +export function redactTranscript(text) { + const findings = scanTranscriptForSecrets(text) + let out = '' + let cursor = 0 + for (const finding of findings) { + out += text.slice(cursor, finding.index) + out += placeholderFor(finding.kind, finding.match.length) + cursor = finding.index + finding.match.length + } + return { text: out + text.slice(cursor), redacted: findings.length } +} + +export function formatFindings(label, findings) { + if (findings.length === 0) { + return `${label}: clean — no account identifier or credential shapes found.` + } + const rows = findings.map( + (finding) => ` ${finding.line}:${finding.column} ${finding.kind} ${preview(finding.match)}` + ) + return [`${label}: ${findings.length} finding(s) — scrub before committing.`, ...rows].join('\n') +} + +// Why a codepoint test and not a character class: a control-byte range written as an escape is +// folded back into raw 0x00-0x1f bytes by the formatter, which makes this file binary to the VCS +// and leaves the one file gating real PTY data into history unreviewable in a diff. +function preview(value) { + const head = value.length <= 24 ? value : `${value.slice(0, 21)}...` + let printable = '' + for (const char of head) { + printable += (char.codePointAt(0) ?? 0) < 0x20 ? '?' : char + } + return printable +} diff --git a/config/scripts/pty-transcript-secret-scan.test.mjs b/config/scripts/pty-transcript-secret-scan.test.mjs new file mode 100644 index 00000000000..2d3cd894da0 --- /dev/null +++ b/config/scripts/pty-transcript-secret-scan.test.mjs @@ -0,0 +1,133 @@ +// The scrub gate is the only thing standing between a live agent transcript and a +// committed account identifier, so it is pinned on the shapes those transcripts carry. +import { readdirSync, readFileSync } from 'node:fs' +import os from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + formatFindings, + placeholderFor, + redactTranscript, + scanTranscriptForSecrets +} from './pty-transcript-secret-scan.mjs' +import { parseArgs, resolveSpawn } from './capture-agent-pty-transcript.mjs' + +describe('pty transcript secret scan', () => { + it('finds the account row of a ready screen', () => { + const findings = scanTranscriptForSecrets('Antigravity CLI 1.1.17\njin.woo@acme.dev (Business)') + expect(findings).toHaveLength(1) + expect(findings[0]).toMatchObject({ kind: 'email', line: 2, column: 1 }) + }) + + it('finds credentials an agent may echo while signing in', () => { + const kinds = scanTranscriptForSecrets( + [ + 'token: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVP', + 'key: AIzaSyA1234567890abcdefghijklmnopqrstu', + 'refresh: 1//0gLm34XyZabcdefghijklmnopqrstuvwx', + 'Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345' + ].join('\n') + ).map((finding) => finding.kind) + expect(kinds).toEqual(['jwt', 'google-api-key', 'google-refresh-token', 'bearer-token']) + }) + + it('flags this machine’s own username, which a prompt line leaks', () => { + const username = os.userInfo().username + const findings = scanTranscriptForSecrets(`~/Users/${username}/orca/repo\n> `) + expect(findings.some((finding) => finding.kind === 'local-username')).toBe(true) + }) + + it('finds the resumable conversation id agy prints on exit', () => { + const findings = scanTranscriptForSecrets( + 'Resume with -c (or command below):\nagy --conversation=26dc1986-9eec-456a-a534-d93e5c1076c2' + ) + expect(findings).toHaveLength(1) + expect(findings[0].kind).toBe('uuid') + expect(placeholderFor('uuid', findings[0].match.length)).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/ + ) + }) + + it('reports a clean transcript as clean', () => { + const findings = scanTranscriptForSecrets('Antigravity CLI 1.1.17\nSonnet 4.6 (High)\n> ') + expect(findings).toEqual([]) + expect(formatFindings('fixture', findings)).toContain('clean') + }) + + it('claims a span once, so a token inside an email is not double-reported', () => { + const findings = scanTranscriptForSecrets('longlivedaccountname@corp.internal') + expect(findings).toHaveLength(1) + }) + + it('passes a fixture that is already scrubbed, so "prove it is gone" can succeed', () => { + const scrubbed = `uuuu@example.com\n${'X'.repeat(44)}` + expect(scanTranscriptForSecrets(scrubbed)).toEqual([]) + }) +}) + +describe('redaction', () => { + it('replaces every finding with the same number of characters', () => { + // Why length matters: the fixture's value is its exact wrapping. A shorter + // replacement reflows the screen and invalidates the capture. + const text = 'Antigravity CLI 1.1.17\njin.woo@acme.dev (Antigravity Business)\n> ' + const { text: redacted, redacted: count } = redactTranscript(text) + expect(count).toBe(1) + expect(redacted).toHaveLength(text.length) + expect(redacted).not.toContain('jin.woo@acme.dev') + expect(scanTranscriptForSecrets(redacted)).toEqual([]) + expect(redactTranscript(redacted).redacted).toBe(0) + }) + + it('keeps a redacted email shaped like an email', () => { + expect(placeholderFor('email', 'a@b.example.com'.length)).toMatch(/^u+@example\.com$/) + }) + + it('leaves the rest of the screen byte-for-byte untouched', () => { + const text = 'line one\nuser@corp.io\nline three' + expect(redactTranscript(text).text.split('\n')[2]).toBe('line three') + }) +}) + +describe('committed transcripts', () => { + // Why in CI and not just in the recorder: a transcript is committed once and read forever. + // The capture-time warning is skippable; this is not. + const fixtureDir = join(import.meta.dirname, '..', '..', 'src', 'main', 'runtime', '__fixtures__') + const transcripts = readdirSync(fixtureDir).filter((entry) => entry.endsWith('.txt')) + + it.each(transcripts)('%s carries no account identifier or credential', (name) => { + const findings = scanTranscriptForSecrets(readFileSync(join(fixtureDir, name), 'utf8')) + expect(formatFindings(name, findings)).toContain('clean') + }) +}) + +describe('capture argv', () => { + it('splits recorder options from the agent command', () => { + const { options, command } = parseArgs([ + '--name', + 'antigravity-ready-personal-non-gemini', + '--cols', + '120', + '--', + 'agy', + '--model', + 'sonnet' + ]) + expect(options.name).toBe('antigravity-ready-personal-non-gemini') + expect(options.cols).toBe(120) + expect(command).toEqual(['agy', '--model', 'sonnet']) + }) + + it('collects a multi-file scan list', () => { + const { options } = parseArgs(['--scan', 'a.txt', 'b.txt', '--redact']) + expect(options.scan).toEqual(['a.txt', 'b.txt']) + expect(options.redact).toBe(true) + }) + + it('routes a Windows shim through cmd.exe, which node-pty cannot spawn directly', () => { + expect(resolveSpawn(['agy.cmd', '--model', 'sonnet'])).toEqual( + process.platform === 'win32' + ? { file: 'cmd.exe', args: ['/c', '"agy.cmd"', '--model', 'sonnet'] } + : { file: 'agy.cmd', args: ['--model', 'sonnet'] } + ) + }) +}) diff --git a/config/scripts/windows-cmd-shim-spawn-boundary.test.mjs b/config/scripts/windows-cmd-shim-spawn-boundary.test.mjs index a8c2cb3f4e7..253605781cf 100644 --- a/config/scripts/windows-cmd-shim-spawn-boundary.test.mjs +++ b/config/scripts/windows-cmd-shim-spawn-boundary.test.mjs @@ -46,6 +46,7 @@ const WINDOWS_SHIM_SPAWN_ALLOWLIST = [ 'config/scripts/electron-builder-config.test.mjs', 'config/scripts/ensure-native-runtime.test.mjs', 'config/scripts/live-remote-freeze-rpc.mjs', + 'config/scripts/pty-transcript-secret-scan.test.mjs', 'config/scripts/remote-agent-session-authority-repro.mjs', // Platform-local build paths; the win32 branch is dead code on both. 'config/scripts/build-mac-local.mjs', diff --git a/docs/assets/readme-downloads.svg b/docs/assets/readme-downloads.svg index 4331989b66d..dce3559fd10 100644 --- a/docs/assets/readme-downloads.svg +++ b/docs/assets/readme-downloads.svg @@ -1,5 +1,5 @@ - - downloads: 47m + + downloads: 48m @@ -15,7 +15,7 @@ downloads downloads - 47m - 47m + 48m + 48m diff --git a/docs/reference/agent-pty-transcript-capture.md b/docs/reference/agent-pty-transcript-capture.md new file mode 100644 index 00000000000..934f0028a93 --- /dev/null +++ b/docs/reference/agent-pty-transcript-capture.md @@ -0,0 +1,129 @@ +# Capturing an agent PTY transcript + +Orca's readiness and blocked-prompt rules are text rules over what an agent CLI paints on a +terminal. They are only as good as the screens they were written against. This is how to record +one, byte for byte, so a rule can be pinned to evidence instead of to a remembered screen. + +Related: [`antigravity-readiness-evidence.md`](./antigravity-readiness-evidence.md) names the +specific Antigravity transcripts that are still missing and what each one decides. + +## The recorder + +``` +node config/scripts/capture-agent-pty-transcript.mjs --name [options] -- [args...] +``` + +It allocates a real PTY, spawns the agent inside it, mirrors the session to your terminal so you +can drive it by hand, and appends every byte it receives to +`src/main/runtime/__fixtures__/.txt`. It does not strip escapes, fold `\r`, rewrap +lines, or normalise anything — the file is what the terminal received. + +- **Ending a capture:** press Ctrl+]. The recorder consumes that key and + never forwards it, which is the only way to end a capture _while a dialog still owns the + screen_. Quitting the agent instead would first dismiss the dialog you came to record. +- `--cols N --rows M` pin the PTY size (default: your terminal's). Wrapping is part of the + evidence, so record the size — the sidecar does it for you. +- `--duration S` stops unattended after S seconds, for a screen that needs no interaction. +- `--send ":"` types into the PTY at a fixed offset, repeatable, with `\r` `\n` `\t` `\e` + escapes. A dialog capture has to be driven, and an unattended run (CI, or an agent) has no TTY to + type into; the keystrokes ride the same PTY a human's would. For example, the committed + `antigravity-dialog-model-picker.txt` was recorded with + `--duration 24 --send "14000:/model" --send "16000:\r"`, which leaves the picker owning the + screen when the capture stops. +- `--note ""` records the account type, plan, model and CLI version in the sidecar. +- `--out ` writes outside the fixture directory (use it for a first dry run). + +Each capture also writes `.meta.json` with the timestamp, platform, command, +PTY size, note and exit code. Commit it with the transcript; the version and account type behind +a screen are not recoverable from the bytes. + +**Prerequisite:** `node-pty` must be built for plain Node: + +``` +node config/scripts/ensure-native-runtime.mjs --runtime=node +``` + +Orca itself does not need to be running, and the recorder never touches Orca state. + +### Platform notes + +- **macOS / Linux:** nothing special. `TERM=xterm-256color` is set for the child. +- **Windows:** run it from Windows Terminal / PowerShell, not a Git Bash (MSYS) pane — MSYS + rewrites arguments that start with `/`, which mangles the `cmd.exe /c` hand-off. A `.cmd` or + `.bat` agent shim cannot be spawned by node-pty directly, so the recorder routes those through + `cmd.exe` for you. +- **WSL:** capture _inside_ the distro (run the recorder from the distro's checkout). Recording + `wsl.exe` from the Windows side adds the login-shell banner to the transcript. +- **SSH:** record on the execution host. A transcript recorded locally is not evidence about what + a remote agent prints. + +## Privacy: scrub before committing + +A live agent screen routinely contains things that must not enter git history: + +| Scrub | Why | +| ---------------------------------------------------------------------- | ---------------------------------------------------- | +| Account email / sign-in identifier | The account row on a ready screen prints it verbatim | +| Org, tenant or team name | Identifies a customer | +| Machine hostname and OS username | Appear in prompts, paths and the OSC title | +| Absolute home paths (`/Users/`, `C:\Users\`) | Contain the username | +| JWTs, `AIza…` keys, `1//…` refresh tokens, `Bearer …`, `sk-…`, `ghp_…` | Live credentials; a sign-in screen can echo one | +| Private repo, branch and ticket names | Leak roadmap detail | +| Anything you pasted into the agent during the capture | You typed it; it is in the transcript | + +The recorder scans the file as soon as the capture ends and prints every hit with a line and +column. To scrub: + +``` +node config/scripts/capture-agent-pty-transcript.mjs --scan src/main/runtime/__fixtures__/.txt --redact +``` + +Redaction replaces each finding with a **same-length** placeholder (`u…u@example.com`, `XXXX…`). +Length matters: a transcript's value is its exact wrapping and column alignment, and a shorter +replacement reflows the screen and destroys the evidence. + +### Verify it is gone + +1. `node config/scripts/capture-agent-pty-transcript.mjs --scan src/main/runtime/__fixtures__/.txt` + must print `clean` and exit `0`. It recognises its own placeholders, so a scrubbed file passes. +2. Grep for the specifics the scanner cannot know: + `rg -n -i -- "$(whoami)|||" src/main/runtime/__fixtures__/.txt` +3. Read it once with escapes visible: `LC_ALL=C cat -v src/main/runtime/__fixtures__/.txt`. + The scanner matches shapes; only a human catches a project name. +4. Check the sidecar too — `--note` text is free-form and is committed. + +`config/scripts/pty-transcript-secret-scan.test.mjs` re-scans every committed +`__fixtures__/*.txt`, so a transcript that skips step 1 fails the suite. + +## Consuming a transcript in a test + +Feed the raw bytes through the runtime rather than into a matcher directly: escape handling, +tail retention and title tracking all live in `onPtyData`, and a rule tested on pre-normalised +text is tested on something no pane ever sees. + +`src/main/runtime/agent-transcript-pane-test-harness.ts` builds the pane; +`src/main/runtime/terminal-interactive-wait-visibility.test.ts` (cursor-agent) and +`src/main/runtime/antigravity-readiness-transcripts.test.ts` (Antigravity) are the two consumers. + +## Worked example: the Antigravity captures + +The six committed `antigravity-*.txt` fixtures were recorded this way on macOS against +`agy` 1.1.25. Two points generalise: + +- **Reach a state without mutating the operator's config.** The ready-screen captures ran in a + directory the CLI already trusted, so no trust answer was written. Where a dialog could only be + reached by signing the operator out or deleting their settings, it was left uncaptured and + recorded as such rather than forced. +- **An environment variable is a legitimate capture knob** where a setting is not. + `AGY_CLI_HIDE_ACCOUNT_INFO=1` produced a second ready screen with no account row, which is + evidence no amount of reasoning about the first screen could have supplied. It changes nothing + on disk. + +## Known gap in the existing captures + +The three `cursor-agent-*.txt` fixtures contain **no escape bytes and no carriage returns**. +Whatever produced them went through a renderer and a clipboard, so they preserve wording and +box-drawing glyphs but not the caret, the cursor moves, the repaints, or whether the CLI uses the +alternate screen buffer. They are good enough for the wording-based rules built on them and are +not evidence for anything else. New captures made with this recorder keep those bytes; the +Antigravity scaffold asserts their presence so a pasted screen cannot pass as a capture. diff --git a/docs/reference/antigravity-readiness-evidence.md b/docs/reference/antigravity-readiness-evidence.md new file mode 100644 index 00000000000..0010fa76ded --- /dev/null +++ b/docs/reference/antigravity-readiness-evidence.md @@ -0,0 +1,263 @@ +# Antigravity readiness: what the transcripts show + +`findAntigravityReadyPromptIndex` in `src/main/runtime/terminal-wait-detection.ts` decides whether +an Antigravity pane is ready for a prompt. It has been written five times, each version tuned +against a five-line screen typed from memory into a `.spec.ts` fixture. Three of the first four +were found worse than the bug they replaced, and the fifth was reverted. + +Real transcripts now exist. They were recorded from a live `agy` on macOS with +[`agent-pty-transcript-capture.md`](./agent-pty-transcript-capture.md) and are committed under +`src/main/runtime/__fixtures__/`. `src/main/runtime/antigravity-readiness-transcripts.test.ts` +replays them through the runtime. + +**Headline: on real output the current detector is inverted.** It refuses a genuinely ready screen +and accepts a live model picker. The five attempts argued about which extra condition to add; none +of them had noticed that the condition they all shared — a line beginning with the model name — +never matches a real Antigravity ready screen at all. + +## Versions + +| Thing | Value | +| ------------------------- | ----------------------------- | +| `agy --version` | `1.1.25` | +| Banner printed by the TUI | `Antigravity CLI 1.2.0` | +| Captured | 2026-09-10, macOS, 120x40 PTY | + +The binary and its own banner disagree. Any rule keyed to a version string must read the banner, +not `--version`, and must tolerate the two disagreeing. + +## What the captures are + +| Fixture | What it is | +| -------------------------------------------- | --------------------------------------------------------- | +| `antigravity-ready-api-key-gemini-model.txt` | Ready screen, API-key identity, Gemini 3.7 Flash (Low) | +| `antigravity-ready-account-info-hidden.txt` | The same ready screen with `AGY_CLI_HIDE_ACCOUNT_INFO=1` | +| `antigravity-dialog-trust-workspace.txt` | Workspace trust dialog, live and unanswered | +| `antigravity-dialog-model-picker.txt` | `/model` picker, live and unanswered | +| `antigravity-dialog-command-palette.txt` | Slash-command palette, live and unanswered | +| `antigravity-dialog-dismissed.txt` | `/model` picker dismissed with esc, then settled | +| `antigravity-busy-mid-turn.txt` | A real turn, recording stopped while the spinner was live | +| `antigravity-busy-turn-ended.txt` | The same turn after it ended and the composer returned | + +## What could not be captured, and why + +Nothing below was faked. Each is a case the recorder could not reach without changing the +operator's account state or configuration, which is out of bounds. + +| Missing | Why | +| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `antigravity-ready-business-non-gemini.txt` | This machine has no OAuth session — the CLI prints _"You are currently not signed in"_ and authenticates from `GEMINI_API_KEY`. Reaching a Business ready screen means signing someone in. | +| A non-Gemini model on any ready screen | `agy models` offers 11 models, all Gemini, and `settings.json` pins `modelProvider: gemini`. A non-Gemini row is not reachable from this account. | +| `antigravity-dialog-sign-in.txt` | Unsetting `GEMINI_API_KEY` does not reach the sign-in dialog; the CLI refuses to start because `modelProvider` is pinned. Reaching it means editing the operator's `settings.json`. | +| `antigravity-dialog-theme-picker.txt` | There is no `/theme` command in 1.2.0 (`Unknown command: /theme`). The picker appears only in first-run onboarding, which means deleting the operator's config. | +| `antigravity-dialog-privacy-notice.txt` | First-run onboarding, as above. | +| `antigravity-dialog-update-banner.txt` | Cannot be forced; no update was pending during the session. | + +Each remains as a named, skipping case in the suite so it is visible rather than forgotten. + +## What the transcripts show + +### 1. The ready screen's model row is not at the start of a line + +The ready screen prints a block-glyph logo down the left, and the identity, model and path rows are +painted **on the same physical lines as the logo**. What Orca derives is: + +``` +▀▀▀▀▀▀ Gemini API key +▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low) +▄▀▀ ▀▀▄ ~ +``` + +The detector requires `normalized.startsWith('gemini', trimmedStart)` on a trimmed line. The +trimmed line starts with `▀`. It never matches. Measured three ways on the real screen: + +| Input | `isKnownReadyPromptPreview` | +| ------------------------------------------------------ | --------------------------- | +| Real ready screen | `false` | +| The same screen with the logo glyphs stripped | `true` | +| Real ready screen followed by the live `/model` picker | `true` | + +So the logo — decoration, and suppressible with `AGY_CLI_HIDE_LOGO` — is what decides readiness +today, and the live dialog is what supplies the model line the ready screen could not. + +### 2. The dialog is what satisfies the model rule + +`/model` prints its options one per line: + +``` +Gemini 3.8 Flash +> Gemini 3.7 Flash (current) +Gemini 3.1 Pro +``` + +Those lines _do_ begin with `Gemini`, and a bare `>` composer line sits earlier in the same tail +from before the picker opened. Both halves of the rule are satisfied **while a dialog owns the +screen**, and the pane reads ready. This is the false-ready hazard the last three attempts were +each trying to close, reproduced from a real capture. + +### 3. `>` is the dialog selection marker, not only the composer caret + +Every dialog uses `>` to mark the highlighted row: `> Yes, I trust this folder`, +`> Gemini 3.7 Flash (current)`, `> /add-dir`. The idle composer is a line whose whole trimmed +content is `>`. That distinction is the only thing separating them, which means the relaxation +proposed in PRs #15840 and #15852 — accept any line _beginning_ with `>` — would make the trust +dialog and the model picker read as ready. On 1.2.0 the idle composer is a bare `>`; those PRs' +1.1.17 mode-banner claim could not be reproduced here and may be mode-specific. + +### 4. There is no email account row, and the row can be switched off entirely + +For an API-key user the identity row reads literally `Gemini API key`. There is no `@`, no +domain, nothing an account-row rule can key on. Separately, `AGY_CLI_HIDE_ACCOUNT_INFO=1` — a +supported environment variable in the binary — removes the row from a fully ready screen, which +`antigravity-ready-account-info-hidden.txt` captures. + +### 5. Dialogs are drawn two different ways, and the banner is never reprinted + +The trust dialog and the sign-in splash take the **alternate screen** (`ESC[?1049h` … `ESC[?1049l`). +The model picker and command palette are drawn **in place on the main screen** with erase-to-EOL. +After dismissal the CLI prints `⎿ Exited /model command` and redraws the composer — it does **not** +reprint the banner. The header stays where it was at startup. + +### 6. Rows are positioned with cursor addressing, not newlines + +The status row is written with absolute and relative moves (`ESC[13;99H`, `ESC[83X ESC[83C`), so +`? for shortcuts` and `Gemini 3.7 Flash · low` end up on one derived line. Any rule that assumes +one screen row equals one `\n`-delimited line is reading a different document than the user sees. + +## 8. Busy frames park the caret exactly like idle frames — the spinner is what differs + +The frame that ends a turn-in-progress and the frame that ends an idle screen park the cursor with +the **same bytes**. Only the hint row differs, and the park erases it: + +``` +idle: ? for shortcuts ESC[83X ESC[83C Gemini 3.7 Flash · low CR ESC[2A ESC[2C ESC[?25h +busy: esc to cancel ESC[85X ESC[85C Gemini 3.7 Flash · low CR ESC[2A ESC[2C ESC[?25h +``` + +So a rule that keys on "the caret is the last thing in the tail" cannot tell busy from idle **on the +frame alone**. What saves it is what comes next. Each spinner tick is its own repaint with its own +park, two rows higher than the frame's: + +``` +ESC[?25l CR ESC[2A ⣯ Generating ESC[11D ESC[?25h +ESC[?25l CR ESC[2A ⣟ Generating. ESC[12D ESC[?25h +``` + +That second `CR ESC[2A` splices the composer row away, so the retained tail during a live turn ends +on the spinner row, not on the caret. Measured on `antigravity-busy-mid-turn.txt`: + +| Capture | last retained line | bare `>` line present | +| -------------------------------------------- | ------------------ | --------------------- | +| `antigravity-ready-api-key-gemini-model.txt` | `>` | **yes** | +| `antigravity-busy-mid-turn.txt` | `⣟ Generating...` | **no** | + +**Consequence for a caret-based rule:** it already answers "not ready" for a real mid-turn capture, +because there is no bare caret in the tail to match. A constructed input that keeps the park bytes +and only edits the status text is not faithful to a live turn — a live turn has a spinner row +repainting _below_ the composer. + +**The residual window, and the clause it implies.** Between a frame park and the next spinner tick +the tail does end on the bare caret and is indistinguishable from idle. The gap is one tick +interval. Any readiness path gated on sustained quiescence is safe, because ticks keep arriving and +the pane is never quiet; a path that only inspects retained text is not. For those paths the +evidence supports one clause, and only one: + +> **A braille glyph (U+2800–U+28FF) on the last visible line of the retained tail means working.** + +That predicate already exists in this file for cursor-agent (`CURSOR_BUSY_SPINNER_RE`) and should be +reused rather than reinvented. It must be scoped to the **last visible line**, not the whole tail: +a first-run transcript prints `⠾ Signing in...` during startup, which would otherwise pin a ready +screen as busy forever. + +Nothing else in the capture distinguishes the two states. The hint row (`esc to cancel` versus +`? for shortcuts`) is erased by the park in both cases, the park offsets are identical, and +`ESC[?25l`/`ESC[?25h` fencing appears around every repaint, idle or busy. + +## Confirmed / refuted, by attempt + +Evidence column names the fixture; all quoted text is from the committed transcripts. + +### Attempt 1 — the rule at HEAD + +| # | Claim | Verdict | Evidence | +| ---- | -------------------------------------------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1.1 | A ready screen prints the banner `Antigravity CLI` | **Confirmed** | `Antigravity CLI 1.2.0` in both ready fixtures | +| 1.1b | …and its last occurrence in the tail is the live one | **Refuted** | The trust dialog's own body says _"Antigravity CLI requires permission to read, edit, and execute files here"_, so `lastIndexOf` lands inside the dialog | +| 1.2 | The model row begins with the vendor word `Gemini` | **Refuted** | `▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low)` — the logo precedes it; never at line start | +| 1.3 | The caret line's whole trimmed content is `>` | **Confirmed** on 1.2.0 idle | bare `>` in both ready fixtures | +| 1.3b | …and only the composer prints `>` | **Refuted** | `> Yes, I trust this folder`, `> Gemini 3.7 Flash (current)`, `> /add-dir` | +| 1.4 | A ready screen prints the workspace path on its own line | **Refuted** | the path shares its line with logo glyphs (`▄▀▀ ▀▀▄ ~`) | + +### Attempt 2 (loop 1) — blacklist the model line + +| # | Claim | Verdict | Evidence | +| --- | ------------------------------------------ | ----------- | ---------------------------------------------------------------------------------------------------------------- | +| 2.1 | Dialog model-row wording is enumerable | **Refuted** | the palette lists 50+ commands with free-form descriptions; the picker prints whatever models the account offers | +| 2.2 | A dialog never reproduces a real model row | **Refuted** | the `/model` picker prints four real model rows, one per line, at line start | + +### Attempt 3 (loop 2) — structural ordering on `headerIndex` + +| # | Claim | Verdict | Evidence | +| --- | -------------------------------------------------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------- | +| 3.1 | A live dialog is printed below the ready chrome | **Confirmed** for in-place dialogs | picker and palette append below the composer | +| 3.2 | The banner is reprinted when a dialog is dismissed | **Refuted** | `antigravity-dialog-dismissed.txt` shows `⎿ Exited /model command` and a redrawn composer, no banner | +| 3.3 | Antigravity does not use the alternate screen | **Refuted** | `ESC[?1049h` opens the trust dialog and the sign-in splash | +| 3.4 | No full repaint per keystroke | **Partly refuted** | typing `/mod` repaints the palette region on each keystroke with `ESC[K` | + +Because of 3.2, `headerIndex` cannot be the anchor: it never advances. Ordering can only be +expressed against the model/caret positions, which is what 1.2 and 1.3b just invalidated. + +### Attempt 4 (loop 3) — require a positive account row + +| # | Claim | Verdict | Evidence | +| --- | ---------------------------------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| 4.1 | Every ready screen prints an account row | **Refuted, twice** | API-key identity prints `Gemini API key` (no `@`); `AGY_CLI_HIDE_ACCOUNT_INFO=1` removes the row entirely | +| 4.2 | A startup dialog never contains an `@`-and-`.` token | **Not reachable here** | none of the captured dialogs contains one, but the palette shows free-form skill descriptions, which are user-authored text | +| 4.3 | The account row is distinguishable from prose | **Refuted** | the row is not a distinct line; it shares one with the logo | + +### Attempt 5 (PR #19749, reverted) — ordering + account row + +| # | Claim | Verdict | Evidence | +| --- | -------------------------------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 5.1 | Ordering plus an account row separates ready from dialog | **Refuted** | the account row is optional (4.1) and the ordering anchor never moves (3.2) | +| 5.2 | Executing both builds was sufficient verification | **Refuted** | the executed input was the hand-written fixture, so the check reproduced the fixture's assumptions. The real screen disagrees with that fixture on the model row, the path row and the account row | +| 5.3 | The wedge is a model-name problem | **Refuted** | it is a line-start problem. Even `Gemini 3.7 Flash (Low)` — a Gemini model — fails, because a logo glyph precedes it | + +### Cross-cutting + +| # | Question | Answer | +| --- | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| X1 | Does `agy` set an OSC title distinguishing busy from idle? | **No.** Not one OSC title sequence appears in any capture. Title-based readiness is unavailable for this agent | +| X2 | Does it repaint with bare `\r`? | **Yes**, constantly, plus `ESC[K` and absolute cursor moves | +| X3 | Does the caret survive in the tail? | **Yes** — a bare `>` line is present in every ready capture | +| X4 | Banner-to-caret distance | ~8 derived lines on a 120x40 PTY; the banner falls outside the 6-line preview window, so only the full retained tail can see it | +| X5 | Pane title on the trust screen versus ready | Identical: none | + +## Can attempt six be written? + +Yes — but not as a variation on any of the five. Every one of them refined a predicate over +`\n`-delimited lines, and that is the layer where the evidence says the information is not. + +What the captures support: + +- **The one stable, dialog-free ready marker is a line whose entire trimmed content is `>`.** It is + present in every ready capture and absent from every dialog capture, because a dialog's `>` always + carries its selected row's label. This is a much narrower rule than any attempt used, and it is + the only one that survived contact with the transcripts. +- **Drop the model-row requirement.** It matches dialogs and not ready screens. Keeping it inverted + the detector. +- **Do not require an account row.** It is optional by environment variable and carries no email for + API-key users. +- **Do not anchor on `headerIndex`.** The banner is printed once and never reprinted. +- **The blocked-signal path already works** for the trust dialog: `antigravity-dialog-trust-workspace.txt` + is correctly refused today, by wording, not by structure. + +What is still unknown and should be captured before shipping: the sign-in, theme, privacy and +update dialogs, and any ready screen where the composer is not idle (accept-edits and plan mode, +which PRs #15840 and #15852 describe from a screenshot). A bare-`>` rule is only as good as the +claim that those modes still end on a bare `>`; that claim is untested. + +The honest summary is that this is a screen-shaped problem being solved with line-shaped tools. A +rule over the derived tail can be made much better than what ships today, but the durable fix is to +ask the terminal emulator what the bottom row of the screen actually is, rather than inferring it +from a byte stream that was written with cursor addressing. diff --git a/docs/relay-region-correction/RELAY-REGION-CORRECTION-ACCEPTANCE.md b/docs/relay-region-correction/RELAY-REGION-CORRECTION-ACCEPTANCE.md new file mode 100644 index 00000000000..7bb182f3fe1 --- /dev/null +++ b/docs/relay-region-correction/RELAY-REGION-CORRECTION-ACCEPTANCE.md @@ -0,0 +1,114 @@ +# Idle regional correction acceptance + +Updated 2026-09-11. Scope: [idle-cutover plan](RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md). +Local implementation is based on main `74cc9b50390b481009b34823a35eee01a5b90e40`, +with uncommitted changes atop `08c0802089a9186cf48ce0d168cd87565bd92d65`. + +**Local validation is green; the published PRs are not updated or merge-ready.** +Cloud draft20037 still points to `0462a87011d8783f91f9a9e977ea7a68437a0d5b`; +desktop draft20031 still points to `f9e5b0193448e6c524603836b9278878b96cf51c`. +Their older CI is not evidence for this implementation. No commit, push, merge, +deployment, workflow dispatch or production mutation occurred in this continuation. + +## What the implementation proves + +The source permits optimization only when actual client sockets, splices, pending +connections and in-flight admission/control work are absent. It installs the local +gate before awaiting a constrained assignment transaction. Duplicate requests bind +exact authority; ambiguous database outcomes keep the source fenced until locked +reconciliation establishes the result. After commit, it releases the empty control +and uses ordinary reconnect and migration recovery. Emergency drains retain their +existing behavior. No retained-source protocol or restoration loop remains. + +Three real TCP WebSocket scenarios join authenticated local HTTP dispatch, two +cells, the actual desktop origin pool, SQLite and an independent execution child: + +1. Either connected device prevents movement; after both leave, target reconnect + succeeds and the durable append-once mutation oracle shows no replay. +2. A racing arrival receives4409; a definite failed commit restores source admission. +3. The target control connection is actually attempted and fails; source activity + is released and ordinary expiry recovery allows source epoch3 reconnect. + +These use synthetic time and token verification. They do not prove physical mobile +background scheduling, production authentication/network behavior or user latency. + +## Commands and results + +Every test/app command uses `ORCA_BACKGROUND_LAUNCH=1`. Cloud commands run from +`cloud/apps/relay`; root commands run from this worktree. All logs below are under +`.tmp/idle-cutover-review/`. + +| Scope | Command | Result / log | +| --- | --- | --- | +| Cloud | `ORCA_RELAY_TEST_POSTGRES_URL='postgresql://postgres@127.0.0.1:55440/postgres?options=-csearch_path%3Didle_full_root_20260911' ORCA_IDLE_REHOME_POSTGRES_URL='postgresql://postgres@127.0.0.1:55440/postgres' ORCA_REGION_CORRECTION_POSTGRES=1 pnpm exec vitest run --no-file-parallelism` |77 files /682 passed /zero skips; `cloud-full-postgres-idle.log` | +| Cloud | `pnpm run typecheck` | Passed; `cloud-typecheck-after-preview.log` | +| Cloud | `pnpm build` | Passed; `cloud-release-build-idle.log` | +| Root | `pnpm test src/main/runtime/relay` |20 files /177 passed; `desktop-relay-full-idle.log` | +| Root | `pnpm test tests/e2e/relay-region-correction.unit.test.ts tests/e2e/relay-region-compatibility.unit.test.ts` |16 passed; `transport-contract-cleanup.log` | +| Root | `pnpm test src/main/global-fetch-call-site-audit.test.ts tests/e2e/relay-region-correction.unit.test.ts` |4 passed after CI fixes; `ci-gaps-green.log` | +| Root | `pnpm tc:node` | Passed; `node-final-idle.log` | +| Root | `pnpm exec oxlint src/main/runtime/relay tests/e2e/relay-region-correction.unit.test.ts tests/e2e/relay-region-compatibility.unit.test.ts` | Passed; `desktop-lint-idle.log` | +| Root | `pnpm run check:code-quality:changed` | Passed,0 new findings across30 changed files; `code-quality-changed-idle.log` | +| Root | `pnpm run check:reliability-gates` |121 manifest gates passed; `reliability-idle.log` | +| Root | `ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ssh-docker-transport-drop-recovery.spec.ts tests/e2e/paired-remote-terminal-serve-restart-binding.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1` |7 passed after fresh build; `ssh-folder-idle.log` | +| Root | `node --test cloud/dev/scripts/deploy-relay-blue-green.test.mjs cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs` |47 passed; `deployment-guards-idle.log` | + +PostgreSQL16.15 reused the existing `orca-region-release-pg16` container on55440. +The suite used an isolated schema, dropped afterward (`cloud-full-postgres-cleanup.log`); +new concurrency tests create and clean independent schemas. No other PostgreSQL +port was used. Root oxlint ignores cloud; the configured cloud lint is TypeScript. + +The seven background Electron checks cover paired folder-capable binding across +serve restart and six real Docker SSH recovery journeys: live pane, output bounds, +host-proven exit, repeated restarts, frozen-host silence and resumed input. They do +not exercise a physical phone-to-SSH regional cutover or packaged upgrade. + +## Regression and review evidence + +- Reconciliation: four cases fail against constant not-committed, then pass with + locked authority checks (`reconciliation-{red,green}.log`). +- Registry: disabling pre-await admission accounting makes the arrival race fail; + restoring it passes. Five conflicting operation-ID authority tuples fail before + the identity fix, then all48 registry tests pass (`operation-tuple-{red,green}.log`). +- SQLite startup capability upgrade: red before upgrade logic;8 database tests pass + afterward. Legacy controls default to not idle-capable. +- The commit placeholder negative control was run after implementation; it is + counterfactual evidence, not a claim of chronological test-first development. +- Full PostgreSQL verification supersedes intermediate preview/legacy-test failures. + An agent's earlier PostgreSQL safety-latch discrepancy was disproven in an + isolated schema and explicitly withdrawn. +- Fifth GPT-6-astra low audit: **APPROVE within implementation scope**, no new blocker. + Reviewed source barriers, exact duplicates, locked ambiguity and worker progress. + Reviewer had migrated PostgreSQL tests, but did not author the core implementation. + Ledger: `.tmp/idle-cutover-review/review-ledger.md`. No sixth cycle started. + +## CI preparation and review artifacts + +Old desktop CI failed on a stale global-fetch inventory count and missing `pg` for +the transport test. The count reproduces locally; the downstream catalog/probe +consumers already consume/cancel bodies, so the audited count is corrected. +The unit workflow installs locked cloud relay dependencies and builds their contracts. +From `cloud/`, both commands pass (`ci-relay-dependencies.log`): + +- `npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts` +- `npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build` + +Local split patches and draft bodies are in `.tmp/idle-cutover-review/`: +`cloud-idle.patch`, `desktop-idle.patch`, `cloud-pr-body.md`, `desktop-pr-body.md`. +`prepare-split.py` verifies ordered application against the stated main baseline; +`split-manifest.json` identifies the combined tree. The actual index/branches remain +unchanged. The HTML explainer is `.tmp/relay-region-explainer.html`; four stages, +failure toggle, light/dark mobile/desktop layout and browser-error checks pass. + +## Remaining acceptance gaps + +- Update the two draft PRs and verify fresh CI for their exact heads. The original + handoff explicitly prohibited pushes; publication needs authorization. +- Packaged mixed-version desktop/mobile, physical-device lifecycle and platform + transport remain unverified. Pinned wire tests are narrower evidence. +- CI soak and production RTT/interaction benefit remain unmeasured. Correction + defaults off; deployment/enablement require the reviewed rollout procedure. +- Archive of intermediate/superseded evidence: + `.tmp/idle-cutover-review/acceptance-history-before-final.md` and pre-rescope branch + `relay-region-before-idle-implementation`. Earlier retention results do not prove + the idle design or repair the superseded live-retention transport case. diff --git a/docs/relay-region-correction/RELAY-REGION-CORRECTION-API.md b/docs/relay-region-correction/RELAY-REGION-CORRECTION-API.md new file mode 100644 index 00000000000..f3bd8b682c5 --- /dev/null +++ b/docs/relay-region-correction/RELAY-REGION-CORRECTION-API.md @@ -0,0 +1,61 @@ +# Idle regional correction contracts + +Updated 2026-09-11. The [idle-cutover plan](RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md) +is authoritative. This replaces the former retention/restoration protocol. + +## Desktop measurement and capability + +The optional assignment `regionCorrection` namespace supports `issue-window` and +`report`. The server issues a generation, fixed expiry, assignment epoch, incumbent +region and policy version1. A report supplies measurements for both regions or an +inconclusive reason. The first accepted report remains immutable; stale or changed +assignment evidence cannot authorize a move. Legacy placement hints remain separate. + +Desktop advertises `idle-regional-rehome-v1` in its control capability header. +There is no new mobile message, retained-control lease, or restoration notification. +The cell persists the exact activity, generation, assignment and incarnation with +`idle_regional_rehome`. The obsolete `finish_existing` database column is always +written0 for schema compatibility and is not an eligibility signal. + +## Director to source cell + +`POST /v1/admin/host-idle-rehome` requires the configured director identity and +matching source cell/incarnation. The strict request contains: + +- `v:1`, stable UUID `attemptId`, `userId`, `relayHostId`; +- `sourceCellId`, `sourceCellIncarnation`, `sourceAssignmentEpoch`, `sourceGeneration`; +- `targetCellId`, authenticated `cohortPercent`, and fresh `directorSafety`. + +The strict response is `{v:1,outcome}` with `busy`, `committed`, `deferred`, or +`stale`. A lost HTTP reply is ambiguous and does not consume a dispatch-failure +budget. Repeat delivery retains the same operation identity. Cell status advertises +regional protocol3; new selection requires both cells at protocol3 or newer. + +## Store and source ownership + +`selectIdleRegionalRehomeCandidates` is read-only. It checks fresh evidence, +capability, policy, cooldown, telemetry and target capacity; bounded rotating pages +allow progress past busy hosts. Selection does not prove physical idleness. + +`HostSessionRegistry.idleRehome` accounts for accepts, attaches, control commands, +and activation before their first await. Only an idle source installs its admission +gate. New arrivals receive normal retryable routing failure. Conflicting reuse of +an operation ID cannot share another authority tuple's result. + +`commitIdleRegionalRehome` rechecks the exact request under existing locks, including +policy, cohort, capacity, global rate and concurrency. It atomically reserves the +target, advances assignment and records the attempt/source generation. It does not +choose a different host or destination. Completion is recorded at the source; +ordinary migration refresh, completion and expiry recovery remain responsible for +the target-registration lifecycle. + +`reconcileIdleRegionalRehome` locks the assignment before reading the operation. +It distinguishes committed, not-committed with unchanged live source authority, +and stale authority. Missing data after an unlocked read is never rollback proof. +A database error leaves admissions fenced while reconciliation retries. Successful +cutover closes/releases the empty source control; the desktop resolves its normal +assignment and reconnects. Definite rollback reopens only the same source authority. + +Outcome reporting aggregates actual attempts and migrations by cell/state, without +host identities or a retained-source table. A completed idle move does not authorize +closing future clients attached to the target. diff --git a/docs/relay-region-correction/RELAY-REGION-CORRECTION-CHECKLIST.md b/docs/relay-region-correction/RELAY-REGION-CORRECTION-CHECKLIST.md new file mode 100644 index 00000000000..fcb2b989c1a --- /dev/null +++ b/docs/relay-region-correction/RELAY-REGION-CORRECTION-CHECKLIST.md @@ -0,0 +1,47 @@ +# Relay region correction — implementation checklist + +> **Scope: idle-only correction.** Superseded retention history is preserved in +> `.tmp/idle-cutover-review/checklist-history-before-final.md` and the backup branch. +> Follow [idle-cutover plan](RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md). + +## Idle-only implementation tracker + +- [x] Resolve plan review findings and obtain Astra approval (revision 2). +- [x] Preserve pre-rescope revision on `relay-region-before-idle-implementation`. +- [x] Endpoint authentication and incarnation check: deterministic red/green. +- [x] Worker selects before cutover; busy deferral and lost reply tests: red/green. +- [x] Source barrier covers accepts, attaches, commands and control replacement (48 registry tests, including conflicting operation-ID authority tuples; red/green evidence in acceptance). +- [x] Constrained assignment commit and locked ambiguous-outcome reconciliation (11 PostgreSQL 16 tests on 55440, including both replacement orders and a lost commit reply). +- [x] Desktop removes live-retention handling, retains fresh decisions and fallback (53 focused desktop/compatibility tests; node typecheck passes). +- [x] Remove superseded cloud retention protocol/store/cleanup and obsolete tests. The legacy database capability column remains written as0 for existing schema compatibility; it enables no behavior. +- [x] Real two-cell transport: two clients, quiet connection, idle move, arrival race, and observed target-registration failure with ordinary recovery (3 tests pass). +- [x] Focused PostgreSQL transaction/concurrency checks on 55440 (11 passed). +- [x] Full local relevant cloud/desktop suites: 682 cloud tests with PostgreSQL,177 desktop relay tests,16 transport/compatibility tests. +- [x] Local pinned-wire compatibility, Docker SSH/folder continuity (7), types, lint and reliability manifest. Packaged/device/platform gates remain open. +- [x] Fifth Astra low implementation audit: APPROVE within the documented scope. +- [ ] Rewrite and validate cloud/desktop PRs, CI and final acceptance evidence. + +These are merge-readiness tasks. Device/package/platform and production rollout +requirements remain explicit gaps until independently evidenced. + + + +Current transport disposition: **the replacement idle-only transport suite is green +(3 tests), but the PRs are not ready**. Local cloud/desktop verification, final implementation audit, SSH/folder evidence +and reliability docs are complete. PR updates, fresh CI and release evidence remain. +The full cloud suite passes682 tests with PostgreSQL16 and no skips; cloud typecheck passes. Final audit and remaining end-to-end/PR tasks are still open. Exact commands/results are at the top of the acceptance document. The original +live-retention rollback assertion is superseded by the approved product rescope; +these results do not claim that old design was repaired. + +## Publication and release gates + +- [x] Prepare separate cloud/desktop patches and concrete PR descriptions locally. +- [x] Reproduce and address old CI failures: fetch audit count and cloud test dependencies. +- [ ] Obtain authorization to publish under the original no-push handoff constraint. +- [ ] Update cloud draft20037 and desktop draft20031; verify exact-head CI. +- [ ] Packaged mixed-version desktop/mobile and physical-device lifecycle. +- [ ] Linux/Windows transport evidence, CI soak, bounded rollout and measured benefit. + +No production mutation or deployment occurred. Local passing tests and audit approval +are not a claim that the current published PRs are ready or the feature is deployed. +The acceptance document lists commands, evidence scope and remaining gaps. diff --git a/docs/relay-region-correction/RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md b/docs/relay-region-correction/RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md new file mode 100644 index 00000000000..0fcf0459636 --- /dev/null +++ b/docs/relay-region-correction/RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md @@ -0,0 +1,213 @@ +# Relay region correction — idle cutover plan + +Status: **REVISION 2 — DESIGN APPROVED; implementation and verification pending.** + +Independent GPT-6-astra low review found no remaining design blocker. Approval is +conditional on the implementation gates below: pre-await ownership accounting, +shared assignment-row serialization, locked ambiguous-outcome reconciliation, +operation identity deduplication and late-callback fencing must be proven in tests. +Four independent idle-scope audits have been counted conservatively (scope, plan v1, +simplification, revision 2). If another plan revision fails review and a sixth cycle +would be needed, stop for user re-evaluation. Approval does not mean the current +live-retention PRs implement this design or are ready to merge. + +## Requirement and deliberate tradeoffs + +Automatically improve the host's relay region when no relay clients are using it. +A phone and iPad both disconnecting can create an opportunity. Continuous clients +can postpone optimization indefinitely. Returning during a move may incur ordinary +reconnect delay; zero-delay reconnect is NOT a requirement. Do not deliberately +close an established client connection to optimize latency. Direct LAN sessions +and running terminal processes do not themselves count as relay clients. + +The mobile 30-second background grace is not a reliable scheduling event: the OS +can delay it. Use actual source-cell connection state. Desktop control remains +open without phones; do not wait for it to disappear naturally. + +Keep fresh desktop measurements, incumbent-relative 25ms AND 20% improvement, +cohort/enable controls, capacity checks and cooldown. Probe improvement is not +proof of improved application latency. Cloud selects authoritative assignments; +mobile reconnect reads an assignment, it does not choose a faster region. + +## Simplicity constraints + +Reuse the regional worker, assignment transactions, reservations, request identity, +cell admission registry and normal desktop reconnect. No live-source retention, +splice transfer, recurring retained-control lease, new mobile protocol, global idle +poller, or desktop background-disconnect timer. Existing data sockets are never +copied or transferred (the former design did not transfer them either). + +Try an idle opportunity, immediately defer if busy; do not keep a gate waiting for +users to leave. The gate exists only for a short attempted cutover. A retryable +arrival may lose this race and reconnect normally. Do not count that as evidence +of lost execution or replay a previously sent mutation. + +Source cells already share the assignment database. Use one source-owned local +barrier around a constrained assignment transaction; do NOT introduce a distributed +prepare/commit protocol, a durable preparation table, or a second admission service. +The director selects candidates; the source's transaction rechecks director policy. + +## What is idle? + +The source owns one per-host/generation admission barrier covering every accept, +attach and control replacement path. Inventory these paths before changing them. +The quiescence predicate must require: + +- `activeConnIds.size === 0` (includes attaches while persistence awaits). +- `activeSplices.size === 0` and `pendingConns.size === 0`. +- Zero accepts in flight before insertion in those maps. Track ownership before + the first asynchronous operation that can admit this host; every exit releases it. +- Zero in-flight control operations that install credentials, mutate connection + basis or create admissions. Track actual server handlers; do not invent an + approximate count from RPC traffic or old database leases. + +An authenticated open desktop control socket, its ping/pong and activity renewal +are NOT client work and NOT a reason to defer. They can remain until cutover closes +the empty session. A desktop request arriving after the barrier gets a normal +retryable transport failure; finish previously accepted state mutations before +claiming idle. Reconnection must preserve pairing and not blindly replay mutations. + +Missing/expired database activity leases are never proof of idle. Outstanding +reservation cleanup remains owned and must be completed or safely fenced, but +waiting for every lease to expire would wrongly wait on the healthy control lease. + +## Source-owned try-cutover + +1. The director selects candidates read-only, using existing eligibility and pacing. + It sends an authenticated idle-cutover request with stable operation ID, expected + source assignment epoch/incarnation/generation and candidate target. Selection + does not reserve capacity or change assignment. Repeat delivery uses the same ID. +2. Source validates the request and current session. In one synchronous segment, + check all counters and install a per-host barrier. If busy, return `busy` without + closing sockets or holding a barrier. Busy is a deferral, not a dispatch failure. + The next worker pass must progress past busy candidates rather than starve others. +3. While barred, reject new clients and any new source control activation/rebind. + Late asynchronous continuations must recheck barrier/session after awaits and + release abandoned reservations. Install this fence before the first await of + the cutover. No barrier timer may reopen admissions on its own. +4. Source calls a constrained version of the existing assignment transaction. Reuse + its lock order, global rate/concurrency controls, cooldown, capacity reservation, + freshness and safety checks. Recheck exact source epoch/incarnation/generation, + target and operation identity. Reserve target, update assignment/epoch and record + the existing durable migration/attempt atomically. Do not call today's unrestricted + `claimRegionalRehome` and let it choose a different host. No network calls inside + DB locks. Zero sockets is established locally, not inferred from activity leases. +5. If committed, retire the still-empty source session/control via the existing + resolve-director closure path and release its activity. Desktop uses normal + reconnect and registers on target. Reply with the durable operation outcome. +6. If definitively not committed and source authority remains unchanged, remove the + barrier and continue on the original control. No target reservation survives a + rolled-back transaction. If authority changed, retire the obsolete source instead. + A timeout or transport error is NOT definitive rollback. + +## Ambiguous transactions, restarts and failures + +The request operation ID is known BEFORE the transaction and recorded as the +existing attempt ID on commit. Duplicate calls return its outcome and never start +another migration. A concurrent retry, cancellation or definitive-abort check must +serialize under the same host lock as commit; an unlocked absent-row lookup is +insufficient because the original transaction could still commit later. + +Keep a barrier until the database transaction is known terminal and a locked +reconciliation establishes the outcome. If the driver result is ambiguous, retry +status through a per-attempt backoff callback, using the same identity. If durable +access is unavailable, remain fenced; bounded availability cannot be promised +while the assignment's authority is unknown. No timeout-only reopen. Use the +existing DB transaction timeout and request timeout, not a new renewable gate lease. + +A lost director HTTP reply does not interrupt source-owned completion: source +finishes its transaction, reads durable outcome and closes/reopens locally. A +retry from any director sees the same operation. Director crashes do not strand +preparations because no separate preparation exists. + +A source restart/replacement control is fenced by existing authoritative registration +and activity validation. It must serialize against the cutover transaction under +host locks, validate the current assignment and reject old source ownership if the +commit won. If replacement won, the old transaction must fail its generation/ +incarnation recheck. This requires tracing current registration persistence and +proving the shared serialization point, not relying solely on the in-memory fence. +Late callbacks from a closed session cannot reopen admissions for its replacement. +Emergency drain invalidates local authority and participates in this serialization; +a cutover already committed follows its outcome, never reopens the emergency source. + +After commit, target registration failure uses ordinary bounded migration recovery. +The old empty control must be released even if outcome delivery failed; otherwise +current rollback refuses while source activity remains (`assignment-store.ts:6923`). +Source process death is handled by normal activity expiry and cell incarnation +fencing. No live clients were discarded, but a returning client may wait for recovery. +Once clients attach at target, preserve them under ordinary assignment rules: +initial source idleness never authorizes closing future target clients. + +## Compatibility and authority + +Mobile already re-resolves on `WRONG_CELL` (4409) through +`dialRelayThroughDirectorFallback`; use that existing close code for arrivals at a +gated source. Before commit, resolution can still return the old address: existing +backoff must prevent tight retry loops. After commit it returns the target. Pin +old parser/client fixtures and test the actual codes; do not assume every error is +retryable. Do not introduce a new mobile close code or protocol message. + +Empty host control closure uses the existing `DRAINING` / resolve-director path; +verify its reconnect behavior against the baseline desktop implementation. +Negotiate a distinct idle-cutover capability for participating cells and updated +desktops; do not reuse finish-existing capability to imply this new behavior. +Unsupported participants skip optional correction. Preserve old-server HTTP-400 +fallback for measurement fields. Emergency drain/auth enforcement can invalidate +any in-flight cutover; it must fence late commit and preserve existing hard deadlines. +Disabling correction stops new attempts; existing ones still reconcile. + +## Review and implementation gates + +Review must verify the shared-store serialization and identify every admission and +control mutation path before approving implementation. Minimum tests (red before +green for new guarantees): + +1. Phone remains while iPad disconnects: no move. Both disconnect: move possible. + A quiet established socket or expired DB splice lease still prevents a move. +2. Accept before/after barrier, accept awaiting activity persistence, attach awaiting + basis persistence, and credential mutation crossing the barrier. No late attach, + leaked reservation or interrupted established client. +3. Busy attempt leaves source admissions usable; repeated busy hosts do not starve + idle candidates or consume dispatch-failure budget. +4. Commit/replacement race; lost database/HTTP replies; timeout during transaction; + duplicate workers; director crash; source restart; replacement control; stale + epoch/incarnation; database outage and recovery. No timeout-only reopening. +5. Target failure before registration and after new client attachment; source-control + release; ordinary recovery completes without retained-source restoration. +6. Current/old mobile reconnect before and after commit, same-address retry pacing, + pairing preservation and no mutation replay. Old/new desktop/cloud combinations. +7. Emergency drain/auth denial during the cutover; no altered hard-drain behavior. +8. Real TCP WebSockets for two clients, admission race and failed cutover, independent + execution-process identity and append-once mutation evidence. Docker SSH and + folder workspace continuity. Tests use `ORCA_BACKGROUND_LAUNCH=1`. + +Keep tests proportional: deterministic component races first, then real transport, +relevant cloud/desktop suites, types/lint and PR CI. PostgreSQL only on 55440 when +validating authoritative transactions. Do not replace a failing oracle with a weaker +assertion or accumulate tests mirroring implementation. + +After clean review, replace superseded feature code/tests/docs on the two draft PRs, +preserving an immutable backup. Freshness and appropriate compatibility tests stay; +retention-only mechanisms and release requirements must be removed if irrelevant. +Do not claim readiness from tests of the superseded design. + +Release separately from merge readiness: packaged mixed versions, physical device +background timing, Linux/Windows, bounded rollout and measured user benefit remain +explicit evidence requirements. No deployment or enable is authorized by this plan. + +## Implementation notes — 2026-09-11 + +The authenticated director command carries its configured cohort percentage and +fresh process safety snapshot. A cell cannot use its own default-zero director +cohort setting to authorize or reject a selected host; it validates the authenticated +command, combines director/source safety, and rechecks durable policy, the host's +cohort bucket, and fleet/target safety inside the existing transaction. These fields +are on the internal admin endpoint, not the mobile or desktop protocol. + +Candidate selection is read-only and uses a rotating page offset to progress past +busy hosts. A deterministic UUIDv5 derived from the exact source authority and target +keeps operation identity stable across director retries/restarts. Both details still +need final implementation audit and an explicit page-boundary fairness test. + +Current focused and real-transport results are recorded at the top of the acceptance +document. They do not complete the remaining compatibility, cleanup and PR gates. diff --git a/docs/relay-region-correction/RELAY-REGION-CORRECTION-PLAN.md b/docs/relay-region-correction/RELAY-REGION-CORRECTION-PLAN.md new file mode 100644 index 00000000000..283e825c934 --- /dev/null +++ b/docs/relay-region-correction/RELAY-REGION-CORRECTION-PLAN.md @@ -0,0 +1,177 @@ +# Relay automatic region correction — implementation plan v2 + +Status: **IMPLEMENTED AND LOCALLY VALIDATED; RELEASE GATES REMAIN**. Section 5a is normative for first-grant adoption, retained-source rollback, renewal and cleanup. This replaces the idle-only proposal. Historical design reviews are preserved in the combined backup revision; current verification and remaining gaps are recorded in the acceptance document. + +Implementation tracking: [live checklist](RELAY-REGION-CORRECTION-CHECKLIST.md). Completed work and validation evidence are recorded there; this document defines the behavior. + +## Outcome and precise terms + +An updated desktop obtains a reliable region preference. If its assigned region is materially worse, the existing director worker makes the better region the destination for new connections. Existing physical relay data connections continue through the old cell until they close or fail naturally. The desktop then releases the old origin and the durable migration completes. + +“Existing connection” means a physical client-to-desktop relay data connection, represented by a cell splice and desktop connection ownership. It can carry multiple commands and subscriptions; it is not a terminal process, agent run, saved pairing, or recent typing. Tracked pending attachments and basis-bound control requests also protect source retirement. Quiet live connections are retained. No fixed maximum connection duration has been established or proposed for optional optimization. + +For mobile: active use remains on the old cell; after the app actually suspends/closes that connection, the next successful connection resolves to the target. Current mobile schedules background suspension after 30 seconds, and checks an overdue deadline on foreground if the timer did not run. Putting the phone down while the app remains foreground is not a disconnect. No re-pairing is required solely for relocation. + +Promise: optional region optimization does not deliberately terminate pre-existing connections merely because its grace timer expired. This is not a guarantee against network failure, process exit, revoked/expired auth, or emergency maintenance. New connection attempts during target startup may need normal recovery; do not promise zero delay on those attempts. + +## Evidence and reuse + +Source experiments used `origin/main` at `721a2692893ab29f8daee3149965bf5e9adf99a0`. Review must fetch current main and record its SHA, distinguishing changed source from these dated results. + +- Bidirectional worker already exists in #19241 and predates deployed #19915. Do not implement a second placement/migration engine. +- Desktop `relay-origin-pool.ts` already opens a target, retains source connection ownership, and closes the source after final release. Auth refresh already visits every origin. +- Cell `host-session-registry.ts` has drain-only state, live/pending connection maps, and existing control renewal. Director store already retains migration state until source activity releases. +- Two unconditional deadline sites force interruption: desktop origin-pool and cell regional host drain. Removing those scheduling sites in a diagnostic snapshot made both preservation tests pass; restoring source made the identical tests fail again. +- A SQLite store test sustained a registered migration for a simulated hour with both controls renewed, then completed on source release. +- Mobile harness restored credentials/assignment/subscriptions after simulated drain. Sent mutations can become delivery-unknown and are not blindly replayed. The 251ms fake-clock recovery result is not measured real-world downtime. + +Full evidence: [interruption findings](RELAY-INTERRUPTION-FINDINGS.md), [harness and patches](https://github.com/stablyai/orca/blob/0db9fdc486366f7451289f0c0599eed9ae1d94be/tests/tools/relay-rehome-interruption/README.md). The counterfactual patch is NOT production code: it has no negotiation, incorrectly changes normal deadline semantics, and does not validate failure/replay paths. + +## 1. Ordered, expiring region decisions + +Reuse desktop sampling and the existing assignment exchange. Separate legacy placement hints from eligibility to move an existing assignment. + +Preserve cold-start placement: run the existing pre-placement probe and send its placement hint with the first assignment request. Do not create a default-region assignment merely to obtain a measurement window. Obtain the epoch-bound window with or after that first assignment; only a subsequent post-window measurement can certify migration eligibility. Test first placement with and without conclusive probes, plus old-server fallback, so the new migration protocol does not regress initial placement. + +Proposed concrete protocol: an opt-in server-issued measurement window. A supporting desktop requests a window; director returns a per-host monotonic generation, fixed server expiry, and incumbent assignment epoch/region. Issuing a successor invalidates the predecessor for migration. The desktop probes after receiving the window, then reports a conclusive or inconclusive decision for that generation. Repeated delivery cannot extend expiry. An inconclusive outcome is retained as a tombstone; delayed older conclusive reports cannot resurrect eligibility. A restart that cannot reuse a valid cached decision obtains a successor window. Server time is authoritative for expiry. + +Persist window generation/expiry, incumbent basis, supported probe-policy version, and outcome in the existing preference state. Keep the latest supported evidence only; no history of every probe. Duplicate same-generation decisions are idempotent; conflicting same-generation outcomes must not upgrade inconclusive to conclusive. Serialize window issuance/reporting per broker and reject stale assignment basis under claim lock. The exact schema and transaction ordering must be reviewed before implementation. + +Legacy enum-only requests retain placement/reconnect behavior but cannot inherit or overwrite verified migration eligibility. Explicit new inconclusive decisions and missing legacy fields are different operations. Diagnostic region overrides are not measured proof. New request and response fields require explicit opt-in because both schemas are strict. Deploy server support before clients; implement old-server 400 fallback for both request shape and response negotiation without changing a healthy assignment. + +## 2. Compare with the actual assigned region + +Keep the current warm-up, sample count, spread rejection, and requirement that both regions be measurable. Initial placement may pick a best measured region. Moving an existing assignment additionally requires target latency at least **25ms lower AND 20% lower** than the measured incumbent region. These are the existing hysteresis thresholds applied to the correct basis, not a production-validated optimum. + +Report compact comparison evidence (incumbent/target region timing, policy and reason), tied to the window and assignment epoch. Director validates bounds, eligibility and margin. A missing incumbent, incomplete catalog, rejected measurement, nearly tied regions, unsupported policy or stale assignment basis means no move. Clear legacy probe caches on upgrade without treating an unopposed or 1ms winner as migration evidence. + +This optimizes the desktop-to-region path. Sample actual assigned-cell and mobile application behavior during rollout before claiming end-to-end user benefit. + +## 3. Refresh ownership and cadence + +The broker owns one decision deadline and one in-flight refresh/report task. Cancel them on broker close. Reuse the successful 24-hour cache interval and one-hour inconclusive retry interval, with jitter/backoff; eligibility expiry is distinct from retry scheduling. Do not wake offline/sleeping desktops for probes. On resume, check deadlines before using expired eligibility. Debounce network-change refresh only where the existing lifecycle provides a reliable signal. + +Probe/report failure must not fail successful auth renewal or intentionally reconnect healthy controls. Serialize assignment response application with drain/recovery: same cell/epoch updates metadata; a newer assignment follows the existing target activation flow; stale responses are discarded. Report retries reuse the same decision/window rather than re-probing or extending expiry. During an open migration, do not claim another move or replace the retained source; refresh decisions may be stored for later reevaluation only. + +## 4. Extend graceful migration to finish existing connections + +Introduce an explicit opt-in drain mode for optional regional optimization. Persist it on the attempt and propagate it through the director-to-cell host-drain command and cell-to-desktop drain message. The source must learn mode from durable protocol state, not infer it from grace=0, a hostname or a retry count. Maintenance/emergency drains keep their current hard deadlines. + +Eligibility requires supporting desktop and cell capabilities, fresh decision and matching source epoch/incarnation. New protocol/capability values and strict-parser fallbacks need cross-version tests. An unsupported participant defers optional correction rather than falling back to forced close. The reviewer should challenge how host capability remains bound to the currently active source generation, not merely a stale version-bearing report. + +Nominal transition: + +1. Existing claim reserves target capacity and commits target assignment plus migration/attempt state. +2. Source receives the mode-bearing drain and establishes the first valid authorized grant specified in section 5a before acknowledgment/cutover; desktop resolves/registers the target through existing code. +3. Target becomes the desktop's active origin. Existing source connections retain their source ownership; new connections resolve the current assignment. Already-admitted pending source connections may finish attaching and remain there. +4. Optional regional mode installs no forced old-origin/session close deadline. Existing event callbacks retire the source after its last owned connection and pending control operation end. Retain normal auth enforcement and operational emergency-close behavior. +5. Source control release/orphan cleanup removes its remaining activity. Existing completion logic finalizes the migration after source activity is gone and target is live. + +Do not use DB splice lease absence to infer idle: those leases can expire with a live socket. This design instead allows source work to exist. Do not build the proposed pre-move globally atomic idle gate. + +Required integration checks: pending control-RPC completion must trigger retirement when it was the final outstanding item; attach timeout/rejection and late async admission must not leak an origin or attach after retirement. Preserve existing source/target identity and request ownership. Cover two simultaneous mobile clients and a quiet connection; terminal counts and workspace type do not decide transport lifetime. + +## 5. Failure, replay and bounded resource use + +This section is a required implementation contract, not evidence that the current implementation already satisfies it. + +| Condition | Required behavior | +| --- | --- | +| Target registration fails | Retain still-live source work; retry or reconcile using existing migration recovery. Restore source new-admission authority only through section 5a's retained-generation rollback; do not discard a migration that still owns connections. | +| Lost receipt / duplicate drain / zero-grace re-drain | Read durable mode and preserve existing work. Replay must never turn optional mode into forced closure. | +| Desktop restart / source generation replacement | Old process connections may already be gone; reconcile exact generations before retaining or clearing state. Unsupported replacement must not silently hard-drain existing work. | +| Source network/cell failure | Use existing failure recovery; connectivity loss is not proof remote execution exited. This is outside the no-deliberate-interruption promise. | +| Target fails after becoming active | Keep source connections that remain live; reconcile assignment/new connections through existing recovery without starting a third overlapping rehome. | +| Auth expiry/revocation or emergency cell drain | Preserve existing enforcement; optimization does not exempt sessions from security/maintenance lifecycle. | +| Last source data connection closes while control RPC pending | Wait for bounded RPC completion/timeout, then run cleanup without a polling loop. | +| Source remains busy for hours | Keep the migration open while healthy. Long duration alone does not force-close users or spend dispatch-failure budget. | + +Bound **concurrent open migrations** in addition to starts/minute. Count pre-existing attempts; one migration per host remains enforced. Retain both controls' auth/lease renewals and account for source use plus target reservations. Data is not duplicated; extra controls/reservations still consume capacity. + +Ensure fair progress in the existing 100-row lease-refresh and 10-row candidate/sweep pages; waiting old migrations must not starve registration, renewal or cleanup for newer ones. Initially enforce a conservative concurrency bound below the smallest relevant page capacity, counting existing open work, until fair traversal is verified. Filters for cohort/policy/capability belong before LIMIT and are rechecked under locks. + +No user-visible maximum drain duration is claimed. If operations later require one for optimization, forcing closure would change the product promise and needs an explicit decision; a larger timer is not equivalent to finish-existing behavior. + +## 5a. Review corrections: retained-source authority and lifetime + +The independent review found three required contracts. This section supersedes any suggestion above that unchanged auth renewal or generic rollback suffices. Current source-control lifetime is six hours with thirty-minute jitter; auth-refresh does not extend it, and ordinary rotation only renews the active target. Current durable rehome refresh also has a 24-hour age ceiling. The one-hour SQLite experiment proved neither of those paths safe for indefinite live retention. + +### Reuse the cell's existing activity renewal (supersedes the extra wire exchange) + +Follow-up investigation found a smaller mechanism: a successfully validated `renewControlActivity` result can extend the same retained control's in-memory lease to `max(existingExpiry, requestedActivityExpiry)`. The cell already runs this renewal; no new desktop renewal timer, old-source rebind or recurring WebSocket exchange is needed. See [prototype evidence](https://github.com/stablyai/orca/blob/0db9fdc486366f7451289f0c0599eed9ae1d94be/tests/tools/relay-rehome-interruption/RETAINED-CONTROL-LEASE.md). + +Before acknowledging the optional drain and telling desktop to cut over, establish the first short authorized renewal for the exact mode/attempt/source generation/incarnation. Subsequent grants reuse normal heartbeat renewal. Failure or stale state during adoption does not authorize retaining the source; reconcile the provisional target through the rollback contract below. A mode flag, pending database request or activity reacquisition alone is not a grant. + +Narrow the existing atomic database renewal predicate with the retained attempt/mode/source basis, using immutable fields and preserving existing lock order; avoid a new precheck/query that can race mutation. The success callback must still match captured attempt, live session/socket/generation and mode; retirement/rollback/replacement/emergency drain invalidates it. Use the deadline sent at request start (currently 105s ahead), not response time, and do not add six hours on every heartbeat. Keep normal JWT/silence/watchdog enforcement and ordinary control rotation unchanged. Normal mode retains its existing lease; only extra retention needs these short successful grants. + +Prototype evidence: 43 cell-registry tests and package typecheck pass, including >12h simulated retention without extra recurring renewal calls; 6 real Postgres renewal tests execute with zero skips on local 55440. Baseline/revert fails the three extension oracles. The prototype injects the future mode marker and is NOT production-ready: initial adoption ordering, durable mode predicate, wire negotiation and rollback remain integration work. This revision addresses source renewal only, not the whole retention feature. + +### Final renewal-review correction: fence failures as well as success + +Fresh GPT-6-astra / low review: [retained-control review](https://github.com/stablyai/orca/blob/0db9fdc486366f7451289f0c0599eed9ae1d94be/docs/relay-region-correction/RELAY-RETAINED-CONTROL-REVIEW.md), **REVISE one completion-fencing detail; heartbeat reuse supported**. The following correction is incorporated after review, not independently approved or implemented. + +Every renewal completion and awaited recovery continuation must validate its captured socket/session, activity ID, current authority/mode transition and applicable ordering before altering scheduling, extending expiry, closing a socket, or reacquiring activity. In particular, a denial from an aborted retained attempt arriving after same-generation rollback must not close the restored source. A current applicable denial must still enforce closure. An obsolete missing-activity result must not initiate recovery; after awaited recovery, recheck authority and clean up abandoned acquisition as required. Do not use a blanket success-only fence or suppress all failures. + +Add controlled-promise tests for late denial after rollback, after a newer valid authority transition, obsolete missing-activity recovery, and applicable denial. Verify both preserved socket/splice identity and correct rejection, not just expiry values. First-grant adoption must also reject a success whose requested expiry is already past. + +In retained-mode SQL, the ordinary current-assignment authorization alternative must not bypass an aborted attempt check after rollback. If locking attempt rows, preserve assignment -> attempt -> migration -> activity dependency order used by existing rehome operations, rather than appending a late attempt lock. Use the prescribed PostgreSQL 16 environment for implementation concurrency validation on 55440; the earlier six-test PostgreSQL 17 run remains accurately labeled as narrower evidence. + +### Retained-generation rollback + +Generic source reassignment alone is insufficient: the source cell otherwise remains drain-only, and a fresh generation closes old splices. Add a durable, idempotent rollback transition for the exact optional attempt. In one authoritative store transaction, assign a newer source epoch and record that attempt's rollback outcome and retained source generation. Reconcile the source cell to that state: clear only that attempt's optional drain, preserve its socket/splices, update assignment metadata, and restore new admission only after validating current authority. Keep an aborted-attempt tombstone so a late drain cannot reverse rollback. + +Desktop recovery must find/reuse the retained source origin and update its assignment metadata without replacing its control generation or transports. This needs an explicit supported transition, not the current rebind-failed -> fresh-generation fallback. Late target registration cannot override the newer source epoch. Release target reservations when reconciled; keep one open migration per host until cleanup completes. If the source process/generation is gone, use ordinary failure recovery and report that preservation is unavailable; do not infer remote execution exited. + +### Transition table + +| State/event | Authority and action | Existing source work | +| --- | --- | --- | +| Claim optional move | Durable attempt binds mode, source generation/incarnation, target, epoch and supported participants | Retained | +| Target registering | Source receives optional drain; existing target retry/reconciliation proceeds | Retained; do not convert age into forced closure | +| Target registered | Director target assignment is authoritative; desktop activates target | Existing source connections keep their origin | +| Retained source needs renewal | Existing cell activity renewal + exact optional migration authorize a short same-generation lease extension | Retained, source remains drain-only | +| Target failure / rollback | New durable source epoch plus rollback tombstone; source cell and desktop reuse exact retained generation | Retained if that generation still exists | +| Final source work ends | Connection and pending-work callbacks retire source; cancel renewals; release activity and complete | No source work left to preserve | +| Delayed drain/renew/register | Compare durable attempt outcome and epochs; ignore/reject obsolete transition | Must not resurrect draining or replace generation | +| Source failed / emergency drain | Existing authenticated operational/failure semantics apply | Preservation not promised under those failures | + +### Mode-specific durable lifetime and compatibility floor + +For healthy registered optional retained-source attempts, remove the current 24-hour age-only lease-refresh ceiling and exclude them from the age-only zero-grace re-drain lane. Keep bounded target-registration failure/reconciliation; do not extend an unreachable unregistered target forever. Duration alone is not dispatch failure, and active source data must not be dropped to reclaim an optimization slot. Current generic and regional cleanup paths must both understand the optional mode. + +Before enabling optional retention, deploy a director/worker compatibility floor that understands all durable mode and rollback states even when new claims are disabled. Operational rollback must not go below that floor while such attempts exist. A disabled enable flag does not stop older cleanup/redrain code from misinterpreting new rows. Prove safe restart/rollback with existing open attempts and multi-day retention. The minimum revision will be recorded only after the compatible implementation is merged and validated. + +Additional accepted obligations: notify retirement on every final pending-control transition (response, rejection, timeout, close); perform a final local session/generation check after awaited admission work and release abandoned reservations; bind negotiated support to current authenticated source generation rather than a stale measurement report; enforce the concurrent-migration cap in locked shared state, including pre-existing work. None of these requires rebuilding a global idle detector. + +## 6. Rollout and observability + +Keep rehome disabled while implementing/testing. Deploy compatible director/database support, then supporting cells and desktops with feature gated off. Verify readiness and actual capabilities before enabling an authorized bounded cohort. Do not dispatch workflows as part of this review. + +Reuse current rate, cooldown, safety, capacity, durable attempts, and failure-budget controls. Preview must be read-only and share eligibility predicates, report full aggregate counts rather than a capped candidate page, and never claim attempts or consume budget. + +Track eligibility/exclusions by direction, target registration, migration completion/abort, number/age of retained sources, concurrent reservations, deliberate forced-close count by drain mode, and reconnect/error rates. Retain compact sampled comparisons and matched before/after assigned-cell/application latency where available; a registered target alone is not evidence of user benefit. Use an unchanged cohort to detect unrelated network variation. Never log credentials, pairing data, or raw host IDs. + +Acceptance: supported eligible hosts move new connections to their chosen target; old data connections survive optional-drain deadlines and retire on actual release; no forced source close solely due to optimization age; resources clean up; failures remain recoverable; sampled latency/reliability shows benefit without material regression. Specify sample sizes and numerical regression limits before production enable, using available traffic rather than inventing measured thresholds here. + +## 7. Required validation and implementation order + +1. Land freshness/ordering and incumbent-relative eligibility support under disabled control, with strict request/response compatibility tests. Cases: first-ever placement with correction disabled, placement-hint/actual-assignment mismatch, cold-start inconclusive probes and old-server fallback; delayed old reports, duplicates, inconclusive tombstones, clock changes, restarts, legacy writes, overrides, policy upgrades, stale epochs and cache clearing. +2. Implement broker refresh and event-driven origin retirement, test no auth coupling, no reconnect on unchanged assignment, pending-operation completion and sleep/resume. +3. Implement negotiated optional drain mode end-to-end in existing worker/cell/desktop paths, including normal emergency deadlines and replay. Extend the diagnostic oracles into real feature tests; do not merge the timer-removal experiment. +4. Validate real WebSocket traffic across source/target while sending unique stream markers and a mutation with delayed acknowledgment; check no duplicate/replayed mutation and independent host-side execution/output. Then validate mobile background/foreground reconnection and pairing preservation. Mock tests are not an end-to-end substitute. +5. Run actual Postgres integration/concurrency tests on **55440 only**. Require configured database, executed test counts and no conditional skip. Cover registration failure, target failure, concurrent admissions, cleanup, pagination and capacity accounting with long-lived sources. +6. Validate supported/unsupported desktop and cell combinations and old/new director rollback. SSH-hosted execution and folder workspaces remain governed by transport/owning host, not local process assumptions. No visible app tests on the user's desktop; background launch and isolated profiles are required. +7. Run a fresh operational safety gate and capability check only when a reviewed rollout is authorized. Enable a bounded cohort, observe retained-source/resource/benefit evidence, then expand. Disable stops new optional moves while safely reconciling existing ones. + +### Implementation correction: restoration confirmation can retry + +A rollback response can reach the cell while desktop director corroboration fails. +The cell therefore retains an exact pending-restoration authority (aborted attempt, +newer source epoch, original generation/incarnation/activity) until ordinary +same-generation rebind confirms restoration. Each successful existing heartbeat +renews only its short request-start deadline and replays `region-restored`. +Retained and restored authority are mutually exclusive; the old retained authority +remains rejected after rollback. Rebind, replacement, emergency drain, and applicable +denial fence outstanding callbacks. No extra timer or six-hour heartbeat grant is +introduced. This avoids losing long-lived source connections merely because the +first director confirmation failed. diff --git a/docs/relay-region-correction/RELAY-REGION-CORRECTION-ROLLOUT.md b/docs/relay-region-correction/RELAY-REGION-CORRECTION-ROLLOUT.md new file mode 100644 index 00000000000..66cd9b8efd5 --- /dev/null +++ b/docs/relay-region-correction/RELAY-REGION-CORRECTION-ROLLOUT.md @@ -0,0 +1,60 @@ +# Remaining release work and proposed rollout + +Status: implementation and local validation; no deployment authorization used. + +## Ordered release actions + +1. Review the local changes/PR and CI results. Record the immutable merged commit + and image for deployment and rollback. All directors must run the reviewed idle + worker before enabling correction; verify mixed-version deployment behavior + while correction remains disabled. +2. Publish the relay image and deploy the director with correction cohort0 and + durable rehome disabled. Verify image, health, preview route, migration schema, + pool pressure and cleanup. This requires the authorized deployment workflow. +3. Roll supporting cell images using their existing cell workflow; verify protocol3, + correct incarnation and telemetry. Release the updated desktop normally. + The phone protocol is unchanged; no mobile update is required for correction. +4. Validate a packaged desktop with a physical phone (foreground, actual background + suspension, resume, quiet connection, and target failure), and existing clients + against new cells. Run the same transport gate on Linux/Windows in CI; validate + an actual SSH-owned terminal survives client disconnection and normal reconnect. +5. Read the authenticated aggregate preview. Estimate eligible population by + direction; use a matching unchanged comparison cohort. Choose the initial + cohort and record approvals before changing settings or enabling. +6. Configure the approved cohort through the existing director workflow input, + keeping the durable control disabled during deployment. Verify the serving revision and tagged rollback + revision, then enable through the existing regional-rehome control workflow. +7. Stop new claims on a regression while ordinary migration cleanup and recovery continue. + Investigate existing work rather than forcing a timer-based source closure. + +## Proposed numerical acceptance criteria (must be approved before enable) + +These are rollout proposals, not measurements of production baseline or authorization. + +- First phase:1% deterministic host cohort; global cap remains8 open migrations. + Observe at least24h and30 completed moves. If traffic cannot supply30, extend + observation; do not treat a small sample as success. +- Immediate stop: any optimization-induced close of an established client, admission + reopening on ambiguous source authority, duplicated mutation, authorization bypass, or more than8 + optimization migrations admitted (pre-existing work also consumes cap). +- Reliability stop: compared with an unchanged cohort over matching15-minute + windows, assignment/connect failure rate rises by>=1 percentage point or2x + (require>=100 attempts in each comparison group); investigate lower-count failures + individually. Existing production incident limits always take precedence. +- Performance acceptance: matched post-move assigned-cell control RTT improves by + >=25ms AND>=20% median per host for at least80% of evaluable moved hosts; require + two independent samples before and after. Identify samples using + assignment epoch/cell identity. Log sample insufficiency as unevaluable. +- Client connection setup p95 must not regress by>10% versus its matched baseline + after accounting for the unchanged cohort. Setup is not application command + latency; separately record physical-phone interaction timings on validation runs. +- Expansion requires healthy registration/completion, stable reservation usage, no growing stuck-recovery backlog, and numerical criteria + above. Continuously connected clients may postpone optimization indefinitely. + +## Evidence boundaries + +Local tests use real socket traffic and independent host execution, but synthetic +clock/authentication. Neither the build nor schema fixtures prove distribution, +production latency, a physical phone or a signed desktop upgrade. Separate Docker SSH tests validate +SSH-provider recovery; they are not physical mobile-to-SSH cutover evidence. +The checklist leaves these release gates open deliberately. diff --git a/mobile/src/components/NewWorktreeModalController.tsx b/mobile/src/components/NewWorktreeModalController.tsx index 4b6812fc9ff..a7f2ce1c2b6 100644 --- a/mobile/src/components/NewWorktreeModalController.tsx +++ b/mobile/src/components/NewWorktreeModalController.tsx @@ -16,7 +16,7 @@ type Props = { openExternalUrl: (url: string) => Promise onVisibleChange?: (visible: boolean) => void onRouteVisibleChange: (visible: boolean) => void - onCreated: (worktreeId: string, name: string) => void + onCreated: (worktreeId: string, name: string, warning?: string) => void } export const NewWorktreeModalController = forwardRef( diff --git a/mobile/src/components/new-worktree-modal-types.ts b/mobile/src/components/new-worktree-modal-types.ts index 8dae250cd71..7e5000ccc11 100644 --- a/mobile/src/components/new-worktree-modal-types.ts +++ b/mobile/src/components/new-worktree-modal-types.ts @@ -24,7 +24,7 @@ export type NewWorktreeModalProps = { existingWorktreePaths?: readonly string[] existingWorktrees?: readonly { repoId: string; branch: string }[] openExternalUrl: (url: string) => Promise - onCreated: (worktreeId: string, name: string) => void + onCreated: (worktreeId: string, name: string, warning?: string) => void onClose: () => void } diff --git a/mobile/src/components/use-new-workspace-create-submit.ts b/mobile/src/components/use-new-workspace-create-submit.ts index 1ab3ac5d5c4..1a7ba078b27 100644 --- a/mobile/src/components/use-new-workspace-create-submit.ts +++ b/mobile/src/components/use-new-workspace-create-submit.ts @@ -58,7 +58,7 @@ export function useNewWorkspaceCreateSubmit(args: { getWorktreeCreateCutoverSupport: () => Promise transitionDrawer: (view: Exclude) => void setError: Dispatch> - onCreated: (worktreeId: string, name: string) => void + onCreated: (worktreeId: string, name: string, warning?: string) => void onClose: () => void }): { creating: boolean @@ -181,7 +181,7 @@ export function useNewWorkspaceCreateSubmit(args: { return } args.onClose() - args.onCreated(result.worktreeId, result.name) + args.onCreated(result.worktreeId, result.name, result.warning) } catch (error) { args.setError(error instanceof Error ? error.message : 'Failed to create workspace') } finally { diff --git a/mobile/src/host-route-action-state.test.ts b/mobile/src/host-route-action-state.test.ts index 7f0ff6375a1..fd2e75d311f 100644 --- a/mobile/src/host-route-action-state.test.ts +++ b/mobile/src/host-route-action-state.test.ts @@ -19,6 +19,30 @@ describe('host route action state', () => { ) }) + // Why: the host reports a create that succeeded with a failed startup terminal via `warning`; + // dropping it here is what lands the phone on an unexplained empty session. + it('carries a host create warning into the session route', () => { + expect( + hostNewWorktreeSessionRoute( + 'local', + 'wt-1', + 'Hammerhead', + 'Failed to create the startup terminal' + ) + ).toBe( + '/h/local/session/wt-1?name=Hammerhead&created=1&warning=Failed+to+create+the+startup+terminal' + ) + }) + + it('omits an absent or blank create warning', () => { + expect(hostNewWorktreeSessionRoute('local', 'wt-1', 'Hammerhead', ' ')).toBe( + '/h/local/session/wt-1?name=Hammerhead&created=1' + ) + expect(hostNewWorktreeSessionRoute('local', 'wt-1', 'Hammerhead')).toBe( + '/h/local/session/wt-1?name=Hammerhead&created=1' + ) + }) + it('opens new worktree modal on an initial newWorktree action', () => { expect(createInitialHostRouteActionState('newWorktree')).toEqual({ routeAction: 'newWorktree', diff --git a/mobile/src/host-route-action-state.ts b/mobile/src/host-route-action-state.ts index a89c03fb472..5c2588b1e89 100644 --- a/mobile/src/host-route-action-state.ts +++ b/mobile/src/host-route-action-state.ts @@ -10,9 +10,14 @@ export function hostNewWorktreeRoute(hostId: string): `/h/${string}?action=newWo export function hostNewWorktreeSessionRoute( hostId: string, worktreeId: string, - worktreeName: string + worktreeName: string, + /** Host-reported create warning (e.g. the startup terminal failed to spawn). */ + warning?: string ): `/h/${string}/session/${string}?${string}` { const params = new URLSearchParams({ name: worktreeName, created: '1' }) + if (warning?.trim()) { + params.set('warning', warning) + } return `/h/${encodeURIComponent(hostId)}/session/${encodeURIComponent(worktreeId)}?${params}` } diff --git a/mobile/src/host-screen/host-screen-overlays.tsx b/mobile/src/host-screen/host-screen-overlays.tsx index 0f15f9d4e61..35a140fec7a 100644 --- a/mobile/src/host-screen/host-screen-overlays.tsx +++ b/mobile/src/host-screen/host-screen-overlays.tsx @@ -219,10 +219,10 @@ export function HostScreenOverlays({ controller }: { controller: HostScreenContr onVisibleChange={(visible) => { state.newWorktreeModalVisibleRef.current = visible }} - onCreated={(worktreeId, worktreeName) => { + onCreated={(worktreeId, worktreeName, warning) => { void catalog.fetchWorktrees({ allowDuringModal: true }) actions.navigateFromHostList( - hostNewWorktreeSessionRoute(hostId, worktreeId, worktreeName) + hostNewWorktreeSessionRoute(hostId, worktreeId, worktreeName, warning) ) }} onRouteVisibleChange={actions.setShowNewWorktreeVisible} diff --git a/mobile/src/session/MobileNativeChatMessage.test.ts b/mobile/src/session/MobileNativeChatMessage.test.ts index e09d1631a8d..67e90132866 100644 --- a/mobile/src/session/MobileNativeChatMessage.test.ts +++ b/mobile/src/session/MobileNativeChatMessage.test.ts @@ -9,6 +9,7 @@ vi.mock('react-native', async () => { const Text = ({ children, ...props }: { children?: unknown }): unknown => React.createElement('Text', props, children) return { + ActivityIndicator: 'ActivityIndicator', Animated: { Text, Value: class { @@ -268,12 +269,12 @@ describe('MobileNativeChatMessage', () => { expect(tree.root.findAllByType('Wrench' as never)).toHaveLength(0) }) - it('renders the turn status row under a user message', () => { + it('renders the settled turn status row under a user message', () => { const tree = render(userMessage([{ type: 'text', text: 'go' }]), { structuredActivityUi: true, - turnStatus: { startedAt: Date.now(), thinking: true, workedSeconds: null } + turnStatus: { startedAt: Date.now() - 3_000, thinking: false, workedSeconds: 3 } }) - expect(textIn(tree.root)).toContain('Thinking') + expect(textIn(tree.root)).toContain('Worked for 3s') }) it('does not render a turn status row without one', () => { diff --git a/mobile/src/session/MobileNativeChatMessage.tsx b/mobile/src/session/MobileNativeChatMessage.tsx index 5d013688249..9b480fdd7d9 100644 --- a/mobile/src/session/MobileNativeChatMessage.tsx +++ b/mobile/src/session/MobileNativeChatMessage.tsx @@ -82,7 +82,7 @@ function MobileNativeChatMessageImpl({ /** Multiplies all chat text sizes for pinch-to-zoom (1 = no change). */ fontScale?: number onOpenFile?: (relativePath: string) => void - /** This turn's status row, rendered under a user message (desktop parity). */ + /** This settled turn's status row, rendered under its user message. */ turnStatus?: NativeChatTurnStatus | null /** Whether the turn caret has disclosed this turn's activity. */ turnExpanded?: boolean diff --git a/mobile/src/session/MobileNativeChatOverlay.tsx b/mobile/src/session/MobileNativeChatOverlay.tsx index 997083cb26e..389beb8eaad 100644 --- a/mobile/src/session/MobileNativeChatOverlay.tsx +++ b/mobile/src/session/MobileNativeChatOverlay.tsx @@ -73,6 +73,7 @@ export function MobileNativeChatOverlay({ agentWorking={controller.nativeChatAgentWorking} canStop={controller.nativeChatCanStop} structuredActivityUi={controller.nativeChatStructured} + turnIndicator={controller.nativeChatTurnIndicator} workingStartedAt={controller.nativeChatWorkingStartedAt} settledTurns={controller.nativeChatSettledTurns} streaming={streaming} diff --git a/mobile/src/session/MobileNativeChatTurnStatus.test.ts b/mobile/src/session/MobileNativeChatTurnStatus.test.ts index 78ac01e0d37..6b6a1d49e07 100644 --- a/mobile/src/session/MobileNativeChatTurnStatus.test.ts +++ b/mobile/src/session/MobileNativeChatTurnStatus.test.ts @@ -7,18 +7,8 @@ vi.mock('react-native', async () => { const Text = ({ children, ...props }: { children?: unknown }): unknown => React.createElement('Text', props, children) return { - Animated: { - Text, - Value: class { - constructor(private value: number) {} - setValue(next: number): void { - this.value = next - } - }, - loop: (animation: unknown) => animation, - sequence: () => ({ start: vi.fn(), stop: vi.fn() }), - timing: () => ({ start: vi.fn(), stop: vi.fn() }) - }, + ActivityIndicator: (props: Record) => + React.createElement('ActivityIndicator', props), Pressable: ({ children, ...props }: { children?: unknown }) => React.createElement('Pressable', props, children), Text, @@ -49,6 +39,7 @@ describe('MobileNativeChatTurnStatus', () => { startedAt: number | null thinking: boolean workedSeconds?: number | null + activityText?: string | null expanded?: boolean onToggleExpanded?: () => void }): ReactTestRenderer { @@ -61,12 +52,16 @@ describe('MobileNativeChatTurnStatus', () => { const labels = (node: ReactTestInstance): string[] => node.findAllByType('Text' as never).map((text) => String(text.children.join(''))) - it('reads "Thinking" before the turn produces output', () => { + const spinners = (node: ReactTestInstance): ReactTestInstance[] => + node.findAllByType('ActivityIndicator' as never) + + it('reads "Thinking" beside one spinner while the turn reasons', () => { const tree = render({ startedAt: Date.now(), thinking: true }) expect(labels(tree.root)).toEqual(['Thinking']) + expect(spinners(tree.root)).toHaveLength(1) }) - it('counts up once the turn is producing output', () => { + it('counts up on that same single row when the turn is not reasoning', () => { const startedAt = Date.now() const tree = render({ startedAt, thinking: false }) expect(labels(tree.root)).toEqual(['Working for 0s']) @@ -74,6 +69,19 @@ describe('MobileNativeChatTurnStatus', () => { vi.advanceTimersByTime(12_000) }) expect(labels(tree.root)).toEqual(['Working for 12s']) + expect(spinners(tree.root)).toHaveLength(1) + }) + + it('lets provider activity text beat both fallbacks and hold the clock', () => { + const tree = render({ + startedAt: Date.now(), + thinking: true, + activityText: 'Running pnpm test' + }) + expect(labels(tree.root)).toEqual(['Running pnpm test']) + expect(spinners(tree.root)).toHaveLength(1) + // No label consumes the duration, so nothing schedules a tick for it. + expect(vi.getTimerCount()).toBe(0) }) it('settles to a tappable "Worked for" row that toggles the turn', () => { @@ -98,9 +106,10 @@ describe('MobileNativeChatTurnStatus', () => { expect(labels(tree.root)).toEqual(['Worked for 5s']) }) - it('holds no interval once the turn has settled', () => { - render({ startedAt: Date.now(), thinking: false, workedSeconds: 5 }) + it('holds no interval, and no spinner, once the turn has settled', () => { + const tree = render({ startedAt: Date.now(), thinking: false, workedSeconds: 5 }) expect(vi.getTimerCount()).toBe(0) + expect(spinners(tree.root)).toHaveLength(0) }) it('announces the live row to assistive tech', () => { diff --git a/mobile/src/session/MobileNativeChatTurnStatus.tsx b/mobile/src/session/MobileNativeChatTurnStatus.tsx index 4ce73cdcd38..acf922265f6 100644 --- a/mobile/src/session/MobileNativeChatTurnStatus.tsx +++ b/mobile/src/session/MobileNativeChatTurnStatus.tsx @@ -1,7 +1,8 @@ -import { useEffect, useRef, useState } from 'react' -import { Animated, Pressable, StyleSheet, Text, View } from 'react-native' +import { useEffect, useState } from 'react' +import { ActivityIndicator, Pressable, StyleSheet, Text, View } from 'react-native' import { ChevronRight } from 'lucide-react-native' import { + formatNativeChatActiveTurnLabel, formatNativeChatTurnStatusLabel, NATIVE_CHAT_TURN_STATUS_COPY, nativeChatElapsedSeconds @@ -25,48 +26,38 @@ function useElapsedSeconds(startedAt: number | null, counting: boolean): number return counting ? nativeChatElapsedSeconds(startedAt, mountedAt, now) : 0 } -/** The per-turn status row — "Thinking", then "Working for 12s" while the turn - * runs, settling to a tappable "Worked for 3m 4s" that discloses the turn's - * tool activity. Desktop parity: `NativeChatWorkingStatus`. */ +/** The per-turn status row. While the turn runs it is the one live indicator — a + * spinner beside what the provider says it is doing, else "Thinking", else + * "Working for 12s". It settles to a tappable "Worked for 3m 4s" that discloses + * the turn's tool activity. Desktop parity: `NativeChatTurnActivityLine` for the + * live row, `NativeChatWorkingStatus` for the settled one. */ export function MobileNativeChatTurnStatus({ startedAt, thinking, workedSeconds, + activityText, expanded = false, onToggleExpanded }: { startedAt: number | null thinking: boolean workedSeconds?: number | null + /** Provider activity copy for a live turn; outranks the other two labels. */ + activityText?: string | null expanded?: boolean onToggleExpanded?: () => void }): React.JSX.Element { - const counting = !thinking && workedSeconds == null + const settled = workedSeconds != null + const counting = !settled && !thinking && !activityText?.trim() const elapsedSeconds = useElapsedSeconds(startedAt, counting) - const label = formatNativeChatTurnStatusLabel({ thinking, workedSeconds, elapsedSeconds }) + const label = settled + ? formatNativeChatTurnStatusLabel({ thinking, workedSeconds, elapsedSeconds }) + : formatNativeChatActiveTurnLabel({ activityText, thinking, elapsedSeconds }) - const pulse = useRef(new Animated.Value(1)).current - useEffect(() => { - if (!thinking) { - pulse.setValue(1) - return - } - const animation = Animated.loop( - Animated.sequence([ - Animated.timing(pulse, { toValue: 0.45, duration: 700, useNativeDriver: true }), - Animated.timing(pulse, { toValue: 1, duration: 700, useNativeDriver: true }) - ]) - ) - animation.start() - return () => animation.stop() - }, [pulse, thinking]) - - const rowStyle = [styles.row, thinking ? null : styles.rowSettled] - - if (workedSeconds != null && onToggleExpanded) { + if (settled && onToggleExpanded) { return ( [...rowStyle, pressed && styles.pressed]} + style={({ pressed }) => [styles.row, styles.rowSettled, pressed && styles.pressed]} onPress={onToggleExpanded} hitSlop={6} accessibilityRole="button" @@ -83,11 +74,14 @@ export function MobileNativeChatTurnStatus({ return ( - {label} + {settled ? null : } + + {label} + ) } @@ -109,7 +103,8 @@ const styles = StyleSheet.create({ }, label: { color: colors.textMuted, - fontSize: typography.bodySize + fontSize: typography.bodySize, + flexShrink: 1 }, caretOpen: { transform: [{ rotate: '90deg' }] diff --git a/mobile/src/session/MobileNativeChatView.test.ts b/mobile/src/session/MobileNativeChatView.test.ts index 63bfb715445..c573d89cc89 100644 --- a/mobile/src/session/MobileNativeChatView.test.ts +++ b/mobile/src/session/MobileNativeChatView.test.ts @@ -73,6 +73,7 @@ type Overrides = { onSend?: (text: string) => Promise pending?: Parameters[0]['pending'] structuredActivityUi?: boolean + turnIndicator?: Parameters[0]['turnIndicator'] agentWorking?: boolean canStop?: boolean sendSurfaceId?: string @@ -273,20 +274,74 @@ describe('MobileNativeChatView', () => { return (renderedRow(id) as { props: Record }).props } + function footerProps(): Record | null { + const list = renderer!.root.find((node) => node.type === 'FlatList') + const footer = list.props.ListFooterComponent as + | { props: Record } + | null + | undefined + return footer?.props ?? null + } + function workingIndicators(): ReactTestInstance[] { return renderer!.root.findAll((node) => node.type === 'WorkingIndicator') } - it('gives the live user turn a status row and drops the three-dot indicator', async () => { - const folded = [userTurn('u1', 'go')] + it('puts the live status at the turn tail and drops the three-dot indicator', async () => { + const folded = [userTurn('u1', 'go'), assistantTurn('a1', 'still working')] await render({ messages: folded, folded, structuredActivityUi: true, agentWorking: true }) const props = rowProps('u1') expect(props.structuredActivityUi).toBe(true) - expect(props.turnStatus).toMatchObject({ thinking: true, workedSeconds: null }) + expect(props.turnStatus).toBeNull() + // Nothing reports reasoning, so the one live footer counts instead of guessing. + expect(footerProps()).toMatchObject({ thinking: false, workedSeconds: null }) + expect(listIds().at(-1)).toBe('a1') expect(props.activeTurnIsWorking).toBe(true) expect(workingIndicators()).toHaveLength(0) }) + it('reports the live turn as thinking only when its journal says it is reasoning', async () => { + const folded = [userTurn('u1', 'go')] + await render({ + messages: folded, + folded, + structuredActivityUi: true, + agentWorking: true, + turnIndicator: { thinking: true, activityText: null } + }) + expect(rowProps('u1').turnStatus).toBeNull() + expect(footerProps()).toMatchObject({ thinking: true, workedSeconds: null }) + }) + + it('hands the live row the provider activity copy that outranks its fallbacks', async () => { + const folded = [userTurn('u1', 'go')] + await render({ + messages: folded, + folded, + structuredActivityUi: true, + agentWorking: true, + turnIndicator: { thinking: true, activityText: 'Running pnpm test' } + }) + expect(footerProps()).toMatchObject({ + thinking: true, + activityText: 'Running pnpm test' + }) + }) + + it('keeps the activity copy on the live footer instead of a historical row', async () => { + const folded = [userTurn('u1', 'go'), userTurn('u2', 'again')] + await render({ + messages: folded, + folded, + structuredActivityUi: true, + agentWorking: true, + turnIndicator: { thinking: false, activityText: 'Running pnpm test' } + }) + expect(rowProps('u1')).not.toHaveProperty('turnActivityText') + expect(rowProps('u2')).not.toHaveProperty('turnActivityText') + expect(footerProps()).toMatchObject({ activityText: 'Running pnpm test' }) + }) + it('keeps the bridge lane on the three-dot indicator with no turn status', async () => { const folded = [userTurn('u1', 'go')] await render({ messages: folded, folded, agentWorking: true }) @@ -294,13 +349,15 @@ describe('MobileNativeChatView', () => { expect(props.structuredActivityUi).toBe(false) expect(props.turnStatus).toBeNull() expect(props.activeTurnIsWorking).toBe(false) + expect(footerProps()).toBeNull() expect(workingIndicators()).toHaveLength(1) }) it('settles the finished turn to a tappable duration', async () => { const folded = [userTurn('u1', 'go'), assistantTurn('a1', 'done')] await render({ messages: folded, folded, structuredActivityUi: true, agentWorking: true }) - expect(rowProps('u1').turnStatus).toMatchObject({ thinking: false, workedSeconds: null }) + expect(rowProps('u1').turnStatus).toBeNull() + expect(footerProps()).toMatchObject({ thinking: false, workedSeconds: null }) await update({ messages: folded, folded, structuredActivityUi: true, agentWorking: false }) const settled = rowProps('u1') expect(settled.turnStatus).toMatchObject({ thinking: false }) @@ -309,6 +366,7 @@ describe('MobileNativeChatView', () => { ) expect(settled.onToggleTurn).toBeTypeOf('function') expect(settled.activeTurnIsWorking).toBe(false) + expect(footerProps()).toBeNull() }) it('hangs no status row on an assistant row', async () => { @@ -317,6 +375,7 @@ describe('MobileNativeChatView', () => { expect(rowProps('a1').turnStatus).toBeNull() // The assistant row still belongs to the live turn, so its tool row stays visible. expect(rowProps('a1').activeTurnIsWorking).toBe(true) + expect(footerProps()).toMatchObject({ workedSeconds: null }) }) it('does not carry a running turn clock across chat surfaces', async () => { @@ -331,7 +390,7 @@ describe('MobileNativeChatView', () => { agentWorking: true, sendSurfaceId: 'host\0worktree\0tab-a' }) - expect(rowProps('u1').turnStatus).toMatchObject({ startedAt: 1_000 }) + expect(footerProps()).toMatchObject({ startedAt: 1_000 }) vi.setSystemTime(12_000) const secondTab = [userTurn('u2', 'second')] @@ -343,7 +402,7 @@ describe('MobileNativeChatView', () => { sendSurfaceId: 'host\0worktree\0tab-b' }) - expect(rowProps('u2').turnStatus).toMatchObject({ startedAt: 12_000 }) + expect(footerProps()).toMatchObject({ startedAt: 12_000 }) } finally { vi.useRealTimers() } diff --git a/mobile/src/session/MobileNativeChatView.tsx b/mobile/src/session/MobileNativeChatView.tsx index 440e05e6b9d..67fc93506a9 100644 --- a/mobile/src/session/MobileNativeChatView.tsx +++ b/mobile/src/session/MobileNativeChatView.tsx @@ -13,7 +13,10 @@ import { GestureDetector, GestureHandlerRootView } from 'react-native-gesture-ha import { ArrowDown, ChevronsDownUp, ChevronsUpDown, Square } from 'lucide-react-native' import type { AskAnswerSelection, AskPrompt } from '../../../src/shared/native-chat-ask' import type { NativeChatMessage } from '../../../src/shared/native-chat-types' -import type { NativeChatSettledTurns } from '../../../src/shared/native-chat-turn-status' +import type { + NativeChatLiveTurnIndicator, + NativeChatSettledTurns +} from '../../../src/shared/native-chat-turn-status' import { colors } from '../theme/mobile-theme' import { styles } from './mobile-native-chat-view-styles' import { @@ -53,6 +56,8 @@ type Props = { /** Structured lane: per-turn "Working for N" status plus live tool progress, * replacing the bridge lane's static three-dot working row (desktop parity). */ structuredActivityUi?: boolean + /** What labels the live turn's one indicator row (structured lane only). */ + turnIndicator?: NativeChatLiveTurnIndicator | null /** Structured lane: host-recorded turn timing feeding the per-turn status rows. */ workingStartedAt?: number | null settledTurns?: NativeChatSettledTurns | null @@ -136,6 +141,7 @@ export function MobileNativeChatView({ agentWorking, canStop = agentWorking, structuredActivityUi = false, + turnIndicator = null, workingStartedAt, settledTurns, onStop, @@ -259,14 +265,17 @@ export function MobileNativeChatView({ [hasMore, loadingEarlier, onLoadEarlier] ) - // Per-turn "Thinking / Working for N / Worked for N" rows. The structured lane - // owns them; the bridge lane keeps its three-dot indicator. + // Per-turn status rows: one live indicator while the turn runs, then a settled + // "Worked for N" row. The structured lane owns them; the bridge lane keeps its + // three-dot indicator. const turns = useMobileNativeChatTurnDisclosure({ messages: data, enabled: structuredActivityUi, isWorking: agentWorking === true, workingStartedAt, settledTurns, + thinking: turnIndicator?.thinking === true, + activityText: turnIndicator?.activityText ?? null, scopeKey: sendSurfaceId }) @@ -331,11 +340,12 @@ export function MobileNativeChatView({ ) : null } ListFooterComponent={ - turns.activeTurnIsUnanchored && turns.active ? ( + structuredActivityUi && agentWorking && turns.active ? ( ) : null } diff --git a/mobile/src/session/mobile-native-chat-controller-contract.ts b/mobile/src/session/mobile-native-chat-controller-contract.ts index da1b91acb25..36c07215e8e 100644 --- a/mobile/src/session/mobile-native-chat-controller-contract.ts +++ b/mobile/src/session/mobile-native-chat-controller-contract.ts @@ -6,7 +6,10 @@ import type { } from '../../../src/shared/native-chat-ask' import type { detectAgentPermission } from './mobile-native-chat-permission' import type { parseAgentQuestion } from './mobile-native-chat-question' -import type { NativeChatSettledTurns } from '../../../src/shared/native-chat-turn-status' +import type { + NativeChatLiveTurnIndicator, + NativeChatSettledTurns +} from '../../../src/shared/native-chat-turn-status' import type { MobileNativeChatSendOutcome } from './mobile-native-chat-send' import type { MobileNativeChatPendingMessage } from './use-mobile-native-chat-drafts' import type { useMobileNativeChatSession } from './use-mobile-native-chat-session' @@ -29,6 +32,8 @@ export type MobileNativeChatController = { /** Structured lane: drives the per-turn status row and live tool progress. */ nativeChatStructured: boolean nativeChatAgentWorking: boolean + /** What labels the live turn's one indicator row; null off the structured lane. */ + nativeChatTurnIndicator: NativeChatLiveTurnIndicator | null /** Structured lane: host-recorded turn timing for the per-turn status rows. */ nativeChatWorkingStartedAt: number | null nativeChatSettledTurns: NativeChatSettledTurns | null diff --git a/mobile/src/session/mobile-session-route-parity.test.ts b/mobile/src/session/mobile-session-route-parity.test.ts index 4bf325a930c..5486fde6ee5 100644 --- a/mobile/src/session/mobile-session-route-parity.test.ts +++ b/mobile/src/session/mobile-session-route-parity.test.ts @@ -73,7 +73,7 @@ const HEAD_CONTENT_HOOK_SHA256 = '9c3b612fef3f370d66873aefdbe1d701f20cb64ded31fe // after the anchor it sent the host instead of appending. Only that body changed; the other 11 // extracted functions are byte-identical. const HEAD_NESTED_FUNCTION_SHA256 = - 'e54db6971e298dc27b59da9cbb2c455b85f0e1224cdf00482410ac43d179b02d' + '5999a4118d60b40935e6fd61a5457a12c110c1108479d56439b8e186b863f8d8' const HEAD_NATIVE_REGISTRATION_SHA256 = 'cab85e4e4a3f43289ba93ddea9ccce57aea83e0bf14fd1620a965aad0c1cb49e' const HEAD_NATIVE_REMOVAL_SHA256 = @@ -82,7 +82,7 @@ const HEAD_TIMER_CREATION_SHA256 = '1a31b625e2174c3db77272249843196d2b6b06ab1e654a96d8f7858e3082e66b' const HEAD_TIMER_CLEANUP_SHA256 = 'c73f1d1c2cc89642f3d727d6f3b6b81860a9d6f34234541a2065ec3d1a8cd116' const HEAD_RUNTIME_STRING_SHA256 = - '31951b0b83be01ebfa659c4b94df9ad7eaff6404df5338fbade89eb7473a3cb4' + '57ef354b97fb4fd3776fd1b09a34305d84022c04c43c6391bd130517bf6e37af' const HEAD_HOST_JSX_SHA256 = '390405926b1695fa3a33686f0bc192b432f5468d8576499d7cafbb4922defbb5' const HEAD_LEAF_JSX_SHA256 = '21dba981875e173f692590bf910d60964660c5f4cbb79f3a377c7e54f6a1f016' const HEAD_STYLE_REFERENCE_SHA256 = diff --git a/mobile/src/session/use-mobile-bridge-chat-prompt-writes.ts b/mobile/src/session/use-mobile-bridge-chat-prompt-writes.ts new file mode 100644 index 00000000000..2dddad48e16 --- /dev/null +++ b/mobile/src/session/use-mobile-bridge-chat-prompt-writes.ts @@ -0,0 +1,65 @@ +import type { MutableRefObject } from 'react' +import type { RpcClient } from '../transport/rpc-client' +import { useMobileNativeChatPermissionSend } from './mobile-native-chat-permission-send' +import { useMobileNativeChatAnswerSend } from './use-mobile-native-chat-answer-send' +import { useMobileNativeChatCancelAsk } from './use-mobile-native-chat-cancel-ask' +import { useMobileNativeChatStop } from './use-mobile-native-chat-stop' +import type { MobileNativeChatAnswerSend } from './use-mobile-native-chat-answer-send' + +/** The bridge lane's four prompt/interrupt write seams. They share one enable + * gate and chain through the answer seam's `cancelPending`, so a caller cannot + * wire one of them to a different lane or forget to drop in-flight answer + * writes before an Escape. The structured lane answers over RPC instead. */ +export function useMobileBridgeChatPromptWrites(args: { + client: RpcClient | null + enabled: boolean + handleRef: MutableRefObject + deviceTokenRef: MutableRefObject + agentRef: MutableRefObject + /** Changes on chat session swap; cancels pending writes when it does. */ + sessionId: string | null + streamIdentity: string + onSendError: (message: string) => void +}): { + answerAsk: MobileNativeChatAnswerSend['answerAsk'] + cancelAsk: () => Promise + respondPermission: (send: string) => Promise + stop: () => void +} { + const { client, enabled, handleRef, deviceTokenRef, streamIdentity, onSendError } = args + const { answerAsk, cancelPending } = useMobileNativeChatAnswerSend({ + client, + enabled, + handleRef, + deviceTokenRef, + agentRef: args.agentRef, + sessionId: args.sessionId, + streamIdentity, + onSendError + }) + const cancelAsk = useMobileNativeChatCancelAsk({ + client, + enabled, + handleRef, + deviceTokenRef, + cancelPending, + onSendError + }) + const respondPermission = useMobileNativeChatPermissionSend({ + client, + enabled, + handleRef, + deviceTokenRef, + onSendError + }) + const stop = useMobileNativeChatStop({ + client, + enabled, + handleRef, + deviceTokenRef, + streamIdentity, + cancelPending, + onSendError + }) + return { answerAsk, cancelAsk, respondPermission, stop } +} diff --git a/mobile/src/session/use-mobile-native-chat-controller.ts b/mobile/src/session/use-mobile-native-chat-controller.ts index 5b941367451..4087694b567 100644 --- a/mobile/src/session/use-mobile-native-chat-controller.ts +++ b/mobile/src/session/use-mobile-native-chat-controller.ts @@ -2,10 +2,7 @@ import { useLayoutEffect, useRef, type MutableRefObject } from 'react' import type { RpcClient } from '../transport/rpc-client' import type { ConnectionState } from '../transport/types' import type { MobileNativeChatTab } from './mobile-native-chat-eligibility' -import { useMobileNativeChatPermissionSend } from './mobile-native-chat-permission-send' -import { useMobileNativeChatAnswerSend } from './use-mobile-native-chat-answer-send' import { useMobileNativeChatAskDismiss } from './use-mobile-native-chat-ask-dismiss' -import { useMobileNativeChatCancelAsk } from './use-mobile-native-chat-cancel-ask' import { useMobileNativeChatDrafts } from './use-mobile-native-chat-drafts' import { useMobileNativeChatFileSearch } from './use-mobile-native-chat-file-search' import { useMobileNativeChatMessageSend } from './use-mobile-native-chat-message-send' @@ -14,10 +11,10 @@ import { useMobileNativeChatSessionOptionController } from './use-mobile-native- import { useMobileNativeChatSessionLane } from './use-mobile-native-chat-session-lane' import { useMobileStructuredNativeChatSendBridge } from './use-mobile-structured-native-chat-send-bridge' import { useMobileNativeChatPrompts } from './use-mobile-native-chat-prompts' -import { useMobileNativeChatStop } from './use-mobile-native-chat-stop' import { useNativeChatAcceptedAction } from './use-native-chat-action-outcomes' import { useThrottledLatestValue } from './use-throttled-latest-value' import type { MobileNativeChatController } from './mobile-native-chat-controller-contract' +import { useMobileBridgeChatPromptWrites } from './use-mobile-bridge-chat-prompt-writes' import { useMobileNativeChatActiveResolution } from './use-mobile-native-chat-active-resolution' export type { MobileNativeChatController } from './mobile-native-chat-controller-contract' @@ -171,42 +168,19 @@ export function useMobileNativeChatController(args: { ? client != null && activeChatSessionId != null && connState === 'connected' : nativeChatInputLeaseReady && connState === 'connected' - const { answerAsk: handleNativeChatAnswerAsk, cancelPending: cancelNativeChatAnswer } = - useMobileNativeChatAnswerSend({ - client, - enabled: inputSendable && !activeChatStructured, - handleRef: activeHandleRef, - deviceTokenRef, - agentRef: activeChatAgentRef, - sessionId: activeChatSessionId, - streamIdentity, - onSendError - }) - - const handleNativeChatCancelAsk = useMobileNativeChatCancelAsk({ - client, - enabled: inputSendable && !activeChatStructured, - handleRef: activeHandleRef, - deviceTokenRef, - cancelPending: cancelNativeChatAnswer, - onSendError - }) - - const legacyHandleNativeChatRespondPermission = useMobileNativeChatPermissionSend({ - client, - enabled: inputSendable && !activeChatStructured, - handleRef: activeHandleRef, - deviceTokenRef, - onSendError - }) - - const handleNativeChatStop = useMobileNativeChatStop({ + const { + answerAsk: handleNativeChatAnswerAsk, + cancelAsk: handleNativeChatCancelAsk, + respondPermission: legacyHandleNativeChatRespondPermission, + stop: handleNativeChatStop + } = useMobileBridgeChatPromptWrites({ client, enabled: inputSendable && !activeChatStructured, handleRef: activeHandleRef, deviceTokenRef, + agentRef: activeChatAgentRef, + sessionId: activeChatSessionId, streamIdentity, - cancelPending: cancelNativeChatAnswer, onSendError }) @@ -299,6 +273,7 @@ export function useMobileNativeChatController(args: { /** Structured lane: drives the per-turn status row and live tool progress. */ nativeChatStructured: activeChatStructured, nativeChatAgentWorking, + nativeChatTurnIndicator: activeChatStructured ? structuredNativeChat.turnIndicator : null, nativeChatWorkingStartedAt: activeChatStructured ? structuredNativeChat.workingStartedAt : null, nativeChatSettledTurns: activeChatStructured ? structuredNativeChat.settledTurns : null, nativeChatCanStop: activeChatStructured diff --git a/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts b/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts index 5620c1390b2..a67a6a88663 100644 --- a/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts +++ b/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts @@ -28,6 +28,8 @@ export function useMobileNativeChatTurnDisclosure({ isWorking, workingStartedAt, settledTurns, + thinking = false, + activityText = null, scopeKey }: { messages: readonly NativeChatMessage[] @@ -36,12 +38,16 @@ export function useMobileNativeChatTurnDisclosure({ workingStartedAt?: number | null /** Host-recorded durations; they outrank whatever this client observed. */ settledTurns?: NativeChatSettledTurns | null + /** Whether the turn is reasoning right now, derived from its journal content. */ + thinking?: boolean + /** What the provider says the live turn is doing; outranks the other labels. */ + activityText?: string | null /** Host/worktree/tab identity for timing and disclosure isolation. */ scopeKey: string }): { active: NativeChatTurnStatus | null - /** True when the live turn has no user message to hang its status row under. */ - activeTurnIsUnanchored: boolean + /** The live turn's provider activity copy, for the footer row. */ + activeActivityText: string | null onToggleTurn: (turnKey: string) => void resolveRow: (index: number, message: NativeChatMessage) => MobileNativeChatTurnRow } { @@ -51,6 +57,7 @@ export function useMobileNativeChatTurnDisclosure({ isWorking, workingStartedAt, settledTurns, + thinking, scopeKey }) const [expandedTurns, setExpandedTurns] = useState<{ @@ -93,17 +100,16 @@ export function useMobileNativeChatTurnDisclosure({ }, [enabled, messages]) const { active, activeTurnKey, completedByTurn } = turnStatuses + const activeActivityText = enabled && isWorking ? (activityText ?? null) : null const resolveRow = useCallback( (index: number, message: NativeChatMessage): MobileNativeChatTurnRow => { const turnKey = turnKeys[index] const turnStatus = !enabled || message.role !== 'user' ? null - : turnKey === activeTurnKey - ? active - : turnKey - ? (completedByTurn[turnKey] ?? null) - : null + : turnKey + ? (completedByTurn[turnKey] ?? null) + : null return { turnStatus, turnExpanded: turnKey ? expandedTurnIds.has(turnKey) : false, @@ -120,15 +126,14 @@ export function useMobileNativeChatTurnDisclosure({ (turnKey === undefined && activeTurnKey === MOBILE_UNANCHORED_TURN_KEY)) } }, - [turnKeys, enabled, activeTurnKey, active, completedByTurn, expandedTurnIds, isWorking] + [turnKeys, enabled, activeTurnKey, completedByTurn, expandedTurnIds, isWorking] ) return { active, + activeActivityText, /** Stable for a given chat scope, so it never disturbs a row's memo. */ onToggleTurn: toggleExpandedTurn, - activeTurnIsUnanchored: - enabled && active != null && activeTurnKey === MOBILE_UNANCHORED_TURN_KEY, resolveRow } } diff --git a/mobile/src/session/use-mobile-native-chat-turn-status.ts b/mobile/src/session/use-mobile-native-chat-turn-status.ts index 5686acde515..f7cbb2dd887 100644 --- a/mobile/src/session/use-mobile-native-chat-turn-status.ts +++ b/mobile/src/session/use-mobile-native-chat-turn-status.ts @@ -1,7 +1,6 @@ import { useEffect, useMemo, useRef, useState } from 'react' import type { NativeChatMessage } from '../../../src/shared/native-chat-types' import { - nativeChatTurnHasResponse, reduceNativeChatTurnTiming, selectNativeChatTurnStatuses, type NativeChatSettledTurns, @@ -27,6 +26,7 @@ export function useMobileNativeChatTurnStatus({ isWorking, workingStartedAt, settledTurns, + thinking = false, scopeKey }: { messages: readonly NativeChatMessage[] @@ -35,6 +35,8 @@ export function useMobileNativeChatTurnStatus({ workingStartedAt?: number | null /** Host-recorded durations; they outrank whatever this client observed. */ settledTurns?: NativeChatSettledTurns | null + /** Whether the turn is reasoning right now, derived from its journal content. */ + thinking?: boolean /** Host/worktree/tab identity. Timings never carry across chat surfaces. */ scopeKey: string }): { @@ -45,7 +47,6 @@ export function useMobileNativeChatTurnStatus({ const latestUserIndex = enabled ? messages.findLastIndex((message) => message.role === 'user') : -1 - const hasCurrentTurnResponse = enabled && nativeChatTurnHasResponse(messages, latestUserIndex) const latestUserId = latestUserIndex !== -1 ? (messages[latestUserIndex]?.id ?? null) : null const activeTurnKey = latestUserId ?? MOBILE_UNANCHORED_TURN_KEY const [scopedTiming, setScopedTiming] = useState(() => ({ @@ -95,6 +96,7 @@ export function useMobileNativeChatTurnStatus({ // turn re-renders ~20x/s. Without this, every settled turn's row gets fresh // props each tick and the memoized message rows all re-render. const turnIsWorking = enabled && isWorking + const turnIsThinking = enabled && thinking const settledByTurn = enabled ? (settledTurns ?? undefined) : undefined const statuses = useMemo( () => @@ -102,17 +104,10 @@ export function useMobileNativeChatTurnStatus({ activeTurnKey, isWorking: turnIsWorking, workingStartedAt, - hasCurrentTurnResponse, + thinking: turnIsThinking, settledByTurn }), - [ - timingByTurn, - activeTurnKey, - turnIsWorking, - workingStartedAt, - hasCurrentTurnResponse, - settledByTurn - ] + [timingByTurn, activeTurnKey, turnIsWorking, workingStartedAt, turnIsThinking, settledByTurn] ) return { ...statuses, activeTurnKey } } diff --git a/mobile/src/session/use-mobile-session-terminal-create-actions.test.ts b/mobile/src/session/use-mobile-session-terminal-create-actions.test.ts index 4e097a1fc1c..f087dae338f 100644 --- a/mobile/src/session/use-mobile-session-terminal-create-actions.test.ts +++ b/mobile/src/session/use-mobile-session-terminal-create-actions.test.ts @@ -263,6 +263,51 @@ describe('mobile + Codex tab creation routing', () => { expect(scope.showToast).toHaveBeenCalledWith('create outcome ambiguous', 1800) } ) + // Why: pty exhaustion, a disabled agent and an unresolved worktree owner all arrived as the + // same 'Failed to create terminal', leaving the empty session with nothing to act on. + it('surfaces the host reason instead of a generic terminal-create error', async () => { + const client = clientReturning({ + ok: false, + error: { + code: 'runtime_error', + message: 'Your system cannot allocate any more pty devices.' + } + }) + const scope = createScope(client) + let actions: ReturnType | undefined + function Harness() { + actions = useMobileSessionTerminalCreateActions(scope as never) + return null + } + await act(async () => { + renderer = create(createElement(Harness)) + }) + await act(async () => { + await actions?.handleCreateTerminal() + }) + + expect(scope.setCreateError).toHaveBeenCalledWith( + 'Your system cannot allocate any more pty devices.' + ) + }) + + it('falls back to the generic message when the host gives no reason', async () => { + const client = clientReturning({ ok: false, error: { code: 'runtime_error', message: '' } }) + const scope = createScope(client) + let actions: ReturnType | undefined + function Harness() { + actions = useMobileSessionTerminalCreateActions(scope as never) + return null + } + await act(async () => { + renderer = create(createElement(Harness)) + }) + await act(async () => { + await actions?.handleCreateTerminal() + }) + + expect(scope.setCreateError).toHaveBeenCalledWith('Failed to create terminal') + }) }) describe('optimistic placement of a created tab', () => { diff --git a/mobile/src/session/use-mobile-session-terminal-create-actions.ts b/mobile/src/session/use-mobile-session-terminal-create-actions.ts index a3d61f9ca32..76b5256d6c5 100644 --- a/mobile/src/session/use-mobile-session-terminal-create-actions.ts +++ b/mobile/src/session/use-mobile-session-terminal-create-actions.ts @@ -66,6 +66,17 @@ export function useMobileSessionTerminalCreateActions(scope: MobileSessionAttach .toString(36) .slice(2, 10)}` + // Why: the host names the real cause (pty exhaustion, disabled agent, unresolved worktree); + // collapsing every failure to 'Failed to create terminal' left the phone undiagnosable. + function reportCreateFailure(hostReason: string): void { + const reason = hostReason.trim() + setCreateError(reason || options?.errorToast || 'Failed to create terminal') + if (options?.errorToast) { + triggerError() + showToast(options.errorToast, 1800) + } + } + try { // Bare structured-provider launches follow host createSupport; prompted launches keep their startup semantics. if (isAgentSessionHandleProvider(agent) && options === undefined) { @@ -209,20 +220,10 @@ export function useMobileSessionTerminalCreateActions(scope: MobileSessionAttach } scheduleDelayedAction(() => void fetchSessionTabs(), 500) } else { - const message = options?.errorToast ?? 'Failed to create terminal' - setCreateError(message) - if (options?.errorToast) { - triggerError() - showToast(message, 1800) - } - } - } catch { - const message = options?.errorToast ?? 'Failed to create terminal' - setCreateError(message) - if (options?.errorToast) { - triggerError() - showToast(message, 1800) + reportCreateFailure((response as RpcFailure).error.message) } + } catch (error) { + reportCreateFailure(error instanceof Error ? error.message : '') } finally { creatingTerminalRef.current = false setCreating(false) diff --git a/mobile/src/session/use-mobile-structured-agent-session.ts b/mobile/src/session/use-mobile-structured-agent-session.ts index 5c458bbdd9f..938e881aec7 100644 --- a/mobile/src/session/use-mobile-structured-agent-session.ts +++ b/mobile/src/session/use-mobile-structured-agent-session.ts @@ -11,10 +11,12 @@ import { import { encodeNativeChatTranscriptIdentity } from '../../../src/shared/native-chat-transcript-retention' import type { MobileNativeChatSendOutcome } from './mobile-native-chat-send' import { projectStructuredAgentSessionMessages } from '../../../src/shared/structured-agent-session-message-projection' +import { hasUnansweredStructuredAgentSessionDispatch } from '../../../src/shared/structured-agent-session-projection' import { activeStructuredAgentSessionTurnId, - hasUnansweredStructuredAgentSessionDispatch -} from '../../../src/shared/structured-agent-session-projection' + isStructuredAgentSessionThinking +} from '../../../src/shared/structured-agent-session-live-turn' +import { selectStructuredAgentTurnActivity } from '../../../src/shared/native-chat-turn-activity' import { pendingStructuredApproval, pendingStructuredQuestion, @@ -31,6 +33,7 @@ import type { RpcClient } from '../transport/rpc-client' import type { MobileChatPermission } from './mobile-native-chat-permission' import type { MobileChatQuestion } from './mobile-native-chat-question' import type { MobileNativeChatSession } from './use-mobile-native-chat-session' +import type { NativeChatLiveTurnIndicator } from '../../../src/shared/native-chat-turn-status' import { useMobileStructuredAgentState } from './use-mobile-structured-agent-state' import { useMobileStructuredPromptResponses } from './use-mobile-structured-prompt-responses' import { useMobileStructuredAgentOptions } from './use-mobile-structured-agent-options' @@ -43,6 +46,8 @@ type StructuredMobileSession = ReturnType ({ thinking, activityText }), [thinking, activityText]) const status = state.status === 'idle' ? 'idle' : state.status const approvalPrompt = useMemo( () => state.items.find(pendingStructuredApproval) ?? null, @@ -296,6 +307,7 @@ export function useMobileStructuredAgentSession(args: { turnId !== null || hasUnansweredStructuredAgentSessionDispatch(state.submissions, state.fence), turnId, + turnIndicator, ...turnTiming, sendWithOutcome, cancel, diff --git a/mobile/src/session/use-mobile-structured-turn-indicator.test.tsx b/mobile/src/session/use-mobile-structured-turn-indicator.test.tsx new file mode 100644 index 00000000000..8369ff03a17 --- /dev/null +++ b/mobile/src/session/use-mobile-structured-turn-indicator.test.tsx @@ -0,0 +1,138 @@ +import { createElement } from 'react' +import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalRenderItem } from '../../../src/shared/agent-session-journal-types' +import type { AgentSessionSubscribeEvent } from '../../../src/shared/agent-session-wire' +import type { RpcClient } from '../transport/rpc-client' +import { useMobileStructuredAgentSession } from './use-mobile-structured-agent-session' + +function journalItem( + sequence: number, + body: AgentJournalRenderItem['body'] +): AgentJournalRenderItem { + return { itemId: `item-${sequence}`, revision: 1, sequence, observedAt: sequence, body } +} + +function snapshot(items: AgentJournalRenderItem[], fence: number): AgentSessionSubscribeEvent { + const newest = items.length + return { + type: 'snapshot', + sessionId: 'session-1', + fence, + page: { + sessionId: 'session-1', + epoch: 'epoch-1', + fence, + direction: 'tail', + items, + removedItemIds: [], + submissions: [], + window: { + oldest: { epoch: 'epoch-1', sequence: 1 }, + newest: { epoch: 'epoch-1', sequence: newest }, + nextCursor: { epoch: 'epoch-1', sequence: newest + 1 } + }, + liveCursor: { epoch: 'epoch-1', sequence: newest }, + hasOlder: false, + hasNewer: false + } + } as AgentSessionSubscribeEvent +} + +/** What the one live indicator row reads, resolved off the session journal. */ +describe('useMobileStructuredAgentSession turn indicator', () => { + let renderer: ReactTestRenderer | null = null + let hook: ReturnType | null = null + let listener: ((value: unknown) => void) | null = null + const sendRequest = vi.fn(async (method: string) => ({ + ok: true, + result: + method === 'agentSession.options' + ? { + models: [{ id: 'gpt-fast', label: 'GPT Fast', isDefault: true, efforts: [] }], + current: { model: 'gpt-fast' } + } + : {}, + _meta: { runtimeId: 'r1' } + })) + const subscribe = vi.fn((_method: string, _params: unknown, onData: (value: unknown) => void) => { + listener = onData + return vi.fn() + }) + const client = { sendRequest, subscribe } as unknown as RpcClient + // Stable across renders: a fresh callback would re-run the hold/subscribe effect + // and release the session out from under the test. + const onSendError = vi.fn() + + function Harness(): null { + hook = useMobileStructuredAgentSession({ + client, + sessionId: 'session-1', + sourceIdentity: 'host-a\0workspace-a', + enabled: true, + connected: true, + agent: 'codex', + onSendError + } as never) + return null + } + + beforeEach(() => { + vi.clearAllMocks() + listener = null + }) + + afterEach(() => { + act(() => renderer?.unmount()) + renderer = null + hook = null + }) + + const runningTurn = journalItem(1, { kind: 'turn', turnId: 'turn-1', state: 'running' }) + const reasoning = journalItem(2, { + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'Weighing two approaches' }] + }) + + it('reads the live turn as reasoning while reasoning is its newest content', async () => { + act(() => { + renderer = create(createElement(Harness)) + }) + await vi.waitFor(() => expect(listener).not.toBeNull()) + + act(() => { + listener?.(snapshot([runningTurn, reasoning], 3)) + }) + + expect(hook?.turnIndicator).toEqual({ thinking: true, activityText: null }) + }) + + it('hands the row the provider copy once real content ends the reasoning', async () => { + act(() => { + renderer = create(createElement(Harness)) + }) + await vi.waitFor(() => expect(listener).not.toBeNull()) + + act(() => { + listener?.( + snapshot( + [ + runningTurn, + reasoning, + journalItem(3, { + kind: 'tool-call', + name: 'shell', + input: { command: 'pnpm lint' }, + state: 'running' + }), + journalItem(4, { kind: 'status', text: 'Updating the plan' }) + ], + 3 + ) + ) + }) + + expect(hook?.turnIndicator).toEqual({ thinking: false, activityText: 'Updating the plan' }) + }) +}) diff --git a/mobile/src/tasks/worktree-create-retry.test.ts b/mobile/src/tasks/worktree-create-retry.test.ts index beb9d463e32..1611ed74c19 100644 --- a/mobile/src/tasks/worktree-create-retry.test.ts +++ b/mobile/src/tasks/worktree-create-retry.test.ts @@ -55,7 +55,7 @@ async function flush(): Promise { // cutover). Records every call so tests can assert on the clientMutationId. function scriptedClient( outcomes: Array< - | { id: string; displayName?: string } + | { id: string; displayName?: string; warning?: string } | { errorMessage: string } // takesMs models how long the ambiguity took to SURFACE — a clean close is // instant, a half-open socket waits out the liveness watchdog or the timeout. @@ -107,7 +107,8 @@ function scriptedClient( worktree: { id: outcome.id, ...(outcome.displayName !== undefined ? { displayName: outcome.displayName } : {}) - } + }, + ...(outcome.warning !== undefined ? { warning: outcome.warning } : {}) }, _meta: { runtimeId: 'r' } } @@ -116,6 +117,41 @@ function scriptedClient( } describe('createWorktreeWithNameRetry', () => { + // Why: `worktree.create` succeeds even when the startup terminal failed to spawn (pty + // exhaustion), and `warning` is the only place the host says so. + it('returns the host create warning alongside the worktree', async () => { + const attempts: Attempt[] = [] + const client = scriptedClient( + [{ id: 'wt-warned', warning: 'Failed to create the startup terminal for /w: no pty' }], + attempts + ) + await expect( + createWorktreeWithNameRetry({ + client, + baseName: 'puffin', + buildParams: (name) => ({ repo: 'id:r', name }), + worktreeCreateIdempotency: Promise.resolve(IDEMPOTENT_CREATE_SUPPORT) + }) + ).resolves.toEqual({ + worktreeId: 'wt-warned', + name: 'puffin', + warning: 'Failed to create the startup terminal for /w: no pty' + }) + }) + + it('omits a blank create warning', async () => { + const attempts: Attempt[] = [] + const client = scriptedClient([{ id: 'wt-clean', warning: ' ' }], attempts) + await expect( + createWorktreeWithNameRetry({ + client, + baseName: 'puffin', + buildParams: (name) => ({ repo: 'id:r', name }), + worktreeCreateIdempotency: Promise.resolve(IDEMPOTENT_CREATE_SUPPORT) + }) + ).resolves.toEqual({ worktreeId: 'wt-clean', name: 'puffin' }) + }) + it('waits for capability detection before sending a create', async () => { const attempts: Attempt[] = [] const client = scriptedClient([{ id: 'wt-ready' }], attempts) diff --git a/mobile/src/tasks/worktree-create-retry.ts b/mobile/src/tasks/worktree-create-retry.ts index a3fa8ae2e1b..b0f8f618773 100644 --- a/mobile/src/tasks/worktree-create-retry.ts +++ b/mobile/src/tasks/worktree-create-retry.ts @@ -21,7 +21,9 @@ import { // branches outlive worktrees in git, and remote branches/PRs aren't visible from // worktree.ps. Retry by appending -2, -3, ... mirroring the desktop createWorktree // loop in src/renderer/src/store/slices/worktrees.ts. -export type WorktreeCreateResult = { worktreeId: string; name: string } | { error: string } +export type WorktreeCreateResult = + | { worktreeId: string; name: string; warning?: string } + | { error: string } // Why: a create in flight when the mobile transport migrates (relay/direct // hand-off on shoddy cellular, relay lease rotation) rejects with a cutover error @@ -84,14 +86,19 @@ export async function createWorktreeWithNameRetry( if (response.ok) { const result = (response as RpcSuccess).result as { worktree: { id: string; displayName?: string } + warning?: string } const authoritativeName = result.worktree.displayName + // Why: a create can succeed with the startup terminal failing (pty exhaustion); dropping + // `warning` here is what lands the phone on an unexplained empty session. + const warning = typeof result.warning === 'string' ? result.warning.trim() : '' return { worktreeId: result.worktree.id, name: typeof authoritativeName === 'string' && authoritativeName.trim() ? authoritativeName - : candidateName + : candidateName, + ...(warning ? { warning } : {}) } } lastError = response.error.message diff --git a/package.json b/package.json index 9feaad74882..4f03d793aa6 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "test": "node config/scripts/ensure-native-runtime.mjs --runtime=node && vitest run --config config/vitest.config.ts", "test:skill-sharing:release": "vitest run --config config/vitest.config.ts src/main/skills src/main/runtime/rpc/methods/skills.test.ts src/relay/skill-install-handler.test.ts src/shared/skill-bundle-install-contract.test.ts src/shared/skill-install-contract.test.ts src/shared/skill-install-failure.test.ts src/shared/skill-package-manifest.test.ts", "test:repro:remote-agent-session": "pnpm run build:cli && pnpm run build:electron-vite && node config/scripts/remote-agent-session-authority-repro.mjs", + "capture:agent-transcript": "node config/scripts/ensure-native-runtime.mjs --runtime=node && node config/scripts/capture-agent-pty-transcript.mjs", "check:reliability-gates": "node config/scripts/check-reliability-gates.mjs", "check:max-lines-ratchet": "node config/scripts/check-max-lines-ratchet.mjs", "check:ts-nocheck-ratchet": "node config/scripts/check-ts-nocheck-ratchet.mjs", diff --git a/src/cli/terminal-format.test.ts b/src/cli/terminal-format.test.ts index 42c036471eb..294fbc09cee 100644 --- a/src/cli/terminal-format.test.ts +++ b/src/cli/terminal-format.test.ts @@ -1,5 +1,16 @@ import { describe, expect, it } from 'vitest' -import { formatTerminalClose, formatTerminalFocus, formatTerminalSend } from './terminal-format' +import type { + RuntimeTerminalShow, + RuntimeTerminalWait, + RuntimeTerminalWaitBlockedReason +} from '../shared/runtime-terminal-contracts' +import { + formatTerminalClose, + formatTerminalFocus, + formatTerminalSend, + formatTerminalShow, + formatTerminalWait +} from './terminal-format' describe('formatTerminalFocus', () => { it('distinguishes superseded navigation from a winning focus', () => { @@ -171,3 +182,73 @@ describe('formatTerminalSend', () => { expect(output).toContain('--retry-request prompt-swallowed --wait-submit ') }) }) + +// Why: an older host still publishes the codex-* tokens for dialogs its matcher never proved were +// Codex's, so a Gemini/Cursor/Antigravity user reads a Codex label unless the CLI names the neutral one. +describe('blocked-reason rendering against a mixed-version host', () => { + function showResult(reason?: RuntimeTerminalWaitBlockedReason): { + terminal: RuntimeTerminalShow + } { + return { + terminal: { + handle: 'term_agy', + ptyId: 'pty-1', + paneRuntimeId: 1, + rendererGraphEpoch: 1, + worktreeId: 'worktree-1', + worktreePath: '/tmp/w', + branch: 'main', + tabId: 'tab-1', + leafId: 'leaf-1', + title: 'Antigravity', + connected: true, + writable: true, + lastOutputAt: null, + preview: 'Do you trust the files in this folder?', + agentWait: { source: 'prompt-text', reason } + } + } + } + + function waitResult(blockedReason: RuntimeTerminalWaitBlockedReason): { + wait: RuntimeTerminalWait + } { + return { + wait: { + handle: 'term_agy', + condition: 'tui-idle', + satisfied: false, + status: 'running', + exitCode: null, + blockedReason + } + } + } + + // Why one assertion over every reason: a test that only asserts the *absence* of an alias suffix + // passes when the aliasing code is deleted, so each case is paired with a legacy token that must + // gain one. + it.each([ + ['codex-trust-workspace', 'codex-trust-workspace (agent-trust-workspace)'], + ['codex-update-prompt', 'codex-update-prompt (agent-update-prompt)'], + ['codex-cwd-prompt', 'codex-cwd-prompt (agent-cwd-prompt)'], + ['codex-hooks-review-prompt', 'codex-hooks-review-prompt (agent-hooks-review-prompt)'], + ['codex-interactive-prompt', 'codex-interactive-prompt (agent-interactive-prompt)'], + // This build published these itself, so there is nothing to reinterpret. + ['agent-trust-workspace', 'agent-trust-workspace'], + ['codex-model-migration-prompt', 'codex-model-migration-prompt'] + ] as const)('renders %s as %s on both wait and show', (reason, rendered) => { + expect(formatTerminalWait(waitResult(reason)).split('\n').at(-1)).toBe( + `blockedReason: ${rendered}` + ) + expect(formatTerminalShow(showResult(reason))).toContain( + `agentWait: ${rendered} (via prompt-text)` + ) + }) + + it('still describes a wait with no reason at all', () => { + expect(formatTerminalShow(showResult(undefined))).toContain( + 'agentWait: interactive prompt (via prompt-text)' + ) + }) +}) diff --git a/src/cli/terminal-format.ts b/src/cli/terminal-format.ts index 46c26556889..06d897828fe 100644 --- a/src/cli/terminal-format.ts +++ b/src/cli/terminal-format.ts @@ -1,5 +1,6 @@ import { PTY_LIVE_NOTE, describeUnconfirmedStop } from '../shared/pty-liveness-verdict' import { structuredChatPtyWriteRefusalCopy } from '../shared/agent-session-pty-write-refusal-copy' +import { describeTerminalWaitBlockedReason } from '../shared/terminal-wait-blocked-reason-legacy-alias' import { formatListingHostScope, type WithAnnotatedHostScope } from './omitted-host-scope-selectors' import type { RuntimeTerminalClose, @@ -118,7 +119,10 @@ function formatAgentWait(agentWait: RuntimeTerminalShow['agentWait']): string { if (!agentWait) { return 'none' } - return `${agentWait.reason ?? 'interactive prompt'} (via ${agentWait.source})` + if (!agentWait.reason) { + return `interactive prompt (via ${agentWait.source})` + } + return `${describeTerminalWaitBlockedReason(agentWait.reason)} (via ${agentWait.source})` } export function formatTerminalRead(result: { terminal: RuntimeTerminalRead }): string { @@ -278,7 +282,7 @@ export function formatTerminalWait(result: { wait: RuntimeTerminalWait }): strin `exitCode: ${result.wait.exitCode ?? 'null'}` ] if (result.wait.blockedReason) { - lines.push(`blockedReason: ${result.wait.blockedReason}`) + lines.push(`blockedReason: ${describeTerminalWaitBlockedReason(result.wait.blockedReason)}`) } return lines.join('\n') } diff --git a/src/main/claude/claude-structured-journal-translation.test.ts b/src/main/claude/claude-structured-journal-translation.test.ts index 44d7e3a1251..7a8ecf54344 100644 --- a/src/main/claude/claude-structured-journal-translation.test.ts +++ b/src/main/claude/claude-structured-journal-translation.test.ts @@ -569,8 +569,11 @@ describe('Claude structured journal translation', () => { translator.handle(message('assistant', 'assistant-thinking', [{ type: 'thinking', thinking }])) expect(state.items.at(-1)?.body).toEqual({ - kind: 'status', - text: boundInlineText(thinking, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text + kind: 'message', + role: 'reasoning', + blocks: [ + { type: 'text', text: boundInlineText(thinking, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text } + ] }) }) diff --git a/src/main/claude/claude-structured-journal-translation.ts b/src/main/claude/claude-structured-journal-translation.ts index 478645d62ed..8b71149cba2 100644 --- a/src/main/claude/claude-structured-journal-translation.ts +++ b/src/main/claude/claude-structured-journal-translation.ts @@ -180,8 +180,11 @@ export function createClaudeJournalTranslator( const thinking = claudeThinkingText(outputEnvelope) if (thinking) { deps.sink.appendItem(claudeThinkingIdentity(envelope.sessionId, envelope.uuid), { - kind: 'status', - text: boundInlineText(thinking, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text + kind: 'message', + role: 'reasoning', + blocks: [ + { type: 'text', text: boundInlineText(thinking, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text } + ] }) changed = true } diff --git a/src/main/codex/codex-goal-journal-identity.ts b/src/main/codex/codex-goal-journal-identity.ts new file mode 100644 index 00000000000..5203b7b62d5 --- /dev/null +++ b/src/main/codex/codex-goal-journal-identity.ts @@ -0,0 +1,47 @@ +import { createHash } from 'node:crypto' +import { parseAgentJournalItemKey } from '../../shared/agent-session-journal-item-key' +import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types' + +export type CodexGoalJournalState = { + thread: string + signature: string + occurrence: string +} + +const GOAL_IDENTITY_PREFIX = 'codex-goal' +const DIGEST_PATTERN = /^[0-9a-f]{64}$/ + +export function codexGoalJournalDigest(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +export function codexGoalJournalIdentity( + thread: string, + signature: string, + occurrence: string +): AgentJournalItemIdentity { + return { + provider: 'orca', + clientMessageId: `${GOAL_IDENTITY_PREFIX}:${thread}:${signature}:${occurrence}` + } +} + +/** Recognizes only the host-owned rows used to record Codex goal lifecycle state. */ +export function parseCodexGoalJournalItemId(itemId: string): CodexGoalJournalState | null { + const identity = parseAgentJournalItemKey(itemId) + if (identity?.provider !== 'orca') { + return null + } + const [prefix, thread, signature, occurrence, ...rest] = identity.clientMessageId.split(':') + return prefix === GOAL_IDENTITY_PREFIX && + DIGEST_PATTERN.test(thread ?? '') && + DIGEST_PATTERN.test(signature ?? '') && + DIGEST_PATTERN.test(occurrence ?? '') && + rest.length === 0 + ? { + thread: thread as string, + signature: signature as string, + occurrence: occurrence as string + } + : null +} diff --git a/src/main/codex/codex-goal-journal-rows.test.ts b/src/main/codex/codex-goal-journal-rows.test.ts new file mode 100644 index 00000000000..dcbc11a816f --- /dev/null +++ b/src/main/codex/codex-goal-journal-rows.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from 'vitest' +import { unhandledProviderFrameJournalItem } from '../native-chat/agent-session-wire/unhandled-provider-frame' +import { codexGoalRowSignature, codexGoalRowText } from './codex-goal-journal-rows' + +/** The shape a live Codex app-server session emits for `thread/goal/updated`. */ +function goalFrame(overrides: { goal?: Record } = {}): Record { + return { + threadId: '01a08cc2-f96e-76d0-bb74-88b9bc0b03fc', + turnId: '01a08cc2-fa6a-7541-a4c7-67d98a6e40c2', + goal: { + threadId: '01a08cc2-f96e-76d0-bb74-88b9bc0b03fc', + objective: 'Keep the current scratch directory tidy.', + status: 'active', + tokenBudget: null, + tokensUsed: 0, + timeUsedSeconds: 0, + createdAt: 1789067988, + updatedAt: 1789067988, + ...overrides.goal + } + } +} + +describe('codexGoalRowText', () => { + it('leads with the objective the goal actually carries', () => { + expect(codexGoalRowText('thread/goal/updated', goalFrame())).toBe( + 'Goal set: Keep the current scratch directory tidy.' + ) + }) + + it.each([ + ['paused', 'Goal paused'], + ['blocked', 'Goal blocked'], + ['complete', 'Goal complete'], + ['usageLimited', 'Goal stopped — usage limit'], + ['budgetLimited', 'Goal stopped — token budget spent'] + ])('says what %s means rather than echoing the status', (status, prefix) => { + expect(codexGoalRowText('thread/goal/updated', goalFrame({ goal: { status } }))).toBe( + `${prefix}: Keep the current scratch directory tidy.` + ) + }) + + it('still says something true for a status this build does not know', () => { + expect( + codexGoalRowText('thread/goal/updated', goalFrame({ goal: { status: 'somethingNew' } })) + ).toBe('Goal updated: Keep the current scratch directory tidy.') + }) + + it('reports a cleared goal, and ignores unrelated methods', () => { + expect(codexGoalRowText('thread/goal/cleared', {})).toBe('Goal cleared') + expect(codexGoalRowText('thread/tokenUsage/updated', goalFrame())).toBeNull() + }) + + it('falls back to the prefix alone when no objective survives', () => { + expect(codexGoalRowText('thread/goal/updated', goalFrame({ goal: { objective: ' ' } }))).toBe( + 'Goal set' + ) + expect(codexGoalRowText('thread/goal/updated', {})).toBe('Goal updated') + }) +}) + +describe('codexGoalRowSignature', () => { + it('ignores the counters that climb on every turn', () => { + // Two frames one live turn apart: only accounting moved. + const first = codexGoalRowSignature('thread/goal/updated', goalFrame()) + const later = codexGoalRowSignature( + 'thread/goal/updated', + goalFrame({ goal: { tokensUsed: 25999, timeUsedSeconds: 8, updatedAt: 1789067996 } }) + ) + expect(later).toBe(first) + }) + + it('separates visible objective and status changes', () => { + const base = codexGoalRowSignature('thread/goal/updated', goalFrame()) + expect( + codexGoalRowSignature('thread/goal/updated', goalFrame({ goal: { status: 'complete' } })) + ).not.toBe(base) + expect( + codexGoalRowSignature('thread/goal/updated', goalFrame({ goal: { objective: 'Ship it.' } })) + ).not.toBe(base) + }) + + it('does not append an identical visible row for a budget-only change', () => { + const base = codexGoalRowSignature('thread/goal/updated', goalFrame()) + expect( + codexGoalRowSignature('thread/goal/updated', goalFrame({ goal: { tokenBudget: 50_000 } })) + ).toBe(base) + }) + + it('has no signature for a frame that is not a goal', () => { + expect(codexGoalRowSignature('thread/tokenUsage/updated', goalFrame())).toBeNull() + }) +}) + +describe('goal frames as journal rows', () => { + it('journals the goal instead of dropping it as chrome', () => { + const row = unhandledProviderFrameJournalItem( + 'codex', + 'notification:thread/goal/updated', + goalFrame() + ) + + expect(row?.classification).toBe('timeline-substantive') + expect(row?.body.text).toBe('Goal set: Keep the current scratch directory tidy.') + // The raw frame stays available behind the row's disclosure. + expect(row?.body.providerFrame?.kind).toBe('notification:thread/goal/updated') + }) + + it('journals a cleared goal', () => { + const row = unhandledProviderFrameJournalItem('codex', 'notification:thread/goal/cleared', { + threadId: '01a08cc2-f96e-76d0-bb74-88b9bc0b03fc' + }) + + expect(row?.body.text).toBe('Goal cleared') + }) + + it('never shows the bare opcode, which is what a plain reclassify would have done', () => { + const row = unhandledProviderFrameJournalItem( + 'codex', + 'notification:thread/goal/updated', + goalFrame() + ) + + expect(row?.body.text).not.toContain('notification:') + expect(row?.body.text).not.toContain('codex · ') + }) +}) diff --git a/src/main/codex/codex-goal-journal-rows.ts b/src/main/codex/codex-goal-journal-rows.ts new file mode 100644 index 00000000000..36dac339901 --- /dev/null +++ b/src/main/codex/codex-goal-journal-rows.ts @@ -0,0 +1,74 @@ +/** + * Codex thread goals reach us only as notifications: the `create_goal` tool call the + * model makes is never emitted as an item, so `thread/goal/updated` is the single + * truthful signal that a goal exists. The model narrates goals in prose either way, + * and that prose can be wrong — it claims "Goal created" in sessions where no goal + * was ever set — so the row below is what lets a reader tell the two apart. + */ + +const GOAL_UPDATED_METHOD = 'thread/goal/updated' +const GOAL_CLEARED_METHOD = 'thread/goal/cleared' + +/** Status values Codex can report, mapped to how a reader would say them. */ +const GOAL_STATUS_PREFIX: Record = { + active: 'Goal set', + paused: 'Goal paused', + blocked: 'Goal blocked', + complete: 'Goal complete', + usageLimited: 'Goal stopped — usage limit', + budgetLimited: 'Goal stopped — token budget spent' +} + +function goalRecord(payload: unknown): Record | null { + if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) { + return null + } + const goal = (payload as Record).goal + return typeof goal === 'object' && goal !== null && !Array.isArray(goal) + ? (goal as Record) + : null +} + +export function isCodexGoalFrameMethod(method: string): boolean { + return method === GOAL_UPDATED_METHOD || method === GOAL_CLEARED_METHOD +} + +/** The sentence for a goal frame, or null when the frame is not one. */ +export function codexGoalRowText(method: string, payload: unknown): string | null { + if (method === GOAL_CLEARED_METHOD) { + return 'Goal cleared' + } + if (method !== GOAL_UPDATED_METHOD) { + return null + } + const goal = goalRecord(payload) + const objective = typeof goal?.objective === 'string' ? goal.objective.trim() : '' + const status = typeof goal?.status === 'string' ? goal.status : '' + // An unknown future status still says something true rather than falling back to + // the bare opcode. + const prefix = GOAL_STATUS_PREFIX[status] ?? 'Goal updated' + return objective ? `${prefix}: ${objective}` : prefix +} + +/** + * What changes the visible sentence. Counters and budget stay in the raw disclosure but + * cannot append another row with identical copy. + */ +export function codexGoalRowSignature(method: string, payload: unknown): string | null { + if (method === GOAL_CLEARED_METHOD) { + return GOAL_CLEARED_METHOD + } + if (method !== GOAL_UPDATED_METHOD) { + return null + } + const goal = goalRecord(payload) + const objective = typeof goal?.objective === 'string' ? goal.objective.trim() : '' + const status = typeof goal?.status === 'string' ? goal.status : '' + return `${GOAL_UPDATED_METHOD}\u0000${status}\u0000${objective}` +} + +/** Provider-owned goal generation, stable while accounting counters change. */ +export function codexGoalGeneration(payload: unknown): string | null { + const createdAt = goalRecord(payload)?.createdAt + return typeof createdAt === 'number' && Number.isFinite(createdAt) ? String(createdAt) : null +} diff --git a/src/main/codex/codex-notice-item-translation.test.ts b/src/main/codex/codex-notice-item-translation.test.ts index 15f638c4015..14a87cb8cc5 100644 --- a/src/main/codex/codex-notice-item-translation.test.ts +++ b/src/main/codex/codex-notice-item-translation.test.ts @@ -22,11 +22,16 @@ describe('plan document translation', () => { expect( codexItemBody({ id: 'r', type: 'reasoning', summary: ['Thinking through the problem.'] }) ).toEqual({ - kind: 'status', - text: 'Thinking through the problem.' + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'Thinking through the problem.' }] }) expect(codexStreamingJournalItem({ id: 'r', type: 'reasoning' }, 'Thinking…')).toEqual({ - body: { kind: 'status', text: 'Thinking…' }, + body: { + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'Thinking…' }] + }, handled: true }) }) diff --git a/src/main/codex/codex-structured-item-translation.test.ts b/src/main/codex/codex-structured-item-translation.test.ts index 3f8ec524002..53cf94e265c 100644 --- a/src/main/codex/codex-structured-item-translation.test.ts +++ b/src/main/codex/codex-structured-item-translation.test.ts @@ -10,6 +10,7 @@ import { codexItemIdentity, codexJournalItem, codexMessageBlocks, + codexStreamingJournalItem, CodexTurnOrdinals, MAX_CODEX_TURN_ORDINAL_BYTES, MAX_CODEX_TURN_ORDINAL_ENTRIES, @@ -601,12 +602,18 @@ describe('codex item bodies', () => { body: { kind: 'status', text, presentation: 'plan-document' }, handled: true }) + // A plan is a durable artifact, so it must never read as the model reasoning now. + expect(codexItemBody({ type: 'plan', id: 'plan-document', text })).not.toMatchObject({ + kind: 'message', + role: 'reasoning' + }) }) - it('renders reasoning as status and exposes an unknown item as a provider frame', () => { + it('renders reasoning as a typed message and exposes an unknown item as a provider frame', () => { expect(codexItemBody({ type: 'reasoning', id: 'r', text: 'thinking' })).toEqual({ - kind: 'status', - text: 'thinking' + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'thinking' }] }) expect(codexItemBody({ type: 'reasoning', id: 'r' })).toBeNull() expect(codexItemBody({ type: 'agentMessage', id: 'm', text: '' })).toBeNull() @@ -617,6 +624,15 @@ describe('codex item bodies', () => { }) }) + it('keeps non-reasoning item streams as status activity', () => { + expect( + codexStreamingJournalItem({ type: 'somethingCodexAddedLater', id: 'x' }, 'still working') + ).toEqual({ + body: { kind: 'status', text: 'still working' }, + handled: true + }) + }) + it('gives an mcp tool call a typed body with its own arguments as input', () => { expect( codexItemBody({ @@ -843,7 +859,11 @@ describe('codex item bodies', () => { summary: ['first', 'second'], content: [{ text: 'fallback' }] }) - ).toEqual({ kind: 'status', text: 'first\nsecond' }) + ).toEqual({ + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'first\nsecond' }] + }) }) it('refuses a value that is not a thread item at all', () => { diff --git a/src/main/codex/codex-structured-item-translation.ts b/src/main/codex/codex-structured-item-translation.ts index 52d7d5ae47f..34f07eefda7 100644 --- a/src/main/codex/codex-structured-item-translation.ts +++ b/src/main/codex/codex-structured-item-translation.ts @@ -85,6 +85,10 @@ export type CodexJournalItem = { handled: boolean } +function reasoningMessageBody(text: string): AgentJournalItemBody { + return { kind: 'message', role: 'reasoning', blocks: [{ type: 'text', text }] } +} + function commandItem(item: CodexThreadItem): CodexJournalItem { const output = readFirstString(item, ['aggregatedOutput', 'aggregated_output']) const bounded = output === null ? null : boundInlineText(output, DEFAULT_JOURNAL_PAYLOAD_LIMITS) @@ -272,7 +276,7 @@ export function codexJournalItem(item: CodexThreadItem): CodexJournalItem { handled: true } } - if (item.type === 'reasoning' || item.type === 'plan') { + if (item.type === 'reasoning') { const text = readTextContent(item, 'text') ?? readTextContent(item, 'summary') ?? @@ -281,7 +285,7 @@ export function codexJournalItem(item: CodexThreadItem): CodexJournalItem { body: text === null ? null - : { kind: 'status', text: boundInlineText(text, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text }, + : reasoningMessageBody(boundInlineText(text, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text), handled: true } } @@ -331,5 +335,11 @@ export function codexStreamingJournalItem(item: CodexThreadItem, text: string): } } const bounded = boundInlineText(text, DEFAULT_JOURNAL_PAYLOAD_LIMITS) - return { body: { kind: 'status', text: bounded.text }, handled: true } + return { + body: + item.type === 'reasoning' + ? reasoningMessageBody(bounded.text) + : { kind: 'status', text: bounded.text }, + handled: true + } } diff --git a/src/main/codex/codex-structured-journal-goal-admission.test.ts b/src/main/codex/codex-structured-journal-goal-admission.test.ts new file mode 100644 index 00000000000..5fe926d18c9 --- /dev/null +++ b/src/main/codex/codex-structured-journal-goal-admission.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' +import type { + AgentJournalItemBody, + AgentJournalItemIdentity +} from '../../shared/agent-session-journal-types' +import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { CodexJournalGoals } from './codex-structured-journal-goals' +import { + createCodexJournalTranslator, + MAX_CODEX_GENERIC_ROWS_PER_TURN +} from './codex-structured-journal-translation' +import { MAX_CODEX_GOAL_THREADS } from './codex-structured-journal-limits' + +const THREAD = '01a08cc2-f96e-76d0-bb74-88b9bc0b03fc' + +function goalFrame(goal: Record = {}): Record { + return { + threadId: THREAD, + turnId: 'turn-1', + goal: { + threadId: THREAD, + objective: 'Keep the current scratch directory tidy.', + status: 'active', + tokenBudget: null, + tokensUsed: 0, + timeUsedSeconds: 0, + createdAt: 1789067988, + updatedAt: 1789067988, + ...goal + } + } +} + +function texts(rows: readonly AgentJournalItemBody[]): string[] { + return rows.map((row) => (row.kind === 'status' ? row.text : '')) +} + +describe('codex goal lifecycle admission', () => { + it.each(['append', 'publish'] as const)( + 'retries the same goal after rejected %s without losing or duplicating its row', + (stage) => { + let reject = true + let successfulPublishes = 0 + const rows = new Map() + const identities: string[] = [] + const lifecycleOptions: boolean[] = [] + const sink = { + appendItem: () => {}, + appendTombstone: () => {}, + publish: () => {}, + tryAppendItem: (identity, body, options) => { + if (stage === 'append' && reject) { + return { accepted: false, reason: 'backpressure' } as const + } + const key = agentJournalItemKey(identity) + identities.push(key) + rows.set(key, body) + lifecycleOptions.push(options?.lifecycle === true) + return { accepted: true } as const + }, + tryPublish: (options) => { + if (stage === 'publish' && reject) { + return { accepted: false, reason: 'backpressure' } as const + } + successfulPublishes += 1 + lifecycleOptions.push(options?.lifecycle === true) + return { accepted: true } as const + } + } satisfies StructuredAgentSessionEventSink + const translator = createCodexJournalTranslator({ sink }) + const event = { + type: 'notification' as const, + sessionId: 'session', + threadId: THREAD, + method: 'thread/goal/updated', + params: goalFrame() + } + + expect(translator.handle(event)).toEqual({ accepted: false, reason: 'backpressure' }) + reject = false + expect(translator.handle(event)).toEqual({ accepted: true }) + + expect(rows.size).toBe(1) + expect(new Set(identities)).toHaveLength(1) + expect(successfulPublishes).toBe(1) + expect(lifecycleOptions.every(Boolean)).toBe(true) + translator.dispose() + } + ) + + it('does not let the generic-row cap permanently hide the first goal evidence', () => { + const rows: AgentJournalItemBody[] = [] + const sink = { + appendItem: (_identity: AgentJournalItemIdentity, body: AgentJournalItemBody) => + rows.push(body), + appendTombstone: () => {}, + publish: () => {} + } satisfies StructuredAgentSessionEventSink + const translator = createCodexJournalTranslator({ sink }) + for (let index = 0; index < MAX_CODEX_GENERIC_ROWS_PER_TURN; index += 1) { + translator.handle({ + type: 'notification', + sessionId: 'session', + threadId: THREAD, + method: 'process/exited', + params: { threadId: THREAD, turnId: 'turn-1', processId: `process-${index}` } + }) + } + + translator.handle({ + type: 'notification', + sessionId: 'session', + threadId: THREAD, + method: 'thread/goal/updated', + params: goalFrame() + }) + translator.handle({ + type: 'notification', + sessionId: 'session', + threadId: THREAD, + method: 'thread/goal/updated', + params: { ...goalFrame({ tokensUsed: 1 }), turnId: 'turn-2' } + }) + + expect(texts(rows).filter((text) => text.startsWith('Goal '))).toEqual([ + 'Goal set: Keep the current scratch directory tidy.' + ]) + translator.dispose() + }) + + it('keeps repeated lifecycle states distinct across status cycles and goal recreation', () => { + const rows = new Map() + const sink = { + appendItem: (identity: AgentJournalItemIdentity, body: AgentJournalItemBody) => + rows.set(agentJournalItemKey(identity), body), + appendTombstone: () => {}, + publish: () => {} + } satisfies StructuredAgentSessionEventSink + const goals = new CodexJournalGoals(sink) + const update = (goal: Record = {}) => + goals.handle({ threadId: THREAD, method: 'thread/goal/updated', params: goalFrame(goal) }) + const clear = () => + goals.handle({ + threadId: THREAD, + method: 'thread/goal/cleared', + params: { threadId: THREAD } + }) + + update() + update({ status: 'paused' }) + update() + clear() + update() + clear() + clear() + + expect(texts([...rows.values()])).toEqual([ + 'Goal set: Keep the current scratch directory tidy.', + 'Goal paused: Keep the current scratch directory tidy.', + 'Goal set: Keep the current scratch directory tidy.', + 'Goal cleared', + 'Goal set: Keep the current scratch directory tidy.', + 'Goal cleared' + ]) + goals.dispose() + }) + + it('bounds thread state with LRU eviction while stable identities keep one history row', () => { + const writes: string[] = [] + const rows = new Map() + const sink = { + appendItem: (identity: AgentJournalItemIdentity, body: AgentJournalItemBody) => { + const key = agentJournalItemKey(identity) + writes.push(key) + rows.set(key, body) + }, + appendTombstone: () => {}, + publish: () => {} + } satisfies StructuredAgentSessionEventSink + const goals = new CodexJournalGoals(sink) + const send = (threadId: string) => + goals.handle({ threadId, method: 'thread/goal/updated', params: goalFrame() }) + + for (let index = 0; index < MAX_CODEX_GOAL_THREADS; index += 1) { + send(`thread-${index}`) + } + const threadZeroIdentity = writes[0] + const threadOneIdentity = writes[1] + send('thread-0') + send('thread-over-cap') + expect(writes).toHaveLength(MAX_CODEX_GOAL_THREADS + 1) + + send('thread-1') + expect(writes).toHaveLength(MAX_CODEX_GOAL_THREADS + 2) + expect(writes.at(-1)).toBe(threadOneIdentity) + expect(rows).toHaveLength(MAX_CODEX_GOAL_THREADS + 1) + + send('thread-0') + expect(writes.at(-1)).toBe(threadOneIdentity) + expect(writes.filter((identity) => identity === threadZeroIdentity)).toHaveLength(1) + goals.dispose() + }) + + it('releases duplicate-suppression state on session clear and dispose', () => { + const identities: string[] = [] + const sink = { + appendItem: (identity: AgentJournalItemIdentity) => { + identities.push(agentJournalItemKey(identity)) + }, + appendTombstone: () => {}, + publish: () => {} + } satisfies StructuredAgentSessionEventSink + const goals = new CodexJournalGoals(sink) + const event = { threadId: THREAD, method: 'thread/goal/updated', params: goalFrame() } + + goals.handle(event) + goals.handle(event) + expect(identities).toHaveLength(1) + + goals.clear() + goals.handle(event) + expect(identities).toHaveLength(2) + expect(new Set(identities)).toHaveLength(1) + + goals.dispose() + goals.handle(event) + expect(identities).toHaveLength(3) + expect(new Set(identities)).toHaveLength(1) + }) +}) diff --git a/src/main/codex/codex-structured-journal-goal-resume.test.ts b/src/main/codex/codex-structured-journal-goal-resume.test.ts new file mode 100644 index 00000000000..4cf87b790bb --- /dev/null +++ b/src/main/codex/codex-structured-journal-goal-resume.test.ts @@ -0,0 +1,365 @@ +import { describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' +import type { + AgentJournalItemBody, + AgentJournalItemIdentity, + AgentJournalRenderItem +} from '../../shared/agent-session-journal-types' +import { + createDeferredStructuredAgentSessionEventSink, + type StructuredAgentSessionEventTarget +} from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { CodexJournalGoals } from './codex-structured-journal-goals' +import { MAX_CODEX_GOAL_THREADS } from './codex-structured-journal-limits' + +const THREAD = '01a08cc2-f96e-76d0-bb74-88b9bc0b03fc' + +function goalFrame(goal: Record = {}): Record { + return { + threadId: THREAD, + turnId: 'turn-1', + goal: { + threadId: THREAD, + objective: 'Keep the current scratch directory tidy.', + status: 'active', + tokenBudget: null, + tokensUsed: 0, + timeUsedSeconds: 0, + createdAt: 1789067988, + updatedAt: 1789067988, + ...goal + } + } +} + +function goalJournal( + options: Parameters[0] = {} +) { + let rowSequence = 0 + let publishes = 0 + let epochNumber = 1 + let visits = 0 + let visitedItems = 0 + const rows = new Map() + const writes: string[] = [] + const deferred = createDeferredStructuredAgentSessionEventSink(options) + const journal = { + get epoch() { + return `epoch-${epochNumber}` + }, + appendItem: async (identity: AgentJournalItemIdentity, body: AgentJournalItemBody) => { + rowSequence += 1 + const itemId = agentJournalItemKey(identity) + const existing = rows.get(itemId) + const revision = (existing?.revision ?? 0) + 1 + writes.push(itemId) + rows.set(itemId, { + itemId, + body, + revision, + sequence: existing?.sequence ?? rowSequence, + observedAt: existing?.observedAt ?? rowSequence + }) + return { cursor: { epoch: `epoch-${epochNumber}`, sequence: rowSequence }, itemId, revision } + }, + snapshot: () => ({ + sessionId: 'session', + cursor: { epoch: `epoch-${epochNumber}`, sequence: rowSequence }, + items: [...rows.values()].sort((left, right) => left.sequence - right.sequence), + submissions: [] + }), + visitItems: (visit: (itemId: string, sequence: number) => void) => { + visits += 1 + for (const item of rows.values()) { + visitedItems += 1 + visit(item.itemId, item.sequence) + } + } + } as unknown as StructuredAgentSessionEventTarget['journal'] + const target = { + journal, + fence: 1, + publish: () => { + publishes += 1 + } + } + deferred.bind(target) + return { + sink: deferred.sink, + writes, + rows: () => journal.snapshot().items, + publishes: () => publishes, + visits: () => visits, + visitedItems: () => visitedItems, + seedProviderItems: (count: number) => { + for (let index = 0; index < count; index += 1) { + rowSequence += 1 + const identity = { + provider: 'codex' as const, + threadId: THREAD, + turnId: `seed-${index}`, + ordinal: 0 + } + const itemId = agentJournalItemKey(identity) + rows.set(itemId, { + itemId, + body: { kind: 'message', role: 'assistant', blocks: [] }, + revision: 1, + sequence: rowSequence, + observedAt: rowSequence + }) + } + }, + replaceEpoch: () => { + epochNumber += 1 + rowSequence = 0 + rows.clear() + }, + rebind: () => deferred.bind(target), + unbind: deferred.unbind, + drained: deferred.drained + } +} + +function texts(rows: readonly AgentJournalItemBody[]): string[] { + return rows.map((row) => (row.kind === 'status' ? row.text : '')) +} + +describe('codex goal lifecycle resume', () => { + it('does not append a cleared snapshot when the journal has no prior goal occurrence', async () => { + const journal = goalJournal() + journal.unbind() + const resumed = new CodexJournalGoals(journal.sink) + + expect( + resumed.handle({ + threadId: THREAD, + method: 'thread/goal/cleared', + params: { threadId: THREAD, turnId: null, clearedAt: 1789068999 } + }) + ).toEqual({ accepted: true }) + expect(journal.writes).toHaveLength(0) + + journal.rebind() + await journal.drained() + + expect(journal.writes).toHaveLength(0) + expect(journal.publishes()).toBe(0) + resumed.dispose() + }) + + it('does not revisit durable history for accounting-only updates', async () => { + const journal = goalJournal() + const goals = new CodexJournalGoals(journal.sink) + goals.handle({ threadId: THREAD, method: 'thread/goal/updated', params: goalFrame() }) + await journal.drained() + const visits = journal.visits() + + for (let index = 1; index <= 10; index += 1) { + goals.handle({ + threadId: THREAD, + method: 'thread/goal/updated', + params: goalFrame({ + tokensUsed: index * 1_000, + timeUsedSeconds: index, + updatedAt: 1789067988 + index + }) + }) + } + await journal.drained() + + expect(journal.visits()).toBe(visits) + expect(journal.writes).toHaveLength(1) + goals.dispose() + }) + + it('rebuilds dedupe state after the journal epoch is replaced', async () => { + const journal = goalJournal() + const goals = new CodexJournalGoals(journal.sink) + const event = { threadId: THREAD, method: 'thread/goal/updated', params: goalFrame() } + + goals.handle(event) + await journal.drained() + expect(journal.writes).toHaveLength(1) + + journal.replaceEpoch() + goals.handle(event) + await journal.drained() + + expect(journal.writes).toHaveLength(2) + expect(texts(journal.rows().map((row) => row.body))).toEqual([ + 'Goal set: Keep the current scratch directory tidy.' + ]) + expect(journal.visits()).toBe(2) + goals.dispose() + }) + + it('visits a large journal once per epoch when thread churn exceeds the transient LRU', async () => { + const journal = goalJournal() + journal.seedProviderItems(10_000) + const goals = new CodexJournalGoals(journal.sink) + const threadCount = MAX_CODEX_GOAL_THREADS + 1 + const sendRound = () => { + for (let index = 0; index < threadCount; index += 1) { + goals.handle({ + threadId: `thread-${index}`, + method: 'thread/goal/updated', + params: goalFrame() + }) + } + } + + sendRound() + await journal.drained() + for (let round = 0; round < 10; round += 1) { + sendRound() + } + await journal.drained() + + expect(journal.visits()).toBe(1) + expect(journal.visitedItems()).toBe(10_000) + expect(journal.writes).toHaveLength(threadCount) + goals.dispose() + }) + + it('resolves queued thread transitions from one shared durable projection', async () => { + const journal = goalJournal() + journal.seedProviderItems(10_000) + journal.unbind() + const goals = new CodexJournalGoals(journal.sink) + + for (let index = 0; index < MAX_CODEX_GOAL_THREADS; index += 1) { + goals.handle({ + threadId: `thread-${index}`, + method: 'thread/goal/updated', + params: goalFrame() + }) + } + expect(journal.visits()).toBe(0) + + journal.rebind() + await journal.drained() + + expect(journal.visits()).toBe(1) + expect(journal.visitedItems()).toBe(10_000) + expect(journal.writes).toHaveLength(MAX_CODEX_GOAL_THREADS) + goals.dispose() + }) + + it('retries a journal-derived transition after lifecycle backpressure', async () => { + const journal = goalJournal({ watermarks: { maxLifecycleQueuedOperations: 1 } }) + const goals = new CodexJournalGoals(journal.sink) + journal.unbind() + + expect( + goals.handle({ + threadId: THREAD, + method: 'thread/goal/updated', + params: goalFrame() + }) + ).toEqual({ accepted: true }) + expect( + goals.handle({ + threadId: THREAD, + method: 'thread/goal/updated', + params: goalFrame({ status: 'paused' }) + }) + ).toEqual({ accepted: false, reason: 'backpressure' }) + + journal.rebind() + await journal.drained() + expect( + goals.handle({ + threadId: THREAD, + method: 'thread/goal/updated', + params: goalFrame({ status: 'paused' }) + }) + ).toEqual({ accepted: true }) + await journal.drained() + + expect(texts(journal.rows().map((row) => row.body))).toEqual([ + 'Goal set: Keep the current scratch directory tidy.', + 'Goal paused: Keep the current scratch directory tidy.' + ]) + goals.dispose() + }) + + it.each([ + { + name: 'paused', + beforeResume: ['active', 'paused'] as const, + resumed: { method: 'thread/goal/updated', goal: { status: 'paused' } }, + expected: ['Goal set', 'Goal paused'] + }, + { + name: 'cleared', + beforeResume: ['active', 'cleared'] as const, + resumed: { method: 'thread/goal/cleared', goal: {} }, + expected: ['Goal set', 'Goal cleared'] + }, + { + name: 'active after a pause', + beforeResume: ['active', 'paused', 'active'] as const, + resumed: { method: 'thread/goal/updated', goal: { status: 'active' } }, + expected: ['Goal set', 'Goal paused', 'Goal set'] + } + ])('does not duplicate a $name snapshot after translator recreation', async (scenario) => { + const journal = goalJournal() + const send = ( + goals: CodexJournalGoals, + state: (typeof scenario.beforeResume)[number] + ): void => { + goals.handle({ + threadId: THREAD, + method: state === 'cleared' ? 'thread/goal/cleared' : 'thread/goal/updated', + params: state === 'cleared' ? { threadId: THREAD } : goalFrame({ status: state }) + }) + } + + const prior = new CodexJournalGoals(journal.sink) + for (const state of scenario.beforeResume) { + send(prior, state) + } + await journal.drained() + const acceptedOccurrence = journal.writes.at(-1) + const writesBeforeResume = journal.writes.length + const publishesBeforeResume = journal.publishes() + const acceptedBody = journal.rows().find((row) => row.itemId === acceptedOccurrence)?.body + prior.dispose() + journal.unbind() + + const resumed = new CodexJournalGoals(journal.sink) + resumed.handle({ + threadId: THREAD, + method: scenario.resumed.method, + params: + scenario.resumed.method === 'thread/goal/cleared' + ? { threadId: THREAD, turnId: null, clearedAt: 1789068999 } + : { + ...goalFrame({ + ...scenario.resumed.goal, + tokensUsed: 12_345, + timeUsedSeconds: 42, + updatedAt: 1789068999 + }), + turnId: null + } + }) + expect(journal.writes).toHaveLength(writesBeforeResume) + journal.rebind() + await journal.drained() + + expect(texts(journal.rows().map((row) => row.body))).toEqual( + scenario.expected.map((prefix) => + prefix === 'Goal cleared' ? prefix : `${prefix}: Keep the current scratch directory tidy.` + ) + ) + expect(journal.writes).toHaveLength(writesBeforeResume) + expect(journal.publishes()).toBe(publishesBeforeResume) + expect(journal.writes.at(-1)).toBe(acceptedOccurrence) + expect(journal.rows().find((row) => row.itemId === acceptedOccurrence)?.body).toEqual( + acceptedBody + ) + resumed.dispose() + }) +}) diff --git a/src/main/codex/codex-structured-journal-goal-rows.test.ts b/src/main/codex/codex-structured-journal-goal-rows.test.ts new file mode 100644 index 00000000000..4c3837b4373 --- /dev/null +++ b/src/main/codex/codex-structured-journal-goal-rows.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it, vi } from 'vitest' +import type { AgentJournalItemBody } from '../../shared/agent-session-journal-types' +import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { CodexJournalGenericFrames } from './codex-structured-journal-generic-frames' +import { CodexJournalGoals } from './codex-structured-journal-goals' + +const THREAD = '01a08cc2-f96e-76d0-bb74-88b9bc0b03fc' + +function goalFrame(goal: Record): Record { + return { + threadId: THREAD, + turnId: '01a08cc2-fa6a-7541-a4c7-67d98a6e40c2', + goal: { + threadId: THREAD, + objective: 'Keep the current scratch directory tidy.', + status: 'active', + tokenBudget: null, + tokensUsed: 0, + timeUsedSeconds: 0, + createdAt: 1789067988, + updatedAt: 1789067988, + ...goal + } + } +} + +function frames(): { + rows: AgentJournalItemBody[] + frames: Pick +} { + const rows: AgentJournalItemBody[] = [] + const sink = { + appendItem: (_identity: unknown, body: AgentJournalItemBody) => { + rows.push(body) + }, + publish: vi.fn() + } as unknown as StructuredAgentSessionEventSink + const goals = new CodexJournalGoals(sink) + const generic = new CodexJournalGenericFrames({ sink }, () => null) + return { + rows, + frames: { + appendUnhandled: (kind, payload, threadId = 'session') => { + const method = kind.startsWith('notification:') ? kind.slice('notification:'.length) : kind + return ( + goals.handle({ threadId, method, params: payload }) ?? + generic.appendUnhandled(kind, payload, threadId) + ) + } + } + } +} + +function texts(rows: AgentJournalItemBody[]): string[] { + return rows.map((row) => (row as { text?: string }).text ?? '') +} + +describe('codex goal frames as journal rows', () => { + it('writes one row when the goal appears', () => { + const { rows, frames: generic } = frames() + + generic.appendUnhandled('notification:thread/goal/updated', goalFrame({}), THREAD) + + expect(texts(rows)).toEqual(['Goal set: Keep the current scratch directory tidy.']) + }) + + it('does not repeat the row while only the counters climb', () => { + const { rows, frames: generic } = frames() + + // Codex re-sends the goal through the turn as accounting ticks; a live session + // emitted these two seconds apart with nothing else changed. + generic.appendUnhandled('notification:thread/goal/updated', goalFrame({}), THREAD) + generic.appendUnhandled( + 'notification:thread/goal/updated', + goalFrame({ tokensUsed: 23869, timeUsedSeconds: 8, updatedAt: 1789067905 }), + THREAD + ) + generic.appendUnhandled( + 'notification:thread/goal/updated', + goalFrame({ tokensUsed: 25999, timeUsedSeconds: 12, updatedAt: 1789067912 }), + THREAD + ) + + expect(rows).toHaveLength(1) + }) + + it('writes a second row when the status changes', () => { + const { rows, frames: generic } = frames() + + generic.appendUnhandled('notification:thread/goal/updated', goalFrame({}), THREAD) + generic.appendUnhandled( + 'notification:thread/goal/updated', + goalFrame({ status: 'complete', tokensUsed: 31_000 }), + THREAD + ) + + expect(texts(rows)).toEqual([ + 'Goal set: Keep the current scratch directory tidy.', + 'Goal complete: Keep the current scratch directory tidy.' + ]) + }) + + it('writes a row when the objective is replaced', () => { + const { rows, frames: generic } = frames() + + generic.appendUnhandled('notification:thread/goal/updated', goalFrame({}), THREAD) + generic.appendUnhandled( + 'notification:thread/goal/updated', + goalFrame({ objective: 'Ship the parser.' }), + THREAD + ) + + expect(texts(rows)).toEqual([ + 'Goal set: Keep the current scratch directory tidy.', + 'Goal set: Ship the parser.' + ]) + }) + + it('writes a row when the goal is cleared, and again if a new goal follows', () => { + const { rows, frames: generic } = frames() + + generic.appendUnhandled('notification:thread/goal/updated', goalFrame({}), THREAD) + generic.appendUnhandled('notification:thread/goal/cleared', { threadId: THREAD }, THREAD) + generic.appendUnhandled( + 'notification:thread/goal/updated', + goalFrame({ createdAt: 1789067989, updatedAt: 1789067989 }), + THREAD + ) + + expect(texts(rows)).toEqual([ + 'Goal set: Keep the current scratch directory tidy.', + 'Goal cleared', + 'Goal set: Keep the current scratch directory tidy.' + ]) + }) + + it('keeps each thread’s goal separate', () => { + const { rows, frames: generic } = frames() + const other = '01a08cc3-0000-7000-8000-000000000000' + + generic.appendUnhandled('notification:thread/goal/updated', goalFrame({}), THREAD) + generic.appendUnhandled('notification:thread/goal/updated', goalFrame({}), other) + + expect(rows).toHaveLength(2) + }) + + it('leaves non-goal frames to the existing path', () => { + const { rows, frames: generic } = frames() + + generic.appendUnhandled('notification:warning', { message: 'disk almost full' }, THREAD) + generic.appendUnhandled('notification:warning', { message: 'disk almost full' }, THREAD) + + // No goal dedupe applies, so both warnings still land. + expect(rows).toHaveLength(2) + }) +}) diff --git a/src/main/codex/codex-structured-journal-goals.ts b/src/main/codex/codex-structured-journal-goals.ts new file mode 100644 index 00000000000..83bb66551ae --- /dev/null +++ b/src/main/codex/codex-structured-journal-goals.ts @@ -0,0 +1,177 @@ +import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types' +import { unhandledProviderFrameJournalItem } from '../native-chat/agent-session-wire/unhandled-provider-frame' +import type { + StructuredAgentSessionEventSink, + StructuredAgentSessionLifecycleJournal +} from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { + codexGoalJournalDigest, + codexGoalJournalIdentity, + parseCodexGoalJournalItemId, + type CodexGoalJournalState +} from './codex-goal-journal-identity' +import { + codexGoalGeneration, + codexGoalRowSignature, + isCodexGoalFrameMethod +} from './codex-goal-journal-rows' +import { + CODEX_JOURNAL_ADMITTED, + type CodexJournalTranslationAdmission +} from './codex-structured-journal-contracts' +import { MAX_CODEX_GOAL_THREADS } from './codex-structured-journal-limits' +import { appendCodexLifecycleTransition } from './codex-structured-journal-sink' + +type GoalThreadState = { + signature: string + occurrence: string +} + +/** Persists provider-owned goal lifecycle notifications outside generic-row policy. */ +export class CodexJournalGoals { + private readonly stateByThread = new Map() + private readonly durableStateByThread = new Map() + private durableJournal: StructuredAgentSessionLifecycleJournal | null = null + private durableEpoch: string | null = null + private transientEpoch: string | null = null + + constructor(private readonly sink: StructuredAgentSessionEventSink) {} + + handle(event: { + threadId: string + method: string + params: unknown + }): CodexJournalTranslationAdmission | null { + if (!isCodexGoalFrameMethod(event.method)) { + return null + } + const signature = codexGoalRowSignature(event.method, event.params) + if (signature === null) { + return null + } + this.synchronizeTransientEpoch() + const thread = codexGoalJournalDigest(event.threadId) + const reportedGeneration = codexGoalGeneration(event.params) + const providerGeneration = + reportedGeneration === null ? null : codexGoalJournalDigest(`provider:${reportedGeneration}`) + const signatureKey = codexGoalJournalDigest(`${signature}\u0000${providerGeneration ?? ''}`) + const previous = this.stateByThread.get(thread) + if (previous?.signature === signatureKey) { + this.remember(thread, previous) + return CODEX_JOURNAL_ADMITTED + } + const occurrence = previous + ? codexGoalJournalDigest(JSON.stringify([previous.occurrence, signatureKey])) + : codexGoalJournalDigest(JSON.stringify([thread, signatureKey])) + const state = { signature: signatureKey, occurrence } + const translated = unhandledProviderFrameJournalItem( + 'codex', + `notification:${event.method}`, + event.params + ) + if (!translated) { + return { accepted: false, reason: 'untranslated' } + } + const admission = appendCodexLifecycleTransition( + this.sink, + codexGoalJournalIdentity(thread, signatureKey, occurrence), + translated.body, + (journal) => + this.persistedGoalIdentity( + journal, + thread, + signatureKey, + event.method === 'thread/goal/cleared' + ) + ) + if (!admission.accepted) { + return admission + } + this.remember(thread, state) + return CODEX_JOURNAL_ADMITTED + } + + clear(): void { + this.stateByThread.clear() + this.durableStateByThread.clear() + this.durableJournal = null + this.durableEpoch = null + this.transientEpoch = null + } + + dispose(): void { + this.clear() + } + + private remember(thread: string, state: GoalThreadState): void { + this.stateByThread.delete(thread) + this.stateByThread.set(thread, state) + while (this.stateByThread.size > MAX_CODEX_GOAL_THREADS) { + const oldest = this.stateByThread.keys().next().value + if (typeof oldest !== 'string') { + break + } + this.stateByThread.delete(oldest) + } + } + + private synchronizeTransientEpoch(): void { + const epoch = this.sink.journalEpoch?.() ?? null + if (epoch === null) { + return + } + if (this.transientEpoch !== null && this.transientEpoch !== epoch) { + this.stateByThread.clear() + } + this.transientEpoch = epoch + } + + private persistedGoalIdentity( + journal: StructuredAgentSessionLifecycleJournal, + thread: string, + signature: string, + requirePrevious: boolean + ): AgentJournalItemIdentity | null { + this.seedDurableState(journal) + const previous = this.durableStateByThread.get(thread) ?? null + if (previous?.signature === signature) { + return null + } + // Codex sends a cleared snapshot while resuming threads that never had a goal. + if (previous === null && requirePrevious) { + return null + } + const occurrence = previous + ? codexGoalJournalDigest(JSON.stringify([previous.occurrence, signature])) + : codexGoalJournalDigest(JSON.stringify([thread, signature])) + this.durableStateByThread.set(thread, { signature, occurrence }) + return codexGoalJournalIdentity(thread, signature, occurrence) + } + + private seedDurableState(journal: StructuredAgentSessionLifecycleJournal): void { + if (this.durableJournal === journal && this.durableEpoch === journal.epoch) { + return + } + const latest = new Map() + journal.visitItems((itemId, sequence) => { + const state = parseCodexGoalJournalItemId(itemId) + const previous = state ? latest.get(state.thread) : undefined + if (state && (!previous || sequence > previous.sequence)) { + latest.set(state.thread, { state, sequence }) + } + }) + this.durableStateByThread.clear() + for (const [thread, { state }] of latest) { + this.durableStateByThread.set(thread, { + signature: state.signature, + occurrence: state.occurrence + }) + } + this.durableJournal = journal + this.durableEpoch = journal.epoch + if (this.transientEpoch !== null && this.transientEpoch !== journal.epoch) { + this.stateByThread.clear() + } + this.transientEpoch = journal.epoch + } +} diff --git a/src/main/codex/codex-structured-journal-limits.ts b/src/main/codex/codex-structured-journal-limits.ts index 5137ea8dd16..4f56cd0e282 100644 --- a/src/main/codex/codex-structured-journal-limits.ts +++ b/src/main/codex/codex-structured-journal-limits.ts @@ -2,6 +2,8 @@ export const MAX_CODEX_GENERIC_ROWS_PER_TURN = 8 export const MAX_CODEX_GENERIC_TURN_BUCKETS = 64 export const MAX_CODEX_GENERIC_BOOKKEEPING_ENTRIES = 128 export const MAX_CODEX_GENERIC_BOOKKEEPING_BYTES = 32 * 1024 +/** Goal duplicate-suppression state is LRU-bounded per live translator. */ +export const MAX_CODEX_GOAL_THREADS = 64 export const MAX_CODEX_ACTIVE_ITEMS = 256 export const MAX_CODEX_PENDING_PROMPTS = 128 export const MAX_CODEX_IDENTITY_ENTRIES = 512 diff --git a/src/main/codex/codex-structured-journal-sink.ts b/src/main/codex/codex-structured-journal-sink.ts index 5c4ecec9658..7da381def41 100644 --- a/src/main/codex/codex-structured-journal-sink.ts +++ b/src/main/codex/codex-structured-journal-sink.ts @@ -4,6 +4,7 @@ import type { } from '../../shared/agent-session-journal-types' import type { StructuredAgentSessionEventSink, + StructuredAgentSessionLifecycleIdentityResolver, StructuredAgentSessionSinkAdmission } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' import type { CodexPendingJournalPrompt } from './codex-structured-journal-settlement' @@ -28,6 +29,21 @@ export function appendCodexLifecycleItem( return CODEX_JOURNAL_ADMITTED } +export function appendCodexLifecycleTransition( + sink: StructuredAgentSessionEventSink, + identitySizeBound: AgentJournalItemIdentity, + body: AgentJournalItemBody, + resolveIdentity: StructuredAgentSessionLifecycleIdentityResolver +): CodexJournalTranslationAdmission { + if (sink.tryAppendLifecycleTransition) { + return criticalAdmission( + sink.tryAppendLifecycleTransition(identitySizeBound, body, resolveIdentity) + ) + } + const admission = appendCodexLifecycleItem(sink, identitySizeBound, body) + return admission.accepted ? publishCodexLifecycle(sink) : admission +} + export function publishCodexLifecycle( sink: StructuredAgentSessionEventSink ): CodexJournalTranslationAdmission { diff --git a/src/main/codex/codex-structured-journal-translation-settlement.test.ts b/src/main/codex/codex-structured-journal-translation-settlement.test.ts index f8a5c7a1671..b5e60699ce1 100644 --- a/src/main/codex/codex-structured-journal-translation-settlement.test.ts +++ b/src/main/codex/codex-structured-journal-translation-settlement.test.ts @@ -799,8 +799,9 @@ describe('codex journal translation', () => { const reduced = new Map(tap.rows.map((row) => [row.key, row.body])) expect(reduced.get('orca:codex-item%3Athread-abc%3Ar-1')).toEqual({ - kind: 'status', - text: 'thinking' + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'thinking' }] }) expect(reduced.get('orca:codex-item%3Athread-abc%3Apatch-1')).toMatchObject({ kind: 'diff', diff --git a/src/main/codex/codex-structured-journal-translation.ts b/src/main/codex/codex-structured-journal-translation.ts index 2907b34bc70..5c1e97310bc 100644 --- a/src/main/codex/codex-structured-journal-translation.ts +++ b/src/main/codex/codex-structured-journal-translation.ts @@ -7,6 +7,7 @@ import { CodexSubagentRoster } from './codex-subagent-roster' import { readCodexThreadItem } from './codex-structured-item-translation' import { CodexJournalGenericFrames } from './codex-structured-journal-generic-frames' import { CodexJournalCompactions } from './codex-structured-journal-compactions' +import { CodexJournalGoals } from './codex-structured-journal-goals' import { CodexJournalItems } from './codex-structured-journal-items' import { CodexJournalPrompts } from './codex-structured-journal-prompts' import { @@ -51,6 +52,7 @@ export function createCodexJournalTranslator( const genericFrames = new CodexJournalGenericFrames(deps, (threadId) => activeTurns.current(threadId) ) + const goals = new CodexJournalGoals(deps.sink) const items = new CodexJournalItems( deps, (threadId) => activeTurns.current(threadId), @@ -178,6 +180,7 @@ export function createCodexJournalTranslator( prompts.pending.clear() activeTurns.clear() compactions.clear() + goals.clear() return CODEX_JOURNAL_ADMITTED } if (event.type === 'notification') { @@ -221,6 +224,10 @@ export function createCodexJournalTranslator( if (compaction) { return publishActivity(event, compaction) } + const goal = goals.handle(event) + if (goal) { + return publishActivity(event, goal) + } if (event.method === CODEX_TOKEN_USAGE_METHOD) { // Classified `status-chrome`, so the generic-frame path swallows it // before the journal. The roster consumes it as a typed notification. @@ -274,6 +281,7 @@ export function createCodexJournalTranslator( subagents.dispose() activeTurns.clear() compactions.clear() + goals.dispose() } } } diff --git a/src/main/daemon/headless-osc-link-ranges.test.ts b/src/main/daemon/headless-osc-link-ranges.test.ts new file mode 100644 index 00000000000..cf8f9f757e2 --- /dev/null +++ b/src/main/daemon/headless-osc-link-ranges.test.ts @@ -0,0 +1,63 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { HeadlessEmulator } from './headless-emulator' + +// Why this suite: collectHeadlessOscLinkRanges skips its per-cell scan when +// xterm holds no OSC 8 registration. That skip is only safe if it can never +// fire while a link is reachable, so each case below pins one way it could. +let emulator: HeadlessEmulator | undefined + +const link = (uri: string, text: string): string => `\x1b]8;;${uri}\x1b\\${text}\x1b]8;;\x1b\\` + +afterEach(() => { + emulator?.dispose() + emulator = undefined +}) + +describe('headless OSC link ranges', () => { + it('finds a link written into the buffer', async () => { + emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + await emulator.write(`before ${link('https://example.com/a', 'CLICK')} after`) + + const ranges = emulator.getSnapshot().oscLinks ?? [] + expect(ranges).toHaveLength(1) + expect(ranges[0]).toMatchObject({ row: 0, uri: 'https://example.com/a' }) + }) + + it('returns nothing for a buffer that never emitted a link', async () => { + emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + await emulator.write('plain output with no hyperlink\r\n'.repeat(50)) + + expect(emulator.getSnapshot().oscLinks).toEqual([]) + }) + + // The dangerous case: restored ranges are seeded without xterm registering + // anything, so an early-out keyed only on the registry would drop them. + it('still maps restored ranges when the buffer itself has no link', async () => { + emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + await emulator.write('restored row') + const restored = { row: 0, startCol: 0, endCol: 4, uri: 'https://example.com/restored' } + emulator.setRestoredOscLinks([restored]) + + expect(emulator.getSnapshot().oscLinks).toEqual([restored]) + }) + + it('finds links far down a long scrollback, not just the visible screen', async () => { + emulator = new HeadlessEmulator({ cols: 80, rows: 24, scrollback: 5_000 }) + await emulator.write(`${link('https://example.com/top', 'TOP')}\r\n`) + await emulator.write('filler\r\n'.repeat(2_000)) + + const ranges = emulator.getSnapshot({ scrollbackRows: 5_000 }).oscLinks ?? [] + expect(ranges.map((range) => range.uri)).toContain('https://example.com/top') + }) + + it('keeps every distinct link when several are present', async () => { + emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + await emulator.write( + `${link('https://example.com/1', 'ONE')} ${link('https://example.com/2', 'TWO')}` + ) + + const uris = (emulator.getSnapshot().oscLinks ?? []).map((range) => range.uri) + expect(uris).toContain('https://example.com/1') + expect(uris).toContain('https://example.com/2') + }) +}) diff --git a/src/main/daemon/headless-osc-link-ranges.ts b/src/main/daemon/headless-osc-link-ranges.ts index 418a0c65166..ea017a7b928 100644 --- a/src/main/daemon/headless-osc-link-ranges.ts +++ b/src/main/daemon/headless-osc-link-ranges.ts @@ -1,10 +1,14 @@ -import type { Terminal } from '@xterm/headless' +import type { IBufferCell, IBufferLine, Terminal } from '@xterm/headless' import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges' type TerminalWithOscLinks = Terminal & { _core?: { _oscLinkService?: { getLinkData: (linkId: number) => { uri?: string } | undefined + // Why read it: xterm registers every OSC 8 id here, so an empty registry + // proves the buffer holds no hyperlink and the per-cell scan can be skipped. + // Optional because it is private — an xterm that renames it just scans. + _dataByLinkId?: { size?: number } } } } @@ -14,6 +18,11 @@ type CellWithOscLink = { hasExtendedAttrs?: () => boolean } +/** True when xterm holds no OSC 8 registration at all, so no cell can carry one. */ +function hasNoRegisteredOscLinks(service: { _dataByLinkId?: { size?: number } }): boolean { + return service._dataByLinkId?.size === 0 +} + export function collectHeadlessOscLinkRanges( terminal: Terminal, scrollbackRows: number | undefined, @@ -26,9 +35,19 @@ export function collectHeadlessOscLinkRanges( return [] } const buffer = terminal.buffer.active + // Why before the scan: the walk below reads every cell of every row, and a + // session that never emitted a hyperlink — the overwhelming majority — would + // pay that for a guaranteed-empty result. `restoredLinks` still needs mapping. + if (hasNoRegisteredOscLinks(service) && restoredLinks.length === 0) { + return [] + } const startRow = scrollbackRows === undefined ? 0 : Math.max(0, buffer.length - terminal.rows - scrollbackRows) const ranges: TerminalOscLinkRange[] = [] + // Why one cell for the whole walk: xterm's getCell allocates a fresh CellData + // per call unless handed a target, which is a per-cell allocation across the + // entire scrollback. See the IBufferLine.getCell docs. + const scratchCell = buffer.getNullCell() for (let row = startRow; row < buffer.length; row += 1) { const line = buffer.getLine(row) if (!line) { @@ -38,7 +57,7 @@ export function collectHeadlessOscLinkRanges( let currentUrlId = 0 let currentStart = -1 for (let col = 0; col <= lineLength; col += 1) { - const urlId = col < lineLength ? getOscLinkIdAtCell(line, col) : 0 + const urlId = col < lineLength ? getOscLinkIdAtCell(line, col, scratchCell) : 0 if (urlId === currentUrlId) { continue } @@ -83,8 +102,8 @@ function dedupeOscLinkRanges(ranges: TerminalOscLinkRange[]): TerminalOscLinkRan }) } -function getOscLinkIdAtCell(line: { getCell: (col: number) => unknown }, col: number): number { - const cell = line.getCell(col) as CellWithOscLink | undefined +function getOscLinkIdAtCell(line: IBufferLine, col: number, scratchCell: IBufferCell): number { + const cell = line.getCell(col, scratchCell) as (IBufferCell & CellWithOscLink) | undefined // Why: OSC link IDs live in extended cell attrs; missing attrs means no link. return cell?.hasExtendedAttrs?.() && cell.extended?.urlId ? cell.extended.urlId : 0 } diff --git a/src/main/global-fetch-call-site-audit.test.ts b/src/main/global-fetch-call-site-audit.test.ts index 39bd157fc1d..a16171ea10c 100644 --- a/src/main/global-fetch-call-site-audit.test.ts +++ b/src/main/global-fetch-call-site-audit.test.ts @@ -26,7 +26,8 @@ const AUDITED_GLOBAL_FETCH_LINES = new Map([ ['main/runtime/push/push-gateway-client.ts', 1], ['main/runtime/relay/relay-http-client.ts', 2], ['main/runtime/relay/relay-region-catalog-fetch.ts', 1], - ['main/runtime/relay/relay-region-preference.ts', 2], + // Measurement reuses the audited catalog/probe consumers, which consume or cancel every body. + ['main/runtime/relay/relay-region-preference.ts', 3], ['main/runtime/relay/relay-region-probe.ts', 1], ['main/source-control/hosted-review-api-request.ts', 1], ['main/speech/openai-transcription-client.ts', 1], diff --git a/src/main/ipc/runtime-environment-capability-evidence.test.ts b/src/main/ipc/runtime-environment-capability-evidence.test.ts index 4671326ef02..8c9bce5f272 100644 --- a/src/main/ipc/runtime-environment-capability-evidence.test.ts +++ b/src/main/ipc/runtime-environment-capability-evidence.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it } from 'vitest' import type { PairingOffer } from '../../shared/pairing' import { advanceRuntimeEnvironmentCapabilityIncarnation, @@ -17,7 +17,6 @@ describe('runtime environment capability evidence', () => { it('accepts evidence by dispatch order instead of completion order', () => { const older = captureRuntimeEnvironmentCapabilityEvidence('env', pairing()) const newer = captureRuntimeEnvironmentCapabilityEvidence('env', pairing()) - const pause = vi.fn() expect( applyRuntimeEnvironmentCapabilityVerdict({ @@ -30,12 +29,10 @@ describe('runtime environment capability evidence', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence: older, verdict: 'absent', - runtimeId: 'runtime-old', - onAbsent: pause + runtimeId: 'runtime-old' }) ).toBe(false) - expect(pause).not.toHaveBeenCalled() expect(isRuntimeEnvironmentCapabilityPaused('env')).toBe(false) }) diff --git a/src/main/ipc/runtime-environment-capability-evidence.ts b/src/main/ipc/runtime-environment-capability-evidence.ts index d32bda584e9..197ec71f4f8 100644 --- a/src/main/ipc/runtime-environment-capability-evidence.ts +++ b/src/main/ipc/runtime-environment-capability-evidence.ts @@ -68,8 +68,6 @@ export function applyRuntimeEnvironmentCapabilityVerdict(args: { evidence: RuntimeEnvironmentCapabilityEvidence verdict: RuntimeEnvironmentCapabilityVerdict runtimeId: string - onCapable?: () => void - onAbsent?: () => void }): boolean { const state = stateFor(args.evidence.environmentId) if ( @@ -83,11 +81,6 @@ export function applyRuntimeEnvironmentCapabilityVerdict(args: { verdict: args.verdict, runtimeId: args.runtimeId } - if (args.verdict === 'capable') { - args.onCapable?.() - } else { - args.onAbsent?.() - } return true } diff --git a/src/main/ipc/runtime-environment-connectivity-handlers.ts b/src/main/ipc/runtime-environment-connectivity-handlers.ts index 1e267d8675a..bfbc63847c4 100644 --- a/src/main/ipc/runtime-environment-connectivity-handlers.ts +++ b/src/main/ipc/runtime-environment-connectivity-handlers.ts @@ -20,6 +20,8 @@ import { verifyAndAddRuntimeEnvironmentFromPairingCode } from './runtime-environ import { clearRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence' import { closeRemoteRuntimeRequestConnection, + getRuntimeEnvironmentStatusOwner, + getRuntimeEnvironmentStatusSnapshots, retryRemoteRuntimeSharedControlConnectionNow } from './runtime-environment-request-connections' import { @@ -29,7 +31,6 @@ import { } from './runtime-environment-manual-disconnect' import { callRuntimeEnvironment, - clearSharedControlSupport, getRuntimeEnvironmentStatus } from './runtime-environment-transport-routing' @@ -60,6 +61,9 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ getUserDataPath, invalidateTransport }: ConnectivityHandlerOptions): void { + ipcMain.handle('runtimeEnvironments:getStatusSnapshots', () => + getRuntimeEnvironmentStatusSnapshots() + ) ipcMain.handle('runtimeEnvironments:list', () => listEnvironments(getUserDataPath()).map(redactRuntimeEnvironment) ) @@ -80,6 +84,12 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ const result = await verifyAndAddRuntimeEnvironmentFromPairingCode(getUserDataPath(), args) if (result.ok) { clearRuntimeEnvironmentManualDisconnect(result.environment.id) + getRuntimeEnvironmentStatusOwner(getUserDataPath(), result.environment.id).acceptVerified({ + id: 'status.get', + ok: true, + result: result.runtimeStatus, + _meta: { runtimeId: result.runtimeStatus.runtimeId } + }) } return result } @@ -121,6 +131,8 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ markRuntimeEnvironmentManuallyDisconnected(environment.id) invalidateTransport(environment.id) closeLegacySelectorTransport(args.selector, environment.id) + // Retain disconnected evidence for renderers that missed the teardown event. + getRuntimeEnvironmentStatusOwner(getUserDataPath(), environment.id) return { disconnected: redactRuntimeEnvironment(environment) } } ) @@ -132,7 +144,9 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ ): Promise> => { const environment = resolveEnvironment(getUserDataPath(), args.selector) clearRuntimeEnvironmentManualDisconnect(environment.id) - return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs) + return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs, { + reconnect: true + }) } ) ipcMain.handle( @@ -156,7 +170,6 @@ function closeLegacySelectorTransport(selector: string, environmentId: string): return } closeRemoteRuntimeRequestConnection(selector) - clearSharedControlSupport(selector) } function registerPassiveStatusHandler(getUserDataPath: () => string): void { diff --git a/src/main/ipc/runtime-environment-federated-read-routing.test.ts b/src/main/ipc/runtime-environment-federated-read-routing.test.ts index c58a404fd39..51c77510a01 100644 --- a/src/main/ipc/runtime-environment-federated-read-routing.test.ts +++ b/src/main/ipc/runtime-environment-federated-read-routing.test.ts @@ -1,3 +1,5 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -20,14 +22,17 @@ vi.mock('../../shared/remote-runtime-client', () => ({ sendRemoteRuntimeRequest: sendRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: vi.fn(), - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - reconnectRemoteRuntimeSharedControlConnection: vi.fn(), - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn() -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: vi.fn(), + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + reconnectRemoteRuntimeSharedControlConnection: vi.fn(), + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn() + }) +}) import { callRuntimeEnvironment, @@ -55,6 +60,7 @@ describe('federated read RPC transport routing', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environment-handler-channels.ts b/src/main/ipc/runtime-environment-handler-channels.ts index 0b63dea943a..23b40fe5183 100644 --- a/src/main/ipc/runtime-environment-handler-channels.ts +++ b/src/main/ipc/runtime-environment-handler-channels.ts @@ -9,6 +9,7 @@ export const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [ 'runtimeEnvironments:retryControlConnection', 'runtimeEnvironments:prepareBrowserClientHostPlacement', 'runtimeEnvironments:getStatus', + 'runtimeEnvironments:getStatusSnapshots', 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', 'runtimeEnvironments:unsubscribe' diff --git a/src/main/ipc/runtime-environment-request-connections.test.ts b/src/main/ipc/runtime-environment-request-connections.test.ts index 750d1becc6a..b02d1b06fb5 100644 --- a/src/main/ipc/runtime-environment-request-connections.test.ts +++ b/src/main/ipc/runtime-environment-request-connections.test.ts @@ -47,9 +47,9 @@ describe('runtime environment shared-control connection cache', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence: absent, verdict: 'absent', - runtimeId: 'runtime-test', - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) + runtimeId: 'runtime-test' }) + pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) expect(getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state).toBe('closed') await delay(400) expect(server.connectionCount()).toBe(1) @@ -58,12 +58,10 @@ describe('runtime environment shared-control connection cache', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence: capable, verdict: 'capable', - runtimeId: 'runtime-test', - onCapable: () => { - ensureRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID, server.pairing) - reconnectRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID) - } + runtimeId: 'runtime-test' }) + ensureRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID, server.pairing) + reconnectRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID) await waitFor(() => server.connectionCount() === 2) }) @@ -119,9 +117,9 @@ describe('runtime environment shared-control connection cache', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence, verdict: 'absent', - runtimeId: 'runtime-test', - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) + runtimeId: 'runtime-test' }) + pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) expect(getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state).toBe('reconnecting') await waitFor(() => server.connectionCount() === 2) diff --git a/src/main/ipc/runtime-environment-request-connections.ts b/src/main/ipc/runtime-environment-request-connections.ts index c1f855697e5..6ba9f273c1e 100644 --- a/src/main/ipc/runtime-environment-request-connections.ts +++ b/src/main/ipc/runtime-environment-request-connections.ts @@ -1,4 +1,9 @@ import type { PairingOffer } from '../../shared/pairing' +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { getPreferredPairingOffer } from '../../shared/runtime-environments' +import type { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner' +import type { RuntimeStatus } from '../../shared/runtime-types' +import { createRuntimeEnvironmentStatusOwner } from './runtime-environment-status-owner' import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/protocol-version' import type { RuntimeOrchestrationEnvelope, @@ -30,6 +35,56 @@ type CachedSharedControlConnection = { const requestConnections = new Map() const sharedControlConnections = new Map() +const statusOwners = new Map() + +export function getRuntimeEnvironmentStatusOwner( + userDataPath: string, + selector: string +): RuntimeHostStatusOwner { + const environment = resolveEnvironment(userDataPath, selector) + const pairing = getPreferredPairingOffer(environment) + const key = `${userDataPath}\0${environment.pairingRevision ?? environment.createdAt}\0${getPairingKey(pairing)}` + let cached = statusOwners.get(environment.id) + if (!cached || cached.key !== key || cached.owner.read().retired) { + if (cached) { + closeRemoteRuntimeRequestConnection(environment.id) + } + const owner = createRuntimeEnvironmentStatusOwner(userDataPath, environment, { + isReady: () => getRemoteRuntimeSharedControlDiagnostics(environment.id)?.state === 'ready', + request: (signal) => + sendRemoteRuntimeSharedControlRequest( + environment.id, + pairing, + 'status.get', + undefined, + 15_000, + undefined, + signal + ), + establish: () => { + ensureRemoteRuntimeSharedControlConnection(environment.id, pairing) + reconnectRemoteRuntimeSharedControlConnection(environment.id) + }, + pause: () => pauseRemoteRuntimeSharedControlRetry(environment.id) + }) + cached = { key, owner } + statusOwners.set(environment.id, cached) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + owner.dispose() + } + } + return cached.owner +} + +export function resetRuntimeEnvironmentStatusOwners(): void { + for (const id of statusOwners.keys()) { + closeRemoteRuntimeRequestConnection(id) + } +} + +export function getRuntimeEnvironmentStatusSnapshots() { + return [...statusOwners.values()].map(({ owner }) => owner.read()) +} export function sendRemoteRuntimeConnectionRequest( environmentId: string, @@ -56,6 +111,9 @@ export function sendRemoteRuntimeConnectionRequest( } export function closeRemoteRuntimeRequestConnection(environmentId: string): void { + const status = statusOwners.get(environmentId) + statusOwners.delete(environmentId) + status?.owner.dispose() const cached = requestConnections.get(environmentId) requestConnections.delete(environmentId) cached?.connection.close() @@ -166,6 +224,16 @@ function getSharedControlConnection( transportGeneration, diagnostics }) + statusOwners + .get(environmentId) + ?.owner.connectionChanged( + diagnostics.state === 'ready' + ? 'ready' + : diagnostics.state === 'closed' || diagnostics.state === 'reconnecting' + ? 'disconnected' + : 'connecting', + diagnostics + ) } }) } diff --git a/src/main/ipc/runtime-environment-shared-control-support.ts b/src/main/ipc/runtime-environment-shared-control-support.ts index 29513e2970a..0de6a35e0f6 100644 --- a/src/main/ipc/runtime-environment-shared-control-support.ts +++ b/src/main/ipc/runtime-environment-shared-control-support.ts @@ -1,39 +1,23 @@ -import { - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY -} from '../../shared/protocol-version' -import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' -import { markEnvironmentUsed } from '../../shared/runtime-environment-store' import type { getPreferredPairingOffer, KnownRuntimeEnvironment } from '../../shared/runtime-environments' -import type { RuntimeStatus } from '../../shared/runtime-types' +import { RemoteRuntimeClientError } from '../../shared/remote-runtime-client-error' import { - applyRuntimeEnvironmentCapabilityVerdict, - captureRuntimeEnvironmentCapabilityEvidence, getAcceptedRuntimeEnvironmentCapabilityOutcome, - isRuntimeEnvironmentCapabilityOutcomeCurrent, - runtimeEnvironmentCapabilityOutcome, resetRuntimeEnvironmentCapabilityEvidence, type RuntimeEnvironmentCapabilityOutcome } from './runtime-environment-capability-evidence' -import { pauseRemoteRuntimeSharedControlRetry } from './runtime-environment-request-connections' - -const sharedControlSupport = new Map< - string, - { cacheKey: string; check: Promise } ->() +import { + getRuntimeEnvironmentStatusOwner, + resetRuntimeEnvironmentStatusOwners +} from './runtime-environment-request-connections' export function resetSharedControlSupport(): void { - sharedControlSupport.clear() + resetRuntimeEnvironmentStatusOwners() resetRuntimeEnvironmentCapabilityEvidence() } -export function clearSharedControlSupport(environmentId: string): void { - sharedControlSupport.delete(environmentId) -} - export async function supportsSharedControl( userDataPath: string, environment: KnownRuntimeEnvironment, @@ -48,85 +32,17 @@ export async function supportsSharedControl( if (accepted) { return accepted } - const cacheKey = getSharedControlSupportCacheKey(environment, pairing) - const cached = sharedControlSupport.get(environment.id) - if (cached?.cacheKey === cacheKey) { - const outcome = await cached.check - if (isRuntimeEnvironmentCapabilityOutcomeCurrent(outcome)) { - return outcome - } - if (sharedControlSupport.get(environment.id)?.check === cached.check) { - sharedControlSupport.delete(environment.id) - } - return { kind: 'stale_incarnation' } + const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({ + timeoutMs + }) + if (!response.ok) { + throw new RemoteRuntimeClientError(response.error.code, response.error.message) } - let resolvedCacheKey = cacheKey - const evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) - const check = (async () => { - const response = await sendRemoteRuntimeRequest( + return ( + getAcceptedRuntimeEnvironmentCapabilityOutcome( + environment.id, pairing, - 'status.get', - undefined, - timeoutMs, - undefined, - undefined, - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES - ) - if (response.ok === true) { - const verdict = response.result.capabilities?.includes( - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY - ) - ? 'capable' - : 'absent' - const acceptedEvidence = applyRuntimeEnvironmentCapabilityVerdict({ - evidence, - verdict, - runtimeId: response._meta.runtimeId, - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(environment.id) - }) - if (!acceptedEvidence) { - return { kind: 'stale_incarnation' } as const - } - markEnvironmentUsed(userDataPath, environment.id, { runtimeId: response._meta.runtimeId }) - resolvedCacheKey = getSharedControlSupportCacheKey( - environment, - pairing, - response._meta.runtimeId - ) - return runtimeEnvironmentCapabilityOutcome(evidence, verdict, response._meta.runtimeId) - } - return runtimeEnvironmentCapabilityOutcome( - evidence, - 'absent', - environment.runtimeId ?? 'unknown-runtime' - ) - })() - // Why: support belongs to the saved pairing/runtime identity, not its mutable display name. - sharedControlSupport.set(environment.id, { cacheKey, check }) - try { - const outcome = await check - const cachedAfterCheck = sharedControlSupport.get(environment.id) - if (cachedAfterCheck?.check === check && cachedAfterCheck.cacheKey !== resolvedCacheKey) { - sharedControlSupport.set(environment.id, { cacheKey: resolvedCacheKey, check }) - } - return outcome - } catch (error) { - if (sharedControlSupport.get(environment.id)?.check === check) { - sharedControlSupport.delete(environment.id) - } - throw error - } -} - -function getSharedControlSupportCacheKey( - environment: KnownRuntimeEnvironment, - pairing: ReturnType, - runtimeId = environment.runtimeId -): string { - return [ - runtimeId ?? 'unknown-runtime', - pairing.endpoint, - pairing.deviceToken, - pairing.publicKeyB64 - ].join('\0') + response._meta.runtimeId + ) ?? { kind: 'stale_incarnation' } + ) } diff --git a/src/main/ipc/runtime-environment-status-connection.test.ts b/src/main/ipc/runtime-environment-status-connection.test.ts new file mode 100644 index 00000000000..8d4fad6f9b9 --- /dev/null +++ b/src/main/ipc/runtime-environment-status-connection.test.ts @@ -0,0 +1,65 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import { encodePairingOffer } from '../../shared/pairing' +import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' +import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' +import { + createSharedControlTestServer, + closeSharedControlTestServers +} from '../../shared/remote-runtime-shared-control-test-server' +import { getRuntimeEnvironmentStatus } from './runtime-environment-transport-routing' +import { + getRuntimeEnvironmentStatusOwner, + resetRuntimeEnvironmentStatusOwners +} from './runtime-environment-request-connections' + +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) +const profiles: string[] = [] +afterEach(async () => { + resetRuntimeEnvironmentStatusOwners() + await closeSharedControlTestServers() + profiles.splice(0).forEach((profile) => rmSync(profile, { recursive: true, force: true })) +}) + +it('publishes real same-socket verification after every authenticated reconnect', async () => { + let runtimeId = 'host-before' + const server = await createSharedControlTestServer({ + resultForRequest: () => ({ + runtimeId, + capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] + }) + }) + const profile = mkdtempSync(join(tmpdir(), 'orca-status-socket-')) + profiles.push(profile) + const environment = addEnvironmentFromPairingCode(profile, { + name: 'host', + pairingCode: encodePairingOffer(server.pairing) + }) + await getRuntimeEnvironmentStatus(profile, environment.id) + const owner = getRuntimeEnvironmentStatusOwner(profile, environment.id) + await vi.waitFor( + () => { + expect(owner.read()).toMatchObject({ transport: 'ready', verification: 'verified' }) + expect(server.requests).toHaveLength(2) + }, + { timeout: 3_000 } + ) + expect(server.connectionCount()).toBe(2) // Bootstrap plus persistent control. + runtimeId = 'host-after' + server.closeClients() + await vi.waitFor( + () => { + expect(owner.read().status?.runtimeId).toBe('host-after') + expect(owner.read().verification).toBe('verified') + }, + { timeout: 3_000 } + ) + expect(server.connectionCount()).toBe(3) + expect(server.requests.map((request) => request.method)).toEqual([ + 'status.get', + 'status.get', + 'status.get' + ]) +}) diff --git a/src/main/ipc/runtime-environment-status-owner.ts b/src/main/ipc/runtime-environment-status-owner.ts new file mode 100644 index 00000000000..4ac3c067f74 --- /dev/null +++ b/src/main/ipc/runtime-environment-status-owner.ts @@ -0,0 +1,89 @@ +import { BrowserWindow } from 'electron' +import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' +import { + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, + REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY +} from '../../shared/protocol-version' +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { markEnvironmentUsed } from '../../shared/runtime-environment-store' +import { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner' +import { + RUNTIME_HOST_STATUS_CHANNEL, + type RuntimeHostStatusResponse +} from '../../shared/runtime-host-status' +import { + applyRuntimeEnvironmentCapabilityVerdict, + getAcceptedRuntimeEnvironmentCapabilityOutcome, + captureRuntimeEnvironmentCapabilityEvidence +} from './runtime-environment-capability-evidence' +import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' + +export function createRuntimeEnvironmentStatusOwner( + userDataPath: string, + environment: KnownRuntimeEnvironment, + transport: { + isReady: () => boolean + request: (signal: AbortSignal) => Promise + establish: () => void + pause: () => void + } +): RuntimeHostStatusOwner { + const pairing = getPreferredPairingOffer(environment) + let evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) + return new RuntimeHostStatusOwner({ + environmentId: environment.id, + pairingRevision: environment.pairingRevision ?? environment.createdAt, + request: (signal) => { + evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) + return transport.isReady() && + getAcceptedRuntimeEnvironmentCapabilityOutcome(environment.id, pairing, null)?.kind === + 'supported' + ? transport.request(signal) + : sendRemoteRuntimeRequest( + pairing, + 'status.get', + undefined, + 15_000, + undefined, + signal, + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES + ) + }, + verified: (response, active) => { + const capable = + response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) ?? false + const accepted = applyRuntimeEnvironmentCapabilityVerdict({ + evidence, + verdict: capable ? 'capable' : 'absent', + runtimeId: response._meta.runtimeId + }) + if (accepted && active && !isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + markEnvironmentUsed(userDataPath, environment.id, { + runtimeId: response._meta.runtimeId, + pairedDeviceId: response.result.pairedDeviceId + }) + if (capable) { + transport.establish() + } else { + transport.pause() + } + } + return capable && active + }, + publish: (snapshot) => { + for (const window of BrowserWindow.getAllWindows()) { + if (window.isDestroyed()) { + continue + } + try { + window.webContents.send(RUNTIME_HOST_STATUS_CHANNEL, snapshot) + } catch { + /* A renderer can close during publication. */ + } + } + } + }) +} diff --git a/src/main/ipc/runtime-environment-status-recovery.test.ts b/src/main/ipc/runtime-environment-status-recovery.test.ts new file mode 100644 index 00000000000..82e94ea62e0 --- /dev/null +++ b/src/main/ipc/runtime-environment-status-recovery.test.ts @@ -0,0 +1,89 @@ +import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' +import { pairingCode } from './runtime-environments-ipc-test-harness' +import { + getRuntimeEnvironmentStatus, + resetSharedControlSupport +} from './runtime-environment-transport-routing' + +const { request, publish } = vi.hoisted(() => ({ request: vi.fn(), publish: vi.fn() })) +vi.mock('../../shared/remote-runtime-client', () => ({ + sendRemoteRuntimeRequest: request, + subscribeRemoteRuntimeRequest: vi.fn() +})) +vi.mock('electron', () => ({ + BrowserWindow: { + getAllWindows: () => [ + { + isDestroyed: () => false, + webContents: { send: publish } + } + ] + } +})) + +let profile: string +beforeEach(() => { + vi.useFakeTimers() + request.mockReset() + publish.mockReset() + profile = mkdtempSync(join(tmpdir(), 'orca-status-recovery-')) +}) +afterEach(() => { + resetSharedControlSupport() + vi.useRealTimers() + rmSync(profile, { recursive: true, force: true }) +}) + +it('recovers a saved host after its first status check fails, without another UI request', async () => { + const environment = addEnvironmentFromPairingCode(profile, { + name: 'offline-at-startup', + pairingCode: pairingCode() + }) + request + .mockRejectedValueOnce( + Object.assign(new Error('host offline'), { code: 'runtime_unavailable' }) + ) + .mockResolvedValue({ + id: 'status', + ok: true, + result: { runtimeId: 'host-1', graphStatus: 'ready', capabilities: [] }, + _meta: { runtimeId: 'host-1' } + }) + expect((await getRuntimeEnvironmentStatus(profile, environment.id)).ok).toBe(false) + await vi.advanceTimersByTimeAsync(3_000) + expect(request).toHaveBeenCalledTimes(2) + expect(publish).toHaveBeenCalledWith( + 'runtimeEnvironments:statusChanged', + expect.objectContaining({ + environmentId: environment.id, + verification: 'verified', + status: expect.objectContaining({ runtimeId: 'host-1' }) + }) + ) + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledTimes(2) +}) + +it('a passive capability check does not strand later active bootstrap recovery', async () => { + const environment = addEnvironmentFromPairingCode(profile, { + name: 'passive-first', + pairingCode: pairingCode() + }) + request + .mockResolvedValueOnce({ + id: 'status', + ok: true, + result: { runtimeId: 'host-1', capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] }, + _meta: { runtimeId: 'host-1' } + }) + .mockRejectedValue(new Error('host offline')) + await getRuntimeEnvironmentStatus(profile, environment.id, undefined, { observeOnly: true }) + await getRuntimeEnvironmentStatus(profile, environment.id) + await vi.advanceTimersByTimeAsync(3_000) + expect(request).toHaveBeenCalledTimes(3) +}) diff --git a/src/main/ipc/runtime-environment-support-routing.test.ts b/src/main/ipc/runtime-environment-support-routing.test.ts index feb09ee481a..8fdde15a81c 100644 --- a/src/main/ipc/runtime-environment-support-routing.test.ts +++ b/src/main/ipc/runtime-environment-support-routing.test.ts @@ -57,7 +57,6 @@ describe('runtime environment support routing', () => { ).resolves.toMatchObject({ ok: true }) expect(supportsMock).toHaveBeenCalledTimes(2) - expect(clearSupportMock).toHaveBeenCalledOnce() expect(supported).toHaveBeenCalledOnce() expect(unsupported).not.toHaveBeenCalled() }) diff --git a/src/main/ipc/runtime-environment-support-routing.ts b/src/main/ipc/runtime-environment-support-routing.ts index e2503ad4445..9566b2fc1b7 100644 --- a/src/main/ipc/runtime-environment-support-routing.ts +++ b/src/main/ipc/runtime-environment-support-routing.ts @@ -18,10 +18,7 @@ import { type RuntimeEnvironmentCapabilityOutcome } from './runtime-environment-capability-evidence' import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' -import { - clearSharedControlSupport, - supportsSharedControl -} from './runtime-environment-shared-control-support' +import { supportsSharedControl } from './runtime-environment-shared-control-support' import { sendRemoteRuntimeRequestAbortable, sendRemoteRuntimeSharedControlRequestAbortable @@ -205,7 +202,6 @@ export async function routeRuntimeEnvironmentCallBySupport(args: { } return response } - clearSharedControlSupport(environment.id) environment = resolveEnvironment(args.userDataPath, environment.id) } return runtimeEnvironmentChangedFailure(environment, args.method) diff --git a/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts b/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts index a44f5df84c6..bb2bbdac322 100644 --- a/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts +++ b/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts @@ -1,20 +1,23 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { generateKeyPair, publicKeyToBase64 } from '../../shared/e2ee-crypto' import { encodePairingOffer, type PairingOffer } from '../../shared/pairing' import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' import { callRuntimeEnvironment, getRuntimeEnvironmentStatus, - subscribeRuntimeEnvironment + subscribeRuntimeEnvironment, + resetSharedControlSupport } from './runtime-environment-transport-routing' // Why: prove the wiring, not just the helper — an unreachable endpoint exercises // the real WebSocket failure → reject → Tailscale-hint join points the settings // probe (returned ok:false) and in-use calls (thrown) actually use. +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) + let userDataPath: string function seedEnvironment(name: string, endpoint: string): string { @@ -39,6 +42,7 @@ beforeEach(() => { }) afterEach(() => { + resetSharedControlSupport() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environment-transport-routing.ts b/src/main/ipc/runtime-environment-transport-routing.ts index 70f49816603..b19b0d9e376 100644 --- a/src/main/ipc/runtime-environment-transport-routing.ts +++ b/src/main/ipc/runtime-environment-transport-routing.ts @@ -1,8 +1,5 @@ import { getPreferredPairingOffer } from '../../shared/runtime-environments' -import { - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY -} from '../../shared/protocol-version' +import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/protocol-version' import { resolveEnvironment, markEnvironmentUsed } from '../../shared/runtime-environment-store' import { isOrchestrationMutation } from '../../shared/orchestration-rpc-contract' import type { @@ -11,33 +8,22 @@ import type { } from '../../shared/runtime-rpc-envelope' import type { RuntimeStatus } from '../../shared/runtime-types' import { - sendRemoteRuntimeRequest, subscribeRemoteRuntimeRequest, type RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' import { withRemoteRuntimeTailscaleHint } from '../../shared/remote-runtime-tailscale-hint' import { enqueueRuntimeCall } from './runtime-environment-call-queue' -import { - ensureRemoteRuntimeSharedControlConnection, - pauseRemoteRuntimeSharedControlRetry, - reconnectRemoteRuntimeSharedControlConnection -} from './runtime-environment-request-connections' +import { getRuntimeEnvironmentStatusOwner } from './runtime-environment-request-connections' import { sendRemoteRuntimeConnectionRequestAbortable, sendRemoteRuntimeRequestAbortable } from './runtime-environment-abortable-requests' import { attachRemoteControlDiagnostics } from './runtime-environment-status-diagnostics' -import { - applyRuntimeEnvironmentCapabilityVerdict, - captureRuntimeEnvironmentCapabilityEvidence -} from './runtime-environment-capability-evidence' + import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' import { withTailscaleHintForResponse } from './runtime-environment-tailscale-response' -import { - clearSharedControlSupport, - resetSharedControlSupport -} from './runtime-environment-shared-control-support' +import { resetSharedControlSupport } from './runtime-environment-shared-control-support' import { executeSupportRoutedCall, shouldRouteCallBySupport, @@ -47,72 +33,31 @@ import { const DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS = 15_000 -export { clearSharedControlSupport, resetSharedControlSupport } +export { resetSharedControlSupport } export async function getRuntimeEnvironmentStatus( userDataPath: string, selector: string, timeoutMs?: number, - options?: { observeOnly?: true } + options?: { observeOnly?: true; signal?: AbortSignal; reconnect?: true } ): Promise> { const environment = resolveEnvironment(userDataPath, selector) - const pairing = getPreferredPairingOffer(environment) - const evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) - let response: RuntimeRpcResponse - try { - response = await sendRemoteRuntimeRequest( - pairing, - 'status.get', - undefined, - timeoutMs ?? DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS, - undefined, - undefined, - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES - ) - } catch (error) { - // Why: the status UI needs shared-control diagnostics most when the - // fresh status probe failed and the host is reconnecting/offline. - return attachRemoteControlDiagnostics( - withTailscaleHintForResponse( - { - id: 'status.get', - ok: false, - error: { - code: 'runtime_unavailable', - message: error instanceof Error ? error.message : String(error) - }, - _meta: { runtimeId: environment.runtimeId } - }, - pairing.endpoint - ), - environment.id - ) - } - if (response.ok === true) { - const verdict = response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) - ? 'capable' - : 'absent' - const accepted = applyRuntimeEnvironmentCapabilityVerdict({ - evidence, - verdict, - runtimeId: response._meta.runtimeId, - onCapable: () => { - if (!options?.observeOnly && !isRuntimeEnvironmentManuallyDisconnected(environment.id)) { - ensureRemoteRuntimeSharedControlConnection(environment.id, pairing) - reconnectRemoteRuntimeSharedControlConnection(environment.id) - } - }, - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(environment.id) - }) - if (accepted && !options?.observeOnly) { - markEnvironmentUsed(userDataPath, environment.id, { - runtimeId: response._meta.runtimeId, - pairedDeviceId: response.result.pairedDeviceId - }) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + return { + id: 'status.get', + ok: false, + error: { + code: 'runtime_manually_disconnected', + message: 'Runtime environment is manually disconnected.' + } } } + const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({ + timeoutMs, + ...options + }) return attachRemoteControlDiagnostics( - withTailscaleHintForResponse(response, pairing.endpoint), + withTailscaleHintForResponse(response, getPreferredPairingOffer(environment).endpoint), environment.id ) } @@ -127,6 +72,15 @@ export async function callRuntimeEnvironment( envelope?: RuntimeOrchestrationEnvelope, options?: { signal?: AbortSignal } ): Promise> { + if (method === 'status.get') { + const environment = resolveEnvironment(userDataPath, selector) + const failure = runtimeEnvironmentRevisionFailure( + environment, + expectedEnvironmentPairingRevision, + method + ) + return failure ?? getRuntimeEnvironmentStatus(userDataPath, selector, timeoutMs, options) + } const environment = resolveEnvironment(userDataPath, selector) // Why: connection failures reject (they don't resolve as ok:false), so the // Tailscale hint is applied to the thrown error here — wrapping the resolved diff --git a/src/main/ipc/runtime-environments-call-routing.test.ts b/src/main/ipc/runtime-environments-call-routing.test.ts index ef92dc66826..e6f8946527d 100644 --- a/src/main/ipc/runtime-environments-call-routing.test.ts +++ b/src/main/ipc/runtime-environments-call-routing.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -44,6 +45,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -58,18 +60,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -112,6 +119,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -339,7 +347,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { undefined, 15_000, undefined, - undefined, + expect.any(AbortSignal), ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES ) expect(sendRemoteRuntimeSharedControlRequestMock).toHaveBeenCalledWith( @@ -451,7 +459,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { } ) - it('keeps uncoded call failures on the rejected IPC fallback path', async () => { + it('returns uncoded status failures through the owner response', async () => { registerRuntimeEnvironmentHandlers(store as never) sendRemoteRuntimeRequestMock.mockRejectedValue(new Error('shared down')) @@ -464,9 +472,10 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:call' ) - await expect(call(null, { selector: 'desk', method: 'status.get' })).rejects.toThrow( - 'shared down' - ) + await expect(call(null, { selector: 'desk', method: 'status.get' })).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_unavailable', message: 'shared down' } + }) }) it('does not fall back after a shared-control request fails on a supported runtime', async () => { diff --git a/src/main/ipc/runtime-environments-capability-cache.test.ts b/src/main/ipc/runtime-environments-capability-cache.test.ts index 8ac11d69dd1..32f724df981 100644 --- a/src/main/ipc/runtime-environments-capability-cache.test.ts +++ b/src/main/ipc/runtime-environments-capability-cache.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -37,6 +38,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -51,18 +53,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -105,6 +112,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -182,9 +190,10 @@ describe('registerRuntimeEnvironmentHandlers', () => { { selector: string; method: string; params?: unknown; timeoutMs?: number }, { ok: true; result: unknown } >('runtimeEnvironments:call') - await expect(call(null, { selector: 'desk', method: 'repo.list' })).rejects.toThrow( - 'probe failed' - ) + await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_unavailable', message: 'probe failed' } + }) await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({ ok: true, result: { repos: [] } diff --git a/src/main/ipc/runtime-environments-ipc-test-harness.ts b/src/main/ipc/runtime-environments-ipc-test-harness.ts index 016352793cb..e97e45ede1a 100644 --- a/src/main/ipc/runtime-environments-ipc-test-harness.ts +++ b/src/main/ipc/runtime-environments-ipc-test-harness.ts @@ -1,6 +1,62 @@ import { expect } from 'vitest' import type { Mock } from 'vitest' +import { getPreferredPairingOffer } from '../../shared/runtime-environments' import { encodePairingOffer } from '../../shared/pairing' +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { createRuntimeEnvironmentStatusOwner } from './runtime-environment-status-owner' +import type { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner' +import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' + +/** Keep IPC tests on the production owner while replacing only its transport. */ +export function withRuntimeStatusOwners>(transport: T) { + const owners = new Map() + return { + ...transport, + getRuntimeEnvironmentStatusOwner: (profile: string, selector: string) => { + const environment = resolveEnvironment(profile, selector) + let owner = owners.get(environment.id) + if (!owner || owner.read().retired) { + owner = createRuntimeEnvironmentStatusOwner(profile, environment, { + isReady: () => + transport.getRemoteRuntimeSharedControlDiagnostics?.(environment.id)?.state === 'ready', + request: (signal) => + transport.sendRemoteRuntimeSharedControlRequest( + environment.id, + undefined, + 'status.get', + undefined, + 15_000, + undefined, + signal + ), + establish: () => { + transport.ensureRemoteRuntimeSharedControlConnection?.( + environment.id, + getPreferredPairingOffer(environment) + ) + transport.reconnectRemoteRuntimeSharedControlConnection?.(environment.id) + }, + pause: () => transport.pauseRemoteRuntimeSharedControlRetry?.(environment.id) + }) + owners.set(environment.id, owner) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + owner.dispose() + } + } + return owner + }, + getRuntimeEnvironmentStatusSnapshots: () => [...owners.values()].map((owner) => owner.read()), + resetRuntimeEnvironmentStatusOwners: () => { + owners.forEach((owner) => owner.dispose()) + owners.clear() + }, + closeRemoteRuntimeRequestConnection: (...args: unknown[]) => { + owners.get(args[0] as string)?.dispose() + owners.delete(args[0] as string) + transport.closeRemoteRuntimeRequestConnection(...args) + } + } +} export function pairingCode(endpoint = 'ws://127.0.0.1:6768'): string { return encodePairingOffer({ diff --git a/src/main/ipc/runtime-environments-pairing.test.ts b/src/main/ipc/runtime-environments-pairing.test.ts index 87d6c2698ab..ce492a7a740 100644 --- a/src/main/ipc/runtime-environments-pairing.test.ts +++ b/src/main/ipc/runtime-environments-pairing.test.ts @@ -1,3 +1,5 @@ +import type { RuntimeHostStatusSnapshot } from '../../shared/runtime-host-status' +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -44,6 +46,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -58,18 +61,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: retryRemoteRuntimeSharedControlConnectionNowMock, - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: retryRemoteRuntimeSharedControlConnectionNowMock, + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -125,6 +133,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -132,6 +141,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { registerRuntimeEnvironmentHandlers(store as never) expect(handleMock.mock.calls.map((call) => call[0])).toEqual([ + 'runtimeEnvironments:getStatusSnapshots', 'runtimeEnvironments:list', 'runtimeEnvironments:addFromPairingCode', 'runtimeEnvironments:verifyAndAddFromPairingCode', @@ -166,6 +176,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:retryControlConnection', 'runtimeEnvironments:prepareBrowserClientHostPlacement', 'runtimeEnvironments:getStatus', + 'runtimeEnvironments:getStatusSnapshots', 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', 'runtimeEnvironments:unsubscribe', @@ -467,6 +478,13 @@ describe('registerRuntimeEnvironmentHandlers', () => { ok: false, error: { code: 'runtime_manually_disconnected' } }) + const getSnapshots = handler( + 'runtimeEnvironments:getStatusSnapshots' + ) + // A new renderer only has the snapshot read, not the earlier disconnect event. + expect(await getSnapshots(null, undefined)).toMatchObject([ + { environmentId: added.environment.id, retired: true, transport: 'disconnected' } + ]) const call = handler< { selector: string; method: string }, { ok: boolean; error?: { code: string } } @@ -492,6 +510,10 @@ describe('registerRuntimeEnvironmentHandlers', () => { result: { runtimeId: 'runtime-remote' } }) expect(sendRemoteRuntimeRequestMock).toHaveBeenCalledOnce() + expect(await getSnapshots(null, undefined)).toMatchObject([ + { environmentId: added.environment.id, verification: 'verified' } + ]) + expect((await getSnapshots(null, undefined))[0].retired).not.toBe(true) }) it('marks environments owned by ephemeral VM runtimes in the public list', async () => { diff --git a/src/main/ipc/runtime-environments-status-diagnostics.test.ts b/src/main/ipc/runtime-environments-status-diagnostics.test.ts index b210e7c209b..9fe3d6baf0e 100644 --- a/src/main/ipc/runtime-environments-status-diagnostics.test.ts +++ b/src/main/ipc/runtime-environments-status-diagnostics.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -44,6 +45,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -58,18 +60,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: ensureRemoteRuntimeSharedControlConnectionMock, - pauseRemoteRuntimeSharedControlRetry: pauseRemoteRuntimeSharedControlRetryMock, - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: ensureRemoteRuntimeSharedControlConnectionMock, + pauseRemoteRuntimeSharedControlRetry: pauseRemoteRuntimeSharedControlRetryMock, + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -114,6 +121,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -148,9 +156,9 @@ describe('registerRuntimeEnvironmentHandlers', () => { expect.objectContaining({ endpoint: 'ws://127.0.0.1:6768', deviceToken: 'device-token' }), 'status.get', undefined, - 50, - undefined, + 15_000, undefined, + expect.any(AbortSignal), ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES ) expect(reconnectRemoteRuntimeSharedControlConnectionMock).toHaveBeenCalledWith( @@ -319,36 +327,41 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) }) - it('returns shared-control diagnostics when saved remote runtime status throws', async () => { - registerRuntimeEnvironmentHandlers(store as never) - getRemoteRuntimeSharedControlDiagnosticsMock.mockReturnValue({ - state: 'reconnecting', - pendingRequestCount: 0, - subscriptionCount: 1, - reconnectAttempt: 2, - lastConnectedAt: 123, - lastClose: { code: 1006, reason: '' }, - lastError: 'closed' - }) - sendRemoteRuntimeRequestMock.mockRejectedValue(new Error('socket closed')) + it.each(['runtimeEnvironments:getStatus', 'runtimeEnvironments:connect'])( + 'preserves failure diagnostics and guidance on %s', + async (channel) => { + registerRuntimeEnvironmentHandlers(store as never) + getRemoteRuntimeSharedControlDiagnosticsMock.mockReturnValue({ + state: 'reconnecting', + pendingRequestCount: 0, + subscriptionCount: 1, + reconnectAttempt: 2, + lastConnectedAt: 123, + lastClose: { code: 1006, reason: '' }, + lastError: 'closed' + }) + sendRemoteRuntimeRequestMock.mockRejectedValue( + new Error('Could not connect to the remote Orca runtime.') + ) - const add = handler< - { name: string; pairingCode: string }, - { environment: { id: string; name: string } } - >('runtimeEnvironments:addFromPairingCode') - await add(null, { name: 'desk', pairingCode: pairingCode() }) + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + await add(null, { name: 'desk', pairingCode: pairingCode() }) - const getStatus = handler< - { selector: string; timeoutMs?: number }, - { ok: false; error: { message: string; data?: { remoteControl?: { state: string } } } } - >('runtimeEnvironments:getStatus') + const getStatus = handler< + { selector: string; timeoutMs?: number }, + { ok: false; error: { message: string; data?: { remoteControl?: { state: string } } } } + >(channel) - await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({ - ok: false, - error: { - message: 'socket closed', - data: { remoteControl: { state: 'reconnecting' } } - } - }) - }) + await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({ + ok: false, + error: { + message: expect.stringContaining('connect both devices to Tailscale'), + data: { remoteControl: { state: 'reconnecting' } } + } + }) + } + ) }) diff --git a/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts b/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts index 53a07442c3b..ed4dcd62182 100644 --- a/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts +++ b/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -38,6 +39,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -52,18 +54,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { invalidateRuntimeEnvironmentTransport, @@ -109,6 +116,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environments-subscription-routing.test.ts b/src/main/ipc/runtime-environments-subscription-routing.test.ts index 494f0b9ea6b..0ef70f7ac96 100644 --- a/src/main/ipc/runtime-environments-subscription-routing.test.ts +++ b/src/main/ipc/runtime-environments-subscription-routing.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -40,6 +41,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -54,18 +56,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -108,6 +115,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environments-subscription-teardown.test.ts b/src/main/ipc/runtime-environments-subscription-teardown.test.ts index 13a98e1057a..afb1adf457a 100644 --- a/src/main/ipc/runtime-environments-subscription-teardown.test.ts +++ b/src/main/ipc/runtime-environments-subscription-teardown.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -38,6 +39,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -52,18 +54,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) vi.mock('../browser/paired-runtime-browser-client-host-runtime', () => ({ retirePairedRuntimeBrowserClientHostEnvironment: retirePairedRuntimeBrowserClientHostEnvironmentMock @@ -115,6 +122,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environments.ts b/src/main/ipc/runtime-environments.ts index 7d9a261eef9..ac7a4107bc8 100644 --- a/src/main/ipc/runtime-environments.ts +++ b/src/main/ipc/runtime-environments.ts @@ -1,6 +1,6 @@ import { app, ipcMain } from 'electron' import { randomUUID } from 'node:crypto' -import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { listEnvironments, resolveEnvironment } from '../../shared/runtime-environment-store' import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' import type { Store } from '../persistence' import { @@ -8,14 +8,16 @@ import { registerRuntimeEnvironmentConnectivityHandlers, registerRuntimeEnvironmentPassiveHandlers } from './runtime-environment-connectivity-handlers' -import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections' +import { + closeRemoteRuntimeRequestConnection, + getRuntimeEnvironmentStatusOwner +} from './runtime-environment-request-connections' import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler' import { advanceRuntimeEnvironmentTransportGeneration, getRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' import { - clearSharedControlSupport, resetSharedControlSupport, subscribeRuntimeEnvironment } from './runtime-environment-transport-routing' @@ -64,7 +66,6 @@ export function invalidateRuntimeEnvironmentTransport(environmentId: string): Pr advanceRuntimeEnvironmentCapabilityIncarnation(environmentId) advanceRuntimeEnvironmentTransportGeneration(environmentId) closeRemoteRuntimeRequestConnection(environmentId) - clearSharedControlSupport(environmentId) closeSubscriptionsForEnvironment(environmentId) return retirePairedRuntimeBrowserClientHostEnvironment( environmentId, @@ -97,6 +98,11 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { }) registerRuntimeEnvironmentRecoveryHandler() registerRuntimeEnvironmentPassiveHandlers(getUserDataPath) + for (const environment of listEnvironments(getUserDataPath())) { + if (!isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + getRuntimeEnvironmentStatusOwner(getUserDataPath(), environment.id).activate() + } + } ipcMain.handle( 'runtimeEnvironments:subscribe', async ( diff --git a/src/main/ipc/worktrees/removal/worktree-removal-ownership.ts b/src/main/ipc/worktrees/removal/worktree-removal-ownership.ts index ec2198bcdb0..ecc982076c5 100644 --- a/src/main/ipc/worktrees/removal/worktree-removal-ownership.ts +++ b/src/main/ipc/worktrees/removal/worktree-removal-ownership.ts @@ -40,11 +40,17 @@ export async function stopPtysForDestructiveWorktreeRemoval( ...(allowUnverifiedStop ? { allowUnverifiedStop: true } : {}), ...(connectionId ? { includeLocalRegistry: false } : {}) }) + // Structured sessions are counted here too: closing a user's chat is now an ordinary outcome + // of this verb, and a removal that closed one but no PTY would otherwise log nothing at all. + const structuredStopped = teardownResult.structuredStopped ?? 0 const total = - teardownResult.runtimeStopped + teardownResult.providerStopped + teardownResult.registryStopped + teardownResult.runtimeStopped + + teardownResult.providerStopped + + teardownResult.registryStopped + + structuredStopped if (total > 0) { console.info( - `[worktree-teardown] ${worktreeId} killed runtime=${teardownResult.runtimeStopped} provider=${teardownResult.providerStopped} registry=${teardownResult.registryStopped}` + `[worktree-teardown] ${worktreeId} killed runtime=${teardownResult.runtimeStopped} provider=${teardownResult.providerStopped} registry=${teardownResult.registryStopped} structured=${structuredStopped}` ) } } diff --git a/src/main/native-chat/agent-session-journal/journal-store.test.ts b/src/main/native-chat/agent-session-journal/journal-store.test.ts index 97eac02fe75..3592947faaa 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.test.ts @@ -8,6 +8,7 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { + agentJournalItemKey, boundJournalKeyComponent, MAX_JOURNAL_KEY_COMPONENT_CHARS } from '../../../shared/agent-session-journal-item-key' @@ -96,6 +97,21 @@ describe('sequences', () => { expect(journal.snapshot().items[0]?.revision).toBe(3) }) + it('visits reduced items at their creation sequence without promoting an older revision', async () => { + const journal = await open() + await journal.appendItem(item(0), body('first'), { fence: 1 }) + const latest = await journal.appendItem(item(1), body('second'), { fence: 1 }) + await journal.appendItem(item(0), body('first revised'), { fence: 1 }) + const visited: { itemId: string; sequence: number }[] = [] + + journal.visitItems((itemId, sequence) => visited.push({ itemId, sequence })) + + expect(visited).toEqual([ + { itemId: agentJournalItemKey(item(0)), sequence: 2 }, + { itemId: latest.itemId, sequence: latest.cursor.sequence } + ]) + }) + it('preserves an oversized identity and its raw digest-form mimic across reopen', async () => { const oversizedTurnId = 'a'.repeat(MAX_JOURNAL_KEY_COMPONENT_CHARS + 1) const digestFormMimic = boundJournalKeyComponent(oversizedTurnId) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 3be64d9ceca..2e372f9abae 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -162,6 +162,13 @@ export class AgentSessionJournal { snapshot = (): AgentJournalSnapshot => renderJournalState(this.state) + /** Visits reduced items without allocating and sorting a full snapshot. */ + visitItems = (visit: (itemId: string, sequence: number) => void): void => { + for (const item of this.state.items.values()) { + visit(item.itemId, item.sequence) + } + } + /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt diff --git a/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts b/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts index d1bc7d0e7d3..548a719ceb1 100644 --- a/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts +++ b/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts @@ -25,8 +25,10 @@ export const PROVIDER_FRAME_CLASSIFICATIONS = { 'thread/closed': 'status-chrome', 'skills/changed': 'status-chrome', 'thread/name/updated': 'status-chrome', - 'thread/goal/updated': 'status-chrome', - 'thread/goal/cleared': 'status-chrome', + // The goal tool call is never emitted as an item, so these two frames are the only + // truthful evidence a goal exists; the model's prose about goals can be wrong. + 'thread/goal/updated': 'timeline-substantive', + 'thread/goal/cleared': 'timeline-substantive', 'thread/environment/connected': 'status-chrome', 'thread/environment/disconnected': 'status-chrome', 'thread/settings/updated': 'status-chrome', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts index 510c1df2f4a..c9a32533db4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts @@ -60,6 +60,8 @@ export class StructuredAgentSessionSinkQueue { failed: this.failure !== null }) + journalEpoch = (): string | null => this.target?.journal.epoch ?? null + bindReadingControl(control: StructuredAgentSessionReadingControl): () => void { this.readingControl = control if (this.backpressured) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts index d1b7ea533a1..3d0a5e8a269 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts @@ -54,7 +54,8 @@ function target( appendLifecycleBatch: vi.fn(async (input: { settlementId: string }) => { log.push({ call: 'appendLifecycleBatch', fence, settlementId: input.settlementId }) return { epoch: 'e', sequence: 0 } - }) + }), + latestItemMatching: vi.fn(() => null) } as unknown as AgentSessionJournal return { journal, @@ -115,6 +116,25 @@ describe('deferred structured agent-session event sink', () => { expect(log).toEqual([{ call: 'appendItem', fence: 2, ordinal: 0 }]) }) + it('resolves a lifecycle transition after journal bind and skips an existing state', async () => { + const log: Recorded[] = [] + const deferred = createDeferredStructuredAgentSessionEventSink() + + expect( + deferred.sink.tryAppendLifecycleTransition?.(identity(0), BODY, () => identity(1)) + ).toEqual({ accepted: true }) + expect(deferred.sink.tryAppendLifecycleTransition?.(identity(0), BODY, () => null)).toEqual({ + accepted: true + }) + deferred.bind(target(2, log)) + await deferred.drained() + + expect(log).toEqual([ + { call: 'appendItem', fence: 2, ordinal: 1 }, + { call: 'publish', fence: 2 } + ]) + }) + it('drops buffered and later writes once closed, and refuses to rebind', async () => { const log: Recorded[] = [] const deferred = createDeferredStructuredAgentSessionEventSink() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index 784e3aa7b20..857aa118fe8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -31,6 +31,15 @@ export type StructuredAgentSessionAppendOptions = { observedAt?: number } +export type StructuredAgentSessionLifecycleJournal = Pick< + AgentSessionJournal, + 'epoch' | 'visitItems' +> + +export type StructuredAgentSessionLifecycleIdentityResolver = ( + journal: StructuredAgentSessionLifecycleJournal +) => AgentJournalItemIdentity | null + export type StructuredAgentSessionEventSink = { appendItem( identity: AgentJournalItemIdentity, @@ -52,6 +61,14 @@ export type StructuredAgentSessionEventSink = { body: AgentJournalItemBody, options?: StructuredAgentSessionAppendOptions ): StructuredAgentSessionSinkAdmission + /** Queues one journal-derived lifecycle append; a null resolution is a no-op. */ + tryAppendLifecycleTransition?( + identitySizeBound: AgentJournalItemIdentity, + body: AgentJournalItemBody, + resolveIdentity: StructuredAgentSessionLifecycleIdentityResolver + ): StructuredAgentSessionSinkAdmission + /** Current durable epoch, when this deferred sink is bound to its journal. */ + journalEpoch?(): string | null appendLifecycleBatch?( settlementId: string, mutations: readonly JournalLifecycleMutationInput[], @@ -186,6 +203,28 @@ export function createDeferredStructuredAgentSessionEventSink( }, options ), + tryAppendLifecycleTransition: (identitySizeBound, body, resolveIdentity) => { + const bytes = estimateStructuredAgentSessionItemBytes(identitySizeBound, body) + return queue.submit( + { + bytes, + lifecycle: true, + run: async (bound) => { + const identity = resolveIdentity(bound.journal) + if (identity === null) { + return + } + if (estimateStructuredAgentSessionItemBytes(identity, body) > bytes) { + throw new Error('structured agent-session item identity exceeded its reserved size') + } + await bound.journal.appendItem(identity, body, { fence: bound.fence }) + bound.publish() + } + }, + { lifecycle: true } + ) + }, + journalEpoch: queue.journalEpoch, appendLifecycleBatch: (settlementId, mutations, options = {}) => { const admission = appendLifecycleBatch(settlementId, mutations, options) if (!admission.accepted) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index ac6dc23385a..4d0e0faab30 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -13,6 +13,7 @@ import type { AgentSessionMutationResult, AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { AGENT_SESSION_UNATTACHED_REFUSAL_CODE } from '../../../shared/structured-agent-session-read-refusal' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' @@ -21,8 +22,10 @@ import { runSettledAgentSessionMutation } from './structured-agent-session-opera import { resolveAgentSessionReplayOutcome } from './structured-agent-session-replay-outcome' import type { AgentSessionTurnContext } from './structured-agent-session-turns' +// The code is shared with the client so a read that refuses this way can be told apart from a +// transcript that failed to load; the two must never drift apart. export const AGENT_SESSION_NOT_ATTACHED: AgentSessionWireRefusal = { - code: 'agent_session_ownership_unknown', + code: AGENT_SESSION_UNATTACHED_REFUSAL_CODE, message: 'This host holds no attached session by that id.' } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts index c75301c6461..e022f640403 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts @@ -420,7 +420,7 @@ describe('host rewind', () => { expect(await host.rewind(caller, params(target))).toMatchObject({ ok: true }) }) - it('keeps the host-stamped turn rows before the boundary through a Codex provider hydration', async () => { + it('keeps host-stamped turn and goal rows through a Codex provider hydration', async () => { expect(await host.attach(caller, hostTestAttachParams(null))).toMatchObject({ ok: true }) const message = (turnId: string) => ({ provider: 'codex' as const, @@ -434,6 +434,19 @@ describe('host rewind', () => { sessionId: HOST_TEST_SESSION, recordId: `turn-lifecycle:${turnId}` }) + const goalRow = { + provider: 'orca' as const, + clientMessageId: `codex-goal:${'a'.repeat(64)}:${'b'.repeat(64)}:${'c'.repeat(64)}` + } + const goalBody = { + kind: 'status' as const, + text: 'Goal set: Keep the retained evidence.', + providerFrame: { + provider: 'codex', + kind: 'notification:thread/goal/updated', + payload: { head: '{}', byteLength: 2, digest: 'd'.repeat(64), truncated: false } + } + } const keptTurn = { kind: 'turn' as const, turnId: 'kept', @@ -444,6 +457,7 @@ describe('host rewind', () => { durationMs: 5_000 } sink.appendItem(message('kept'), hostTestMessage('kept')) + sink.appendItem(goalRow, goalBody) sink.appendItem(turnRow('kept'), keptTurn) sink.appendItem(message('drop'), hostTestMessage('drop')) sink.appendItem(turnRow('drop'), { ...keptTurn, turnId: 'drop', durationMs: 1_000 }) @@ -465,11 +479,66 @@ describe('host rewind', () => { host.journalSnapshot(HOST_TEST_SESSION).items.map(({ itemId, body }) => ({ itemId, body })) ).toEqual([ { itemId: agentJournalItemKey(message('kept')), body: hostTestMessage('kept from provider') }, + { itemId: agentJournalItemKey(goalRow), body: goalBody }, { itemId: agentJournalItemKey(turnRow('kept')), body: keptTurn } ]) expect(store.getRecord(HOST_TEST_SESSION)?.rewind?.phase).toBe('completed') }) + it('keeps a host goal row when interrupted Codex rewind recovery rebuilds provider history', async () => { + expect(await host.attach(caller, hostTestAttachParams(null))).toMatchObject({ ok: true }) + const message = (turnId: string) => ({ + provider: 'codex' as const, + threadId: HOST_TEST_THREAD, + turnId, + ordinal: 0 + }) + const goalRow = { + provider: 'orca' as const, + clientMessageId: `codex-goal:${'1'.repeat(64)}:${'2'.repeat(64)}:${'3'.repeat(64)}` + } + const goalBody = { + kind: 'status' as const, + text: 'Goal set: Survive recovery.', + providerFrame: { + provider: 'codex', + kind: 'notification:thread/goal/updated', + payload: { head: '{}', byteLength: 2, digest: '4'.repeat(64), truncated: false } + } + } + sink.appendItem(message('kept'), hostTestMessage('kept')) + sink.appendItem(goalRow, goalBody) + sink.appendItem(message('drop'), hostTestMessage('drop')) + sink.appendItem(message('tip'), { ...hostTestMessage('tip'), role: 'assistant' }) + await host.flushStreamedEvents(HOST_TEST_SESSION) + rewind.mockImplementationOnce(async (input) => { + await input.onReverted?.() + throw new Error('lost after provider revert') + }) + + await expect(host.rewind(caller, params(agentJournalItemKey(message('drop'))))).rejects.toThrow( + 'lost after provider revert' + ) + recoverRewind.mockResolvedValueOnce({ + ok: true, + items: [{ identity: message('kept'), body: hostTestMessage('kept from recovery') }] + }) + expect( + await host.attach( + caller, + hostTestAttachParams(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) + ) + ).toMatchObject({ ok: true }) + + expect( + host.journalSnapshot(HOST_TEST_SESSION).items.map(({ itemId, body }) => ({ itemId, body })) + ).toEqual([ + { itemId: agentJournalItemKey(message('kept')), body: hostTestMessage('kept from recovery') }, + { itemId: agentJournalItemKey(goalRow), body: goalBody } + ]) + expect(store.getRecord(HOST_TEST_SESSION)?.rewind?.phase).toBe('completed') + }) + it('recovers against the complete provider preflight when the local journal omitted an older turn', async () => { const target = await seed() const items = ['older', 'kept'].map((turnId) => ({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index 417a42fe414..cb7cbb1f20b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -20,7 +20,7 @@ import { conversationCommandBlocked } from './structured-conversation-command-ad import { rewindRefusal } from './structured-rewind-refusal' import { persistRewindRecord, recoverStructuredRewind } from './structured-rewind-recovery' import { replaceClaudeRewindOwner } from './structured-rewind-claude-owner' -import { mergeRetainedTurnRows } from './structured-rewind-retained-turns' +import { mergeRetainedHostLifecycleRows } from './structured-rewind-retained-host-rows' export async function rewindStructuredAgentSession( context: StructuredAgentSessionMutationContext, @@ -174,7 +174,7 @@ export async function rewindStructuredAgentSession( fence: ctx.fence, beforeTurnId: key.provider === 'codex' ? key.turnId : '', onPrepared: async (items) => { - const retained = mergeRetainedTurnRows( + const retained = mergeRetainedHostLifecycleRows( prepared.retained, items.map(({ identity, body }) => ({ itemId: agentJournalItemKey(identity), @@ -220,7 +220,7 @@ export async function rewindStructuredAgentSession( return rewindRefusal(reason) } const confirmed = provider.items - ? mergeRetainedTurnRows( + ? mergeRetainedHostLifecycleRows( prepared.retained, provider.items.map(({ identity, body }) => ({ itemId: agentJournalItemKey(identity), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index 1cb8439c391..678b790ed4e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -14,6 +14,7 @@ import type { AgentSessionMutationEnvelope, AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { AGENT_SESSION_UNATTACHED_REFUSAL_CODE } from '../../../shared/structured-agent-session-read-refusal' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' @@ -180,6 +181,25 @@ describe('a chat that closes', () => { expect(host.hasSession(SESSION)).toBe(true) }) + // The pane outlives the close by a few frames — a workspace delete closes the chats inside it + // while their panes are still mounted — so whatever a read raises in that window is what the user + // sees. This is the code the client narrows on to keep that window off the pane; a host that + // starts raising a different one there puts the red error back. + it('answers a read from the pane that outlived it with the code the client treats as transitional', async () => { + await attach() + await host.hold(SESSION, SURFACE) + + await host.close(SESSION) + + expect(host.hasSession(SESSION)).toBe(false) + expect(() => host.history({ sessionId: SESSION, direction: 'tail' })).toThrow( + AGENT_SESSION_UNATTACHED_REFUSAL_CODE + ) + expect(() => + host.subscribe({ id: 'sub-1', sessionId: SESSION, emit: () => undefined }) + ).toThrow(AGENT_SESSION_UNATTACHED_REFUSAL_CODE) + }) + it('does not lose the session to a release the client sent twice', async () => { await attach() await host.hold(SESSION, SURFACE) diff --git a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts index 8f5a8942a86..4916710e9c1 100644 --- a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts @@ -1,5 +1,5 @@ import { restoreRewindJournalBody } from './structured-rewind-journal-body' -import { isRetainedTurnRow, mergeRetainedTurnRows } from './structured-rewind-retained-turns' +import { mergeRetainedHostLifecycleRows } from './structured-rewind-retained-host-rows' import { isDeepStrictEqual } from 'node:util' import { agentJournalItemKey, @@ -61,9 +61,13 @@ export async function recoverStructuredRewind( } throw new Error(`agent_session_rewind:${recovered?.reason ?? 'outcome-unknown'}`) } - // Turn rows are the host's, never the provider's; the proof covers provider items only. const expectedItems = new Set( - rewind.retained.filter((item) => !isRetainedTurnRow(item)).map((item) => item.itemId) + rewind.retained + .filter((item) => { + const identity = parseAgentJournalItemKey(item.itemId) + return identity?.provider === 'codex' && identity.threadId === target.threadId + }) + .map((item) => item.itemId) ) const observedItems = new Set() for (const { identity } of recovered.items) { @@ -80,7 +84,7 @@ export async function recoverStructuredRewind( if (observedItems.size !== expectedItems.size) { throw new Error('agent_session_rewind:proof-mismatch') } - const retained = mergeRetainedTurnRows( + const retained = mergeRetainedHostLifecycleRows( rewind.retained, recovered.items.map(({ identity, body }) => ({ itemId: agentJournalItemKey(identity), diff --git a/src/main/native-chat/agent-session-wire/structured-rewind-retained-turns.ts b/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts similarity index 57% rename from src/main/native-chat/agent-session-wire/structured-rewind-retained-turns.ts rename to src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts index aa0753b502c..35307566599 100644 --- a/src/main/native-chat/agent-session-wire/structured-rewind-retained-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts @@ -1,20 +1,23 @@ -// The Codex preflight returns provider items only. The host's turn rows are its own record, so a -// rewind that takes the provider's list as the new epoch would drop every duration before the -// boundary unless those rows are spliced back beside the item each one followed. +// Provider preflight returns provider items only. The host's lifecycle rows are its own record, so +// a rewind that takes the provider list as the new epoch must splice those rows back beside the +// provider item each one followed. +import { parseCodexGoalJournalItemId } from '../../codex/codex-goal-journal-identity' import type { AgentJournalItemBody } from '../../../shared/agent-session-journal-types' import type { AgentSessionRewindRecord } from '../../../shared/agent-session-rewind' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' type RetainedRow = AgentSessionRewindRecord['retained'][number] -export function isRetainedTurnRow(item: Pick): boolean { - return readAgentJournalTurn(item.body as AgentJournalItemBody) !== null +export function isRetainedHostLifecycleRow(item: RetainedRow): boolean { + return ( + readAgentJournalTurn(item.body as AgentJournalItemBody) !== null || + parseCodexGoalJournalItemId(item.itemId) !== null + ) } -/** `reference` fixes where each turn row sits; the provider items are the spine and keep their - * own order, including turns the local journal never saw. */ -export function mergeRetainedTurnRows( +/** `reference` fixes where each host row sits; provider items are the ordered spine. */ +export function mergeRetainedHostLifecycleRows( reference: readonly RetainedRow[], providerItems: readonly RetainedRow[] ): RetainedRow[] { @@ -22,7 +25,7 @@ export function mergeRetainedTurnRows( const rowsAfter = new Map() let anchor = -1 for (const item of reference) { - if (!isRetainedTurnRow(item)) { + if (!isRetainedHostLifecycleRow(item)) { anchor = spineIndex.get(item.itemId) ?? anchor } else if (!spineIndex.has(item.itemId)) { rowsAfter.set(anchor, [...(rowsAfter.get(anchor) ?? []), item]) diff --git a/src/main/native-chat/agent-session-wire/unhandled-provider-frame.test.ts b/src/main/native-chat/agent-session-wire/unhandled-provider-frame.test.ts index 61544fd56ab..aefceacbca4 100644 --- a/src/main/native-chat/agent-session-wire/unhandled-provider-frame.test.ts +++ b/src/main/native-chat/agent-session-wire/unhandled-provider-frame.test.ts @@ -50,9 +50,6 @@ describe('unhandled provider frame journal fallback', () => { expect( unhandledProviderFrameJournalItem('codex', 'notification:thread/tokenUsage/updated', {}) ).toBeNull() - expect( - unhandledProviderFrameJournalItem('codex', 'notification:thread/goal/cleared', {}) - ).toBeNull() expect(unhandledProviderFrameJournalItem('claude', 'message:system:init', {})).toBeNull() expect( unhandledProviderFrameJournalItem('claude', 'message:result', { diff --git a/src/main/native-chat/agent-session-wire/unhandled-provider-frame.ts b/src/main/native-chat/agent-session-wire/unhandled-provider-frame.ts index 272e960b838..22ca3d89542 100644 --- a/src/main/native-chat/agent-session-wire/unhandled-provider-frame.ts +++ b/src/main/native-chat/agent-session-wire/unhandled-provider-frame.ts @@ -5,6 +5,7 @@ import { DEFAULT_JOURNAL_PAYLOAD_LIMITS, type JournalPayloadLimits } from '../agent-session-journal/journal-payload-bounds' +import { codexGoalRowText } from '../../codex/codex-goal-journal-rows' import { classifyProviderFrame } from './provider-frame-disposition' export type UnhandledProviderFrameJournalItem = { @@ -115,11 +116,15 @@ export function unhandledProviderFrameJournalItem( .filter((part): part is string => typeof part === 'string' && part.trim().length > 0) .join('\n\n') || message } + const goalText = provider === 'codex' ? codexGoalRowText(method, payload) : null const display = message ? boundInlineText(message, limits) : null + const goalDisplay = goalText ? boundInlineText(goalText, limits) : null return { body: { kind: 'status', - text: compaction ? 'Context compacted' : (display?.text ?? `${provider} · ${kind}`), + text: compaction + ? 'Context compacted' + : (goalDisplay?.text ?? display?.text ?? `${provider} · ${kind}`), ...(compaction ? { presentation: 'compaction' } : {}), ...(tone ? { tone } : {}), providerFrame: { provider, kind, payload: bounded } diff --git a/src/main/pi/agent-status-extension-test-harness.ts b/src/main/pi/agent-status-extension-test-harness.ts index eec2b615017..810bc3d04d5 100644 --- a/src/main/pi/agent-status-extension-test-harness.ts +++ b/src/main/pi/agent-status-extension-test-harness.ts @@ -24,6 +24,7 @@ type FakeCurlChild = { } export type AgentStatusExtensionHarness = { + killMock: ReturnType fetchMock: ReturnType spawnMock: ReturnType spawnedChildren: FakeCurlChild[] @@ -57,6 +58,7 @@ export const AGENT_STATUS_EXTENSION_SELF_PID = 4242 export function createAgentStatusExtensionHarness(args: { kind: 'pi' | 'omp' | 'prime-agent' + killImpl?: (pid: number, signal: number) => void env?: Record pid?: number title?: string @@ -115,7 +117,9 @@ export function createAgentStatusExtensionHarness(args: { throw new Error(`unexpected require(${specifier})`) }) + const killMock = vi.fn(args.killImpl ?? (() => undefined)) const processMock = { + kill: killMock, env: { ...BASE_ENV, ...(args.kind === 'prime-agent' ? { PRIME_AGENT_INTERNAL_DAEMON_WORKER: '1' } : {}), @@ -172,6 +176,7 @@ export function createAgentStatusExtensionHarness(args: { return { fetchMock, + killMock, spawnMock, spawnedChildren, fsMock, diff --git a/src/main/pi/agent-status-handler-source.ts b/src/main/pi/agent-status-handler-source.ts index 9d02abbd78d..5a778a1c81f 100644 --- a/src/main/pi/agent-status-handler-source.ts +++ b/src/main/pi/agent-status-handler-source.ts @@ -88,13 +88,31 @@ export function getPiAgentStatusHandlerSourceLines(kind: PiAgentKind): string[] '// etc.), so we forward the raw object verbatim under the same field', '// names Claude uses (tool_name / tool_input) and let the server pick the', '// preview. Keeps tool-name knowledge centralized on the receiver side.', + '// Why: a restarted agent inherits the previous owner PID through env, so a', + '// dead owner must be claimable or the pane goes silent for good. Only ESRCH', + '// proves the owner is gone -- every other probe result keeps suppression, so', + '// a live foreign owner still cannot double-report. Mirrors the tri-state in', + '// main/agent-hooks/managed-hook-owner-identity.ts, which this runtime cannot', + '// import (the extension loads inside pi/omp with no Orca deps).', + 'function isStatusOwnerAlive(pid: string): boolean {', + ' const parsed = Number(pid)', + ' if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 0x7fffffff) return false', + " if (typeof process.kill !== 'function') return true", + ' try {', + ' process.kill(parsed, 0)', + ' return true', + ' } catch (err: unknown) {', + " return (err as { code?: string } | null)?.code !== 'ESRCH'", + ' }', + '}', + '', "// Why: child agents inherit the lead's pane env; only its process may", '// register status hooks. PID identity keeps in-process reloads reporting.', 'export default function (pi): void {', ...primeDaemonWorkerGuard, ` const ownerPid = process.env.${ownerEnv}`, ' const selfPid = String(process.pid)', - ' if (ownerPid && ownerPid !== selfPid) return', + ' if (ownerPid && ownerPid !== selfPid && isStatusOwnerAlive(ownerPid)) return', ` process.env.${ownerEnv} = selfPid`, ...sessionStartHandler, ` pi.on('before_agent_start', (event${ctxParam}) => {`, diff --git a/src/main/pi/agent-status-owner-recovery.test.ts b/src/main/pi/agent-status-owner-recovery.test.ts new file mode 100644 index 00000000000..d176bcb8dae --- /dev/null +++ b/src/main/pi/agent-status-owner-recovery.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'vitest' +import { + createAgentStatusExtensionHarness as createHarness, + AGENT_STATUS_EXTENSION_SELF_PID as SELF_PID +} from './agent-status-extension-test-harness' + +describe('Pi status owner recovery', () => { + it.each(['pi', 'omp', 'prime-agent'] as const)( + 'claims the pane for a restarted %s agent whose inherited owner PID is dead', + async (kind) => { + // Why: STA-5245 -- a restart leaves a dead owner PID in the inherited env. + // Without a liveness probe the guard suppresses every later load, so the + // pane never reports status again. + const ownerKey = + kind === 'prime-agent' ? 'ORCA_PRIME_AGENT_STATUS_OWNED' : 'ORCA_PI_STATUS_OWNED' + const harness = createHarness({ + kind, + pid: SELF_PID, + env: { [ownerKey]: String(SELF_PID - 1) }, + killImpl: () => { + throw Object.assign(new Error('ESRCH'), { code: 'ESRCH' }) + } + }) + + expect(harness.killMock).toHaveBeenCalledWith(SELF_PID - 1, 0) + expect(harness.handlers.agent_end).toBeTypeOf('function') + expect(harness.processEnv[ownerKey]).toBe(String(SELF_PID)) + + await harness.callHook('agent_end') + expect(harness.fetchMock).toHaveBeenCalledTimes(1) + } + ) + + it.each(['EPERM', 'EACCES', 'EINVAL', undefined])( + 'keeps suppression for unverifiable probe error %s', + (code) => { + // Why: EPERM means the owner exists but belongs to another user, so + // claiming the pane there would reintroduce double-reporting. + const harness = createHarness({ + kind: 'pi', + pid: SELF_PID, + env: { ORCA_PI_STATUS_OWNED: String(SELF_PID - 1) }, + killImpl: () => { + throw Object.assign(new Error('probe failed'), { code }) + } + }) + + expect(harness.handlers).toEqual({}) + expect(harness.processEnv.ORCA_PI_STATUS_OWNED).toBe(String(SELF_PID - 1)) + } + ) + + it('claims the pane when the inherited owner PID is not a usable pid', () => { + // Why: a truncated/garbage marker is not evidence of a live owner. + const harness = createHarness({ + kind: 'pi', + pid: SELF_PID, + env: { ORCA_PI_STATUS_OWNED: 'not-a-pid' } + }) + + expect(harness.killMock).not.toHaveBeenCalled() + expect(harness.handlers.agent_end).toBeTypeOf('function') + expect(harness.processEnv.ORCA_PI_STATUS_OWNED).toBe(String(SELF_PID)) + }) + + it('claims the pane when the inherited owner PID exceeds safe integer precision', () => { + const harness = createHarness({ + kind: 'pi', + pid: SELF_PID, + env: { ORCA_PI_STATUS_OWNED: '99999999999999999999999' } + }) + + expect(harness.killMock).not.toHaveBeenCalled() + expect(harness.handlers.agent_end).toBeTypeOf('function') + expect(harness.processEnv.ORCA_PI_STATUS_OWNED).toBe(String(SELF_PID)) + }) + + it('claims the pane when the inherited owner PID exceeds the process API range', () => { + const harness = createHarness({ + kind: 'pi', + pid: SELF_PID, + env: { ORCA_PI_STATUS_OWNED: String(2 ** 31) } + }) + + expect(harness.killMock).not.toHaveBeenCalled() + expect(harness.handlers.agent_end).toBeTypeOf('function') + expect(harness.processEnv.ORCA_PI_STATUS_OWNED).toBe(String(SELF_PID)) + }) +}) diff --git a/src/main/pi/titlebar-extension-lifetime-source.ts b/src/main/pi/titlebar-extension-lifetime-source.ts new file mode 100644 index 00000000000..1d7c4abd304 --- /dev/null +++ b/src/main/pi/titlebar-extension-lifetime-source.ts @@ -0,0 +1,30 @@ +export function getPiTitlebarLifetimeSourceLines(): string[] { + return [ + ' // Why: replacement factories share the process realm; retire the old owner before painting.', + " const ownersKey = Symbol.for('orca.pi.titlebar.owners')", + ' const owners = globalThis[ownersKey] ??= new Map()', + ' const paneKey = process.env.ORCA_PANE_KEY', + ' owners.get(paneKey)?.()', + ' let disposed = false', + ' function clearOwnedTimers() {', + ' clearPendingAgentEndCheck()', + ' clearAnimation()', + ' stopMarkerReassert()', + ' }', + '', + ' function dispose() {', + ' disposed = true', + ' clearOwnedTimers()', + ' resetPromptState()', + ' if (owners.get(paneKey) === dispose) owners.delete(paneKey)', + ' }', + ' owners.set(paneKey, dispose)', + '', + ' function on(name, handler) {', + ' pi.on(name, (event, ctx) => {', + ' if (!disposed) return handler(event, ctx)', + ' })', + ' }', + '' + ] +} diff --git a/src/main/pi/titlebar-extension-service.test.ts b/src/main/pi/titlebar-extension-service.test.ts index 3ad73bba6eb..69884d6591b 100644 --- a/src/main/pi/titlebar-extension-service.test.ts +++ b/src/main/pi/titlebar-extension-service.test.ts @@ -39,6 +39,7 @@ vi.mock('os', async (importOriginal) => { }) import { PiTitlebarExtensionService, isSafeDescendCandidate } from './titlebar-extension-service' +import { getPiTitlebarExtensionSource } from './titlebar-extension-source' function legacyOverlayPath(kind: 'pi' | 'omp', ptyId: string): string { const rootDir = kind === 'pi' ? 'pi-agent-overlays' : 'omp-agent-overlays' @@ -458,6 +459,16 @@ describe('PiTitlebarExtensionService', () => { expectPiHomeIntact() }) + it('refreshes a managed spinner in an explicitly selected senpi home', () => { + const agentDir = join(userDataDir, '.omo', 'agent') + const extensionPath = join(agentDir, 'extensions', 'orca-titlebar-spinner.ts') + mkdirSync(join(agentDir, 'extensions'), { recursive: true }) + writeFileSync(extensionPath, '// @orca-managed-pi-extension\nstale spinner') + const svc = new PiTitlebarExtensionService() + svc.buildPtyEnv('pty-senpi', agentDir, 'pi') + expect(readFileSync(extensionPath, 'utf8')).toContain(getPiTitlebarExtensionSource()) + }) + it('rebuilding updates Orca-owned extensions while preserving user files', () => { const svc = new PiTitlebarExtensionService() svc.buildPtyEnv('pty-refresh-1', piHome, 'pi') diff --git a/src/main/pi/titlebar-extension-source.test.ts b/src/main/pi/titlebar-extension-source.test.ts index be21f8c6a16..eb826c1903e 100644 --- a/src/main/pi/titlebar-extension-source.test.ts +++ b/src/main/pi/titlebar-extension-source.test.ts @@ -35,6 +35,8 @@ function createHarness( processTitle?: string cwdImpl?: () => string sessionNameImpl?: () => string + setTitle?: (title: string) => void + globals?: Record env?: Record } = {} ): Harness { @@ -42,6 +44,7 @@ function createHarness( const ctx: TitlebarContext = { ui: { setTitle: (title: string) => { + options.setTitle?.(title) titles.push(title) } }, @@ -75,7 +78,7 @@ function createHarness( setTimeout: (...args: Parameters) => setTimeout(...args), clearTimeout: (timer: ReturnType) => clearTimeout(timer) } as Record - context.globalThis = context + context.globalThis = options.globals ?? context const output = ts.transpileModule(getPiTitlebarExtensionSource(options.kind ?? 'pi'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } @@ -588,4 +591,148 @@ describe('getPiTitlebarExtensionSource', () => { expect(harness.handlers.ui_prompt_start).toBeDefined() expect(() => harness.handlers.ui_prompt_start?.({}, undefined)).not.toThrow() }) + + it.each(['getter', 'title'] as const)( + 'retires a stale %s during animation without throwing or rescheduling', + async (failure) => { + let stale = false + const harness = createHarness({ + sessionNameImpl: () => { + if (stale && failure === 'getter') { + throw new Error('expired session') + } + return SESSION + }, + setTitle: () => { + if (stale && failure === 'title') { + throw new Error('expired UI') + } + } + }) + await harness.callHook('agent_start') + stale = true + expect(() => vi.advanceTimersByTime(80)).not.toThrow() + expect(vi.getTimerCount()).toBe(0) + await harness.callHook('agent_start') + expect(vi.getTimerCount()).toBe(0) + stale = false + await harness.callHook('agent_start') + expect(vi.getTimerCount()).toBe(1) + } + ) + + it.each(['getter', 'title'] as const)('contains stale %s during shutdown', async (failure) => { + let stale = false + const harness = createHarness({ + sessionNameImpl: () => { + if (stale && failure === 'getter') { + throw new Error('expired session') + } + return SESSION + }, + setTitle: () => { + if (stale && failure === 'title') { + throw new Error('expired UI') + } + }, + isIdle: () => false + }) + await harness.callHook('agent_start') + await harness.callHook('agent_end') + stale = true + await expect(harness.callHook('session_shutdown')).resolves.toBeUndefined() + expect(vi.getTimerCount()).toBe(0) + }) + + it('does not schedule a timer when the first frame fails', async () => { + const harness = createHarness({ + sessionNameImpl: () => { + throw new Error('expired') + } + }) + await harness.callHook('agent_start') + expect(vi.getTimerCount()).toBe(0) + }) + + it('retires animation when an idle recheck loses its session', async () => { + const harness = createHarness({ + isIdle: () => { + throw new Error('expired') + } + }) + await harness.callHook('agent_start') + await harness.callHook('agent_end') + expect(() => vi.advanceTimersByTime(1)).not.toThrow() + expect(vi.getTimerCount()).toBe(0) + }) + + it('clears animation and pending idle checks on session replacement', async () => { + const harness = createHarness({ isIdle: () => false }) + await harness.callHook('agent_start') + await harness.callHook('agent_end') + await harness.callHook('session_shutdown') + await harness.callHook('session_start') + expect(vi.getTimerCount()).toBe(0) + await harness.callHook('agent_start') + expect(vi.getTimerCount()).toBe(1) + }) + + it('reload replaces only its pane owner and ignores late old-generation events', async () => { + const globals = {} + const old = createHarness({ globals, isIdle: () => false }) + const other = createHarness({ globals, paneKey: 'pane-2' }) + await old.callHook('agent_start') + await old.callHook('ui_prompt_start') + await old.callHook('agent_end') + await other.callHook('agent_start') + const replacement = createHarness({ globals }) + expect(vi.getTimerCount()).toBe(1) + await replacement.callHook('agent_start') + const oldCount = old.titles.length + await old.callHook('session_shutdown') + await old.callHook('agent_start') + vi.advanceTimersByTime(80) + expect(old.titles).toHaveLength(oldCount) + expect(vi.getTimerCount()).toBe(2) + expect(replacement.lastTitle()).toMatch(BRAILLE_RE) + expect(other.lastTitle()).toMatch(BRAILLE_RE) + const third = createHarness({ globals }) + expect(vi.getTimerCount()).toBe(1) + await third.callHook('agent_start') + await replacement.callHook('session_shutdown') + expect(vi.getTimerCount()).toBe(2) + }) + + it('stops spinner, prompt reassertion and idle recheck together on invalidation', async () => { + let stale = false + const harness = createHarness({ + isIdle: () => false, + sessionNameImpl: () => { + if (stale) { + throw new Error('stale generation') + } + return SESSION + } + }) + await harness.callHook('agent_start') + await harness.callHook('ui_prompt_start') + await harness.callHook('agent_end') + expect(vi.getTimerCount()).toBe(3) + stale = true + await vi.advanceTimersByTimeAsync(80) + expect(vi.getTimerCount()).toBe(0) + }) + + it('clears prompt and idle timers at session_start without needing shutdown', async () => { + const harness = createHarness({ isIdle: () => false }) + await harness.callHook('agent_start') + await harness.callHook('ui_prompt_start') + await harness.callHook('agent_end') + expect(vi.getTimerCount()).toBe(3) + await harness.callHook('session_start') + expect(vi.getTimerCount()).toBe(0) + await harness.callHook('agent_start') + expect(harness.lastTitle()).toMatch(BRAILLE_RE) + expect(vi.getTimerCount()).toBe(1) + }) }) diff --git a/src/main/pi/titlebar-extension-source.ts b/src/main/pi/titlebar-extension-source.ts index 7fc15c191bc..570d23a15d3 100644 --- a/src/main/pi/titlebar-extension-source.ts +++ b/src/main/pi/titlebar-extension-source.ts @@ -1,3 +1,4 @@ +import { getPiTitlebarLifetimeSourceLines } from './titlebar-extension-lifetime-source' import type { PiAgentKind } from '../../shared/pi-agent-kind' import { getPiOmpRuntimeDetectionSourceLines } from './agent-status-runtime-detection-source' @@ -10,7 +11,7 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { const uiPromptHandlers = kind === 'pi' ? [ - " pi.on('ui_prompt_start', async (_event, ctx) => {", + " on('ui_prompt_start', async (_event, ctx) => {", ' if (isOmpRuntime() || !ownsMarker) return', ' promptDepth++', ' // Why: retry on every open rather than only the outermost, so an outer ctx', @@ -25,7 +26,7 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' startMarkerReassert(painter)', ' })', '', - " pi.on('ui_prompt_end', async (_event, ctx) => {", + " on('ui_prompt_end', async (_event, ctx) => {", ' if (isOmpRuntime() || !ownsMarker || promptDepth === 0) return', ' promptDepth--', ' if (promptDepth > 0) return', @@ -98,20 +99,6 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' }', '}', '', - '// Why: buildTitle runs inside the try because it is not safe either — getSessionName()', - '// calls assertActive() and process.cwd() throws ENOENT once the worktree is deleted.', - '// Most call sites are timer callbacks, where an escape is an uncaught exception and pi', - '// exits(1) through its own uncaughtException handler.', - 'function paintTitle(ctx, buildTitle) {', - ' if (!ctx) return false', - ' try {', - ' ctx.ui.setTitle(buildTitle())', - ' return true', - ' } catch {', - ' return false', - ' }', - '}', - '', 'export default function (pi) {', ' if (!process.env.ORCA_PANE_KEY) return', ...(kind === 'pi' @@ -125,6 +112,7 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ] : []), + ...getPiTitlebarLifetimeSourceLines(), ' let timer = null', ' let frameIndex = 0', ' // Why: only idle maintenance owns a spinner of its own. A threshold compaction runs', @@ -144,6 +132,21 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' let pendingAgentEndContext = null', ' let agentEndIdleRecheckMs = AGENT_END_IDLE_RECHECK_MS', '', + '// Why: buildTitle runs inside the try because it is not safe either — getSessionName()', + '// calls assertActive() and process.cwd() throws ENOENT once the worktree is deleted.', + '// Most call sites are timer callbacks, where an escape is an uncaught exception and pi', + '// exits(1) through its own uncaughtException handler.', + ' function paintTitle(ctx, buildTitle) {', + ' if (disposed || !ctx) return false', + ' try {', + ' ctx.ui.setTitle(buildTitle())', + ' return true', + ' } catch {', + ' clearOwnedTimers()', + ' return false', + ' }', + ' }', + '', ' function resetPromptState() {', ' stopMarkerReassert()', ' promptDepth = 0', @@ -199,23 +202,24 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' // otherwise wipe the marker with nothing to restore it. The frame still counts,', ' // so the cap above keeps accruing in wall-clock.', ' if (markerPainted) {', - " paintTitle(ctx, () => getMarkedTitle(pi, '!'))", + " const painted = paintTitle(ctx, () => getMarkedTitle(pi, '!'))", ' frameIndex++', - ' return', + ' return painted', ' }', - ' paintTitle(ctx, () => {', + ' const painted = paintTitle(ctx, () => {', ' const frame = BRAILLE_FRAMES[frameIndex % BRAILLE_FRAMES.length]', ' const cwd = process.cwd().split(/[\\\\/]/).filter(Boolean).at(-1) || process.cwd()', ' const session = pi.getSessionName()', ' return session ? `${frame} \\u03c0 - ${session} - ${cwd}` : `${frame} \\u03c0 - ${cwd}`', ' })', ' frameIndex++', + ' return painted', ' }', '', ' function startAnimation(ctx) {', ' clearPendingAgentEndCheck()', ' clearAnimation()', - ' renderFrame(ctx)', + ' if (!renderFrame(ctx)) return', ' timer = setInterval(() => renderFrame(ctx), FRAME_INTERVAL_MS)', ' }', '', @@ -230,7 +234,7 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' return', ' }', ' } catch {', - ' pendingAgentEndContext = null', + ' clearOwnedTimers()', ' return', ' }', ' pendingAgentEndCheck = setTimeout(checkPendingAgentEnd, agentEndIdleRecheckMs)', @@ -238,7 +242,7 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' agentEndIdleRecheckMs = Math.min(agentEndIdleRecheckMs * 2, AGENT_END_IDLE_RECHECK_MAX_MS)', ' }', '', - " pi.on('agent_start', async (_event, ctx) => {", + " on('agent_start', async (_event, ctx) => {", ' resetPromptState()', ' startAnimation(ctx)', ' })', @@ -246,17 +250,18 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' // Why: pi drops an open dialog through resetExtensionUI without resolving its promise,', ' // so a replaced or reloaded session never sends the matching close. Both boundaries', ' // prove no dialog from the old session is still on screen.', - " pi.on('session_start', async () => {", + " on('session_start', async () => {", + ' clearOwnedTimers()', ' resetPromptState()', ' })', '', ' // Why: modern Pi/OMP emit agent_end mid-run and only settle later, so settlement is the', ' // authoritative completion boundary. Legacy runtimes never emit it, so agent_end stays.', - " pi.on('agent_settled', async (_event, ctx) => {", + " on('agent_settled', async (_event, ctx) => {", ' stopAnimation(ctx)', ' })', '', - " pi.on('agent_end', async (event, ctx) => {", + " on('agent_end', async (event, ctx) => {", ' if (event?.willContinue === true) {', ' clearPendingAgentEndCheck()', ' return', @@ -273,7 +278,7 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' })', '', ...uiPromptHandlers, - " pi.on('auto_compaction_start', async (event, ctx) => {", + " on('auto_compaction_start', async (event, ctx) => {", " if (event?.reason !== 'idle') return", ' // Why: the idle worker can fire against a turn that just started, and reason alone does', ' // not prove the pane is idle. Adopting a live agent spinner would let the matching', @@ -283,12 +288,12 @@ export function getPiTitlebarExtensionSource(kind: PiAgentKind = 'pi'): string { ' idleCompactionOwnsSpinner = true', ' })', '', - " pi.on('auto_compaction_end', async (_event, ctx) => {", + " on('auto_compaction_end', async (_event, ctx) => {", ' if (!idleCompactionOwnsSpinner) return', ' stopAnimation(ctx)', ' })', '', - " pi.on('session_shutdown', async (_event, ctx) => {", + " on('session_shutdown', async (_event, ctx) => {", ' resetPromptState()', ' stopAnimation(ctx)', ' })', diff --git a/src/main/runtime/__fixtures__/antigravity-busy-mid-turn.meta.json b/src/main/runtime/__fixtures__/antigravity-busy-mid-turn.meta.json new file mode 100644 index 00000000000..4e875eb047a --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-busy-mid-turn.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T06:10:52.713Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "agy TUI 1.2.0; recording stopped ~0.3s after submit, while the spinner was live; no shutdown repaint in the file", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-busy-mid-turn.txt b/src/main/runtime/__fixtures__/antigravity-busy-mid-turn.txt new file mode 100644 index 00000000000..8f3645800f7 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-busy-mid-turn.txt @@ -0,0 +1,38 @@ +[?2026$p[?2027$p[>4m[=0;1u[?1049h[?25l[?5W[?2004h[>4;2m[=1;1u[?u +▄▀▀▄ +▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀ + ▄▀▀ ▀▀▄ + ▄▀▀ ▀▀▄ + + Welcome to the Antigravity CLI. You are currently not signed in. + + ⣾ Signing in... No authentication methods available. + + Press ctrl+c or ctrl+d twice to exit.[>4m[=0;1u[?1049l[>4;2m[=1;1u[?u[0 q  +▄▀▀▄ Antigravity CLI 1.2.0 +▀▀▀▀▀▀ Gemini API key +▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low) + ▄▀▀ ▀▀▄ ~ + ▄▀▀ ▀▀▄ + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +? for shortcutsGemini 3.7 Flash · low [?25h[?25lI[?25h[?25ln ab + + G[?25h[?25lout 8[?25h[?25l0 wo[?25h[?25lrds,[?25h[?25lexpla[?25h[?25lin w[?25h[?25lhat a[?25h[?25l pse[?25h[?25lud[?25h[?25loter[?25h[?25lminal[?25h[?25l is.[?25h[?25l[?25h[?25l + +? for shortcuts[?25h[?25lM +> In about 80 words, explain what a pseudoterminal is. +⣷ Generating... +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +esc to cancelGemini 3.7 Flash · low [?25h[?25lng + +[?25h[?25l ⣯ Generating + +[?25h[?25l ⣟ Generating. + +[?25h \ No newline at end of file diff --git a/src/main/runtime/__fixtures__/antigravity-busy-turn-ended.meta.json b/src/main/runtime/__fixtures__/antigravity-busy-turn-ended.meta.json new file mode 100644 index 00000000000..084be8e54bc --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-busy-turn-ended.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T06:13:00.364Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "agy TUI 1.2.0; recording stopped after the turn ended and the composer returned, with the process still alive. This account's API key cannot complete a turn, so the turn ends in a backend error", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-busy-turn-ended.txt b/src/main/runtime/__fixtures__/antigravity-busy-turn-ended.txt new file mode 100644 index 00000000000..e10de85d361 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-busy-turn-ended.txt @@ -0,0 +1,42 @@ +[?2026$p[?2027$p[?5W[?2004h[>4;2m[=1;1u[?u[0 q  +▄▀▀▄ Antigravity CLI 1.2.0 +▀▀▀▀▀▀ Gemini API key +▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low) + ▄▀▀ ▀▀▄ ~ + ▄▀▀ ▀▀▄ + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +? for shortcutsGemini 3.7 Flash · low [?25h[?25lIn + + G[?25h[?25labo[?25h[?25lut 80[?25h[?25l wo[?25h[?25lrds[?25h[?25l, ex[?25h[?25lpla[?25h[?25lin wh[?25h[?25lat a[?25h[?25lpseudo[?25h[?25ltermi[?25h[?25lnal is[?25h[?25l.[?25h[?25l + +? for shortcuts[?25h[?25lM +> In about 80 words, explain what a pseudoterminal is. +⣾ Generating... +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +esc to cancelGemini 3.7 Flash · low [?25h[?25l ⣷ Generatin + +[?25h[?25l ⣯ Generating + +[?25h[?25l ⣟ Generating. + +[?25h[?25l ⡿ Generating... + +[?25h[?25l ⢿ Generatin + +[?25h[?25l  +⚠ Agent execution terminated due to error. +Error ID: 00000000-0000-4000-8000-000000000000-2 +⢿ Generating... +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +esc to cancelGemini 3.7 Flash · low [?25h[?25l  + + + +? for shortcuts[?25h \ No newline at end of file diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-command-palette.meta.json b/src/main/runtime/__fixtures__/antigravity-dialog-command-palette.meta.json new file mode 100644 index 00000000000..e098a1677ab --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-command-palette.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T04:34:32.974Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "agy TUI 1.2.0; slash-command palette live, unanswered", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-command-palette.txt b/src/main/runtime/__fixtures__/antigravity-dialog-command-palette.txt new file mode 100644 index 00000000000..9bf02cc0ff9 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-command-palette.txt @@ -0,0 +1,41 @@ +[?2026$p[?2027$p[?5W[?2004h[>4;2m[=1;1u[?u[0 q  +▄▀▀▄ Antigravity CLI 1.2.0 +▀▀▀▀▀▀ Gemini API key +▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low) + ▄▀▀ ▀▀▄ ~ + ▄▀▀ ▀▀▄ + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +? for shortcutsGemini 3.7 Flash · low [?25h[?25l/ + +> /add-dir  Add a directory to the workspace + /agents List available custom agents + /artifact View and review artifacts + /btw Ask a side question without interrupting the current task + /changelog Show release notes and changes + ↓ 50 more + + ↑/↓ Navigate · enter Select · tab Complete + Gemini 3.7 Flash · low [?25h[?25l + + + + + + + + + +esc to cancel[?25h[>4m[=0;1u + + + + + + + + + +[?2004l[0 q \ No newline at end of file diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-dismissed.meta.json b/src/main/runtime/__fixtures__/antigravity-dialog-dismissed.meta.json new file mode 100644 index 00000000000..8e8d5043fdf --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-dismissed.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T04:35:06.866Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "agy TUI 1.2.0; /model picker opened then dismissed with esc, settled before stop", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-dismissed.txt b/src/main/runtime/__fixtures__/antigravity-dialog-dismissed.txt new file mode 100644 index 00000000000..bb35ae33af2 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-dismissed.txt @@ -0,0 +1,54 @@ +[?2026$p[?2027$p[?5W[?2004h[>4;2m[=1;1u[?u[0 q  +▄▀▀▄ Antigravity CLI 1.2.0 +▀▀▀▀▀▀ Gemini API key +▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low) + ▄▀▀ ▀▀▄ ~ + ▄▀▀ ▀▀▄ + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +? for shortcutsGemini 3.7 Flash · low [?25h[?25l/mod + +> /model Set a model, or run a single prompt on another model + /permissioned-github Guidelines for interacting with GitHub and request permissions from the user when commands f... + + ↑/↓ Navigate · enter Select · tab Complete +esc to cancelGemini 3.7 Flash · low [?25h[?25l + + + + +/model + +  + + ↑/↓ Navigate · enter Select · tab Complete +esc to cancelGemini 3.7 Flash · low [?25h[?25l[0 q + +Switch Model + + Gemini 3.8 Flash +> Gemini 3.7 Flash (current) + Gemini 3.6 Flash + Gemini 3.1 Pro + + Effort ◂  ◉──────────────○──────────────○  ▸ +  low  medium high  + Faster responses, lighter reasoning — great for simpler tasks + +Keyboard: ↑/↓ Navigate ←/→ Effort enter Select esc Go Back + + Gemini 3.7 Flash · low [0 q> /model + ⎿ Exited /model command + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +Gemini 3.7 Flash · low [?25h[?25l + +? for shortcuts[?25h[>4m[=0;1u + +[?2004l[0 q +Resume with -c (or command below): +agy --conversation=00000000-0000-4000-8000-000000000000 diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-model-picker.meta.json b/src/main/runtime/__fixtures__/antigravity-dialog-model-picker.meta.json new file mode 100644 index 00000000000..9a4e5c0c8e1 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-model-picker.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T04:34:10.855Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "agy TUI 1.2.0; /model picker live, unanswered, killed while it owns the screen", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-model-picker.txt b/src/main/runtime/__fixtures__/antigravity-dialog-model-picker.txt new file mode 100644 index 00000000000..6a09f6082f8 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-model-picker.txt @@ -0,0 +1,56 @@ +[?2026$p[?2027$p[?5W[?2004h[>4;2m[=1;1u[?u[0 q  +▄▀▀▄ Antigravity CLI 1.2.0 +▀▀▀▀▀▀ Gemini API key +▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low) + ▄▀▀ ▀▀▄ ~ + ▄▀▀ ▀▀▄ + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +? for shortcutsGemini 3.7 Flash · low [?25h[?25l/mo + +> /model Set a model, or run a single prompt on another model + /migrate-workflows Automatically migrate legacy workflows to modern skills across global and workspace configur... + /permissions Manage tool permissions + /agy-customizations Comprehensive guide and reference for the Antigravity Customization System. Use to explain h... + /permissioned-github Guidelines for interacting with GitHub and request permissions from the user when commands f... + + ↑/↓ Navigate · enter Select · tab Complete +? for shortcutsGemini 3.7 Flash · low [?25h[?25l + + + + +/model + +  + + ↑/↓ Navigate · enter Select · tab Complete +esc to cancelGemini 3.7 Flash · low [?25h[?25l[0 q + +Switch Model + +> Gemini 3.8 Flash + Gemini 3.7 Flash (current) + Gemini 3.6 Flash + Gemini 3.1 Pro + + Effort ◂  ●━━━━━━━━━━━━━━◉──────────────○  ▸ +  low  medium  high  + Balanced speed and reasoning quality for most tasks + +Keyboard: ↑/↓ Navigate ←/→ Effort enter Select esc Go Back + +? for shortcutsGemini 3.7 Flash · low  Gemini 3.8 Flash +> Gemini 3.7 Flash + + + +◂  ◉──────────────○ + low  medium  +Faster responses, lighter reasoning — great for simpler tasks + + + +  G[>4m[=0;1u [?25h[?2004l \ No newline at end of file diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-trust-workspace.meta.json b/src/main/runtime/__fixtures__/antigravity-dialog-trust-workspace.meta.json new file mode 100644 index 00000000000..07fb15ab6f7 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-trust-workspace.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T04:35:20.989Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "agy TUI 1.2.0; workspace trust dialog live and unanswered in a throwaway untrusted directory", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-dialog-trust-workspace.txt b/src/main/runtime/__fixtures__/antigravity-dialog-trust-workspace.txt new file mode 100644 index 00000000000..b2e1b342199 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-dialog-trust-workspace.txt @@ -0,0 +1,12 @@ +[?2026$p[?2027$p[>4m[=0;1u[?1049h[?25l[?5W[?2004h[>4;2m[=1;1u[?uAccessing workspace: + +/private/tmp/agy-trust-scratch-77950 + +Do you trust the contents of this project? + +Antigravity CLI requires permission to read, edit, and execute files here. + +> Yes, I trust this folder + No, exit + + ↑/↓ Navigate · enter ConfirmGemini 3.7 Flash · low[>4m[=0;1u [?1049l[?25h[?2004l \ No newline at end of file diff --git a/src/main/runtime/__fixtures__/antigravity-ready-account-info-hidden.meta.json b/src/main/runtime/__fixtures__/antigravity-ready-account-info-hidden.meta.json new file mode 100644 index 00000000000..9607841cf6e --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-ready-account-info-hidden.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T04:33:34.954Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "same session as antigravity-ready-api-key-gemini-model but with AGY_CLI_HIDE_ACCOUNT_INFO=1", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-ready-account-info-hidden.txt b/src/main/runtime/__fixtures__/antigravity-ready-account-info-hidden.txt new file mode 100644 index 00000000000..b93514374e0 --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-ready-account-info-hidden.txt @@ -0,0 +1,13 @@ +[?2026$p[?2027$p[?5W[?2004h[>4;2m[=1;1u[?u[0 q  +▄▀▀▄ Antigravity CLI 1.2.0 +▀▀▀▀▀▀ Gemini 3.7 Flash (Low) +▀▀▀▀▀▀▀▀ ~ + ▄▀▀ ▀▀▄ + ▄▀▀ ▀▀▄ + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +? for shortcutsGemini 3.7 Flash · low [?25h[>4m[=0;1u + +[?2004l[0 q \ No newline at end of file diff --git a/src/main/runtime/__fixtures__/antigravity-ready-api-key-gemini-model.meta.json b/src/main/runtime/__fixtures__/antigravity-ready-api-key-gemini-model.meta.json new file mode 100644 index 00000000000..97a54e107dc --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-ready-api-key-gemini-model.meta.json @@ -0,0 +1,9 @@ +{ + "capturedAt": "2026-09-11T04:33:14.819Z", + "platform": "darwin", + "command": ["agy"], + "cols": 120, + "rows": 40, + "note": "agy binary 1.1.25, TUI banner 1.2.0; Gemini API key identity (no OAuth sign-in); model Gemini 3.7 Flash (Low); workspace ~", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/antigravity-ready-api-key-gemini-model.txt b/src/main/runtime/__fixtures__/antigravity-ready-api-key-gemini-model.txt new file mode 100644 index 00000000000..c9501f1caac --- /dev/null +++ b/src/main/runtime/__fixtures__/antigravity-ready-api-key-gemini-model.txt @@ -0,0 +1,13 @@ +[?2026$p[?2027$p[?5W[?2004h[>4;2m[=1;1u[?u[0 q  +▄▀▀▄ Antigravity CLI 1.2.0 +▀▀▀▀▀▀ Gemini API key +▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low) + ▄▀▀ ▀▀▄ ~ + ▄▀▀ ▀▀▄ + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +> +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +? for shortcutsGemini 3.7 Flash · low [?25h[>4m[=0;1u + +[?2004l[0 q \ No newline at end of file diff --git a/src/main/runtime/agent-transcript-pane-test-harness.ts b/src/main/runtime/agent-transcript-pane-test-harness.ts new file mode 100644 index 00000000000..f3a9a64793c --- /dev/null +++ b/src/main/runtime/agent-transcript-pane-test-harness.ts @@ -0,0 +1,79 @@ +// One pane builder for every suite that replays a captured agent transcript through the runtime. +import { vi } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' + +const TRANSCRIPT_PANE_LEAF_ID = '11111111-1111-4111-8111-111111111111' +const TRANSCRIPT_PANE_TAB_ID = 'tab-1' +const TRANSCRIPT_PANE_WORKTREE_ID = 'wt-1' +export const TRANSCRIPT_PANE_PTY_ID = 'pty-1' + +export type TranscriptPaneOptions = { + paneTitle: string + foregroundProcess: string | null + data: string + /** Set for a pane whose PTY lives on an SSH host or WSL distro rather than locally. */ + connectionId?: string + /** Simulates a PTY controller whose foreground probe never settles. */ + foregroundProbeHangs?: boolean + onForegroundProbe?: () => void +} + +export async function createTranscriptPane( + options: TranscriptPaneOptions +): Promise<{ runtime: OrcaRuntimeService; handle: string }> { + const runtime = new OrcaRuntimeService(null) + const internals = runtime as unknown as { + resolveTerminalWorkspaceLaunchScope: (selector: string) => Promise + } + vi.spyOn(internals, 'resolveTerminalWorkspaceLaunchScope').mockResolvedValue({ + id: TRANSCRIPT_PANE_WORKTREE_ID, + path: '/repo/app', + connectionId: options.connectionId ?? null, + repo: null, + folderWorkspace: null + }) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: TRANSCRIPT_PANE_PTY_ID, incarnationId: 'inc-1' }), + write: () => true, + kill: () => true, + getForegroundProcess: (): Promise => { + options.onForegroundProbe?.() + return options.foregroundProbeHangs === true + ? new Promise(() => {}) + : Promise.resolve(options.foregroundProcess) + } + }) + const terminal = await runtime.createTerminal(`id:${TRANSCRIPT_PANE_WORKTREE_ID}`, { + tabId: TRANSCRIPT_PANE_TAB_ID, + leafId: TRANSCRIPT_PANE_LEAF_ID, + title: 'Terminal' + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: TRANSCRIPT_PANE_TAB_ID, + worktreeId: TRANSCRIPT_PANE_WORKTREE_ID, + title: 'Terminal', + activeLeafId: TRANSCRIPT_PANE_LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId: TRANSCRIPT_PANE_TAB_ID, + worktreeId: TRANSCRIPT_PANE_WORKTREE_ID, + leafId: TRANSCRIPT_PANE_LEAF_ID, + paneRuntimeId: 1, + ptyId: TRANSCRIPT_PANE_PTY_ID, + paneTitle: options.paneTitle + } + ] + }) + // Why the guard: a restore seed is only applied to a never-written record, so the restore + // cases must not write an empty chunk first. + if (options.data.length > 0) { + runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, options.data, Date.now()) + } + return { runtime, handle: terminal.handle } +} diff --git a/src/main/runtime/antigravity-readiness-transcripts.test.ts b/src/main/runtime/antigravity-readiness-transcripts.test.ts new file mode 100644 index 00000000000..3ac7707565f --- /dev/null +++ b/src/main/runtime/antigravity-readiness-transcripts.test.ts @@ -0,0 +1,281 @@ +/** + * Pins Antigravity readiness to captured transcripts instead of hand-written fixtures. + * + * Five detector attempts were tuned against a five-line screen someone typed from memory, and + * three of them shipped worse behaviour than the bug they replaced. Nothing here asserts what + * Antigravity prints: the transcripts do. Six are recorded from a live `agy`; the rest name + * themselves as skipped until someone can reach them. + * + * Four cases are pinned as KNOWN DEFECT: on real output the shipped detector refuses the ready + * screen and accepts the live model picker. Those assert what it does, not what it should. + * + * Capture protocol: docs/reference/agent-pty-transcript-capture.md + * What each transcript decides: docs/reference/antigravity-readiness-evidence.md + */ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { createTranscriptPane } from './agent-transcript-pane-test-harness' +import { extractLastOscTitle } from '../../shared/osc-title-extraction' + +vi.mock('electron', () => ({ + BrowserWindow: { fromId: vi.fn(() => null) }, + webContents: { fromId: vi.fn(() => null) }, + ipcMain: { on: vi.fn(), removeListener: vi.fn() }, + app: { getPath: vi.fn(() => '/tmp') } +})) + +const FIXTURE_DIR = join(__dirname, '__fixtures__') +const EVIDENCE_DOC = join( + __dirname, + '..', + '..', + '..', + 'docs', + 'reference', + 'antigravity-readiness-evidence.md' +) +// Why asymmetric: a ready verdict has to survive the settle window, while a refusal only has to +// hold for one poll. Keeping the refusal short keeps seven transcripts off the suite's clock. +const READY_TIMEOUT_MS = 2_000 +const REFUSAL_TIMEOUT_MS = 600 +/** Antigravity's binary, as Orca launches and probes it (`tui-agent-config.ts` detectCmd). */ +const ANTIGRAVITY_COMMAND = 'agy' +// String.fromCharCode, not a literal: the formatter rewrites an escape sequence into a raw +// control byte in source, which is unreadable and survives badly in diffs. +const ESC = String.fromCharCode(27) + +type TranscriptCase = { + /** Fixture basename; `.txt` under `__fixtures__/`. */ + name: string + /** Capture in docs/reference/antigravity-readiness-evidence.md. */ + capture: string + what: string + /** What a correct detector must answer. Not what the shipped one answers. */ + expectReady: boolean + /** + * Set where the shipped detector contradicts the transcript. The case then runs inverted, so + * CI pins the defect instead of going permanently red — and flips to failing the moment + * someone fixes it, which is exactly when these expectations need re-reading. + */ + knownDefect?: string +} + +const TRANSCRIPTS: readonly TranscriptCase[] = [ + { + name: 'antigravity-ready-api-key-gemini-model', + capture: 'B', + what: 'ready screen, API-key identity — the account row reads "Gemini API key", not an email', + expectReady: true, + knownDefect: 'refused: the model row never starts a line, the logo shares it' + }, + { + name: 'antigravity-ready-account-info-hidden', + capture: 'B', + what: 'ready screen with AGY_CLI_HIDE_ACCOUNT_INFO=1 — no account row at all', + expectReady: true, + knownDefect: 'refused: same line-start defect, and no account row exists to require' + }, + { + name: 'antigravity-dialog-trust-workspace', + capture: 'C', + what: 'workspace trust dialog owning the screen', + expectReady: false + }, + { + name: 'antigravity-dialog-model-picker', + capture: 'C', + what: 'model picker owning the screen', + expectReady: false, + knownDefect: "accepted: the picker's own `Gemini 3.x Flash` rows satisfy the model rule" + }, + { + name: 'antigravity-dialog-command-palette', + capture: 'C', + what: 'slash-command palette owning the screen', + expectReady: false + }, + { + name: 'antigravity-busy-mid-turn', + capture: 'E', + what: 'mid-turn, spinner live — the pane is working, not waiting for a prompt', + expectReady: false + }, + { + // Expected ready because the turn is over and the composer is back on screen. The captured + // turn ends in a backend error, which is the only ending this account's key can produce. + name: 'antigravity-busy-turn-ended', + capture: 'E', + what: 'the turn has ended and the composer has returned, process still alive', + expectReady: true, + knownDefect: 'refused: the retained tail ends on the error block, with no composer row in it' + }, + { + name: 'antigravity-dialog-dismissed', + capture: 'D', + what: 'the screen immediately after the model picker is dismissed', + expectReady: true, + knownDefect: 'refused: the banner is not reprinted and no model row starts a line' + }, + // Not captured: this machine's agy has no OAuth session and offers only Gemini models, and + // reaching the rest would mean signing the operator out or deleting their config. See + // docs/reference/antigravity-readiness-evidence.md § What could not be captured. + { + name: 'antigravity-ready-business-non-gemini', + capture: 'A', + what: 'ready screen, Business account, non-Gemini model', + expectReady: true + }, + { + name: 'antigravity-dialog-sign-in', + capture: 'C', + what: 'sign-in dialog owning the screen', + expectReady: false + }, + { + name: 'antigravity-dialog-theme-picker', + capture: 'C', + what: 'theme picker owning the screen', + expectReady: false + }, + { + name: 'antigravity-dialog-privacy-notice', + capture: 'C', + what: 'privacy notice owning the screen', + expectReady: false + }, + { + name: 'antigravity-dialog-update-banner', + capture: 'C', + what: 'update banner owning the screen', + expectReady: false + } +] + +function fixturePath(name: string): string { + return join(FIXTURE_DIR, `${name}.txt`) +} + +/** + * A `tui-idle` wait ends three ways, and only one of them is readiness: it resolves satisfied, it + * resolves unsatisfied with a blocked reason, or it rejects with `timeout` because nothing ever + * looked ready. The orchestrator treats the last two identically — no prompt is delivered — so + * they are both `ready: false` here. This is the shape `worker-start` sees. + */ +async function readinessVerdict( + transcript: string, + timeoutMs: number +): Promise<{ ready: boolean; blockedReason: unknown; outcome: string }> { + const { runtime, handle } = await createTranscriptPane({ + // Why the transcript's own title: every attempt guessed at Antigravity's title. A raw + // capture carries the OSC bytes, so the pane wears whatever the CLI actually set. + paneTitle: extractLastOscTitle(transcript) ?? ANTIGRAVITY_COMMAND, + foregroundProcess: ANTIGRAVITY_COMMAND, + data: transcript + }) + try { + const result = (await runtime.waitForTerminal(handle, { + condition: 'tui-idle', + timeoutMs + })) as { satisfied?: boolean; blockedReason?: unknown } + return { + ready: result.satisfied === true, + blockedReason: result.blockedReason ?? null, + outcome: result.satisfied === true ? 'satisfied' : 'unsatisfied' + } + } catch (error) { + return { ready: false, blockedReason: null, outcome: `rejected: ${String(error)}` } + } +} + +describe('Antigravity readiness, decided by captured transcripts', () => { + for (const transcript of TRANSCRIPTS) { + const path = fixturePath(transcript.name) + const captured = existsSync(path) + const label = `capture ${transcript.capture}: ${transcript.what}` + + // A pinned defect asserts what the detector DOES, so CI is honest rather than permanently + // red; fixing the detector flips this case to failing, which is when these expectations + // need re-reading. The correct answer stays in `expectReady` and in the test's name. + const shipped = + transcript.knownDefect === undefined ? transcript.expectReady : !transcript.expectReady + const verdictName = + transcript.knownDefect === undefined + ? `${label} → ${transcript.expectReady ? 'ready' : 'not ready'}` + : `${label} → must be ${transcript.expectReady ? 'ready' : 'not ready'}; KNOWN DEFECT, ${transcript.knownDefect}` + + it.skipIf(!captured)( + verdictName, + async () => { + // A refusal only has to hold for one poll; a ready verdict has to survive the settle + // window. Keeping the refusal short keeps eleven transcripts off the suite's clock. + const verdict = await readinessVerdict( + readFileSync(path, 'utf8'), + transcript.expectReady ? READY_TIMEOUT_MS : REFUSAL_TIMEOUT_MS + ) + // A silent dialog carries no blocked-signal wording, so the assertion is only that Orca + // does not call the pane ready and type a prompt into a dialog that owns the screen. + expect({ ready: verdict.ready, outcome: verdict.outcome }).toMatchObject({ + ready: shipped + }) + }, + READY_TIMEOUT_MS + 10_000 + ) + + it.skipIf(!captured)(`${label} was captured raw, not pasted from a rendered screen`, () => { + const text = readFileSync(path, 'utf8') + // Why: a transcript with no escape bytes went through a terminal's renderer and a + // human's clipboard. It cannot answer what the caret or chrome looked like. + expect(text).toContain(ESC) + }) + } + + it('documents every transcript the detector is allowed to depend on', () => { + // Why a test: the doc is the operator's checklist. A name that drifts out of it is a + // transcript nobody will capture, and a case that silently skips forever. + const doc = readFileSync(EVIDENCE_DOC, 'utf8') + for (const transcript of TRANSCRIPTS) { + expect(doc).toContain(`${transcript.name}.txt`) + } + }) + + it('reports how much evidence exists, so a fully skipped run is visible', () => { + const missing = TRANSCRIPTS.filter( + (transcript) => !existsSync(fixturePath(transcript.name)) + ).map((transcript) => `${transcript.name}.txt`) + if (missing.length > 0) { + console.info( + `Antigravity transcripts: ${TRANSCRIPTS.length - missing.length}/${TRANSCRIPTS.length} captured. Missing: ${missing.join(', ')}` + ) + } + expect(missing.length).toBeLessThanOrEqual(TRANSCRIPTS.length) + }) +}) + +describe('scaffold self-check', () => { + // Why these two live here: when a transcript lands and fails, the failure has to mean the + // capture disagreed with the detector — not that the harness or the timeouts are broken. + // Neither case is evidence about Antigravity; both are shapes the current detector already + // decides, used only to prove the plumbing reaches a verdict. + it('reaches a ready verdict through the harness', async () => { + const verdict = await readinessVerdict( + [ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'Gemini 3.5 Flash (High)', + '~/orca/workspaces/orca/agy-dispatch-issue', + '>' + ].join('\n'), + READY_TIMEOUT_MS + ) + expect(verdict.ready).toBe(true) + }) + + it('reaches a not-ready verdict through the harness', async () => { + const verdict = await readinessVerdict( + 'Do you trust this workspace directory?\nPress t to trust\n', + REFUSAL_TIMEOUT_MS + ) + expect(verdict.ready).toBe(false) + }) +}) diff --git a/src/main/runtime/orca-runtime-pty-foreground-process-reads.ts b/src/main/runtime/orca-runtime-pty-foreground-process-reads.ts index a7e10247fed..84161b64789 100644 --- a/src/main/runtime/orca-runtime-pty-foreground-process-reads.ts +++ b/src/main/runtime/orca-runtime-pty-foreground-process-reads.ts @@ -149,13 +149,17 @@ export class OrcaRuntimeWithPtyForegroundProcessReads extends OrcaRuntimeWithSta ...(allowUnverifiedStop ? { allowUnverifiedStop: true } : {}), ...(connectionId ? { includeLocalRegistry: false } : {}) }) + // Structured sessions are counted here too, mirroring the IPC path: closing a user's chat is + // now an ordinary outcome of this verb, and a removal that closed one but no PTY logged nothing. + const structuredStopped = teardownResult.structuredStopped ?? 0 const total = teardownResult.runtimeStopped + teardownResult.providerStopped + - teardownResult.registryStopped + teardownResult.registryStopped + + structuredStopped if (total > 0) { console.info( - `[worktree-teardown] ${worktreeId} killed runtime=${teardownResult.runtimeStopped} provider=${teardownResult.providerStopped} registry=${teardownResult.registryStopped}` + `[worktree-teardown] ${worktreeId} killed runtime=${teardownResult.runtimeStopped} provider=${teardownResult.providerStopped} registry=${teardownResult.registryStopped} structured=${structuredStopped}` ) } } diff --git a/src/main/runtime/orca-runtime-tail-wait-memo.test.ts b/src/main/runtime/orca-runtime-tail-wait-memo.test.ts index f61431b7b86..5ce8d9710cd 100644 --- a/src/main/runtime/orca-runtime-tail-wait-memo.test.ts +++ b/src/main/runtime/orca-runtime-tail-wait-memo.test.ts @@ -134,7 +134,7 @@ describe('onPtyData tail wait memoization', () => { '' ) expect(blocked.fromTail).toBe(true) - expect(blocked.signal?.reason).toBe('codex-update-prompt') + expect(blocked.signal?.reason).toBe('agent-update-prompt') }) it('does not rebuild or repeatedly scan an ordinary saturated tail', () => { diff --git a/src/main/runtime/orca-runtime-tests/agent-status-and-waits.spec.ts b/src/main/runtime/orca-runtime-tests/agent-status-and-waits.spec.ts index 6fed887e741..7c0310554ef 100644 --- a/src/main/runtime/orca-runtime-tests/agent-status-and-waits.spec.ts +++ b/src/main/runtime/orca-runtime-tests/agent-status-and-waits.spec.ts @@ -247,7 +247,7 @@ describe('OrcaRuntimeService', () => { runtime.waitForTerminal(terminal.handle, { condition: 'tui-idle', timeoutMs: 1_000 }) ).resolves.toMatchObject({ satisfied: false, - blockedReason: 'codex-interactive-prompt' + blockedReason: 'agent-interactive-prompt' }) }) diff --git a/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts b/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts index e829be07d9f..682661f0fce 100644 --- a/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts +++ b/src/main/runtime/orca-runtime-tests/terminal-creation-and-readiness-part-07.spec.ts @@ -146,11 +146,11 @@ describe('OrcaRuntimeService', () => { condition: 'tui-idle', satisfied: false, status: 'running', - blockedReason: 'codex-hooks-review-prompt' + blockedReason: 'agent-hooks-review-prompt' }) }) - it('returns a blocked wait result for Codex update prompts', async () => { + it('returns an agent-neutral blocked wait result for update prompts', async () => { const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), @@ -177,11 +177,11 @@ describe('OrcaRuntimeService', () => { condition: 'tui-idle', satisfied: false, status: 'running', - blockedReason: 'codex-update-prompt' + blockedReason: 'agent-update-prompt' }) }) - it('returns a blocked wait result for Codex workspace trust prompts', async () => { + it('returns an agent-neutral blocked wait result for workspace trust prompts', async () => { const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), @@ -203,7 +203,7 @@ describe('OrcaRuntimeService', () => { condition: 'tui-idle', satisfied: false, status: 'running', - blockedReason: 'codex-trust-workspace' + blockedReason: 'agent-trust-workspace' }) }) @@ -270,7 +270,7 @@ describe('OrcaRuntimeService', () => { ).rejects.toThrow('timeout') }) - it('returns a blocked wait result for Codex cwd selection prompts', async () => { + it('returns an agent-neutral blocked wait result for cwd selection prompts', async () => { const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), @@ -297,7 +297,7 @@ describe('OrcaRuntimeService', () => { condition: 'tui-idle', satisfied: false, status: 'running', - blockedReason: 'codex-cwd-prompt' + blockedReason: 'agent-cwd-prompt' }) }) @@ -359,11 +359,11 @@ describe('OrcaRuntimeService', () => { condition: 'tui-idle', satisfied: false, status: 'running', - blockedReason: 'codex-hooks-review-prompt' + blockedReason: 'agent-hooks-review-prompt' }) }) - it('returns a blocked wait result for generic Codex interactive prompts', async () => { + it('returns an agent-neutral blocked wait result for generic interactive prompts', async () => { const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), @@ -390,7 +390,7 @@ describe('OrcaRuntimeService', () => { condition: 'tui-idle', satisfied: false, status: 'running', - blockedReason: 'codex-interactive-prompt' + blockedReason: 'agent-interactive-prompt' }) }) diff --git a/src/main/runtime/orca-runtime-tests/terminal-output-and-worker-recovery-part-02.spec.ts b/src/main/runtime/orca-runtime-tests/terminal-output-and-worker-recovery-part-02.spec.ts index a2d45395b26..ecf2ab53678 100644 --- a/src/main/runtime/orca-runtime-tests/terminal-output-and-worker-recovery-part-02.spec.ts +++ b/src/main/runtime/orca-runtime-tests/terminal-output-and-worker-recovery-part-02.spec.ts @@ -623,7 +623,7 @@ describe('OrcaRuntimeService', () => { runtime.waitForTerminal(terminal.handle, { condition: 'tui-idle', timeoutMs: 100 }) ).resolves.toMatchObject({ satisfied: false, - blockedReason: 'codex-trust-workspace' + blockedReason: 'agent-trust-workspace' }) serializeProviderBuffer.mockImplementationOnce(() => new Promise(() => {})) await expect( diff --git a/src/main/runtime/relay/desktop-relay-service.ts b/src/main/runtime/relay/desktop-relay-service.ts index a9b4f98f3b3..74bbd4e7254 100644 --- a/src/main/runtime/relay/desktop-relay-service.ts +++ b/src/main/runtime/relay/desktop-relay-service.ts @@ -19,7 +19,7 @@ import type { import type { DeviceCredentialInstallAuthorization } from './relay-control-requests' import { deriveRelayHostId } from './relay-http-client' import { RelayDemandLedger } from './relay-demand-ledger' -import { createRelayRegionPreferenceReader } from './relay-region-preference' +import { createRelayRegionPreferenceReader } from './relay-region-preference-reader' type DesktopRelayServiceOptions = { authConfig: OrcaCloudAuthConfig @@ -89,6 +89,7 @@ export class DesktopRelayService { isCurrent, refreshAccessToken, resolvePreferredRegion: regionPreference.resolvePreferredRegion, + measureRegionDecision: regionPreference.measureRegionDecision, onAssignedCellActive: regionPreference.noteAssignedCell, onStatus: options.onStatus }) @@ -327,10 +328,8 @@ export class DesktopRelayService { if (expiresAt !== null) { // Why: an unscanned QR must stop holding a standing control when its // server invite expires, even if no renderer survives to report closure. - this.demandExpiryTimer = setTimeout( - () => this.refreshDemand(), - Math.max(1, expiresAt - Date.now() + 1) - ) + const delay = Math.max(1, expiresAt - Date.now() + 1) + this.demandExpiryTimer = setTimeout(() => this.refreshDemand(), delay) } } } diff --git a/src/main/runtime/relay/relay-control-client-options.ts b/src/main/runtime/relay/relay-control-client-options.ts new file mode 100644 index 00000000000..93d1efc0fc6 --- /dev/null +++ b/src/main/runtime/relay/relay-control-client-options.ts @@ -0,0 +1,22 @@ +import type WebSocket from 'ws' +import type { E2EEKeypair } from '../e2ee-keypair' +import type { RelayConnectionOpenMessage, RelayDrainMessage } from './relay-control-protocol' + +export type RelayControlClientOptions = { + cellUrl: string + relayJwt: string + relayHostId: string + assignmentEpoch: number + identity: { userId: string; profileId: string; organizationId: string } + keypair: E2EEKeypair + appVersion: string + previousGeneration?: number + controlResumeSecret?: string + onConnectionOpen: (message: RelayConnectionOpenMessage) => void + onDrain: (message: RelayDrainMessage) => void + onClose: (code: number) => void + onPendingChanged?: () => void + createSocket?: (url: string, relayJwt: string) => WebSocket + connectDeadlineMs?: number + silenceLimitMs?: number +} diff --git a/src/main/runtime/relay/relay-control-client.test.ts b/src/main/runtime/relay/relay-control-client.test.ts index daa68e0c225..745a79ac84e 100644 --- a/src/main/runtime/relay/relay-control-client.test.ts +++ b/src/main/runtime/relay/relay-control-client.test.ts @@ -221,7 +221,7 @@ describe('RelayControlClient', () => { expect(authorization).toBe('Bearer scoped-token') // Advertised on the upgrade, never in host-hello: a cell that predates the // capability parses host-hello strictly and would refuse the handshake. - expect(capabilities).toBe('pending-conn-details') + expect(capabilities).toBe('pending-conn-details,idle-regional-rehome-v1') expect(path).toBe('/v1/host/control') const hello = await nextJson(socket) expect(hello).toMatchObject({ diff --git a/src/main/runtime/relay/relay-control-client.ts b/src/main/runtime/relay/relay-control-client.ts index 139d63e5640..0c863cce3ad 100644 --- a/src/main/runtime/relay/relay-control-client.ts +++ b/src/main/runtime/relay/relay-control-client.ts @@ -1,8 +1,8 @@ +import type { RelayControlClientOptions } from './relay-control-client-options' import { randomUUID } from 'node:crypto' import WebSocket, { type RawData } from 'ws' import { MOBILE_RELAY_CLOSE_CODE } from '../../../shared/mobile-relay-close-codes' import type { RelayHostCloseReason } from '../../../shared/relay-host-close-reason' -import type { E2EEKeypair } from '../e2ee-keypair' import { RelayConnectionOpenMessageSchema, RelayDrainMessageSchema, @@ -12,8 +12,6 @@ import { RELAY_HOST_CAPABILITY_HEADERS, encodeRelayHostHello, parseRelayControlMessage, - type RelayConnectionOpenMessage, - type RelayDrainMessage, type RelayHostHelloAckMessage, type RelayInviteCreatedMessage } from './relay-control-protocol' @@ -29,24 +27,6 @@ import { controlWebSocketUrl } from './relay-control-url' type RelayControlState = 'idle' | 'opening' | 'proving' | 'active' | 'draining' | 'closed' -type RelayControlClientOptions = { - cellUrl: string - relayJwt: string - relayHostId: string - assignmentEpoch: number - identity: { userId: string; profileId: string; organizationId: string } - keypair: E2EEKeypair - appVersion: string - previousGeneration?: number - controlResumeSecret?: string - onConnectionOpen: (message: RelayConnectionOpenMessage) => void - onDrain: (message: RelayDrainMessage) => void - onClose: (code: number) => void - createSocket?: (url: string, relayJwt: string) => WebSocket - connectDeadlineMs?: number - silenceLimitMs?: number -} - const RELAY_CONTROL_CONNECT_DEADLINE_MS = 15_000 export class RelayControlClient { @@ -54,7 +34,7 @@ export class RelayControlClient { private readonly relayOrigin: string private readonly controlUrl: string private readonly createSocket: NonNullable - private readonly requests = new RelayControlRequests() + private readonly requests: RelayControlRequests private socket: WebSocket | null = null private state: RelayControlState = 'idle' private connectResolve: ((ack: RelayHostHelloAckMessage) => void) | null = null @@ -64,6 +44,7 @@ export class RelayControlClient { constructor(options: RelayControlClientOptions) { this.options = options + this.requests = new RelayControlRequests(options.onPendingChanged) const endpoint = controlWebSocketUrl(options.cellUrl) this.relayOrigin = endpoint.origin this.controlUrl = endpoint.url diff --git a/src/main/runtime/relay/relay-control-origin-options.ts b/src/main/runtime/relay/relay-control-origin-options.ts new file mode 100644 index 00000000000..503f4a7411d --- /dev/null +++ b/src/main/runtime/relay/relay-control-origin-options.ts @@ -0,0 +1,24 @@ +import type WebSocket from 'ws' +import type { E2EEKeypair } from '../e2ee-keypair' +import type { MobileSocketWiring } from '../rpc/mobile-socket-wiring' +import type { RelayIdentity } from './relay-session-broker-contract' +import type { RelayAssignment } from './relay-http-client' +import type { RelayControlOrigin } from './relay-control-origin' +import type { RelayDrainMessage } from './relay-control-protocol' + +export type RelayControlOriginOptions = { + assignment: RelayAssignment + relayJwt: string + relayHostId: string + identity: RelayIdentity + keypair: E2EEKeypair + appVersion: string + mobileSocketWiring: MobileSocketWiring + createControlSocket?: (url: string, relayJwt: string) => WebSocket + createDataSocket?: (url: string) => WebSocket + onConnectionOwned: (connectionId: string, origin: RelayControlOrigin) => void + onConnectionReleased: (connectionId: string, origin: RelayControlOrigin) => void + onDrain: (origin: RelayControlOrigin, message: RelayDrainMessage) => void + onClose: (origin: RelayControlOrigin, code: number) => void + onPendingChanged?: (origin: RelayControlOrigin) => void +} diff --git a/src/main/runtime/relay/relay-control-origin.ts b/src/main/runtime/relay/relay-control-origin.ts index 4145a4b1b6c..7bd34862d9c 100644 --- a/src/main/runtime/relay/relay-control-origin.ts +++ b/src/main/runtime/relay/relay-control-origin.ts @@ -1,39 +1,19 @@ -import type WebSocket from 'ws' -import type { E2EEKeypair } from '../e2ee-keypair' +import type { RelayControlOriginOptions } from './relay-control-origin-options' import { CloudRelayTransport } from '../rpc/relay-transport' -import type { MobileSocketWiring } from '../rpc/mobile-socket-wiring' import { RelayControlClient } from './relay-control-client' import { RELAY_HOST_ATTACH_DEADLINE_MS } from './relay-control-protocol' import type { RelayConnectionOpenMessage, - RelayDrainMessage, RelayHostHelloAckMessage, RelayPendingConnection } from './relay-control-protocol' import type { RelayHostCloseReason } from '../../../shared/relay-host-close-reason' -import type { RelayIdentity } from './relay-session-broker-contract' import type { RelayAssignment } from './relay-http-client' const OBSERVED_OPEN_LIMIT = 16 -type RelayControlOriginOptions = { - assignment: RelayAssignment - relayJwt: string - relayHostId: string - identity: RelayIdentity - keypair: E2EEKeypair - appVersion: string - mobileSocketWiring: MobileSocketWiring - createControlSocket?: (url: string, relayJwt: string) => WebSocket - createDataSocket?: (url: string) => WebSocket - onConnectionOwned: (connectionId: string, origin: RelayControlOrigin) => void - onConnectionReleased: (connectionId: string, origin: RelayControlOrigin) => void - onDrain: (origin: RelayControlOrigin, message: RelayDrainMessage) => void - onClose: (origin: RelayControlOrigin, code: number) => void -} - export class RelayControlOrigin { - readonly assignment: RelayAssignment + assignment: RelayAssignment readonly transport: CloudRelayTransport private readonly options: RelayControlOriginOptions private readonly controls = new Set() @@ -98,6 +78,17 @@ export class RelayControlOrigin { return this.leaseExpiresAt } + get controlGeneration(): number { + return this.generation + } + + updateAssignment(assignment: RelayAssignment): void { + if (assignment.cellUrl !== this.cellUrl || assignment.assignmentEpoch < this.assignmentEpoch) { + throw new Error('relay_assignment_origin_mismatch') + } + this.assignment = assignment + } + get pendingRequestCount(): number { let count = 0 for (const control of this.controls) { @@ -124,6 +115,7 @@ export class RelayControlOrigin { controlResumeSecret: this.controlResumeSecret }) this.activate(control, ack) + this.updateAssignment(assignment) // Why: the resumed control owns the same server generation and splices; // the predecessor remains only long enough for any idempotent reply in flight. if (previous && previous.pendingRequestCount === 0) { @@ -198,6 +190,7 @@ export class RelayControlOrigin { : {}), onConnectionOpen: (message) => this.openConnection(message), onDrain: (message) => this.options.onDrain(this, message), + onPendingChanged: () => this.options.onPendingChanged?.(this), onClose: (code) => { this.controls.delete(control) const timer = this.retiredControlTimers.get(control) diff --git a/src/main/runtime/relay/relay-control-protocol.ts b/src/main/runtime/relay/relay-control-protocol.ts index 5d41498c00f..b2cc7dd0232 100644 --- a/src/main/runtime/relay/relay-control-protocol.ts +++ b/src/main/runtime/relay/relay-control-protocol.ts @@ -32,7 +32,7 @@ const ConnectionKindSchema = z.enum(['invite', 'resume']) // control upgrade rather than host-hello because the cell parses host-hello // strictly: a new hello key is refused by every already-deployed cell. export const RELAY_HOST_CAPABILITY_HEADERS = { - 'x-orca-host-capabilities': 'pending-conn-details' + 'x-orca-host-capabilities': 'pending-conn-details,idle-regional-rehome-v1' } as const // Mirrors RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs in the relay contract: the diff --git a/src/main/runtime/relay/relay-control-request-retirement.test.ts b/src/main/runtime/relay/relay-control-request-retirement.test.ts new file mode 100644 index 00000000000..f2aab8dae93 --- /dev/null +++ b/src/main/runtime/relay/relay-control-request-retirement.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RelayControlRequests } from './relay-control-requests' + +afterEach(() => vi.useRealTimers()) +describe('final source request retirement notification', () => { + it.each(['reply', 'denial', 'timeout', 'send-failed', 'closed'] as const)( + 'notifies final work completion after %s', + async (outcome) => { + vi.useFakeTimers() + const changed = vi.fn() + const requests = new RelayControlRequests(changed) + const result = requests + .confirmResume('req', 'basis', () => { + if (outcome === 'send-failed') { + throw new Error('send-failed') + } + }) + .catch((error: Error) => error.message) + if (outcome === 'reply') { + requests.resolveMessage({ + type: 'device-resume-confirmed', + v: 1, + reqId: 'req', + currentVersion: 1, + acceptedAs: 'current', + renewed: true, + resumeExpiresAt: 123_000 + }) + } else if (outcome === 'denial') { + requests.resolveMessage({ type: 'control-error', reqId: 'req', code: 'denied' }) + } else if (outcome === 'timeout') { + await vi.advanceTimersByTimeAsync(10_000) + } else if (outcome === 'closed') { + requests.rejectAll(new Error('closed')) + } + await result + await vi.advanceTimersByTimeAsync(0) + expect(requests.size).toBe(0) + expect(changed).toHaveBeenCalledOnce() + } + ) +}) diff --git a/src/main/runtime/relay/relay-control-requests.ts b/src/main/runtime/relay/relay-control-requests.ts index 2a96b94e3ec..bbceb067a59 100644 --- a/src/main/runtime/relay/relay-control-requests.ts +++ b/src/main/runtime/relay/relay-control-requests.ts @@ -25,6 +25,8 @@ export type DeviceCredentialInstallAuthorization = export class RelayControlRequests { private readonly pending = new Map() + constructor(private readonly onPendingChanged?: () => void) {} + get size(): number { return this.pending.size } @@ -162,7 +164,7 @@ export class RelayControlRequests { } return new Promise((resolve, reject) => { const timer = setTimeout(() => { - this.pending.delete(reqId) + this.finish(reqId) reject(new Error('relay_control_request_timeout')) }, 10_000) this.pending.set(reqId, { kind, resolve, reject, timer }) @@ -180,6 +182,8 @@ export class RelayControlRequests { if (pending) { clearTimeout(pending.timer) this.pending.delete(reqId) + // Settle the request before its final waiter retires the owning origin. + queueMicrotask(() => this.onPendingChanged?.()) } } } diff --git a/src/main/runtime/relay/relay-control-rotation.ts b/src/main/runtime/relay/relay-control-rotation.ts new file mode 100644 index 00000000000..2f1de94e6c0 --- /dev/null +++ b/src/main/runtime/relay/relay-control-rotation.ts @@ -0,0 +1,64 @@ +import type { RelayControlOrigin } from './relay-control-origin' +import type { RelayAssignment } from './relay-http-client' +import { relayRenewalDelayMs } from './relay-renewal-jitter' + +type RotationOptions = { + current: () => RelayControlOrigin | null + available: () => boolean + token: () => string | null + assignment: () => RelayAssignment | null + busy: () => boolean + now?: () => number + random?: () => number +} +export class RelayControlRotation { + private timer: ReturnType | null = null + constructor(private readonly options: RotationOptions) {} + cancel(): void { + if (this.timer) { + clearTimeout(this.timer) + } + this.timer = null + } + schedule(): void { + this.cancel() + const origin = this.options.current() + if (!origin || !this.options.available()) { + return + } + const delay = relayRenewalDelayMs( + origin.controlLeaseExpiresAt, + (this.options.now ?? Date.now)(), + this.options.random ?? Math.random + ) + this.timer = setTimeout(() => void this.rebind(origin), delay) + } + private async rebind(origin: RelayControlOrigin): Promise { + this.timer = null + if (!this.options.available() || origin !== this.options.current()) { + return + } + if (this.options.busy()) { + this.timer = setTimeout(() => void this.rebind(origin), 5_000) + return + } + const token = this.options.token() + const assignment = this.options.assignment() + if (!token || !assignment) { + return + } + try { + await origin.rebind(token, assignment) + if (this.options.available() && origin === this.options.current()) { + this.schedule() + } + } catch { + if (this.options.available() && origin === this.options.current()) { + this.timer = setTimeout( + () => void this.rebind(origin), + 5_000 + Math.floor((this.options.random ?? Math.random)() * 10_001) + ) + } + } + } +} diff --git a/src/main/runtime/relay/relay-http-client.ts b/src/main/runtime/relay/relay-http-client.ts index b31fe2ff9da..cb0fc519f9d 100644 --- a/src/main/runtime/relay/relay-http-client.ts +++ b/src/main/runtime/relay/relay-http-client.ts @@ -11,6 +11,10 @@ import { type RelayAssignRateGate } from './relay-assign-rate-gate' import type { RelayRegion } from './relay-region-preference' +import { + RelayRegionCorrectionResponseSchema, + type RelayRegionCorrectionRequest +} from './relay-region-correction-protocol' const RELAY_HTTP_REQUEST_DEADLINE_MS = 15_000 const RELAY_RETRY_AFTER_MAX_MS = 5 * 60_000 @@ -33,7 +37,9 @@ const AssignmentResponseSchema = z lease: z .string() .min(1) - .max(8 * 1024) + .max(8 * 1024), + // Optional correction must not make a healthy assignment depend on a future policy. + regionCorrection: RelayRegionCorrectionResponseSchema.optional().catch(undefined) }) .strict() @@ -133,6 +139,7 @@ type RelayAssignmentRequest = { relayHostId: string reconnect?: boolean preferredRegion?: RelayRegion + regionCorrection?: RelayRegionCorrectionRequest fetch?: typeof globalThis.fetch requestDeadlineMs?: number // Fencing for the throttle wait: a superseded caller aborts instead of assigning. @@ -185,6 +192,7 @@ async function sendRelayAssignment( body: JSON.stringify({ v: 1, relayHostId: input.relayHostId, + ...(input.regionCorrection ? { regionCorrection: input.regionCorrection } : {}), ...(input.preferredRegion ? { preferredRegion: input.preferredRegion } : {}), // Declares likely reconnection so the director can verify and admit // through its bounded fast lane instead of the placement queue. @@ -197,6 +205,9 @@ async function sendRelayAssignment( gate.noteRetryAfter(rateKey, retryAfterMs) } await cancelUnreadResponseBody(response) + if (input.regionCorrection && response.status === 400) { + return await sendRelayAssignment({ ...input, regionCorrection: undefined }, gate, rateKey) + } if (input.preferredRegion && response.status === 400) { // A rolled-back director rejects the regional hint; preserve the // reconnect lane while retrying without only that field. diff --git a/src/main/runtime/relay/relay-origin-pool-options.ts b/src/main/runtime/relay/relay-origin-pool-options.ts new file mode 100644 index 00000000000..ffb5455d1f4 --- /dev/null +++ b/src/main/runtime/relay/relay-origin-pool-options.ts @@ -0,0 +1,22 @@ +import type WebSocket from 'ws' +import type { E2EEKeypair } from '../e2ee-keypair' +import type { MobileSocketWiring } from '../rpc/mobile-socket-wiring' +import type { RelayBrokerStatus, RelayIdentity } from './relay-session-broker-contract' +import type { RelayRegion } from './relay-region-preference' + +export type RelayOriginPoolOptions = { + directorUrl: string + relayHostId: string + identity: RelayIdentity + keypair: E2EEKeypair + appVersion: string + mobileSocketWiring: MobileSocketWiring + isCurrent: () => boolean + onStatus: (status: RelayBrokerStatus) => void + resolvePreferredRegion?: () => Promise + fetch?: typeof globalThis.fetch + createControlSocket?: (url: string, relayJwt: string) => WebSocket + createDataSocket?: (url: string) => WebSocket + random?: () => number + now?: () => number +} diff --git a/src/main/runtime/relay/relay-origin-pool.ts b/src/main/runtime/relay/relay-origin-pool.ts index e8fd1d7d82a..ec8bb2f539d 100644 --- a/src/main/runtime/relay/relay-origin-pool.ts +++ b/src/main/runtime/relay/relay-origin-pool.ts @@ -1,50 +1,42 @@ -import type WebSocket from 'ws' -import type { E2EEKeypair } from '../e2ee-keypair' -import type { MobileSocketWiring } from '../rpc/mobile-socket-wiring' +import { RelayOriginRetirement } from './relay-origin-retirement' +import type { RelayOriginPoolOptions } from './relay-origin-pool-options' import { RelayControlOrigin } from './relay-control-origin' import type { RelayControlClient } from './relay-control-client' import type { RelayDrainMessage } from './relay-control-protocol' import type { RelayHostCloseReason } from '../../../shared/relay-host-close-reason' import { RelayDrainRetrySchedule } from './relay-drain-retry-schedule' import { RelayHttpError, requestRelayAssignment, type RelayAssignment } from './relay-http-client' -import { relayRenewalDelayMs } from './relay-renewal-jitter' -import type { RelayBrokerStatus, RelayIdentity } from './relay-session-broker-contract' -import type { RelayRegion } from './relay-region-preference' - -type RelayOriginPoolOptions = { - directorUrl: string - relayHostId: string - identity: RelayIdentity - keypair: E2EEKeypair - appVersion: string - mobileSocketWiring: MobileSocketWiring - isCurrent: () => boolean - onStatus: (status: RelayBrokerStatus) => void - resolvePreferredRegion?: () => Promise - fetch?: typeof globalThis.fetch - createControlSocket?: (url: string, relayJwt: string) => WebSocket - createDataSocket?: (url: string) => WebSocket - random?: () => number - now?: () => number -} +import { RelayControlRotation } from './relay-control-rotation' export class RelayOriginPool { - private readonly options: RelayOriginPoolOptions private activeOrigin: RelayControlOrigin | null = null private readonly origins = new Set() - private readonly drainingOrigins = new Set() - private readonly basisOrigins = new Map() - private readonly drainTimers = new Map>() + private readonly retirement = new RelayOriginRetirement( + () => this.activeOrigin, + (origin) => { + this.origins.delete(origin) + } + ) + private readonly drainingOrigins = this.retirement.draining + private readonly basisOrigins = this.retirement.basis private assignment: RelayAssignment | null = null + private deferredAssignment: RelayAssignment | null = null private relayJwt: string | null = null - private rotationTimer: ReturnType | null = null + private readonly rotation: RelayControlRotation private rotationPromise: Promise | null = null private readonly drainRetry: RelayDrainRetrySchedule private closed = false - constructor(options: RelayOriginPoolOptions) { - this.options = options + constructor(private readonly options: RelayOriginPoolOptions) { this.drainRetry = new RelayDrainRetrySchedule(options.random) + this.rotation = new RelayControlRotation({ + ...options, + current: () => this.activeOrigin, + available: () => this.isCurrent(), + token: () => this.relayJwt, + assignment: () => this.assignment, + busy: () => Boolean(this.rotationPromise) + }) } get activeAssignment(): RelayAssignment | null { @@ -63,6 +55,28 @@ export class RelayOriginPool { return this.activeOrigin?.hasLiveControl() ?? false } + applyAssignmentMetadata(assignment: RelayAssignment): boolean { + const current = this.assignment + if (!this.isCurrent() || !current || assignment.assignmentEpoch < current.assignmentEpoch) { + return false + } + if (assignment.assignmentEpoch > current.assignmentEpoch || this.rotationPromise) { + if ( + !this.deferredAssignment || + assignment.assignmentEpoch >= this.deferredAssignment.assignmentEpoch + ) { + this.deferredAssignment = assignment + } + return true + } + if (assignment.cellUrl !== current.cellUrl) { + return false + } + this.assignment = assignment + this.activeOrigin?.updateAssignment(assignment) + return true + } + async openInitial(assignment: RelayAssignment, relayJwt: string): Promise { this.assignment = assignment this.relayJwt = relayJwt @@ -71,7 +85,7 @@ export class RelayOriginPool { await origin.open() this.assertCurrent() this.activeOrigin = origin - this.scheduleControlRotation() + this.rotation.schedule() } refreshAuthorization(relayJwt: string): void { @@ -86,35 +100,21 @@ export class RelayOriginPool { return } this.closed = true - if (this.rotationTimer) { - clearTimeout(this.rotationTimer) - this.rotationTimer = null - } - this.drainRetry.cancel() - for (const timer of this.drainTimers.values()) { - clearTimeout(timer) - } - this.drainTimers.clear() + this.rotation.cancel() + this.drainRetry.reset() + this.retirement.clear() for (const origin of this.origins) { origin.closeNow(hostCloseReason) } this.origins.clear() - this.drainingOrigins.clear() - this.basisOrigins.clear() this.activeOrigin = null } private createOrigin(assignment: RelayAssignment, relayJwt: string): RelayControlOrigin { return new RelayControlOrigin({ + ...this.options, assignment, relayJwt, - relayHostId: this.options.relayHostId, - identity: this.options.identity, - keypair: this.options.keypair, - appVersion: this.options.appVersion, - mobileSocketWiring: this.options.mobileSocketWiring, - createControlSocket: this.options.createControlSocket, - createDataSocket: this.options.createDataSocket, onConnectionOwned: (connectionId, origin) => { if (this.isCurrent() && this.origins.has(origin)) { this.basisOrigins.set(connectionId, origin) @@ -124,9 +124,10 @@ export class RelayOriginPool { if (this.basisOrigins.get(connectionId) === origin) { this.basisOrigins.delete(connectionId) } - this.maybeCloseDrainedOrigin(origin) + this.retirement.maybeClose(origin) }, onDrain: (origin, message) => this.handleDrain(origin, message), + onPendingChanged: (origin) => this.retirement.maybeClose(origin), onClose: (origin) => { if (origin === this.activeOrigin && this.isCurrent()) { this.options.onStatus('offline') @@ -141,10 +142,12 @@ export class RelayOriginPool { } private handleDrain(origin: RelayControlOrigin, message: RelayDrainMessage): void { - if (!this.isCurrent() || origin !== this.activeOrigin) { + if (!this.isCurrent() || !this.origins.has(origin)) { + return + } + if (!this.retirement.adopt(origin, message)) { return } - this.drainingOrigins.add(origin) this.options.onStatus('draining') if (!this.rotationPromise && !this.drainRetry.pending) { this.rotationPromise = this.resolveDrainTarget(origin, message).finally(() => { @@ -164,7 +167,7 @@ export class RelayOriginPool { const preferredRegion = await this.options.resolvePreferredRegion?.().catch(() => undefined) this.assertCurrent() // Why: only the configured director can choose a migration target. - const assignment = await requestRelayAssignment({ + let assignment = await requestRelayAssignment({ directorUrl: this.options.directorUrl, relayToken: this.relayJwt, relayHostId: this.options.relayHostId, @@ -176,6 +179,13 @@ export class RelayOriginPool { fetch: this.options.fetch }) this.assertCurrent() + if ( + this.deferredAssignment && + this.deferredAssignment.assignmentEpoch > assignment.assignmentEpoch + ) { + assignment = this.deferredAssignment + } + this.deferredAssignment = null if (assignment.cellUrl === origin.cellUrl) { let rebound = false try { @@ -197,7 +207,7 @@ export class RelayOriginPool { } this.options.onStatus('registered') this.drainRetry.reset() - this.scheduleControlRotation() + this.rotation.schedule() } catch (error) { if (this.isCurrent() && origin === this.activeOrigin) { // Why: this retry loop ran silently during the 2026-08 incident while @@ -230,89 +240,9 @@ export class RelayOriginPool { } this.activeOrigin = target this.assignment = assignment - this.scheduleDrainDeadline(origin, graceMs) - this.maybeCloseDrainedOrigin(origin) + this.retirement.schedule(origin, graceMs) + this.retirement.maybeClose(origin) } - - private scheduleControlRotation(): void { - if (this.rotationTimer) { - clearTimeout(this.rotationTimer) - } - const origin = this.activeOrigin - if (!origin || this.closed) { - this.rotationTimer = null - return - } - const now = (this.options.now ?? Date.now)() - const random = this.options.random ?? Math.random - const delay = relayRenewalDelayMs(origin.controlLeaseExpiresAt, now, random) - this.rotationTimer = setTimeout(() => void this.rebindActiveControl(origin), delay) - } - - private async rebindActiveControl(origin: RelayControlOrigin): Promise { - this.rotationTimer = null - if (!this.isCurrent() || origin !== this.activeOrigin || this.rotationPromise) { - return - } - if (!this.relayJwt || !this.assignment) { - return - } - try { - await origin.rebind(this.relayJwt, this.assignment) - this.assertCurrent() - this.scheduleControlRotation() - } catch { - if (this.isCurrent() && origin === this.activeOrigin) { - const random = this.options.random ?? Math.random - this.rotationTimer = setTimeout( - () => void this.rebindActiveControl(origin), - 5_000 + Math.floor(random() * 10_001) - ) - } - } - } - - private scheduleDrainDeadline(origin: RelayControlOrigin, graceMs: number): void { - const existing = this.drainTimers.get(origin) - if (existing) { - clearTimeout(existing) - } - this.drainTimers.set( - origin, - setTimeout(() => this.closeOrigin(origin), graceMs) - ) - } - - private maybeCloseDrainedOrigin(origin: RelayControlOrigin): void { - if ( - !this.drainingOrigins.has(origin) || - origin.pendingRequestCount > 0 || - [...this.basisOrigins.values()].includes(origin) - ) { - return - } - this.closeOrigin(origin) - } - - private closeOrigin(origin: RelayControlOrigin): void { - if (origin === this.activeOrigin) { - return - } - const timer = this.drainTimers.get(origin) - if (timer) { - clearTimeout(timer) - this.drainTimers.delete(origin) - } - for (const [connectionId, owner] of this.basisOrigins) { - if (owner === origin) { - this.basisOrigins.delete(connectionId) - } - } - this.drainingOrigins.delete(origin) - this.origins.delete(origin) - origin.closeNow() - } - private assertCurrent(): void { if (!this.isCurrent()) { throw new Error('stale_relay_origin_pool') diff --git a/src/main/runtime/relay/relay-origin-retirement.ts b/src/main/runtime/relay/relay-origin-retirement.ts new file mode 100644 index 00000000000..bcc2a439ef2 --- /dev/null +++ b/src/main/runtime/relay/relay-origin-retirement.ts @@ -0,0 +1,65 @@ +import type { RelayDrainMessage } from './relay-control-protocol' +import type { RelayControlOrigin } from './relay-control-origin' + +export class RelayOriginRetirement { + readonly draining = new Set() + readonly basis = new Map() + private readonly timers = new Map>() + constructor( + private readonly current: () => RelayControlOrigin | null, + private readonly remove: (origin: RelayControlOrigin) => void + ) {} + adopt(origin: RelayControlOrigin, _message: RelayDrainMessage): boolean { + if (origin !== this.current()) { + return false + } + this.draining.add(origin) + return true + } + schedule(origin: RelayControlOrigin, graceMs: number): void { + const timer = this.timers.get(origin) + if (timer) { + clearTimeout(timer) + } + this.timers.set( + origin, + setTimeout(() => this.close(origin), graceMs) + ) + } + maybeClose(origin: RelayControlOrigin): void { + if ( + !this.draining.has(origin) || + origin.pendingRequestCount > 0 || + [...this.basis.values()].includes(origin) + ) { + return + } + this.close(origin) + } + clear(): void { + for (const timer of this.timers.values()) { + clearTimeout(timer) + } + this.timers.clear() + this.draining.clear() + this.basis.clear() + } + private close(origin: RelayControlOrigin): void { + if (origin === this.current()) { + return + } + const timer = this.timers.get(origin) + if (timer) { + clearTimeout(timer) + this.timers.delete(origin) + } + for (const [id, owner] of this.basis) { + if (owner === origin) { + this.basis.delete(id) + } + } + this.draining.delete(origin) + this.remove(origin) + origin.closeNow() + } +} diff --git a/src/main/runtime/relay/relay-region-correction-protocol.ts b/src/main/runtime/relay/relay-region-correction-protocol.ts new file mode 100644 index 00000000000..59762de33c1 --- /dev/null +++ b/src/main/runtime/relay/relay-region-correction-protocol.ts @@ -0,0 +1,43 @@ +import { z } from 'zod' +import { RelayRegionSchema } from './relay-region-probe' + +const Counter = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const RelayRegionWindowSchema = z + .object({ + generation: Counter, + expiresAt: Counter, + assignmentEpoch: Counter, + incumbentRegion: RelayRegionSchema, + policyVersion: z.literal(1) + }) + .strict() + +export const RelayRegionCorrectionResponseSchema = z + .object({ + v: z.literal(1), + window: RelayRegionWindowSchema.optional(), + reportStatus: z.enum(['accepted', 'duplicate', 'stale', 'expired', 'basis-changed']).optional() + }) + .strict() + +export type RelayRegionWindow = z.infer +export type RelayRegionDecision = + | { outcome: 'conclusive'; measurements: Record, number> } + | { + outcome: 'inconclusive' + reason: + | 'diagnostic-override' + | 'catalog-unavailable' + | 'incomplete-measurement' + | 'insufficient-improvement' + | 'expired-window' + } +export type RelayRegionCorrectionRequest = + | { v: 1; action: 'issue-window' } + | ({ + v: 1 + action: 'report' + generation: number + assignmentEpoch: number + policyVersion: 1 + } & RelayRegionDecision) diff --git a/src/main/runtime/relay/relay-region-correction.test.ts b/src/main/runtime/relay/relay-region-correction.test.ts new file mode 100644 index 00000000000..1a72d3136c9 --- /dev/null +++ b/src/main/runtime/relay/relay-region-correction.test.ts @@ -0,0 +1,132 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RelayRegionPreferenceResolver } from './relay-region-preference' +import { RelayAssignRateGate } from './relay-assign-rate-gate' +import { requestRelayAssignment } from './relay-http-client' +import type { RelayRegionWindow } from './relay-region-correction-protocol' + +const paths: string[] = [] +const US = 'https://us.director.example.test' +const ASIA = 'https://asia.director.example.test' +const DIRECTOR = 'https://director.example.test' +const window: RelayRegionWindow = { + generation: 1, + assignmentEpoch: 5, + incumbentRegion: 'asia-east2', + expiresAt: 1_000_000, + policyVersion: 1 +} +afterEach(() => { + for (const path of paths.splice(0)) { + rmSync(path, { recursive: true, force: true }) + } +}) +function resolver(us: number | null, asia: number | null, override?: string) { + const path = mkdtempSync(join(tmpdir(), 'relay-decision-')) + paths.push(path) + const probe = vi.fn(async (origin: string) => (origin === US ? us : asia)) + const fetch = vi.fn(async () => + Response.json({ + v: 1, + regions: [ + { region: 'us-central1', probeOrigins: [US] }, + { region: 'asia-east2', probeOrigins: [ASIA] } + ] + }) + ) + return { + path, + probe, + instance: new RelayRegionPreferenceResolver({ + directorUrl: DIRECTOR, + userDataPath: path, + probe, + fetch, + now: () => 0, + diagnosticOverride: override + }) + } +} +describe('window-bound region decisions', () => { + it('compares against the actual incumbent despite a previous US placement cache', async () => { + const { instance, path, probe } = resolver(50, 100) + writeFileSync( + join(path, 'orca-relay-region-preference.json'), + JSON.stringify({ v: 2, directorUrl: DIRECTOR, region: 'us-central1', expiresAt: 999_999 }) + ) + expect(await instance.measureDecision(window)).toEqual({ + outcome: 'conclusive', + measurements: { 'us-central1': 50, 'asia-east2': 100 } + }) + expect(probe).toHaveBeenCalledTimes(8) + }) + it.each([ + [76, 100], + [100, 124], + [400, 450] + ])( + 'reports stable insufficient margins as conclusive evidence for director filtering (%i / %i)', + async (us, asia) => { + expect(await resolver(us, asia).instance.measureDecision(window)).toEqual({ + outcome: 'conclusive', + measurements: { 'us-central1': us, 'asia-east2': asia } + }) + } + ) + it('allows the exact inclusive 25ms and 20 percent boundary', async () => { + expect(await resolver(100, 125).instance.measureDecision(window)).toMatchObject({ + outcome: 'conclusive' + }) + }) + it('does not certify a lone measurable region', async () => { + expect(await resolver(40, null).instance.measureDecision(window)).toEqual({ + outcome: 'inconclusive', + reason: 'incomplete-measurement' + }) + }) + it('never converts diagnostic overrides into measured eligibility', async () => { + const { instance, probe } = resolver(40, 100, 'us-central1') + expect(await instance.measureDecision(window)).toEqual({ + outcome: 'inconclusive', + reason: 'diagnostic-override' + }) + expect(probe).not.toHaveBeenCalled() + }) + it('invalidates legacy placement caches on upgrade', async () => { + const { instance, path, probe } = resolver(40, 100) + writeFileSync( + join(path, 'orca-relay-region-preference.json'), + JSON.stringify({ v: 1, directorUrl: DIRECTOR, region: 'asia-east2', expiresAt: 999_999 }) + ) + expect(await instance.resolve()).toBe('us-central1') + expect(probe).toHaveBeenCalledTimes(8) + }) + it('falls back from a strict old director without dropping the cold-start hint', async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 400 })) + .mockResolvedValueOnce( + Response.json({ v: 1, cellUrl: ASIA, assignmentEpoch: 1, lease: 'synthetic' }) + ) + const result = await requestRelayAssignment({ + directorUrl: DIRECTOR, + relayHostId: 'synthetic-host', + relayToken: 'synthetic-token', + preferredRegion: 'asia-east2', + reconnect: true, + regionCorrection: { v: 1, action: 'issue-window' }, + fetch, + assignRateGate: new RelayAssignRateGate() + }) + expect(result.cellUrl).toBe(ASIA) + expect(JSON.parse(String(fetch.mock.calls[1]![1]?.body))).toEqual({ + v: 1, + relayHostId: 'synthetic-host', + preferredRegion: 'asia-east2', + reconnect: true + }) + expect(result.regionCorrection).toBeUndefined() + }) +}) diff --git a/src/main/runtime/relay/relay-region-decision.ts b/src/main/runtime/relay/relay-region-decision.ts new file mode 100644 index 00000000000..677ae6faeb3 --- /dev/null +++ b/src/main/runtime/relay/relay-region-decision.ts @@ -0,0 +1,47 @@ +import type { RelayRegionDecision, RelayRegionWindow } from './relay-region-correction-protocol' +import { + RELAY_REGIONS, + regionMeasurement, + type RegionMeasurement, + type RelayRegionProbeReport +} from './relay-region-probe' + +export async function measureRelayRegionDecision( + window: RelayRegionWindow, + options: { + diagnosticOverride: boolean + now: () => number + measure: () => Promise + } +): Promise { + if (options.diagnosticOverride) { + return { outcome: 'inconclusive', reason: 'diagnostic-override' } + } + if (window.expiresAt <= options.now()) { + return { outcome: 'inconclusive', reason: 'expired-window' } + } + try { + // Placement caches are never evidence for a new server-issued window. + const reports = await options.measure() + const measurements = reports + .map(regionMeasurement) + .filter((entry): entry is RegionMeasurement => entry !== null) + const incumbent = measurements.find((entry) => entry.region === window.incumbentRegion) + if (window.expiresAt <= options.now()) { + return { outcome: 'inconclusive', reason: 'expired-window' } + } + if (!incumbent || measurements.length !== RELAY_REGIONS.length) { + return { outcome: 'inconclusive', reason: 'incomplete-measurement' } + } + // A stable tie is conclusive evidence; the director applies the incumbent margin. + return { + outcome: 'conclusive', + measurements: { + 'us-central1': measurements.find((entry) => entry.region === 'us-central1')!.latencyMs, + 'asia-east2': measurements.find((entry) => entry.region === 'asia-east2')!.latencyMs + } + } + } catch { + return { outcome: 'inconclusive', reason: 'catalog-unavailable' } + } +} diff --git a/src/main/runtime/relay/relay-region-preference-reader.ts b/src/main/runtime/relay/relay-region-preference-reader.ts new file mode 100644 index 00000000000..a856a598c85 --- /dev/null +++ b/src/main/runtime/relay/relay-region-preference-reader.ts @@ -0,0 +1,22 @@ +import { RelayRegionPreferenceResolver } from './relay-region-preference' +import type { RelayRegion } from './relay-region-probe' +import type { RelayRegionDecision, RelayRegionWindow } from './relay-region-correction-protocol' + +export function createRelayRegionPreferenceReader(input: { + authConfig: { relayDirectorUrl: string } + userDataPath: string +}): { + resolvePreferredRegion: () => Promise + measureRegionDecision: (window: RelayRegionWindow) => Promise + noteAssignedCell: (cellUrl: string) => void +} { + const resolver = new RelayRegionPreferenceResolver({ + directorUrl: input.authConfig.relayDirectorUrl, + userDataPath: input.userDataPath + }) + return { + resolvePreferredRegion: () => resolver.resolve(), + measureRegionDecision: (window) => resolver.measureDecision(window), + noteAssignedCell: (cellUrl) => void resolver.invalidateIfAssignedCellIsFar(cellUrl) + } +} diff --git a/src/main/runtime/relay/relay-region-preference.test.ts b/src/main/runtime/relay/relay-region-preference.test.ts index 700517d0b91..b3e2b845600 100644 --- a/src/main/runtime/relay/relay-region-preference.test.ts +++ b/src/main/runtime/relay/relay-region-preference.test.ts @@ -47,7 +47,7 @@ function sampledProbe(samples: Record) { function writeNoHintCache(path: string, expiresAt: number): void { writeFileSync( cachePath(path), - JSON.stringify({ v: 1, directorUrl: DIRECTOR, region: null, expiresAt }) + JSON.stringify({ v: 2, directorUrl: DIRECTOR, region: null, expiresAt }) ) } @@ -58,7 +58,7 @@ function cachePath(path: string): string { function writeCache(path: string, region: string, expiresAt = 999): void { writeFileSync( cachePath(path), - JSON.stringify({ v: 1, directorUrl: DIRECTOR, region, latencyMs: 100, expiresAt }) + JSON.stringify({ v: 2, directorUrl: DIRECTOR, region, latencyMs: 100, expiresAt }) ) } @@ -87,7 +87,7 @@ describe('Relay region preference', () => { expect(calls.filter((origin) => origin === US_SECONDARY)).toHaveLength(4) expect(calls.filter((origin) => origin === ASIA)).toHaveLength(4) expect(JSON.parse(readFileSync(cachePath(path), 'utf8'))).toMatchObject({ - v: 1, + v: 2, directorUrl: DIRECTOR, region: 'asia-east2', latencyMs: 30 @@ -175,7 +175,7 @@ describe('Relay region preference', () => { ).resolves.toBeUndefined() // The withheld hint is remembered briefly so a reconnect does not re-probe. const cached = JSON.parse(readFileSync(cachePath(path), 'utf8')) - expect(cached).toEqual({ v: 1, directorUrl: DIRECTOR, region: null, expiresAt: 3_601_000 }) + expect(cached).toEqual({ v: 2, directorUrl: DIRECTOR, region: null, expiresAt: 3_601_000 }) }) it('reuses the short-lived no-hint cache instead of re-probing on reconnect', async () => { diff --git a/src/main/runtime/relay/relay-region-preference.ts b/src/main/runtime/relay/relay-region-preference.ts index 9ad3743c972..a430e2d8a35 100644 --- a/src/main/runtime/relay/relay-region-preference.ts +++ b/src/main/runtime/relay/relay-region-preference.ts @@ -1,9 +1,11 @@ +import { measureRelayRegionDecision } from './relay-region-decision' import { existsSync, readFileSync, rmSync, statSync } from 'node:fs' import { join } from 'node:path' import { performance } from 'node:perf_hooks' import { z } from 'zod' import { hardenExistingSecureFile, writeSecureJsonFile } from '../../../shared/secure-file' import { fetchRelayRegionCatalog, relayDirectorHost } from './relay-region-catalog-fetch' +import type { RelayRegionDecision, RelayRegionWindow } from './relay-region-correction-protocol' import { logRelayRegionEvent, relayRegionCacheHitEvent, @@ -43,7 +45,7 @@ const FAR_CELL_RATIO = 3 const RelayRegionCacheSchema = z .object({ - v: z.literal(1), + v: z.literal(2), directorUrl: z.string().max(2_048), // Null records a deliberate "no hint"; the field is absent only for a region. region: RelayRegionSchema.nullable(), @@ -75,6 +77,14 @@ export class RelayRegionPreferenceResolver { this.options = options } + measureDecision(window: RelayRegionWindow): Promise { + return measureRelayRegionDecision(window, { + diagnosticOverride: Boolean(this.overrideRegion()), + now: this.options.now ?? Date.now, + measure: () => this.probeCatalog(this.options.fetch ?? globalThis.fetch) + }) + } + async resolve(): Promise { const override = this.overrideRegion() if (override) { @@ -233,7 +243,7 @@ export class RelayRegionPreferenceResolver { ): void { try { writeSecureJsonFile(this.cachePath(), { - v: 1, + v: 2, directorUrl: this.options.directorUrl, region: entry.region, ...(entry.latencyMs === undefined ? {} : { latencyMs: entry.latencyMs }), @@ -269,23 +279,6 @@ export class RelayRegionPreferenceResolver { } } -export function createRelayRegionPreferenceReader(input: { - authConfig: { relayDirectorUrl: string } - userDataPath: string -}): { - resolvePreferredRegion: () => Promise - noteAssignedCell: (cellUrl: string) => void -} { - const resolver = new RelayRegionPreferenceResolver({ - directorUrl: input.authConfig.relayDirectorUrl, - userDataPath: input.userDataPath - }) - return { - resolvePreferredRegion: () => resolver.resolve(), - noteAssignedCell: (cellUrl) => void resolver.invalidateIfAssignedCellIsFar(cellUrl) - } -} - function measuredRegions(reports: RelayRegionProbeReport[]): RegionMeasurement[] { return reports .map(regionMeasurement) diff --git a/src/main/runtime/relay/relay-region-probe-log.test.ts b/src/main/runtime/relay/relay-region-probe-log.test.ts index eb62939d9c7..d4c97866438 100644 --- a/src/main/runtime/relay/relay-region-probe-log.test.ts +++ b/src/main/runtime/relay/relay-region-probe-log.test.ts @@ -48,7 +48,7 @@ function sampledProbe(samples: Record) { function writeCache(path: string, region: string | null, expiresAt: number): void { writeFileSync( join(path, 'orca-relay-region-preference.json'), - JSON.stringify({ v: 1, directorUrl: DIRECTOR, region, expiresAt }) + JSON.stringify({ v: 2, directorUrl: DIRECTOR, region, expiresAt }) ) } diff --git a/src/main/runtime/relay/relay-region-refresh.test.ts b/src/main/runtime/relay/relay-region-refresh.test.ts new file mode 100644 index 00000000000..d6453191602 --- /dev/null +++ b/src/main/runtime/relay/relay-region-refresh.test.ts @@ -0,0 +1,158 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { RelayHttpError, type RelayAssignment } from './relay-http-client' +import type * as RelayHttpClientModule from './relay-http-client' +import type { RelayRegionWindow } from './relay-region-correction-protocol' +const fake = vi.hoisted(() => ({ assign: vi.fn() })) +vi.mock('./relay-http-client', async (original) => ({ + ...(await original()), + requestRelayAssignment: fake.assign +})) +import { RelayRegionRefresh } from './relay-region-refresh' +const HOUR = 60 * 60_000 +const window: RelayRegionWindow = { + generation: 1, + assignmentEpoch: 1, + incumbentRegion: 'asia-east2', + expiresAt: 24 * HOUR, + policyVersion: 1 +} +const assignment: RelayAssignment = { + v: 1, + cellUrl: 'https://source.example.test', + assignmentEpoch: 1, + lease: 'test', + regionCorrection: { v: 1, window } +} +let scheduler: RelayRegionRefresh +function setup(random = 0.5) { + const measure = vi.fn().mockResolvedValue({ + outcome: 'conclusive', + measurements: { 'us-central1': 30, 'asia-east2': 200 } + }) + const applyAssignment = vi.fn(() => true) + const isOnline = vi.fn(() => true) + scheduler = new RelayRegionRefresh({ + directorUrl: 'https://director.example.test', + relayHostId: 'test-host', + token: () => 'test-token', + assignment: () => assignment, + isCurrent: () => true, + isOnline, + applyAssignment, + measure, + random: () => random, + now: () => Date.now() + }) + return { measure, applyAssignment, isOnline } +} +describe('broker-owned region decision refresh', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(0) + fake.assign.mockReset() + }) + afterEach(() => { + scheduler?.close() + vi.useRealTimers() + }) + it('measures only after the server window and reports the complete fixed basis', async () => { + const { measure } = setup() + fake.assign.mockResolvedValue({ + ...assignment, + regionCorrection: { v: 1, reportStatus: 'accepted' } + }) + scheduler.start(assignment) + await vi.advanceTimersByTimeAsync(0) + expect(measure).toHaveBeenCalledWith(window) + expect(fake.assign).toHaveBeenCalledWith( + expect.objectContaining({ + regionCorrection: { + v: 1, + action: 'report', + generation: 1, + assignmentEpoch: 1, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 30, 'asia-east2': 200 } + } + }) + ) + await vi.advanceTimersByTimeAsync(23 * HOUR) + expect(measure).toHaveBeenCalledOnce() + }) + it('never jitters a retry before the director Retry-After minimum', async () => { + setup(0) + fake.assign + .mockRejectedValueOnce(new RelayHttpError('assignment', 429, 120_000)) + .mockResolvedValue({ ...assignment, regionCorrection: { v: 1, reportStatus: 'accepted' } }) + scheduler.start(assignment) + await vi.advanceTimersByTimeAsync(119_999) + expect(fake.assign).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + expect(fake.assign).toHaveBeenCalledTimes(2) + }) + it('retries exactly the same report without probing or extending its window', async () => { + const { measure } = setup() + fake.assign + .mockRejectedValueOnce(new Error('offline')) + .mockResolvedValue({ ...assignment, regionCorrection: { v: 1, reportStatus: 'accepted' } }) + scheduler.start(assignment) + await vi.advanceTimersByTimeAsync(60_000) + expect(measure).toHaveBeenCalledOnce() + expect(fake.assign).toHaveBeenCalledTimes(2) + expect(fake.assign.mock.calls[0]![0].regionCorrection).toEqual( + fake.assign.mock.calls[1]![0].regionCorrection + ) + }) + it('records inconclusive reports and retries measurement after one hour', async () => { + const { measure } = setup() + measure.mockResolvedValue({ outcome: 'inconclusive', reason: 'incomplete-measurement' }) + fake.assign + .mockResolvedValueOnce({ + ...assignment, + regionCorrection: { v: 1, reportStatus: 'accepted' } + }) + .mockResolvedValue(assignment) + scheduler.start(assignment) + await vi.advanceTimersByTimeAsync(HOUR) + expect(measure).toHaveBeenCalledTimes(2) + expect(fake.assign.mock.calls[1]![0].regionCorrection).toEqual({ v: 1, action: 'issue-window' }) + }) + it('does not probe offline and cancels future work on close', async () => { + const { measure, isOnline } = setup() + isOnline.mockReturnValue(false) + scheduler.start(assignment) + await vi.advanceTimersByTimeAsync(60_000) + expect(measure).not.toHaveBeenCalled() + scheduler.close() + isOnline.mockReturnValue(true) + await vi.advanceTimersByTimeAsync(25 * HOUR) + expect(fake.assign).not.toHaveBeenCalled() + }) + it('does not report a measurement that completed after broker close', async () => { + const { measure } = setup() + let resolve!: (value: unknown) => void + measure.mockReturnValue( + new Promise((done) => { + resolve = done + }) + ) + scheduler.start(assignment) + scheduler.close() + resolve({ outcome: 'inconclusive', reason: 'incomplete-measurement' }) + await vi.advanceTimersByTimeAsync(0) + expect(fake.assign).not.toHaveBeenCalled() + }) + it('uses a successor window after an expired report retry', async () => { + setup() + fake.assign.mockRejectedValueOnce(new Error('offline')).mockResolvedValue({ + ...assignment, + regionCorrection: { v: 1, window: { ...window, generation: 2, expiresAt: 48 * HOUR } } + }) + scheduler.start(assignment) + await vi.advanceTimersByTimeAsync(0) + vi.setSystemTime(25 * HOUR) + await vi.advanceTimersByTimeAsync(60_000) + expect(fake.assign.mock.calls[1]![0].regionCorrection).toEqual({ v: 1, action: 'issue-window' }) + }) +}) diff --git a/src/main/runtime/relay/relay-region-refresh.ts b/src/main/runtime/relay/relay-region-refresh.ts new file mode 100644 index 00000000000..c93a9d9618e --- /dev/null +++ b/src/main/runtime/relay/relay-region-refresh.ts @@ -0,0 +1,172 @@ +import { RelayHttpError, requestRelayAssignment, type RelayAssignment } from './relay-http-client' +import type { + RelayRegionCorrectionRequest, + RelayRegionDecision, + RelayRegionWindow +} from './relay-region-correction-protocol' + +type RefreshOptions = { + directorUrl: string + relayHostId: string + token: () => string | undefined + assignment: () => RelayAssignment | null + isCurrent: () => boolean + isOnline: () => boolean + applyAssignment: (assignment: RelayAssignment) => boolean + measure: (window: RelayRegionWindow) => Promise + fetch?: typeof globalThis.fetch + now?: () => number + random?: () => number +} + +const HOUR = 60 * 60_000 + +export class RelayRegionRefresh { + private timer: ReturnType | null = null + private pending: Promise | null = null + private report: Extract | null = null + private window: RelayRegionWindow | null = null + private closed = false + private nextDeadline = 0 + + constructor(private readonly options: RefreshOptions) {} + + start(assignment: RelayAssignment): void { + this.window = assignment.regionCorrection?.window ?? null + if (this.window) { + this.checkDeadline() + } else { + this.schedule(HOUR) + } + } + + checkDeadline(): void { + if (!this.isCurrent() || this.pending) { + return + } + if (this.now() < this.nextDeadline) { + if (!this.timer) { + this.schedule(Math.min(HOUR, this.nextDeadline - this.now())) + } + return + } + if (!this.options.isOnline()) { + this.schedule(60_000) + return + } + this.pending = this.refresh().finally(() => { + this.pending = null + }) + } + + close(): void { + this.closed = true + if (this.timer) { + clearTimeout(this.timer) + } + this.timer = null + this.report = null + this.window = null + } + + private async exchange(regionCorrection: RelayRegionCorrectionRequest): Promise { + const token = this.options.token() + if (!token) { + throw new Error('relay_region_authorization_unavailable') + } + const assignment = await requestRelayAssignment({ + directorUrl: this.options.directorUrl, + relayHostId: this.options.relayHostId, + relayToken: token, + reconnect: true, + regionCorrection, + isCurrent: () => this.isCurrent(), + fetch: this.options.fetch + }) + if (!this.isCurrent()) { + throw new Error('stale_relay_region_refresh') + } + // The mode-bearing source drain owns migration activation; reports never rebind controls. + this.options.applyAssignment(assignment) + return assignment + } + + private async refresh(): Promise { + try { + const assignment = this.options.assignment() + if (!assignment) { + this.schedule(60_000) + return + } + if ( + this.window && + (this.window.expiresAt <= this.now() || + this.window.assignmentEpoch !== assignment.assignmentEpoch) + ) { + this.window = null + this.report = null + } + if (!this.window) { + this.window = + (await this.exchange({ v: 1, action: 'issue-window' })).regionCorrection?.window ?? null + } + const window = this.window + if (!window) { + this.schedule(HOUR) + return + } + if (!this.report) { + const decision = await this.options.measure(window) + if (!this.isCurrent()) { + return + } + this.report = { + v: 1, + action: 'report', + generation: window.generation, + assignmentEpoch: window.assignmentEpoch, + policyVersion: 1, + ...decision + } + } + const report = this.report + const response = await this.exchange(report) + const accepted = response.regionCorrection?.reportStatus + this.report = null + this.window = null + this.schedule( + (accepted === 'accepted' || accepted === 'duplicate') && report.outcome === 'conclusive' + ? 24 * HOUR + : HOUR + ) + } catch (error) { + // Retry the same report/window: auth and healthy sockets are independent of probing. + const retry = error instanceof RelayHttpError ? (error.retryAfterMs ?? 0) : 0 + this.schedule(Math.max(60_000, retry), retry) + } + } + + private schedule(delay: number, minimumDelay = 0): void { + if (!this.isCurrent()) { + return + } + if (this.timer) { + clearTimeout(this.timer) + } + const jitter = 0.9 + (this.options.random ?? Math.random)() * 0.2 + const scheduledDelay = Math.max(minimumDelay, Math.ceil(delay * jitter)) + this.nextDeadline = this.now() + scheduledDelay + this.timer = setTimeout(() => { + this.timer = null + this.checkDeadline() + }, scheduledDelay) + this.timer.unref?.() + } + + private now(): number { + return (this.options.now ?? Date.now)() + } + private isCurrent(): boolean { + return !this.closed && this.options.isCurrent() + } +} diff --git a/src/main/runtime/relay/relay-session-broker-contract.ts b/src/main/runtime/relay/relay-session-broker-contract.ts index 78849f80eba..355bed76360 100644 --- a/src/main/runtime/relay/relay-session-broker-contract.ts +++ b/src/main/runtime/relay/relay-session-broker-contract.ts @@ -4,6 +4,7 @@ import type { MobileRelayStatus } from '../../../shared/mobile-relay-status' import type { E2EEKeypair } from '../e2ee-keypair' import type { MobileSocketWiring } from '../rpc/mobile-socket-wiring' import type { RelayRegion } from './relay-region-preference' +import type { RelayRegionDecision, RelayRegionWindow } from './relay-region-correction-protocol' export type RelayBrokerStatus = MobileRelayStatus @@ -23,6 +24,7 @@ export type RelaySessionBrokerOptions = { isCurrent: () => boolean refreshAccessToken: () => Promise resolvePreferredRegion?: () => Promise + measureRegionDecision?: (window: RelayRegionWindow) => Promise onAssignedCellActive?: (cellUrl: string) => void /** `cellUrl` is absent whenever the host holds no active assignment. */ onStatus: (status: RelayBrokerStatus, cellUrl?: string) => void @@ -32,3 +34,9 @@ export type RelaySessionBrokerOptions = { random?: () => number now?: () => number } + +export class StaleRelayBrokerError extends Error { + constructor() { + super('stale_relay_broker') + } +} diff --git a/src/main/runtime/relay/relay-session-broker.ts b/src/main/runtime/relay/relay-session-broker.ts index e8af020daf8..f32a077885e 100644 --- a/src/main/runtime/relay/relay-session-broker.ts +++ b/src/main/runtime/relay/relay-session-broker.ts @@ -1,3 +1,5 @@ +import { StaleRelayBrokerError } from './relay-session-broker-contract' +export { StaleRelayBrokerError } from './relay-session-broker-contract' import { relayStatusCellUrl } from '../../../shared/mobile-relay-status' import type { PairingRelay } from '../../../shared/mobile-relay-pairing-offer' import type { @@ -17,21 +19,17 @@ import { type RelayAssignment } from './relay-http-client' import { RelayOriginPool } from './relay-origin-pool' +import { RelayRegionRefresh } from './relay-region-refresh' import { relayRenewalDelayMs } from './relay-renewal-jitter' import type { RelayBrokerStatus, RelaySessionBrokerOptions } from './relay-session-broker-contract' export type { RelayBrokerStatus } from './relay-session-broker-contract' -export class StaleRelayBrokerError extends Error { - constructor() { - super('stale_relay_broker') - } -} - export class RelaySessionBroker { private readonly options: RelaySessionBrokerOptions private readonly relayHostId: string private readonly originPool: RelayOriginPool + private readonly regionRefresh: RelayRegionRefresh | null private authorization: RelayAuthorization | null = null private refreshTimer: ReturnType | null = null private closed = false @@ -55,6 +53,21 @@ export class RelaySessionBroker { random: options.random, now: options.now }) + this.regionRefresh = options.measureRegionDecision + ? new RelayRegionRefresh({ + directorUrl: options.authConfig.relayDirectorUrl, + relayHostId: this.relayHostId, + token: () => this.authorization?.relayToken, + assignment: () => this.originPool.activeAssignment, + isCurrent: () => this.isCurrent(), + isOnline: () => this.originPool.hasLiveControl(), + applyAssignment: (assignment) => this.originPool.applyAssignmentMetadata(assignment), + measure: options.measureRegionDecision, + fetch: options.fetch, + now: options.now, + random: options.random + }) + : null } static async connect(options: RelaySessionBrokerOptions): Promise { @@ -194,6 +207,7 @@ export class RelaySessionBroker { this.refreshTimer = null } this.originPool.closeNow(hostCloseReason) + this.regionRefresh?.close() if (publishOffline) { this.options.onStatus('offline') } @@ -220,6 +234,9 @@ export class RelaySessionBroker { // through to the placement lane. reconnect: true, preferredRegion, + ...(this.regionRefresh + ? { regionCorrection: { v: 1 as const, action: 'issue-window' as const } } + : {}), isCurrent: () => this.isCurrent(), fetch: this.options.fetch }) @@ -236,6 +253,7 @@ export class RelaySessionBroker { this.authorization = authorization this.publishStatus('registered') this.scheduleRefresh() + this.regionRefresh?.start(assignment) } private scheduleRefresh(): void { @@ -267,6 +285,7 @@ export class RelaySessionBroker { this.assertCurrent() this.originPool.refreshAuthorization(authorization.relayToken) this.authorization = authorization + this.regionRefresh?.checkDeadline() this.scheduleRefresh() } catch { const expiry = this.authorization?.expiresAt ?? 0 diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation.ts index dbf6a5f4b5e..8d9d92547c8 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation.ts @@ -1,4 +1,5 @@ import type { TuiAgent } from '../../../../../../shared/tui-agent' +import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal-wait-blocked-reason-legacy-alias' import { buildDispatchPreamble } from '../../../../orchestration/preamble' import { OrchestrationError } from '../../../../orchestration/orchestration-error' import { defineMethod } from '../../../core' @@ -222,7 +223,7 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ } throw new Error( wait.blockedReason - ? `Agent startup blocked: ${wait.blockedReason}` + ? `Agent startup blocked: ${describeTerminalWaitBlockedReason(wait.blockedReason)}` : `Agent did not become ready (${wait.status}).` ) } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/composed-workers.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/composed-workers.test.ts index aee45e25259..c9e0f077bc1 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/composed-workers.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/composed-workers.test.ts @@ -476,9 +476,15 @@ describe('orchestration RPC methods', () => { ) }) - it.each(['codex-update-prompt', 'codex-trust-workspace'] as const)( + // Why the second column: an older host still publishes the codex-* token, and this receipt + // reaches the user verbatim -- so it names the neutral spelling the same way the CLI does. + it.each([ + ['codex-update-prompt', 'codex-update-prompt (agent-update-prompt)'], + ['codex-trust-workspace', 'codex-trust-workspace (agent-trust-workspace)'], + ['agent-trust-workspace', 'agent-trust-workspace'] + ] as const)( 'returns a truthful readiness failure for %s', - async (blockedReason) => { + async (blockedReason, expectedReason) => { setup() mockCurrentWorkerStart() vi.mocked(runtime.waitForTerminal).mockResolvedValueOnce({ @@ -500,7 +506,7 @@ describe('orchestration RPC methods', () => { expect(result).toMatchObject({ state: 'failed', failedStage: 'agent_readiness', - lastError: `Agent startup blocked: ${blockedReason}` + lastError: `Agent startup blocked: ${expectedReason}` }) expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts index ec188695f5c..f8cd1033c97 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts @@ -1,4 +1,5 @@ import type { OrcaRuntimeService } from '../../../../orca-runtime' +import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal-wait-blocked-reason-legacy-alias' import type { OrchestrationDb } from '../../../../orchestration/db' import type { RunRow, TaskRow } from '../../../../orchestration/types' import { resolveDispatchCreator } from '../runs/dispatch-creator' @@ -186,7 +187,7 @@ export async function startLocalWorker(args: { } throw new Error( wait.blockedReason - ? `Agent startup blocked: ${wait.blockedReason}` + ? `Agent startup blocked: ${describeTerminalWaitBlockedReason(wait.blockedReason)}` : structuredSession ? `Setup did not finish before the structured worker started (${wait.status}).` : `Agent did not become ready (${wait.status}).` diff --git a/src/main/runtime/structured-agent-session-close.test.ts b/src/main/runtime/structured-agent-session-close.test.ts new file mode 100644 index 00000000000..531ca0aa129 --- /dev/null +++ b/src/main/runtime/structured-agent-session-close.test.ts @@ -0,0 +1,237 @@ +/** + * The chat tab must survive a close that did not land. + * + * `closeStructuredAgentSessionChild` hides the tab BEFORE it issues the close, so every failure + * shape past that point used to leave the user's chat tab pulled out of the durable restore index + * for a session that is still running — a destructive operation that refused, and still took + * something away. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecord } from '../../shared/agent-session-record' + +const hostRef: { current: unknown } = { current: null } + +vi.mock('../native-chat/agent-session-wire/structured-agent-session-registry', () => ({ + getStructuredAgentSessionHost: () => hostRef.current +})) + +const { closeStructuredAgentSessionChild } = await import('./structured-agent-session-close') + +const SESSION = 'session-1' + +function record(sessionId: string): AgentSessionRecord { + return { + sessionId, + provider: 'claude', + location: { + executionHostId: 'local', + wslDistro: null, + workspaceId: 'repo_1::/tmp/wt-a', + workspaceKind: 'folder' + }, + lease: { + sessionId, + runtimeKind: 'native', + claimStatus: 'live', + handoffStage: null, + runtimeFence: 1, + deathEvidence: null + } + } as unknown as AgentSessionRecord +} + +type HostOptions = { + /** Sessions the host keeps holding through a close, so the post-close observation is `live`. */ + stuck?: boolean + /** Rejects the close, without the child going. */ + closeThrows?: Error + /** The child dies and is recorded dead, but the close then fails past that proof. */ + settledThenThrows?: boolean + /** Rejects the visibility write itself, so the hide never lands. */ + visibilityThrows?: Error + /** Sessions already in the persisted visible-tab index. */ + visible?: string[] + /** Blows up the index read, so the rollback cannot prove the tab was ever visible. */ + indexThrows?: boolean +} + +function installHost(options: HostOptions = {}) { + const entry = record(SESSION) + const held = new Set([SESSION]) + const visible = new Set(options.visible ?? [SESSION]) + const setSessionTabVisibility = vi.fn(async (sessionId: string, isVisible: boolean) => { + if (options.visibilityThrows) { + throw options.visibilityThrows + } + if (isVisible) { + visible.add(sessionId) + } else { + visible.delete(sessionId) + } + }) + const close = vi.fn(async (sessionId: string) => { + if (options.closeThrows) { + throw options.closeThrows + } + if (options.stuck) { + return + } + held.delete(sessionId) + entry.lease.claimStatus = 'released' + entry.lease.deathEvidence = { kind: 'exit-observed', detail: 'closed', observedAt: 1 } + if (options.settledThenThrows) { + throw new Error('the event sink could not be flushed') + } + }) + hostRef.current = { + deps: { store: { getRecord: (id: string) => (id === SESSION ? entry : null) } }, + hasSession: (sessionId: string) => held.has(sessionId), + getPersistedVisibleSessionTabIndex: () => { + if (options.indexThrows) { + throw new Error('visible tab index unreadable') + } + return { present: true, sessionIds: [...visible] } + }, + setSessionTabVisibility, + close + } + return { close, setSessionTabVisibility, visible } +} + +describe('closeStructuredAgentSessionChild tab-visibility rollback', () => { + beforeEach(() => { + hostRef.current = null + vi.restoreAllMocks() + }) + + it('retires the tab and reports the close on the success path', async () => { + const host = installHost() + const retire = vi.fn(() => true) + + const outcome = await closeStructuredAgentSessionChild(SESSION, { + runtime: { + retireStructuredAgentSessionTabFromSnapshot: retire + } as never + }) + + expect(outcome).toEqual({ stopped: true, closeAttempted: true }) + expect(host.visible.has(SESSION)).toBe(false) + expect(retire).toHaveBeenCalledWith(SESSION) + // The hide is the only visibility write a settled close performs. + expect(host.setSessionTabVisibility.mock.calls).toEqual([[SESSION, false]]) + }) + + it('restores the tab when the close throws and the child is still there', async () => { + const host = installHost({ closeThrows: new Error('provider round trip failed') }) + + const outcome = await closeStructuredAgentSessionChild(SESSION) + + expect(outcome.stopped).toBe(false) + expect(outcome.closeAttempted).toBe(true) + expect(outcome.reason).toBe('provider round trip failed') + expect(host.visible.has(SESSION)).toBe(true) + expect(host.setSessionTabVisibility.mock.calls).toEqual([ + [SESSION, false], + [SESSION, true] + ]) + }) + + it('restores the tab when the post-close observation is not `exited`', async () => { + const host = installHost({ stuck: true }) + + const outcome = await closeStructuredAgentSessionChild(SESSION) + + expect(outcome.stopped).toBe(false) + expect(outcome.closeAttempted).toBe(true) + expect(host.visible.has(SESSION)).toBe(true) + expect(host.setSessionTabVisibility.mock.calls).toEqual([ + [SESSION, false], + [SESSION, true] + ]) + }) + + it('leaves the tab retired when a close throws PAST a proven exit', async () => { + // `closeStructuredSessionsForWorktree` re-observes and counts this session closed; republishing + // the tab here would resurrect it at the next launch for a workspace that is gone. + const host = installHost({ settledThenThrows: true }) + + const outcome = await closeStructuredAgentSessionChild(SESSION) + + expect(outcome.stopped).toBe(false) + expect(host.visible.has(SESSION)).toBe(false) + expect(host.setSessionTabVisibility.mock.calls).toEqual([[SESSION, false]]) + }) + + it('does not put the tab back when the caller is discarding the workspace anyway', async () => { + // Worktree teardown passes this off for a removal that cannot refuse — force, and the + // folder-workspace paths. A tab put back there is a durable reference to a workspace that is + // about to be gone, so it republishes the chat at the next launch pointing at it. + const host = installHost({ stuck: true }) + + const outcome = await closeStructuredAgentSessionChild(SESSION, { + restoreTabOnUnprovenClose: false + }) + + expect(outcome.stopped).toBe(false) + expect(host.visible.has(SESSION)).toBe(false) + expect(host.setSessionTabVisibility.mock.calls).toEqual([[SESSION, false]]) + }) + + it('does not publish a tab for a session that was already hidden', async () => { + const host = installHost({ closeThrows: new Error('provider round trip failed'), visible: [] }) + + await closeStructuredAgentSessionChild(SESSION) + + expect(host.visible.has(SESSION)).toBe(false) + expect(host.setSessionTabVisibility.mock.calls).toEqual([[SESSION, false]]) + }) + + it('does not roll back a visibility write that never landed', async () => { + const host = installHost({ visibilityThrows: new Error('visibility write failed') }) + + const outcome = await closeStructuredAgentSessionChild(SESSION) + + expect(outcome).toEqual({ + stopped: false, + closeAttempted: false, + reason: 'visibility write failed' + }) + expect(host.close).not.toHaveBeenCalled() + expect(host.setSessionTabVisibility.mock.calls).toEqual([[SESSION, false]]) + }) + + it('keeps the original failure when the restore itself throws', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = installHost({ stuck: true }) + host.setSessionTabVisibility.mockImplementation(async (_sessionId, isVisible) => { + if (isVisible) { + throw new Error('agent_session_identity_required') + } + }) + + const outcome = await closeStructuredAgentSessionChild(SESSION) + + expect(outcome.stopped).toBe(false) + expect(outcome.closeAttempted).toBe(true) + expect(outcome.reason).not.toContain('agent_session_identity_required') + expect(warn).toHaveBeenCalled() + }) + + it('claims nothing when the visible-tab index cannot be read', async () => { + const host = installHost({ indexThrows: true, closeThrows: new Error('boom') }) + + await closeStructuredAgentSessionChild(SESSION) + + expect(host.setSessionTabVisibility.mock.calls).toEqual([[SESSION, false]]) + }) + + it('reports no close attempt when no host is installed', async () => { + hostRef.current = null + + const outcome = await closeStructuredAgentSessionChild(SESSION) + + expect(outcome.stopped).toBe(false) + expect(outcome.closeAttempted).toBe(false) + }) +}) diff --git a/src/main/runtime/structured-agent-session-close.ts b/src/main/runtime/structured-agent-session-close.ts index 756dbdeaef4..f65d1204db9 100644 --- a/src/main/runtime/structured-agent-session-close.ts +++ b/src/main/runtime/structured-agent-session-close.ts @@ -11,6 +11,7 @@ * longer live is proven gone. Anything else is retained rather than settled. */ +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import type { OrcaRuntimeService } from './orca-runtime' import { retireSettledStructuredWorkerTab } from './structured-agent-session-tab-retirement' @@ -35,6 +36,15 @@ export type StructuredAgentSessionCloseOptions = { * keep the child un-evictable for the life of the app. Every settlement has to reach it. */ afterClose?: () => void + /** + * Whether an unproven close may put the chat tab back in the durable restore index. + * + * On by default, which is the retryable case: a stop that refused and still took the user's tab + * away is the loss the rollback exists to undo. A caller that will discard the WORKSPACE + * whatever this close reports passes false — a tab put back there is a durable reference to a + * workspace about to be gone, and it republishes the chat at the next launch pointing at it. + */ + restoreTabOnUnprovenClose?: boolean } export async function closeStructuredAgentSessionChild( @@ -50,6 +60,11 @@ export async function closeStructuredAgentSessionChild( reason: 'The structured agent-session host is not installed; no session was closed.' } } + // Read BEFORE the hide, so a rollback puts the tab back exactly as it was. Restoring + // unconditionally would publish a tab for a session that was already hidden — a worker started + // without a chat tab, or one the user had closed — which is a new side effect, not an undo. + const restoreTabIfCloseFails = + options.restoreTabOnUnprovenClose !== false && readPersistedTabVisibility(host, sessionId) // Set only once the close is actually issued: `setSessionTabVisibility` throwing first leaves a // running child, and a receipt that still said `closed_agent_terminal` for it would be the // close-that-never-happened this flag exists to rule out. @@ -59,6 +74,11 @@ export async function closeStructuredAgentSessionChild( closeAttempted = true await host.close(sessionId) } catch (error) { + // Only `closeAttempted` proves the hide landed: the store transaction restores its own state on + // failure, so a `setSessionTabVisibility` that threw hid nothing and has nothing to undo. + if (closeAttempted) { + await restorePersistedTabVisibility(host, sessionId, restoreTabIfCloseFails) + } return { stopped: false, closeAttempted, @@ -68,6 +88,7 @@ export async function closeStructuredAgentSessionChild( options.afterClose?.() const observation = observeStructuredWorker({ sessionId }) if (observation.status !== 'exited') { + await restorePersistedTabVisibility(host, sessionId, restoreTabIfCloseFails) return { stopped: false, closeAttempted: true, @@ -79,3 +100,52 @@ export async function closeStructuredAgentSessionChild( retireSettledStructuredWorkerTab(sessionId, options.runtime) return { stopped: true, closeAttempted: true } } + +function readPersistedTabVisibility(host: StructuredAgentSessionHost, sessionId: string): boolean { + try { + return host.getPersistedVisibleSessionTabIndex?.().sessionIds.includes(sessionId) ?? false + } catch { + // Unreadable index: claim nothing. A rollback that cannot prove the tab was visible must not + // publish one, for the same reason the read exists at all. + return false + } +} + +/** + * Puts the chat tab back after a close that did not settle. + * + * The hide is the one visible side effect this function performs before the destructive step, so a + * failed close that kept it left the user's chat tab gone from the durable restore index — the + * conversation survived under `userData`, but nothing brought the tab back at the next launch. + * + * Re-observed first rather than restored outright: a close can throw PAST its own proof and still + * have taken the child with it, and `closeStructuredSessionsForWorktree` reads exactly that, + * counting such a session closed and retiring its tab. Republishing there would resurrect a tab for + * a session that is demonstrably gone, at the next launch, pointing at a deleted workspace. + * + * That observation NARROWS the window; it does not close it. This one and the sweep's are taken a + * store write apart, so a child that dies in between is unverifiable here and exited there — which + * is why the sweep re-drops the tab reference when it takes that proof. Do not delete either half + * on the strength of the other. + * + * Never throws: the caller's `reason` is what the user is asked to act on, and a rollback failure + * must not replace it. `agent_session_identity_required` is the expected one — the record can be + * gone by now, which is itself the exit this restore is declining to undo. + */ +async function restorePersistedTabVisibility( + host: StructuredAgentSessionHost, + sessionId: string, + restoreTab: boolean +): Promise { + if (!restoreTab || observeStructuredWorker({ sessionId }).status === 'exited') { + return + } + try { + await host.setSessionTabVisibility?.(sessionId, true) + } catch (error) { + console.warn( + `[structured-session-close] could not restore the chat tab for ${sessionId} after a failed close`, + error + ) + } +} diff --git a/src/main/runtime/structured-session-worktree-teardown.test.ts b/src/main/runtime/structured-session-worktree-teardown.test.ts index a9bdf6aa45c..84977aad4eb 100644 --- a/src/main/runtime/structured-session-worktree-teardown.test.ts +++ b/src/main/runtime/structured-session-worktree-teardown.test.ts @@ -8,18 +8,31 @@ vi.mock('../native-chat/agent-session-wire/structured-agent-session-registry', ( })) const { killAllProcessesForWorktree } = await import('./worktree-teardown') -const { classifyWorktreeForceDeleteReason } = await import('../../shared/worktree/removal') +const { + classifyWorktreeForceDeleteReason, + isProvenLiveStructuredSessionRemovalError, + isUnstoppedPtyRemovalError +} = await import('../../shared/worktree/removal') const { listLiveStructuredSessionsForWorktree } = await import('./structured-session-worktree-teardown') const WORKTREE = 'repo_1::/tmp/wt-a' const OTHER_WORKTREE = 'repo_1::/tmp/wt-b' -function record(sessionId: string, workspaceId: string): AgentSessionRecord { +function record( + sessionId: string, + workspaceId: string, + options: { provider?: 'claude' | 'codex'; executionHostId?: string } = {} +): AgentSessionRecord { return { sessionId, - provider: 'claude', - location: { executionHostId: 'local', wslDistro: null, workspaceId, workspaceKind: 'folder' }, + provider: options.provider ?? 'claude', + location: { + executionHostId: options.executionHostId ?? 'local', + wslDistro: null, + workspaceId, + workspaceKind: 'folder' + }, lease: { sessionId, runtimeKind: 'native', @@ -33,24 +46,66 @@ function record(sessionId: string, workspaceId: string): AgentSessionRecord { function installHost(options: { records: AgentSessionRecord[] - /** Sessions the host still holds; a close removes one unless it is listed as stuck. */ + /** Sessions the host keeps holding through a close, so the post-close observation is `live`. */ stuck?: Set -}): { closed: string[] } { + /** Sessions the host drops without death evidence, so the observation is `unverifiable`. */ + unverifiable?: Set + /** Sessions whose child dies and is recorded dead, but whose close then fails past that point. */ + settledThenThrows?: Set + /** Blocks every close, to exercise the shared sweep budget without fake timers. */ + closeGate?: Promise + /** Blocks ONE session's close, so the serial loop can be caught part-way through. */ + closeGates?: Record> + /** Sessions in the persisted visible-tab index, so a rollback has something to put back. */ + visible?: string[] + /** + * Sessions whose death evidence lands DURING the close's tab-restore write. + * + * `setSessionTabVisibility` is a store transaction — a real disk write — so the close's own + * observation and the sweep's re-read straddle it and can disagree about the same session. + */ + exitsDuringTabRestore?: Set +}): { closed: string[]; visible: Set } { const held = new Set(options.records.map((entry) => entry.sessionId)) const closed: string[] = [] + const visible = new Set(options.visible ?? []) + const recordExit = (sessionId: string): void => { + const entry = options.records.find((candidate) => candidate.sessionId === sessionId) + if (entry) { + entry.lease.claimStatus = 'released' + entry.lease.deathEvidence = { kind: 'exit-observed', detail: 'closed', observedAt: 1 } + } + } hostRef.current = { deps: { store: { listRecords: () => options.records, getRecord: () => null } }, hasSession: (sessionId: string) => held.has(sessionId), - setSessionTabVisibility: async () => {}, + getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: [...visible] }), + setSessionTabVisibility: async (sessionId: string, isVisible: boolean) => { + if (!isVisible) { + visible.delete(sessionId) + return + } + if (options.exitsDuringTabRestore?.has(sessionId)) { + recordExit(sessionId) + } + visible.add(sessionId) + }, close: async (sessionId: string) => { closed.push(sessionId) - if (!options.stuck?.has(sessionId)) { - held.delete(sessionId) - const record = options.records.find((entry) => entry.sessionId === sessionId) - if (record) { - record.lease.claimStatus = 'released' - record.lease.deathEvidence = { kind: 'exit-observed', detail: 'closed', observedAt: 1 } - } + await options.closeGate + await options.closeGates?.[sessionId] + if (options.stuck?.has(sessionId)) { + return + } + held.delete(sessionId) + if (options.unverifiable?.has(sessionId)) { + return + } + if (!options.exitsDuringTabRestore?.has(sessionId)) { + recordExit(sessionId) + } + if (options.settledThenThrows?.has(sessionId)) { + throw new Error('the event sink could not be flushed') } } } @@ -59,7 +114,7 @@ function installHost(options: { hostRef.current as { deps: { store: { getRecord: (id: string) => unknown } } } ).deps.store.getRecord = (sessionId: string) => options.records.find((entry) => entry.sessionId === sessionId) ?? null - return { closed } + return { closed, visible } } const localProvider = { @@ -67,7 +122,7 @@ const localProvider = { shutdown: async () => {} } as never -function destructiveDeps(extra: { allowUnverifiedStop?: boolean } = {}) { +function destructiveDeps(extra: { allowUnverifiedStop?: boolean; timeoutMs?: number } = {}) { return { localProvider, requirePhysicalStop: true, @@ -77,6 +132,15 @@ function destructiveDeps(extra: { allowUnverifiedStop?: boolean } = {}) { } } +/** The structured sweep's own warn — a forced removal can emit a PTY-sweep one onto the same spy. */ +function structuredSessionWarning(warn: { mock: { calls: unknown[][] } }): string { + return ( + warn.mock.calls + .map((call) => String(call[0])) + .find((message) => message.includes('agent session')) ?? '' + ) +} + describe('worktree teardown and structured agent sessions', () => { beforeEach(() => { hostRef.current = null @@ -84,24 +148,98 @@ describe('worktree teardown and structured agent sessions', () => { it('finds sessions by workspace, and ignores a sibling worktree', () => { installHost({ records: [record('s1', WORKTREE), record('s2', OTHER_WORKTREE)] }) - expect(listLiveStructuredSessionsForWorktree(WORKTREE)).toEqual([ + expect(listLiveStructuredSessionsForWorktree(WORKTREE, {})).toEqual([ { sessionId: 's1', agent: 'claude' } ]) }) - it('refuses a destructive removal rather than deleting the checkout under a live child', async () => { - // The defect this pins: all three PTY sweeps enumerate leaves, provider sessions and the local - // registry, and a structured session is on NONE of them. Every sweep answered zero, nothing - // errored, and removal proceeded — leaving the provider child running with its `cwd` deleted - // and the dispatch still reporting the worker live and exact. - installHost({ records: [record('s1', WORKTREE)] }) + it('closes a live session on an ordinary removal instead of refusing it', async () => { + // The defect this pins, and the reason the guard is not simply deleted: all three PTY sweeps + // enumerate leaves, provider sessions and the local registry, and a structured session is on + // NONE of them, so removal used to proceed leaving the provider child running with its `cwd` + // deleted. The stop belongs on the ordinary path — the same one that kills a terminal running + // the same agent — so an idle chat is no harder to delete than that terminal. + const host = installHost({ records: [record('s1', WORKTREE)] }) + await expect(killAllProcessesForWorktree(WORKTREE, destructiveDeps())).resolves.toMatchObject({ + structuredStopped: 1 + }) + expect(host.closed).toEqual(['s1']) + }) + + it('refuses only when the close does not settle', async () => { + installHost({ records: [record('s1', WORKTREE)], stuck: new Set(['s1']) }) await expect(killAllProcessesForWorktree(WORKTREE, destructiveDeps())).rejects.toThrow( - /1 running agent session/ + /still live: 1 agent session \(claude\)/ ) }) + it('puts the chat tab back when the removal refuses over the session', async () => { + // The workspace survives a refusal, so the tab has to survive it too: a destructive operation + // that refused and still took the user's chat tab away is the loss the rollback exists to undo. + const host = installHost({ + records: [record('s1', WORKTREE)], + stuck: new Set(['s1']), + visible: ['s1'] + }) + await expect(killAllProcessesForWorktree(WORKTREE, destructiveDeps())).rejects.toThrow( + /still live: 1 agent session \(claude\)/ + ) + expect([...host.visible]).toEqual(['s1']) + }) + + it('leaves the chat tab dropped when a forced removal deletes the workspace anyway', async () => { + // The other half of the same rollback. Force does not refuse — it warns and goes on to delete + // the checkout — so putting the tab back leaves a DURABLE reference to a workspace that is + // about to be gone, which republishes the chat at the next launch pointing at a deleted + // worktree: the exact outcome this whole sweep exists to remove. + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = installHost({ + records: [record('s1', WORKTREE)], + stuck: new Set(['s1']), + visible: ['s1'] + }) + await killAllProcessesForWorktree(WORKTREE, destructiveDeps({ allowUnverifiedStop: true })) + expect([...host.visible]).toEqual([]) + warn.mockRestore() + }) + + it('leaves the chat tab dropped for a folder-workspace removal, which never refuses', async () => { + // Same reasoning without the force waiver: this caller cannot refuse at all, so the workspace + // is forgotten whatever the close reports. + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = installHost({ + records: [record('s1', WORKTREE)], + stuck: new Set(['s1']), + visible: ['s1'] + }) + await killAllProcessesForWorktree(WORKTREE, { + localProvider, + includeProviderInventory: false as const, + includeLocalRegistry: false as const, + closeStructuredSessions: true + }) + expect([...host.visible]).toEqual([]) + warn.mockRestore() + }) + + it('drops the chat tab for a session the sweep proves exited after the close gave up', async () => { + // `host.close` can return BEFORE the child's exit is recorded, so the close's own observation + // reads unverifiable and puts the tab back — and the sweep's re-read, one store write later, + // proves the exit and counts the session closed. The two observations straddle that write and + // can disagree; the tab must not survive the disagreement, because this removal proceeds. + const host = installHost({ + records: [record('s1', WORKTREE)], + visible: ['s1'], + exitsDuringTabRestore: new Set(['s1']) + }) + await expect(killAllProcessesForWorktree(WORKTREE, destructiveDeps())).resolves.toMatchObject({ + structuredStopped: 1 + }) + expect([...host.visible]).toEqual([]) + }) + it('names the force escape hatch in the refusal, like the unstopped-PTY gate', async () => { - installHost({ records: [record('s1', WORKTREE)] }) + installHost({ records: [record('s1', WORKTREE)], stuck: new Set(['s1']) }) await expect(killAllProcessesForWorktree(WORKTREE, destructiveDeps())).rejects.toThrow(/force/i) }) @@ -109,7 +247,7 @@ describe('worktree teardown and structured agent sessions', () => { // The #11960 dead end, and the shape this file's own comments warn about: the desktop // affordance comes ONLY from the classifier, and an ordinary delete already passes force:true // for the dirty-file skip — so a refusal with no matcher shows raw CLI wording with no button. - installHost({ records: [record('s1', WORKTREE)] }) + installHost({ records: [record('s1', WORKTREE)], stuck: new Set(['s1']) }) const error = await killAllProcessesForWorktree(WORKTREE, destructiveDeps()).catch( (thrown: Error) => thrown.message ) @@ -123,12 +261,12 @@ describe('worktree teardown and structured agent sessions', () => { // A session id is one tab-id hop from the random pane key that gates a worker's mailbox, and // this string reaches CLI output and a desktop toast. A count and the providers are what a // user deciding whether to force actually needs. - installHost({ records: [record('s1', WORKTREE)] }) + installHost({ records: [record('s1', WORKTREE)], stuck: new Set(['s1']) }) const error = await killAllProcessesForWorktree(WORKTREE, destructiveDeps()).catch( (thrown: Error) => thrown.message ) expect(error).not.toContain('s1') - expect(error).toContain('1 running agent session') + expect(error).toContain('1 agent session (claude)') }) it('closes best-effort for a folder-workspace removal, which requires no stop proof', async () => { @@ -166,10 +304,33 @@ describe('worktree teardown and structured agent sessions', () => { destructiveDeps({ allowUnverifiedStop: true }) ) expect(result.structuredStopped).toBeUndefined() - expect(warn).toHaveBeenCalledWith(expect.stringContaining('still attached')) + // The live arm of that record, carrying the verdict the refusal would have shown. + expect(structuredSessionWarning(warn)).toContain('still live: 1 agent session (claude)') warn.mockRestore() }) + it('takes the proof when a failed close is re-observed as exited', async () => { + // `closeStructuredAgentSessionChild` reports `stopped: false` for anything that throws past its + // own observation, and for a record whose death evidence lands after it read. The re-read here + // can still PROVE the exit — refusing a delete over a child that is demonstrably gone is the + // defect this whole sweep exists to remove, so the proof has to win over the close's verdict. + const retired: string[] = [] + const runtime = { + stopTerminalsForWorktree: async () => ({ stopped: 0 }), + retireStructuredAgentSessionTabFromSnapshot: (sessionId: string) => { + retired.push(sessionId) + return true + } + } as never + installHost({ records: [record('s1', WORKTREE)], settledThenThrows: new Set(['s1']) }) + await expect( + killAllProcessesForWorktree(WORKTREE, { ...destructiveDeps(), runtime }) + ).resolves.toMatchObject({ structuredStopped: 1 }) + // Retired here because the close gave up before its own retirement step, and a chat tab left + // behind re-attaches a released session pointing at a workspace that is about to be deleted. + expect(retired).toEqual(['s1']) + }) + it('leaves the best-effort reconciliation paths alone', async () => { // Those callers repair state and delete nothing, so a refusal there would wedge a repair. installHost({ records: [record('s1', WORKTREE)] }) @@ -182,6 +343,262 @@ describe('worktree teardown and structured agent sessions', () => { ).resolves.toMatchObject({ runtimeStopped: 0 }) }) + it('leaves a same-id workspace on another execution host alone', async () => { + // A workspace id is `repoId::path` with no host component, so the local, SSH and paired-runtime + // copies of one id are DIFFERENT workspaces. Unfenced, deleting the local one closed a chat + // running on somebody else's machine — a destructive cross-host act, not a spurious refusal. + const host = installHost({ + records: [record('s1', WORKTREE, { executionHostId: 'ssh:host-a' })] + }) + await expect(killAllProcessesForWorktree(WORKTREE, destructiveDeps())).resolves.toMatchObject({ + runtimeStopped: 0 + }) + expect(host.closed).toEqual([]) + }) + + it('reads an explicit local fence the way the PTY sweeps do', () => { + // This helper reuses the PTY fence's own type, so the two cannot answer `null` differently: + // there it means this machine, and it has to mean this machine here. ABSENT is the one + // deliberate difference — no fence at all for the PTY sweeps, narrowed to local here, because + // a single-host-id comparison cannot express match-all and closing every host's chats is + // destructive. Latent today only because `WorktreeTeardownDeps` cannot yet carry the `null`. + installHost({ + records: [record('s1', WORKTREE, { executionHostId: 'ssh:host-a' }), record('s2', WORKTREE)] + }) + const local = [{ sessionId: 's2', agent: 'claude' }] + expect(listLiveStructuredSessionsForWorktree(WORKTREE, { resolvedConnectionId: null })).toEqual( + local + ) + expect(listLiveStructuredSessionsForWorktree(WORKTREE, {})).toEqual(local) + }) + + it('closes only the session on the host the removal resolved to', async () => { + const host = installHost({ + records: [record('s1', WORKTREE, { executionHostId: 'ssh:host-a' }), record('s2', WORKTREE)] + }) + await expect( + killAllProcessesForWorktree(WORKTREE, { + ...destructiveDeps(), + resolvedConnectionId: 'host-a' + }) + ).resolves.toMatchObject({ structuredStopped: 1 }) + expect(host.closed).toEqual(['s1']) + }) + + it('names only the sessions that stayed, and every provider still there', async () => { + installHost({ + records: [ + record('s1', WORKTREE), + record('s2', WORKTREE, { provider: 'codex' }), + record('s3', WORKTREE) + ], + stuck: new Set(['s2', 's3']) + }) + const error = await killAllProcessesForWorktree(WORKTREE, destructiveDeps()).catch( + (thrown: Error) => thrown.message + ) + expect(error).toContain('still live: 2 agent sessions (claude, codex)') + }) + + it('names the unconfirmed sessions too, instead of counting only the live ones', async () => { + // The PTY sibling may drop everything outside its live list because a fresh inventory PROVED + // those exited. Nothing proves that here: an `unverifiable` session is unclosed as well, so + // naming only the live subset told the user "1 agent session" while two were about to go. + installHost({ + records: [record('s1', WORKTREE), record('s2', WORKTREE, { provider: 'codex' })], + stuck: new Set(['s1']), + unverifiable: new Set(['s2']) + }) + const error = await killAllProcessesForWorktree(WORKTREE, destructiveDeps()).catch( + (thrown: Error) => thrown.message + ) + expect(error).toContain( + 'still live: 1 agent session (claude); could not confirm these closed: 1 agent session (codex)' + ) + // The marker still leads, so the toast keeps showing the stronger of the two warnings. + expect(isProvenLiveStructuredSessionRemovalError(error as string)).toBe(true) + }) + + it('still reports what it closed when a forced removal skips the PTY verdict', async () => { + // A sweep that fails outright short-circuits the per-PTY verdict — but not the structured + // close that already ran, so the count has to survive that return or the removal log claims + // `structured=0` for chats it just ended. + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const runtime = { + stopTerminalsForWorktree: async () => { + throw new Error('the terminal sweep died') + } + } as never + const host = installHost({ records: [record('s1', WORKTREE)] }) + const result = await killAllProcessesForWorktree(WORKTREE, { + ...destructiveDeps({ allowUnverifiedStop: true }), + runtime + }) + expect(host.closed).toEqual(['s1']) + expect(result.structuredStopped).toBe(1) + warn.mockRestore() + }) + + it('separates a close it could not confirm from one it watched stay attached', async () => { + // `src/shared/worktree/removal.ts` keeps these two apart on purpose: a user waiving "we could + // not confirm" is making a different decision than one discarding a conversation Orca just saw + // running. The toast branches on this marker, so flattening them makes one of the two a lie. + installHost({ records: [record('s1', WORKTREE)], unverifiable: new Set(['s1']) }) + const unconfirmed = await killAllProcessesForWorktree(WORKTREE, destructiveDeps()).catch( + (thrown: Error) => thrown.message + ) + expect(unconfirmed).toContain('could not confirm these closed: 1 agent session (claude)') + expect(isProvenLiveStructuredSessionRemovalError(unconfirmed as string)).toBe(false) + + installHost({ records: [record('s1', WORKTREE)], stuck: new Set(['s1']) }) + const live = await killAllProcessesForWorktree(WORKTREE, destructiveDeps()).catch( + (thrown: Error) => thrown.message + ) + expect(isProvenLiveStructuredSessionRemovalError(live as string)).toBe(true) + }) + + it('refuses in agent-session wording when the close outlives the sweep budget', async () => { + // A structured close that runs out of time used to reject with the PTY timeout sentinel, which + // the classifier reads FIRST — so the toast blamed terminals, and the Force Delete meant to + // clear the wedge hit the same rejection again (#11960). + installHost({ records: [record('s1', WORKTREE)], closeGate: new Promise(() => {}) }) + const error = await killAllProcessesForWorktree( + WORKTREE, + destructiveDeps({ timeoutMs: 5 }) + ).catch((thrown: Error) => thrown.message) + expect(error).toContain('could not confirm these closed: 1 agent session (claude)') + expect(isUnstoppedPtyRemovalError(error as string)).toBe(false) + expect(classifyWorktreeForceDeleteReason(error as string, true)).toBe('running-agent-session') + }) + + it('never wedges Force Delete on a close that will not settle', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + installHost({ records: [record('s1', WORKTREE)], closeGate: new Promise(() => {}) }) + await expect( + killAllProcessesForWorktree( + WORKTREE, + destructiveDeps({ allowUnverifiedStop: true, timeoutMs: 5 }) + ) + ).resolves.toMatchObject({ runtimeStopped: 0 }) + const message = structuredSessionWarning(warn) + expect(message).toContain('could not confirm these closed: 1 agent session (claude)') + // The pin: a close that ran out of time was never watched stay attached. This warn is the only + // record a forced removal leaves, and the removal.ts split exists precisely so "we could not + // confirm" is never reported as "we saw it running" — including here. + expect(message).not.toContain('still attached') + warn.mockRestore() + }) + + it('names only the sessions still open when the budget expires mid-close', async () => { + // The close loop is serial, so a deadline can land part-way through it. A fallback assembled + // at the deadline could only name the whole list — so a removal that had already closed the + // first chat still told the user both were still there, which is the exact thing this sweep + // exists to stop doing: never report state nobody observed. + installHost({ + records: [record('s1', WORKTREE), record('s2', WORKTREE, { provider: 'codex' })], + closeGates: { s2: new Promise(() => {}) } + }) + const error = await killAllProcessesForWorktree( + WORKTREE, + destructiveDeps({ timeoutMs: 40 }) + ).catch((thrown: Error) => thrown.message) + expect(error).toContain('could not confirm these closed: 1 agent session (codex)') + expect(error).not.toContain('claude') + }) + + it('counts the closes that landed before the budget expired', async () => { + // The other half of the same fallback: it reported zero closes, so the removal log said + // `structured=0` for a chat it had just ended. + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const slowClose = new Promise((resolve) => { + setTimeout(resolve, 300) + }) + installHost({ + records: [record('s1', WORKTREE), record('s2', WORKTREE, { provider: 'codex' })], + closeGates: { s2: slowClose } + }) + const result = await killAllProcessesForWorktree( + WORKTREE, + destructiveDeps({ allowUnverifiedStop: true, timeoutMs: 40 }) + ) + expect(result.structuredStopped).toBe(1) + expect(structuredSessionWarning(warn)).toContain( + 'could not confirm these closed: 1 agent session (codex)' + ) + warn.mockRestore() + }) + + it('stops issuing new closes once the budget is spent', async () => { + // One slow provider round trip used to starve every session behind it: the outer race had + // already given up on the loop, and it went on issuing closes whose outcome nobody would read. + // The in-flight one is NOT cancelled — nothing here can cancel a provider round trip — so it + // still has to be reported, which is why both sessions are named below. + let releaseFirstClose: () => void = () => {} + const firstClose = new Promise((resolve) => { + releaseFirstClose = resolve + }) + const host = installHost({ + records: [record('s1', WORKTREE), record('s2', WORKTREE)], + closeGates: { s1: firstClose } + }) + const error = await killAllProcessesForWorktree( + WORKTREE, + destructiveDeps({ timeoutMs: 5 }) + ).catch((thrown: Error) => thrown.message) + expect(error).toContain('could not confirm these closed: 2 agent sessions (claude)') + releaseFirstClose() + await new Promise((resolve) => { + setTimeout(resolve, 25) + }) + expect(host.closed).toEqual(['s1']) + }) + + it('leaves the terminals already stopped when it refuses over a stuck session', async () => { + // Pins a tradeoff that was accepted, not an outcome that is wanted. The PTY sweeps now run + // concurrently with the structured close, so a removal that refuses over a session that will + // not close has ALREADY killed that workspace's terminals — the head-first serial order spared + // them. Serialising it back is worse: it spends the whole shared budget before a single PTY is + // asked, and the alternative — refusing before the PTY sweeps — leaves force-delete removing + // files while PTY handles are open. The PTY gate itself already kills first and refuses only + // on what it could not verify stopped. A later change must not flip this back silently. + let terminalSweeps = 0 + const runtime = { + stopTerminalsForWorktree: async () => { + terminalSweeps += 1 + return { stopped: 2 } + } + } as never + installHost({ records: [record('s1', WORKTREE)], stuck: new Set(['s1']) }) + await expect( + killAllProcessesForWorktree(WORKTREE, { ...destructiveDeps(), runtime }) + ).rejects.toThrow(/still live: 1 agent session \(claude\)/) + expect(terminalSweeps).toBe(1) + }) + + it('starts the terminal sweeps while the structured close is still in flight', async () => { + // The close is serial and each one waits on a provider round trip. Awaiting it before the + // sweeps exist spends the shared budget head-first, and the sweeps then report a timeout for + // a stop they never attempted. + let releaseClose: () => void = () => {} + const closeGate = new Promise((resolve) => { + releaseClose = resolve + }) + installHost({ records: [record('s1', WORKTREE)], closeGate }) + let terminalSweepStarted = false + const runtime = { + stopTerminalsForWorktree: async () => { + terminalSweepStarted = true + return { stopped: 0 } + } + } as never + const removal = killAllProcessesForWorktree(WORKTREE, { ...destructiveDeps(), runtime }) + await vi.waitFor(() => { + expect(terminalSweepStarted).toBe(true) + }) + releaseClose() + await expect(removal).resolves.toMatchObject({ structuredStopped: 1 }) + }) + it('does not block removal when no structured host is installed', async () => { // Not being able to look is not evidence a child is there, and reading the persisted store // directly would force-install the host as a side effect of a teardown. diff --git a/src/main/runtime/structured-session-worktree-teardown.ts b/src/main/runtime/structured-session-worktree-teardown.ts index 226f785f039..8b161e4f5a8 100644 --- a/src/main/runtime/structured-session-worktree-teardown.ts +++ b/src/main/runtime/structured-session-worktree-teardown.ts @@ -8,15 +8,29 @@ * kept running with its `cwd` gone, the durable record and chat tab survived to republish at the * next launch pointing at a deleted worktree, and `worker-show` still reported the worker live. * - * Membership is `location.workspaceId`, which every structured session carries — so this covers a - * plain chat session in the worktree as well as a dispatched worker. Liveness is - * `observeStructuredWorker`, the same `live` / `unverifiable` / `exited` vocabulary the rest of the - * structured surface uses; only a PROVEN live child is worth refusing a removal over. + * Membership is `location.workspaceId` PLUS the host fence below, and every structured session + * carries both — so this covers a plain chat session in the worktree as well as a dispatched + * worker. Liveness is `observeStructuredWorker`, the same `live` / `unverifiable` / `exited` + * vocabulary the rest of the structured surface uses. + * + * `live` here is lease state — a provider child is attached — not work in flight, so it says + * nothing about whether the user would lose anything. It selects what to CLOSE, never what to + * refuse over: a removal refuses only on a close that did not settle, exactly as the PTY sweep + * refuses only on a stop it could not verify. */ +import { + LOCAL_EXECUTION_HOST_ID, + toRuntimeExecutionHostId, + toSshExecutionHostId, + type ExecutionHostId +} from '../../shared/execution-host' +import { STILL_LIVE_DETAIL_PREFIX } from '../../shared/worktree/removal' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { observeStructuredWorker } from './structured-worker-authority' import { closeStructuredAgentSessionChild } from './structured-agent-session-close' +import { retireSettledStructuredWorkerTab } from './structured-agent-session-tab-retirement' +import type { WorktreePtyHostFence } from './worktree-pty-host-fence' import type { OrcaRuntimeService } from './orca-runtime' export type LiveStructuredSessionInWorkspace = { @@ -24,13 +38,52 @@ export type LiveStructuredSessionInWorkspace = { agent: 'claude' | 'codex' } +export type UnclosedStructuredSession = LiveStructuredSessionInWorkspace & { + /** Read AFTER the close: `live` is a child watched stay attached, not merely one left unproven. */ + status: 'live' | 'unverifiable' +} + export type StructuredWorktreeSweepRuntime = Pick< OrcaRuntimeService, 'forgetStructuredSessionMail' | 'retireStructuredAgentSessionTabFromSnapshot' > /** - * Structured sessions with a proven-live child in this worktree. + * The two fields every teardown caller already resolves to fence its PTY sweeps to one host. + * + * Deliberately the PTY fence's own type rather than a look-alike: these two helpers are written + * against each other, so a widening on one side must not become a silent disagreement on the + * other. `resolvedConnectionId: null` means this machine on both. + * + * They differ in exactly one reading, and only that one: ABSENT. The PTY fence takes it as no + * fence at all and matches every host, which a single-host-id comparison cannot express — and + * closing every host's chats is destructive, not merely noisy. So this side reads absent as local + * too, the narrower half of that pair. Pinned by test, not left to the next reader to rediscover. + */ +export type StructuredSessionHostFence = WorktreePtyHostFence + +/** + * The one execution host this teardown may touch. + * + * A workspace id is `repoId::path` with no host component, so the local machine, an SSH host and a + * paired runtime can all publish the SAME id and each names a DIFFERENT workspace (STA-4343). The + * PTY sweeps fence on exactly these two fields; a structured session records its host directly, so + * the comparison is on `location.executionHostId` instead of on a pty-id shape. + */ +export function structuredSessionTeardownHostId( + fence: StructuredSessionHostFence +): ExecutionHostId { + if (fence.resolvedRuntimeEnvironmentId !== undefined) { + return toRuntimeExecutionHostId(fence.resolvedRuntimeEnvironmentId) + } + // Both no-connection readings collapse here on purpose — see the fence type. A caller that + // resolved no host, and one that resolved this machine, each close nothing on anyone else's. + const connectionId = fence.resolvedConnectionId ?? null + return connectionId === null ? LOCAL_EXECUTION_HOST_ID : toSshExecutionHostId(connectionId) +} + +/** + * Structured sessions with a proven-live child in this worktree, on the fenced host only. * * An uninstalled host answers empty rather than throwing: no host in this generation means no * provider child was started by this process, and the three PTY sweeps fall through the same way @@ -38,7 +91,8 @@ export type StructuredWorktreeSweepRuntime = Pick< * directly — that would force-install the host, which is itself a side effect on a teardown path. */ export function listLiveStructuredSessionsForWorktree( - worktreeId: string + worktreeId: string, + fence: StructuredSessionHostFence ): LiveStructuredSessionInWorkspace[] { const host = getStructuredAgentSessionHost() if (!host) { @@ -50,58 +104,191 @@ export function listLiveStructuredSessionsForWorktree( } catch { return [] } + const hostId = structuredSessionTeardownHostId(fence) return records .filter( (record) => record.location.workspaceId === worktreeId && + record.location.executionHostId === hostId && observeStructuredWorker({ sessionId: record.sessionId }).status === 'live' ) .map((record) => ({ sessionId: record.sessionId, agent: record.provider })) } /** - * Counts and providers, never session ids. + * A count and its providers — never session ids. * * A session id is one tab-id hop from the random pane key that gates a worker's mailbox, and this * string reaches agent-readable CLI output and a desktop toast. The count and the providers are * what a user deciding whether to force actually needs; the ids identify nothing they can act on. */ -export function describeLiveStructuredSessions( - sessions: readonly LiveStructuredSessionInWorkspace[] -): string { +function countStructuredSessions(sessions: readonly UnclosedStructuredSession[]): string { const noun = sessions.length === 1 ? 'agent session' : 'agent sessions' const providers = [...new Set(sessions.map((session) => session.agent))].sort().join(', ') - return `${sessions.length} running ${noun} (${providers})` + return `${sessions.length} ${noun} (${providers})` } /** - * Closes every live structured session in the worktree, and reports what stayed. + * The two post-close verdicts, each with its own count. * - * Force is the documented escape hatch, so it closes rather than orphaning: a child left running - * against a deleted `cwd` is the exact outcome this whole sweep exists to prevent. + * The split is here for the reason `describeUnstoppedPtys` carries one: "we watched it stay + * attached" and "we could not confirm it went" are different decisions to waive, and the delete + * toast branches on the marker a proven-live session leads with. + * + * Both groups are named, though, which is where this differs from the PTY sibling: there, the + * verdict is a fresh inventory, so anything absent from the live list is PROVEN exited and + * rightly dropped. Here an `unverifiable` session is unclosed too — folding it into the live + * count would overstate what Orca watched, and dropping it said "1 agent session" while three + * were about to be discarded. + */ +export function describeUnclosedStructuredSessions( + sessions: readonly UnclosedStructuredSession[] +): string { + const stillLive = sessions.filter((session) => session.status === 'live') + const unconfirmed = sessions.filter((session) => session.status !== 'live') + if (stillLive.length === 0) { + return `could not confirm these closed: ${countStructuredSessions(unconfirmed)}` + } + const live = `${STILL_LIVE_DETAIL_PREFIX} ${countStructuredSessions(stillLive)}` + return unconfirmed.length === 0 + ? live + : `${live}; could not confirm these closed: ${countStructuredSessions(unconfirmed)}` +} + +/** + * What the close loop has done so far, readable while it is still running. + * + * The loop is serial and every close waits on a provider round trip, so the shared sweep budget can + * expire part-way through it. This is written as it goes rather than returned at the end, because + * the caller's timeout path reads THIS: a fabricated whole-list fallback reported sessions the + * sweep had already closed as unclosed, named them in the refusal the user reads, and logged + * `structured=0` for closes that landed. Saying only what was observed is the point of the sweep. + */ +export type StructuredSweepProgress = { + /** The sessions this sweep closes, in the order the loop reaches them. */ + readonly sessions: readonly LiveStructuredSessionInWorkspace[] + /** Sessions no longer attached after their close — the count this sweep reports. */ + closed: number + /** Attempted closes that did not settle, each carrying the verdict re-read after the attempt. */ + unstopped: UnclosedStructuredSession[] + /** How many of `sessions`, from the front, have an outcome recorded. */ + settled: number +} + +export function createStructuredSweepProgress( + sessions: readonly LiveStructuredSessionInWorkspace[] +): StructuredSweepProgress { + return { sessions, closed: 0, unstopped: [], settled: 0 } +} + +/** + * Everything this sweep did not prove closed. + * + * A session with no recorded outcome — never started, or still in flight — reports `unverifiable`, + * the same verdict as an attempted close that stayed unproven. Chosen, not conflated: the vocabulary is `live` / `unverifiable` / `exited` with no + * synonyms, and "we never asked" and "we asked and could not confirm" are both exactly "not + * observed exited". A fourth bucket would need its own refusal wording and its own toast + * classification for a distinction the user cannot act on any differently — and `live` is the only + * verdict either could be mistaken for, which is the one thing neither is allowed to claim. + */ +export function unclosedStructuredSessions( + progress: StructuredSweepProgress +): UnclosedStructuredSession[] { + return [ + ...progress.unstopped, + ...progress.sessions + .slice(progress.settled) + .map((session) => ({ ...session, status: 'unverifiable' as const })) + ] +} + +/** + * Closes the structured sessions in `progress`, recording what stayed as it goes. + * + * Runs on the ordinary removal too, not just force: a child left running against a deleted `cwd` is + * the outcome this whole sweep exists to prevent, and closing is how you prevent it. What stayed is + * the only thing worth refusing over. + * + * Takes the list rather than re-deriving it, so the refusal can only ever name a session out of + * the set this sweep was handed — re-enumerating would run every liveness observation twice and + * let it name one this call never touched. Not every one of them is a session a close was + * attempted on: the deadline check below can leave the tail of the list unasked, and + * `unclosedStructuredSessions` reports those as `unverifiable` precisely because nobody looked. */ export async function closeStructuredSessionsForWorktree( - worktreeId: string, - runtime?: StructuredWorktreeSweepRuntime -): Promise<{ closed: number; unstopped: LiveStructuredSessionInWorkspace[] }> { + progress: StructuredSweepProgress, + deadline: number, + options: { + runtime?: StructuredWorktreeSweepRuntime + /** + * Whether this removal can still refuse over an unclosed session. + * + * It is the only case where the workspace — and therefore its chat tabs — survives, so it is + * the only case where an unproven close may put a tab back. Force and the folder-workspace + * paths discard the workspace whatever the sweep reports. + */ + mayRefuse?: boolean + } = {} +): Promise { + const { runtime, mayRefuse } = options // No `afterClose` for a dispatched worker: `host.close` drops the holds, so nothing keeps a // provider child un-evictable, but the dispatch's redrive subscription and registry entry do // survive until it settles by another verb. That is a bounded leak, not a hazard — and passing // one here would mean resolving a dispatch id per session on a teardown path that must stay // inside the sweep deadline. - const sessions = listLiveStructuredSessionsForWorktree(worktreeId) - const unstopped: LiveStructuredSessionInWorkspace[] = [] - let closed = 0 - for (const session of sessions) { - const outcome = await closeStructuredAgentSessionChild( - session.sessionId, - runtime ? { runtime } : {} - ) - if (outcome.stopped) { - closed += 1 - } else { - unstopped.push(session) + for (const session of progress.sessions) { + // Stops ISSUING new closes once the budget is spent; an in-flight one is left to finish, since + // nothing here can cancel a provider round trip. Without this, one slow round trip starved + // every session behind it: the caller's race had already given up, and the loop went on + // closing sessions whose outcome nobody would read. + if (Date.now() >= deadline) { + return } + const outcome = await closeStructuredAgentSessionChild(session.sessionId, { + ...(runtime ? { runtime } : {}), + restoreTabOnUnprovenClose: mayRefuse === true + }) + if (outcome.stopped) { + progress.closed += 1 + } else { + // Re-observed rather than reusing the close's own reason string: what the user is asked to + // waive is the state AFTER the attempt, and a close that threw never reached an observation. + const status = observeStructuredWorker({ sessionId: session.sessionId }).status + if (status === 'exited') { + // The re-read can PROVE the exit a failed close could not — it threw past its own + // observation, or the record's death evidence landed after it read. Refusing on a child + // that is demonstrably gone is the defect this sweep exists to remove, so take the proof + // and run the retirement `closeStructuredAgentSessionChild` skipped when it gave up. + // + // Including the hide it UNDID: its rollback ran against an observation taken one store + // write before this one, so a child that died in between left the tab republished for a + // session this sweep is about to count closed. Taking the proof has to take that back. + await dropDurableChatTabReference(session.sessionId) + retireSettledStructuredWorkerTab(session.sessionId, runtime) + progress.closed += 1 + } else { + progress.unstopped.push({ ...session, status }) + } + } + // Advanced only once an outcome is recorded, so a close still in flight when the deadline + // lands stays reported as unclosed instead of falling out of both counts. + progress.settled += 1 + } +} + +/** + * Drops a settled session's durable chat-tab reference, and cannot fail the settlement. + * + * The close's own hide is the ordinary path; this is only for the session whose exit this sweep + * proved after that close had already rolled the hide back. + */ +async function dropDurableChatTabReference(sessionId: string): Promise { + try { + await getStructuredAgentSessionHost()?.setSessionTabVisibility?.(sessionId, false) + } catch (error) { + console.warn( + `[worktree-teardown] could not drop the chat tab reference for ${sessionId}`, + error + ) } - return { closed, unstopped } } diff --git a/src/main/runtime/terminal-interactive-wait-visibility.test.ts b/src/main/runtime/terminal-interactive-wait-visibility.test.ts index 987bdbb15b7..5e652af41f6 100644 --- a/src/main/runtime/terminal-interactive-wait-visibility.test.ts +++ b/src/main/runtime/terminal-interactive-wait-visibility.test.ts @@ -3,7 +3,10 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' -import { OrcaRuntimeService } from './orca-runtime' +import { + createTranscriptPane as createPane, + TRANSCRIPT_PANE_PTY_ID as PTY_ID +} from './agent-transcript-pane-test-harness' import { assertTerminalAgentSendable } from './rpc/terminal-agent-send-guard' vi.mock('electron', () => ({ @@ -13,12 +16,11 @@ vi.mock('electron', () => ({ app: { getPath: vi.fn(() => '/tmp') } })) -const LEAF_ID = '11111111-1111-4111-8111-111111111111' -const TAB_ID = 'tab-1' -const WORKTREE_ID = 'wt-1' -const PTY_ID = 'pty-1' - -// Captured verbatim from cursor-agent 2026.08.11-e8db854 driven through Orca. +// cursor-agent 2026.08.11-e8db854's screens, but NOT raw PTY output: these files contain no +// escape bytes and no carriage returns, so they came through a terminal's renderer and a +// clipboard. They evidence wording, ordering and glyphs — which is all the rules below key on — +// and evidence nothing about the caret, cursor moves, repaints or the alternate screen buffer. +// Record new fixtures with config/scripts/capture-agent-pty-transcript.mjs, which keeps the bytes. function fixture(name: string): string { return readFileSync(join(__dirname, '__fixtures__', `${name}.txt`), 'utf8') } @@ -39,73 +41,6 @@ function agentStatusOsc(state: string): string { return `]9999;${JSON.stringify({ state, prompt: 'ship it', agentType: 'claude' })}` } -async function createPane(options: { - paneTitle: string - foregroundProcess: string | null - data: string - /** Set for a pane whose PTY lives on an SSH host or WSL distro rather than locally. */ - connectionId?: string - /** Simulates a PTY controller whose foreground probe never settles. */ - foregroundProbeHangs?: boolean - onForegroundProbe?: () => void -}): Promise<{ runtime: OrcaRuntimeService; handle: string }> { - const runtime = new OrcaRuntimeService(null) - const internals = runtime as unknown as { - resolveTerminalWorkspaceLaunchScope: (selector: string) => Promise - } - vi.spyOn(internals, 'resolveTerminalWorkspaceLaunchScope').mockResolvedValue({ - id: WORKTREE_ID, - path: '/repo/app', - connectionId: options.connectionId ?? null, - repo: null, - folderWorkspace: null - }) - runtime.setPtyController({ - spawn: vi.fn().mockResolvedValue({ id: PTY_ID, incarnationId: 'inc-1' }), - write: () => true, - kill: () => true, - getForegroundProcess: (): Promise => { - options.onForegroundProbe?.() - return options.foregroundProbeHangs === true - ? new Promise(() => {}) - : Promise.resolve(options.foregroundProcess) - } - }) - const terminal = await runtime.createTerminal(`id:${WORKTREE_ID}`, { - tabId: TAB_ID, - leafId: LEAF_ID, - title: 'Terminal' - }) - runtime.attachWindow(1) - runtime.syncWindowGraph(1, { - tabs: [ - { - tabId: TAB_ID, - worktreeId: WORKTREE_ID, - title: 'Terminal', - activeLeafId: LEAF_ID, - layout: null - } - ], - leaves: [ - { - tabId: TAB_ID, - worktreeId: WORKTREE_ID, - leafId: LEAF_ID, - paneRuntimeId: 1, - ptyId: PTY_ID, - paneTitle: options.paneTitle - } - ] - }) - // Why the guard: a restore seed is only applied to a never-written record, so the restore - // cases must not write an empty chunk first. - if (options.data.length > 0) { - runtime.onPtyData(PTY_ID, options.data, Date.now()) - } - return { runtime, handle: terminal.handle } -} - // cursor-agent renders a braille spinner in its OSC title while it works, and Orca reads // that as `working`; the title is identical whether it is running a command or waiting. const CURSOR_TITLE = '⠇ Cursor Agent' @@ -299,7 +234,7 @@ describe('terminal interactive-wait visibility (STA-4513, STA-3714)', () => { }) await expect(runtime.showTerminal(handle)).resolves.toMatchObject({ - agentWait: { source: 'prompt-text', reason: 'codex-trust-workspace' } + agentWait: { source: 'prompt-text', reason: 'agent-trust-workspace' } }) }) diff --git a/src/main/runtime/terminal-tail-sentinel-index.test.ts b/src/main/runtime/terminal-tail-sentinel-index.test.ts index 2b33b2770ea..cd8bf2e68fb 100644 --- a/src/main/runtime/terminal-tail-sentinel-index.test.ts +++ b/src/main/runtime/terminal-tail-sentinel-index.test.ts @@ -193,7 +193,7 @@ describe('terminal tail sentinel index', () => { expect(tailMayContainBlockedSignal(seeded)).toBe(true) const state = computeTerminalTailWaitState(seeded, '', '') expect(state.fromTail).toBe(true) - expect(state.signal?.reason).toBe('codex-update-prompt') + expect(state.signal?.reason).toBe('agent-update-prompt') const clean = ['boot log', 'no prompt here', 'trailing'] expect(tailMayContainBlockedSignal(clean)).toBe(false) diff --git a/src/main/runtime/terminal-wait-detection.test.ts b/src/main/runtime/terminal-wait-detection.test.ts index eda02e60bbb..e52344c5dc0 100644 --- a/src/main/runtime/terminal-wait-detection.test.ts +++ b/src/main/runtime/terminal-wait-detection.test.ts @@ -98,12 +98,12 @@ const LIVE_CODEX_PROMPTS: { name: string; lines: string[]; reason: string }[] = '2. Trust all and continue', 'Press enter to confirm or esc to go back' ], - reason: 'codex-hooks-review-prompt' + reason: 'agent-hooks-review-prompt' }, { name: 'trust workspace', lines: ['Do you trust this workspace directory?', '1. Yes', '2. No'], - reason: 'codex-trust-workspace' + reason: 'agent-trust-workspace' }, { name: 'update', @@ -113,7 +113,7 @@ const LIVE_CODEX_PROMPTS: { name: string; lines: string[]; reason: string }[] = '2. Skip', 'Press enter to continue' ], - reason: 'codex-update-prompt' + reason: 'agent-update-prompt' }, { name: 'cwd selection', @@ -123,7 +123,7 @@ const LIVE_CODEX_PROMPTS: { name: string; lines: string[]; reason: string }[] = ' Current = your current working directory', ' Press enter to continue' ], - reason: 'codex-cwd-prompt' + reason: 'agent-cwd-prompt' }, { name: 'model migration', @@ -142,7 +142,7 @@ const LIVE_CODEX_PROMPTS: { name: string; lines: string[]; reason: string }[] = '2. No, continue without permissions', 'Press enter to confirm or esc to cancel' ], - reason: 'codex-interactive-prompt' + reason: 'agent-interactive-prompt' }, { name: 'permission required', @@ -153,7 +153,7 @@ const LIVE_CODEX_PROMPTS: { name: string; lines: string[]; reason: string }[] = 'Allow always', 'Reject' ], - reason: 'codex-interactive-prompt' + reason: 'agent-interactive-prompt' } ] @@ -195,6 +195,301 @@ describe('detectTerminalWaitBlockedReason live prompts', () => { 'Press enter to confirm' ]) - expect(detectTerminalWaitBlockedReason(waitText)).toBe('codex-hooks-review-prompt') + expect(detectTerminalWaitBlockedReason(waitText)).toBe('agent-hooks-review-prompt') }) }) + +// Why: these matchers never inspect the pane's agent, so a Codex-named reason on a non-Codex screen +// reaches the user verbatim through the CLI and the worker receipt's "Agent startup blocked:" line. +describe('detectTerminalWaitBlockedReason on non-Codex agents', () => { + const NON_CODEX_PROMPTS: { name: string; lines: string[]; reason: string }[] = [ + { + name: 'an Antigravity workspace trust dialog', + lines: [ + 'Antigravity CLI 1.0.3', + 'Do you trust the files in this folder?', + '1. Yes, I trust this folder', + '2. No, exit' + ], + reason: 'agent-trust-workspace' + }, + { + name: 'a Claude Code trusted-workspace dialog', + lines: [ + 'Claude Code', + 'Trusted workspace?', + 'This directory has not been opened before.', + '1. Yes, proceed', + '2. No, exit' + ], + reason: 'agent-trust-workspace' + }, + { + name: 'a Gemini CLI update banner', + lines: [ + 'Gemini CLI', + 'Update available! 1.4.0 -> 1.5.0', + '1. Update now', + '2. Skip', + 'Press enter to continue' + ], + reason: 'agent-update-prompt' + }, + { + name: 'a Gemini CLI permission dialog', + lines: [ + 'Gemini CLI', + 'Permission required', + 'Running this tool requires permission', + 'Allow once', + 'Allow always', + 'Reject' + ], + reason: 'agent-interactive-prompt' + }, + { + name: 'a Claude Code hooks review dialog', + lines: [ + 'Claude Code', + 'Hooks need review', + 'PreToolUse:Bash .claude/hooks/guard.sh', + 'Press enter to confirm' + ], + reason: 'agent-hooks-review-prompt' + }, + { + name: 'an Antigravity sandbox confirmation', + lines: [ + 'Antigravity CLI 1.0.3', + 'This action runs outside the sandbox.', + 'Press enter to confirm or esc to go back' + ], + reason: 'agent-interactive-prompt' + } + ] + + // Why: the reason was previously picked by looking for 'codex' in 600 chars of scrollback, so any + // agent that merely narrated about Codex handed its user a Codex label. + it('does not borrow a Codex label from scrollback that only mentions Codex', () => { + const waitText = waitTextFor([ + 'Antigravity CLI 1.0.3', + 'I read src/codex-notes.md for you.', + 'This action runs outside the sandbox.', + 'Press enter to confirm or esc to go back' + ]) + + expect(waitText.toLowerCase()).toContain('codex') + expect(detectTerminalWaitBlockedReason(waitText)).toBe('agent-interactive-prompt') + }) + + for (const prompt of NON_CODEX_PROMPTS) { + it(`reports an agent-neutral reason for ${prompt.name}`, () => { + const waitText = waitTextFor(prompt.lines) + const reason = detectTerminalWaitBlockedReason(waitText) + + expect(waitText.toLowerCase()).not.toContain('codex') + expect(reason).toBe(prompt.reason) + expect(reason?.startsWith('codex-')).toBe(false) + }) + } +}) + +// Antigravity readiness, and what this file does NOT claim about it. +// +// The detector recognizes a ready screen by header + a 'gemini'-prefixed model line + a lone '>' +// caret. That is narrow: an Antigravity user on a non-Gemini model never reaches ready and the pane +// wedges. Widening it was attempted and reverted -- every candidate rule was tuned against the +// constructed fixtures below, and the last one let a live sign-in dialog read as ready (the +// orchestrator then types the task prompt into an authentication dialog, which is strictly worse +// than a timeout). No real Antigravity transcript exists in this repo; the cursor-agent rules are +// derived from captures under src/main/runtime/__fixtures__ and Antigravity has no equivalent. +// Widening the model rule needs one first. See the ratchet at the bottom of this block for the +// shapes any replacement has to refuse. +describe('Antigravity readiness does not absorb its own startup dialog', () => { + const TRUST_DIALOG_WITH_CARET = [ + 'Antigravity CLI 1.0.3', + 'Do you trust the files in this folder?', + '1. Yes, I trust this folder', + '2. No, exit', + '>' + ] + + const LIVE_DIALOGS_UNDER_THE_HEADER: { name: string; lines: string[]; reason: string | null }[] = + [ + { + name: 'a bare trust dialog', + lines: TRUST_DIALOG_WITH_CARET, + reason: 'agent-trust-workspace' + }, + { + name: 'a trust dialog with an ordinary sentence in it', + lines: [ + 'Antigravity CLI 1.0.3', + 'This workspace has not been opened before.', + 'Do you trust the files in this folder?', + '1. Yes, I trust this folder', + '2. No, exit', + '>' + ], + reason: 'agent-trust-workspace' + }, + { + name: 'a trust dialog printing the folder on its own line', + lines: [ + 'Antigravity CLI 1.0.3', + 'Do you trust the files in this folder?', + '~/orca/workspaces/orca/agy-dispatch-issue', + '1. Yes', + '2. No', + '>' + ], + reason: 'agent-trust-workspace' + } + ] + + for (const dialog of LIVE_DIALOGS_UNDER_THE_HEADER) { + it(`reports ${dialog.name} drawn under the header and stays unready`, () => { + const waitText = waitTextFor(dialog.lines) + + expect(detectTerminalWaitBlockedReason(waitText)).toBe(dialog.reason) + expect(isKnownReadyPromptPreview(waitText)).toBe(false) + }) + } + + // Discriminating: the Gemini model line and caret satisfy readiness, so only the dialog sitting + // *below* them keeps this unready. Drop the ordering rule and this goes green-to-red. + it('keeps reporting a dialog that opens after a Gemini ready screen', () => { + const waitText = waitTextFor([ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'Gemini 3.5 Flash (High)', + '~/orca/workspaces/orca/agy-dispatch-issue', + '>', + 'Permission required', + 'Allow once', + 'Reject' + ]) + + expect(isKnownReadyPromptPreview(waitText)).toBe(false) + expect(detectTerminalWaitBlockedReason(waitText)).toBe('agent-interactive-prompt') + }) + + // Discriminating: a stale dialog above a reprinted Gemini ready screen must stop being reported, + // which is the whole point of the dismissed-modal rule. + it('clears once a Gemini ready screen replaces the dialog', () => { + const waitText = waitTextFor([ + ...TRUST_DIALOG_WITH_CARET, + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'Gemini 3.5 Flash (High)', + '>' + ]) + + expect(isKnownReadyPromptPreview(waitText)).toBe(true) + expect(detectTerminalWaitBlockedReason(waitText)).toBeNull() + }) + + // Characterization, not a guard: records the wedge this file has not fixed. An Antigravity user on + // a non-Gemini model has no 'gemini' line, so readiness never resolves and the wait times out. + // Flipping this to true is the goal of the follow-up, and needs a captured transcript first. + it('does not yet recognize a non-Gemini ready screen (known wedge)', () => { + const waitText = waitTextFor([ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'Claude Sonnet 4.5 (High)', + '~/orca/workspaces/orca/agy-dispatch-issue', + '>' + ]) + + expect(isKnownReadyPromptPreview(waitText)).toBe(false) + }) + + // Ratchet, not a guard of today's code: these pass now only because none of them prints a 'gemini' + // model line. They exist so the next attempt to widen the model rule has to refuse them -- the + // reverted attempt accepted all five as ready on the strength of the account row alone (and an + // 'x@y.z' anywhere in the dialog body did just as well), and readiness is what gates typing the + // task prompt into the pane. A replacement must rest on positive evidence that the agent's input + // prompt is accepting input, not on absence-of-dialog plus an account row. + const SILENT_STARTUP_DIALOGS: { name: string; lines: string[] }[] = [ + { + name: 'an update banner', + lines: [ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'A new version is available', + '~/orca/workspaces/orca/agy-dispatch-issue', + 'Press enter to continue', + '>' + ] + }, + { + name: 'a sign-in dialog', + lines: [ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'Sign in to continue', + '~/orca/workspaces/orca/agy-dispatch-issue', + '1. Open browser', + '2. Paste an API key', + '>' + ] + }, + { + name: 'a model picker', + lines: [ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'Select a model', + '~/orca/workspaces/orca/agy-dispatch-issue', + '1. Claude Sonnet 4.5', + '2. GPT-5.1', + '>' + ] + }, + { + name: 'a privacy notice', + lines: [ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'We collect usage data to improve the product', + '~/orca/workspaces/orca/agy-dispatch-issue', + '1. Accept', + '2. Decline', + '>' + ] + }, + { + name: 'an onboarding theme picker', + lines: [ + 'Antigravity CLI 1.0.3', + 'user@example.com (Antigravity Business)', + 'Welcome! Choose a theme', + '~/orca/workspaces/orca/agy-dispatch-issue', + '1. Dark', + '2. Light', + '>' + ] + } + ] + + for (const dialog of SILENT_STARTUP_DIALOGS) { + it(`refuses ${dialog.name} whose wording names no blocked reason, account row and all`, () => { + const waitText = waitTextFor(dialog.lines) + + // No blocked-signal rule matches, so the ordering defense cannot reach these: readiness has to + // refuse them on its own or the orchestrator types into a live dialog. + expect(detectTerminalWaitBlockedReason(waitText)).toBeNull() + expect(isKnownReadyPromptPreview(waitText)).toBe(false) + }) + + it(`refuses ${dialog.name} that merely narrates an email address`, () => { + const waitText = waitTextFor([ + ...dialog.lines.slice(0, -1), + 'contact support@antigravity.dev for help', + '>' + ]) + + expect(isKnownReadyPromptPreview(waitText)).toBe(false) + }) + } +}) diff --git a/src/main/runtime/terminal-wait-detection.ts b/src/main/runtime/terminal-wait-detection.ts index ed957d1fe2d..cae25e8bfa6 100644 --- a/src/main/runtime/terminal-wait-detection.ts +++ b/src/main/runtime/terminal-wait-detection.ts @@ -231,11 +231,11 @@ function findBlockedSignalInLiveWindow( const candidates: { reason: RuntimeTerminalWaitBlockedReason; index: number }[] = [] const updateIndex = normalized.lastIndexOf('update available') if (updateIndex !== -1 && normalized.includes('press enter to continue', updateIndex)) { - candidates.push({ reason: 'codex-update-prompt', index: updateIndex }) + candidates.push({ reason: 'agent-update-prompt', index: updateIndex }) } const cwdIndex = normalized.lastIndexOf('choose working directory to') if (cwdIndex !== -1 && normalized.includes('press enter to continue', cwdIndex)) { - candidates.push({ reason: 'codex-cwd-prompt', index: cwdIndex }) + candidates.push({ reason: 'agent-cwd-prompt', index: cwdIndex }) } const modelMigrationIndex = normalized.lastIndexOf('codex just got an upgrade') if ( @@ -246,7 +246,8 @@ function findBlockedSignalInLiveWindow( } const hooksIndex = normalized.lastIndexOf('hooks need review') if (hooksIndex !== -1 && normalized.includes('press enter to confirm', hooksIndex)) { - candidates.push({ reason: 'codex-hooks-review-prompt', index: hooksIndex }) + // Why neutral: this matcher never inspects the agent -- 'hooks need review' is not Codex-only wording. + candidates.push({ reason: 'agent-hooks-review-prompt', index: hooksIndex }) } const trustIndex = Math.max( normalized.lastIndexOf('do you trust'), @@ -261,7 +262,8 @@ function findBlockedSignalInLiveWindow( trustSegment.includes('directory') || trustSegment.includes('repo')) ) { - candidates.push({ reason: 'codex-trust-workspace', index: trustIndex }) + // Why neutral: this matcher never inspects the agent -- every TUI agent ships a workspace-trust dialog. + candidates.push({ reason: 'agent-trust-workspace', index: trustIndex }) } const interactivePromptIndex = Math.max( normalized.lastIndexOf('press enter to confirm'), @@ -274,19 +276,22 @@ function findBlockedSignalInLiveWindow( interactivePromptIndex === -1 ? '' : normalized.slice(Math.max(0, interactivePromptIndex - 600), interactivePromptIndex + 200) - const hasCodexInteractiveContext = + // Why 'codex' only widens detection and never names the reason: the sole Codex evidence here is + // that word somewhere in 600 chars of scrollback, which an agent narrating about Codex satisfies + // on any pane -- enough to suspect a dialog, not enough to label a non-Codex user's pane. + const hasInteractiveDialogContext = interactivePromptContext.includes('codex') || interactivePromptContext.includes('permission') || interactivePromptContext.includes('sandbox') || interactivePromptContext.includes('trust') || interactivePromptContext.includes('hook') - if (interactivePromptIndex !== -1 && hasCodexInteractiveContext) { + if (interactivePromptIndex !== -1 && hasInteractiveDialogContext) { const contextStart = Math.max(0, interactivePromptIndex - 600) const hasSpecificPromptInContext = candidates.some( (candidate) => candidate.index >= contextStart && candidate.index <= interactivePromptIndex ) if (!hasSpecificPromptInContext) { - candidates.push({ reason: 'codex-interactive-prompt', index: interactivePromptIndex }) + candidates.push({ reason: 'agent-interactive-prompt', index: interactivePromptIndex }) } } const cursorApprovalIndex = findCursorApprovalPromptIndex(normalized) @@ -303,8 +308,13 @@ function findBlockedSignalInLiveWindow( permissionSegment.includes(choice) ).length if (decisionCount >= 2) { - // Why: preserve the existing remote receipt value for mixed-version clients. - candidates.push({ reason: 'codex-interactive-prompt', index: permissionPromptIndex }) + // Why neutral: an approval dialog with named choices identifies no agent; older hosts publish + // 'codex-interactive-prompt' here and clients alias the two. Rule 1 additive member -- + // remote-wire-compatibility.md names RuntimeTerminalWaitBlockedReason as Rule 1 because no + // consumer switches exhaustively on it. + // Why alias rather than drop the old spelling: preserve the existing remote receipt value for + // mixed-version clients -- an older host still publishes codex-* on this path. + candidates.push({ reason: 'agent-interactive-prompt', index: permissionPromptIndex }) } } return candidates.length > 0 diff --git a/src/main/runtime/unstopped-pty-verification.ts b/src/main/runtime/unstopped-pty-verification.ts index a2cd83b8eec..7cfd5907797 100644 --- a/src/main/runtime/unstopped-pty-verification.ts +++ b/src/main/runtime/unstopped-pty-verification.ts @@ -2,7 +2,7 @@ import type { IPtyProvider } from '../providers/types' import type { OrcaRuntimeService } from './orca-runtime' import { UNSTOPPED_PTY_DETAIL_SEPARATOR, - UNSTOPPED_PTY_LIVE_DETAIL_PREFIX, + STILL_LIVE_DETAIL_PREFIX, UNSTOPPED_PTY_REMOVAL_PREFIX } from '../../shared/worktree/removal' import { @@ -105,7 +105,7 @@ export function describeUnstoppedPtys( ): string { const detail = verdict.status === 'live' - ? `${UNSTOPPED_PTY_LIVE_DETAIL_PREFIX} ${verdict.ptyIds.join(', ')}` + ? `${STILL_LIVE_DETAIL_PREFIX} ${verdict.ptyIds.join(', ')}` : `could not verify these exited: ${failedPtyIds.join(', ')} (${verdict.reason})` return `${UNSTOPPED_PTY_REMOVAL_PREFIX} ${worktreeId}${UNSTOPPED_PTY_DETAIL_SEPARATOR}${detail}` } diff --git a/src/main/runtime/worktree-pty-host-fence.ts b/src/main/runtime/worktree-pty-host-fence.ts index 855371c4aef..a2a6099bf11 100644 --- a/src/main/runtime/worktree-pty-host-fence.ts +++ b/src/main/runtime/worktree-pty-host-fence.ts @@ -1,8 +1,14 @@ export type WorktreePtyHostFence = { + /** `null` is this machine; ABSENT is no fence at all, so every host matches. */ resolvedConnectionId?: string | null resolvedRuntimeEnvironmentId?: string } +/** + * Also fences the structured sweep, through `structuredSessionTeardownHostId`, which reuses this + * exact type so the two cannot drift. That helper narrows ABSENT to local — the one deliberate + * difference, documented where it is made. + */ export function worktreePtyBelongsToHost( ptyId: string, connectionId: string | null | undefined, diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 82fa055cc3a..84275b40eed 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -15,10 +15,16 @@ import { } from './worktree-pty-surface-sweeps' import { closeStructuredSessionsForWorktree, - describeLiveStructuredSessions, - listLiveStructuredSessionsForWorktree + createStructuredSweepProgress, + describeUnclosedStructuredSessions, + listLiveStructuredSessionsForWorktree, + unclosedStructuredSessions } from './structured-session-worktree-teardown' -import { createWorktreeSweepTracker, settleSweepsForForcedRemoval } from './forced-sweep-settlement' +import { + createWorktreeSweepTracker, + settleSweepsForForcedRemoval, + type WorktreeSweepTracker +} from './forced-sweep-settlement' import { describeError, describeFailedPtySweep, @@ -57,7 +63,7 @@ export type WorktreeTeardownResult = { runtimeStopped: number providerStopped: number registryStopped: number - /** Structured agent sessions closed by the force path; absent when none were found. */ + /** Structured agent sessions this teardown closed; absent when it closed none. */ structuredStopped?: number } @@ -99,12 +105,18 @@ export async function killAllProcessesForWorktree( const deadlineError = new Error( `${WORKTREE_TEARDOWN_TIMEOUT_PREFIX} ${worktreeId}. ${WORKTREE_TEARDOWN_FORCE_HINT}` ) - // FIRST, and before a single PTY sweep starts: a structured agent session is registered on none - // of the three surfaces below, so all three answered zero and removal deleted the checkout out - // from under a running provider child. Refusing costs nothing when there are none, and the check - // is synchronous, so a destructive removal fails fast instead of after the whole sweep budget. - const structuredStopped = await sweepStructuredSessions(worktreeId, deps, deadline, deadlineError) const sweeps = createWorktreeSweepTracker() + // ISSUED first, before a single PTY is touched: a structured agent session is registered on none + // of the three surfaces below, so all three answered zero and removal deleted the checkout out + // from under a running provider child. Asking the agent plane ahead of the terminal plane also + // keeps an intentional stop from reading as a failed process exit. + // + // Not AWAITED first, though. Its close is serial and each one waits on a provider round trip, so + // awaiting here would spend the shared budget before a single PTY was asked — and the sweeps + // would then report a timeout for a stop they never attempted. It is joined below, ahead of the + // PTY verdict, so a structured refusal still outranks one. + const structuredSweep = sweepStructuredSessions(worktreeId, deps, deadline, sweeps) + void structuredSweep.catch(() => undefined) const stopAttempts = new Map>() const stopPty = ( ptyId: string, @@ -196,6 +208,7 @@ export async function killAllProcessesForWorktree( for (const sweep of [runtimeSweep, providerSweep, registrySweep]) { void sweep.catch(() => undefined) } + const structuredStopped = await structuredSweep let runtimeResult: { stopped: number } let providerStopped: number let registryStopped: number @@ -207,7 +220,10 @@ export async function killAllProcessesForWorktree( deadlineError ) if (forced.incomplete) { - return forced.stopped + // Carries the structured count out too: this early return skips the PTY verdict, not the + // sweep that already closed a user's chats, and dropping it makes the log say `structured=0` + // for a removal that closed some. + return { ...forced.stopped, ...(structuredStopped > 0 ? { structuredStopped } : {}) } } runtimeResult = { stopped: forced.stopped.runtimeStopped } providerStopped = forced.stopped.providerStopped @@ -277,59 +293,84 @@ export async function killAllProcessesForWorktree( } /** - * The fourth sweep: structured agent sessions bound to this worktree. + * The fourth sweep: structured agent sessions bound to this worktree, on this host. * - * Refuses rather than auto-closing on the ordinary destructive path. `worktree rm` is the verb - * that deletes a user's work, and a running agent session is exactly the thing they would want to - * be told about before it goes — the same bargain the unstopped-PTY gate already strikes, using - * the same `--force` escape hatch. Force closes them properly instead of orphaning a child against - * a `cwd` that is about to disappear. + * Stops first and refuses only on unproven stops, which is the bargain the unstopped-PTY gate + * actually strikes: that gate kills every PTY — a terminal running an agent included — and refuses + * only for the ones whose exit it could not then verify. Refusing merely because a session is + * attached made an idle chat, which the user is done with, harder to delete than a terminal running + * the same agent. Attachment is lease state, not work in flight, so it was never the right proxy. * - * Two callers participate, for different reasons. A proof-requiring removal (`requirePhysicalStop`) - * refuses, then closes under force. A folder-workspace removal (`closeStructuredSessions`) closes - * best-effort without refusing: it shares its root so no checkout vanishes under the child, and one - * of those paths is a never-throw forget that a refusal would wedge. Reconciliation sweeps set - * neither — they repair state, delete nothing, and must never close a session. + * Two callers participate. A proof-requiring removal (`requirePhysicalStop`) refuses when a close + * does not settle, so nothing deletes a checkout out from under a child that is still there. A + * folder-workspace removal (`closeStructuredSessions`) never refuses: it shares its root so no + * checkout vanishes under the child, and every one of those call sites discards a rejection, so a + * refusal there would be words nobody reads. Reconciliation sweeps set neither — they repair state, + * delete nothing, and must never close a session. */ async function sweepStructuredSessions( worktreeId: string, deps: WorktreeTeardownDeps, deadline: number, - deadlineError: Error + sweeps: WorktreeSweepTracker ): Promise { if (!deps.requirePhysicalStop && !deps.closeStructuredSessions) { return 0 } - const live = listLiveStructuredSessionsForWorktree(worktreeId) + // `deps` carries the same two host fields the PTY sweeps fence on, and a `repoId::path` id names + // a different workspace on every host — so an unfenced list would close a live chat belonging to + // an SSH or paired-runtime copy of the id being removed here. + const live = listLiveStructuredSessionsForWorktree(worktreeId, deps) if (live.length === 0) { return 0 } + // Raced against the same sweep budget every PTY surface is bounded by, because `host.close` + // awaits a provider round trip whose own eviction steps are each bounded well past this budget. + // + // Deliberately NOT fail-closed, unlike the PTY sweeps: their timeout sentinel carries the PTY + // timeout prefix, which the desktop classifier reads as a TERMINAL failure — so a wedged session + // close would refuse in terminal wording, and refuse identically again under the Force Delete + // that is meant to clear it (#11960). A close that ran out of time is a session this removal + // could not confirm closed, which is exactly what the branch below already words. Tracked so a + // forced removal still waits out the abandoned-sweep grace before it deletes files. + // + // The verdict is read off `progress`, which the serial loop fills as it goes, rather than off + // this call's result: the deadline can land mid-loop, and a fallback assembled here could only + // guess — it named every session, including the ones already closed, and reported zero closes. + const progress = createStructuredSweepProgress(live) + await settleBeforeDeadline( + sweeps.track(() => + closeStructuredSessionsForWorktree(progress, deadline, { + ...(deps.runtime ? { runtime: deps.runtime } : {}), + // The only shape of removal that can leave this workspace — and its chat tabs — in place. + mayRefuse: Boolean(deps.requirePhysicalStop) && !deps.allowUnverifiedStop + }) + ), + undefined, + deadline + ) + const closed = progress.closed + const unstopped = unclosedStructuredSessions(progress) + if (unstopped.length === 0) { + return closed + } // Only a proof-requiring removal may refuse. A folder-workspace removal shares its root, so no // checkout disappears under the child — the harm is a session left pointing at a workspace Orca - // has forgotten — and one of those paths is a never-throw forget, which a refusal would wedge. + // has forgotten — and every one of those callers discards a rejection anyway. if (deps.requirePhysicalStop && !deps.allowUnverifiedStop) { // The prefix is what the desktop classifier matches on; without it the toast shows raw CLI // wording and hides the Force Delete button — the #11960 dead end this file already documents. throw new Error( - `${RUNNING_AGENT_SESSION_REMOVAL_PREFIX} ${worktreeId}${UNSTOPPED_PTY_DETAIL_SEPARATOR}${describeLiveStructuredSessions(live)}. ${WORKTREE_TEARDOWN_FORCE_HINT}` + `${RUNNING_AGENT_SESSION_REMOVAL_PREFIX} ${worktreeId}${UNSTOPPED_PTY_DETAIL_SEPARATOR}${describeUnclosedStructuredSessions(unstopped)}. ${WORKTREE_TEARDOWN_FORCE_HINT}` ) } - // Raced against the same sweep budget every PTY surface is bounded by: `host.close` awaits a - // provider round trip, and a wedged one would otherwise hang `worktree rm --force` forever with - // no timeout error at all. On expiry the force path reports the timeout exactly as the PTY - // sweeps do rather than proceeding as if the sessions had closed. - const { closed, unstopped } = await settleBeforeDeadline( - () => closeStructuredSessionsForWorktree(worktreeId, deps.runtime), - { closed: 0, unstopped: live }, - deadline, - deadlineError + // Force is the documented escape hatch, so removal continues — but say so, because the child + // outliving its `cwd` is the failure this sweep exists to make visible. Carries the verdict + // verbatim, like the unstopped-PTY warn above: this line is the only record a forced removal + // leaves, and appending "still attached" asserted the live verdict over sessions the sweep had + // just said it could not confirm either way. + console.warn( + `[worktree-teardown] forcing removal of ${worktreeId}${UNSTOPPED_PTY_DETAIL_SEPARATOR}${describeUnclosedStructuredSessions(unstopped)}` ) - if (unstopped.length > 0) { - // Force is the documented escape hatch, so removal continues — but say so, because the child - // outliving its `cwd` is the failure this sweep exists to make visible. - console.warn( - `[worktree-teardown] forcing removal of ${worktreeId} with ${describeLiveStructuredSessions(unstopped)} still attached` - ) - } return closed } diff --git a/src/preload/api/runtime-api.ts b/src/preload/api/runtime-api.ts index f7f553ec2c0..7fdf229dd88 100644 --- a/src/preload/api/runtime-api.ts +++ b/src/preload/api/runtime-api.ts @@ -1,3 +1,4 @@ +import type { RuntimeHostStatusSnapshot } from '../../shared/runtime-host-status' import type { RuntimeBrowserDriverState, RuntimeRendererSyncWindowGraph, @@ -77,6 +78,8 @@ export type RuntimeApi = { ) => () => void } runtimeEnvironments: { + getStatusSnapshots: () => Promise + onStatusChanged: (callback: (snapshot: RuntimeHostStatusSnapshot) => void) => () => void list: () => Promise addFromPairingCode: (args: { name: string diff --git a/src/preload/api/runtime-environments-bridge.ts b/src/preload/api/runtime-environments-bridge.ts index ddfa498dc74..63c31df52f8 100644 --- a/src/preload/api/runtime-environments-bridge.ts +++ b/src/preload/api/runtime-environments-bridge.ts @@ -1,4 +1,8 @@ import { ipcRenderer } from 'electron' +import { + RUNTIME_HOST_STATUS_CHANNEL, + type RuntimeHostStatusSnapshot +} from '../../shared/runtime-host-status' import type { VerifyAndAddRuntimeEnvironmentResult } from '../../shared/remote-pairing-verification' import type { RuntimeStatus } from '../../shared/runtime-types' import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' @@ -12,6 +16,16 @@ import { import type { PreloadApi } from '../api-types' export const runtimeEnvironmentsApi = { + getStatusSnapshots: (): Promise => + ipcRenderer.invoke('runtimeEnvironments:getStatusSnapshots'), + onStatusChanged: (callback: (snapshot: RuntimeHostStatusSnapshot) => void): (() => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + snapshot: RuntimeHostStatusSnapshot + ): void => callback(snapshot) + ipcRenderer.on(RUNTIME_HOST_STATUS_CHANNEL, listener) + return () => ipcRenderer.removeListener(RUNTIME_HOST_STATUS_CHANNEL, listener) + }, list: (): Promise => ipcRenderer.invoke('runtimeEnvironments:list'), addFromPairingCode: (args: { diff --git a/src/renderer/src/components/NewWorkspaceComposerCard.tsx b/src/renderer/src/components/NewWorkspaceComposerCard.tsx index d17bb8ba56d..7145bed732c 100644 --- a/src/renderer/src/components/NewWorkspaceComposerCard.tsx +++ b/src/renderer/src/components/NewWorkspaceComposerCard.tsx @@ -242,17 +242,11 @@ export default function NewWorkspaceComposerCard( selector: action.environmentId, timeoutMs: 15_000 }) - const runtimeStatus = unwrapRuntimeRpcResult(response) - useAppStore.getState().setRuntimeEnvironmentStatus(action.environmentId, { - status: runtimeStatus, - checkedAt: Date.now() - }) + unwrapRuntimeRpcResult(response) + await useAppStore.getState().readRuntimeHostStatusSnapshots() } catch (error) { if (action.kind === 'runtime') { - useAppStore.getState().setRuntimeEnvironmentStatus(action.environmentId, { - status: null, - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() } toast.error( error instanceof Error diff --git a/src/renderer/src/components/cmd-j/palette-host-badge.test.ts b/src/renderer/src/components/cmd-j/palette-host-badge.test.ts index cd6310c8580..8d15ff8d034 100644 --- a/src/renderer/src/components/cmd-j/palette-host-badge.test.ts +++ b/src/renderer/src/components/cmd-j/palette-host-badge.test.ts @@ -83,8 +83,7 @@ describe('getPaletteHostBadge', () => { repos: [{ executionHostId: 'runtime:env-1' }], sshTargetLabels: new Map(), settings: { activeRuntimeEnvironmentId: 'env-2' }, - // A live status makes the runtime 'available'; without it the host reads - // 'disconnected' and the badge is suppressed (covered below). + // Only verified availability enables unfiltered host badges. runtimeStatusByEnvironmentId: new Map([ [ 'env-1', @@ -145,3 +144,19 @@ describe('getPaletteHostBadge', () => { expect(getPaletteHostBadge(null, hosts)).toBeNull() }) }) + +it.each(['connecting', 'blocked', 'disconnected', 'error'] as const)( + 'does not infer reachability from %s health, but preserves explicit filter labels', + (health) => { + const hosts = buildSidebarHostOptions({ + repos: [{ executionHostId: 'runtime:env-1' }], + sshTargetLabels: new Map(), + settings: { activeRuntimeEnvironmentId: null } + }).map((host) => (host.kind === 'runtime' ? { ...host, health } : host)) + expect(getPaletteHostBadge({ connectionId: null }, hosts)).toBeNull() + expect(getPaletteHostBadge({ executionHostId: 'runtime:env-1' }, hosts, true)).toEqual({ + hostId: 'runtime:env-1', + label: 'env-1' + }) + } +) diff --git a/src/renderer/src/components/cmd-j/palette-host-badge.ts b/src/renderer/src/components/cmd-j/palette-host-badge.ts index 8ac94adeae4..01a9e33d3ac 100644 --- a/src/renderer/src/components/cmd-j/palette-host-badge.ts +++ b/src/renderer/src/components/cmd-j/palette-host-badge.ts @@ -17,7 +17,7 @@ export type PaletteHostBadge = { // unlike the sidebar gate, which lists disconnected hosts so users can connect. function hasActiveRemoteHost(hostOptions: readonly SidebarHostOption[]): boolean { return hostOptions.some( - (host) => host.id !== LOCAL_EXECUTION_HOST_ID && host.health !== 'disconnected' + (host) => host.id !== LOCAL_EXECUTION_HOST_ID && host.health === 'available' ) } diff --git a/src/renderer/src/components/editor/MonacoEditor.tsx b/src/renderer/src/components/editor/MonacoEditor.tsx index 27b1c240e82..31c473bf29f 100644 --- a/src/renderer/src/components/editor/MonacoEditor.tsx +++ b/src/renderer/src/components/editor/MonacoEditor.tsx @@ -10,6 +10,7 @@ import { computeEditorFontSize, resolveEditorFontFamily } from '@/lib/editor-fon import { useContextualCopySetup } from './useContextualCopySetup' import { MonacoGutterContextMenu } from './MonacoGutterContextMenu' import { isLinuxUserAgent } from '../terminal-pane/pane-helpers' +import { MAX_TOKENIZATION_LINE_LENGTH } from '@/lib/monaco-languages/monarch-embed-entry-budget' import { buildFileEditorWordWrapOptions } from './file-editor-word-wrap-options' import { getMonacoAutoHeightForContent, isMonacoAutoHeightCapped } from './monaco-auto-height' import { monacoFindOptions } from './monaco-find-options' @@ -235,6 +236,11 @@ export default function MonacoEditor({ onChange={contentSync.handleChange} onMount={handleMount} options={{ + // `IGlobalEditorOptions`, not per-editor: setting it here pins it for every + // Monaco surface (diff, Peek) too, so this is the only site that needs it. + // Defense-in-depth only — it does NOT guard the Monarch embed recursion, + // which overflowed at ~17_000 chars, under this cap. See the budget module. + maxTokenizationLineLength: MAX_TOKENIZATION_LINE_LENGTH, // Why: only the file editor honors this; Monaco 0.55 DiffEditor hard-overrides minimap.enabled=false on sub-editors (see diffEditorEditors._adjustOptionsForSubEditor). minimap: { enabled: settings?.editorMinimapEnabled ?? false }, scrollBeyondLastLine: false, diff --git a/src/renderer/src/components/native-chat/NativeChatMessageList.test.tsx b/src/renderer/src/components/native-chat/NativeChatMessageList.test.tsx index 9a30aa67eba..d6d74a20304 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageList.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageList.test.tsx @@ -95,221 +95,6 @@ describe('NativeChatMessageList assistant messages', () => { expect(document.querySelector('.text-destructive')).toBeNull() }) - it('keeps a reduced-motion-safe spinner activity line at the tail of a no-tool Codex turn', () => { - render( - - ) - - const activity = screen.getByText('Working…') - const row = activity.closest('[data-native-chat-turn-activity]') - const spinner = row?.querySelector('svg') - expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') - expect(spinner).toHaveClass('size-4', 'animate-spin', 'motion-reduce:animate-none') - expect(row).toHaveAttribute('aria-live', 'polite') - expect(screen.getByText('The answer is still streaming.').compareDocumentPosition(row!)).toBe( - Node.DOCUMENT_POSITION_FOLLOWING - ) - }) - - it('keeps the broad fallback distinct from the running tool row', () => { - render( - - ) - - const toolLabel = screen.getByText('Running pnpm test') - expect(toolLabel).toHaveClass('animate-pulse') - expect(screen.getAllByText('Running pnpm test')).toHaveLength(1) - const activity = screen.getByText('Working…') - expect(activity.textContent).not.toBe(toolLabel.textContent) - expect(activity).not.toHaveTextContent('shell') - expect(activity).not.toHaveTextContent('pnpm test') - const spinner = activity.closest('[data-native-chat-turn-activity]')?.querySelector('svg') - expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') - expect(spinner).toHaveClass('animate-spin', 'motion-reduce:animate-none') - }) - - it('uses the broad fallback after a tool settles', () => { - render( - - ) - - const settledTool = screen.getByText('shell') - const activity = screen.getByText('Working…') - expect(activity.textContent).not.toBe(settledTool.textContent) - expect(activity).not.toHaveTextContent('shell') - expect(activity).not.toHaveTextContent('pnpm test') - expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') - expect(activity.closest('[data-native-chat-turn-activity]')?.querySelector('svg')).toHaveClass( - 'animate-spin' - ) - }) - - it('keeps a completed tool row static while the turn tail spins, then removes the tail', () => { - const workingSession: NativeChatLiveSession = { - ...session, - status: 'working', - messages: [ - { - id: 'assistant-settled-tool', - role: 'assistant', - blocks: [ - { - type: 'tool-call', - name: 'shell', - input: { command: 'pnpm test' }, - state: 'completed' - }, - { type: 'tool-result', output: 'passed' } - ], - timestamp: 1, - source: 'transcript' - } - ] - } - const { container, rerender } = render( - - ) - - const settledTool = screen.getByText('shell') - expect(settledTool).toHaveTextContent('shell pnpm test') - expect(settledTool.closest('button')?.querySelector('.animate-pulse')).toBeNull() - expect(settledTool.closest('button')?.querySelector('.lucide-check')).toBeInTheDocument() - const activity = screen.getByText('Preparing the answer') - expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') - expect(activity.closest('[data-native-chat-turn-activity]')?.querySelector('svg')).toHaveClass( - 'animate-spin' - ) - - rerender( - - ) - - expect(container.querySelector('[data-native-chat-turn-activity]')).toBeNull() - expect(container.querySelector('.animate-pulse')).toBeNull() - expect(container.querySelector('.animate-spin')).toBeNull() - }) - - it('keeps bridge chats on the legacy activity chrome', () => { - render( - - ) - - expect(screen.queryByText('Thinking')).toBeNull() - expect(screen.queryByRole('button', { name: 'Toggle turn details' })).toBeNull() - expect(screen.queryByText('Running sleep 5')).toBeNull() - expect(document.querySelectorAll('.animate-bounce')).toHaveLength(3) - }) - it('keeps the current tool live when a stale completed lifecycle meets active hook state', () => { render( { expect(screen.getByText('Running sleep 5')).toBeInTheDocument() }) - - it('shows a stable thinking status directly below the user message', () => { - const { container } = render( - - ) - - const user = screen.getByText('Start the task') - const thinking = screen.getByText('Thinking') - expect(user.compareDocumentPosition(thinking)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) - expect(thinking.parentElement).not.toHaveClass('border-b') - expect(thinking.parentElement).toHaveClass('text-sm') - expect(container.querySelector('.animate-bounce')).toBeNull() - expect(thinking).toHaveClass('animate-pulse') - expect(container.querySelectorAll('.size-1.5.animate-pulse')).toHaveLength(0) - }) - - it('places the thinking status directly after the latest user message', () => { - render( - - ) - - const user = screen.getByText('Run the checks') - const status = screen.getByText('Working for 0s') - const assistant = screen.getByText('I am checking now.') - expect(user.compareDocumentPosition(status)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) - expect(status.compareDocumentPosition(assistant)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) - expect(status.parentElement).toHaveClass('border-b') - }) - - it('shows elapsed working time once tool activity starts', () => { - render( - - ) - - expect(screen.getByText('Working for 3s')).toBeInTheDocument() - }) - - it('keeps the completed duration below the user message', () => { - const startedAt = Date.now() - 3000 - const turnSession: NativeChatLiveSession = { - ...session, - status: 'working', - messages: [ - { - id: 'user-complete', - role: 'user', - blocks: [{ type: 'text', text: 'Complete this task' }], - timestamp: startedAt, - source: 'transcript' - }, - { - id: 'assistant-complete', - role: 'assistant', - blocks: [{ type: 'text', text: 'Task complete.' }], - timestamp: Date.now(), - source: 'transcript' - } - ] - } - const { rerender } = render( - - ) - - rerender( - - ) - - const user = screen.getByText('Complete this task') - const status = screen.getByText('Worked for 3s') - const assistant = screen.getByText('Task complete.') - expect(user.compareDocumentPosition(status)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) - expect(status.compareDocumentPosition(assistant)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) - - rerender( - - ) - - expect(screen.getByText('Worked for 3s')).toBeInTheDocument() - expect(screen.getByText('Thinking')).toBeInTheDocument() - }) - - it("uses the completed caret to expand that turn's tool details", () => { - const startedAt = Date.now() - 3000 - render( - - ) - - const status = screen.getByRole('button', { name: 'Toggle turn details' }) - expect(status).toHaveAttribute('aria-expanded', 'false') - expect(screen.queryByRole('button', { name: /1× shell/ })).toBeNull() - fireEvent.click(status) - expect(status).toHaveAttribute('aria-expanded', 'true') - const tool = screen.getByRole('button', { name: /1× shell/ }) - expect(tool).toHaveAttribute('aria-expanded', 'true') - expect(screen.getAllByRole('button', { name: /shell pwd/ })[1]).toHaveAttribute( - 'aria-expanded', - 'false' - ) - }) }) // List-level, because every defect this feature has shipped so far lived in the diff --git a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx index 6728aa99439..3da6283057a 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx @@ -9,11 +9,10 @@ import { nativeChatTaskListPredecessors } from './native-chat-task-list-history' import { NativeChatTaskList } from './NativeChatTaskList' import { projectNativeChatTaskListFrames } from './native-chat-task-list-frames' import { shouldShowNativeChatTypingIndicator } from './native-chat-typing-indicator' -import { NativeChatWorkingStatus } from './NativeChatWorkingStatus' import { useNativeChatTurnStatus } from './use-native-chat-turn-status' import { NativeChatTypingIndicatorRow } from './NativeChatTypingIndicatorRow' import type { RuntimeFileOperationArgs } from '@/runtime/runtime-file-client' -import type { NativeChatTurnActivity } from './native-chat-turn-activity' +import type { NativeChatTurnActivity } from '../../../../shared/native-chat-turn-activity' import { NativeChatTurnActivityLine } from './NativeChatTurnActivityLine' import { NativeChatDisclosureContext, @@ -29,6 +28,7 @@ import { useNativeChatTranscriptWindow } from './use-native-chat-transcript-wind import { useNativeChatTranscriptScroll } from './use-native-chat-transcript-scroll' import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' +import { isStructuredAgentSessionThinking } from '../../../../shared/structured-agent-session-live-turn' import type { NativeChatSettledTurns } from '../../../../shared/native-chat-turn-status' import { nativeChatTurnDiffs, @@ -150,12 +150,19 @@ export function NativeChatMessageList({ : new Map(), [journalItems, messages, turnKeys] ) + // "Thinking" is real reasoning content at the tail of the turn, not the absence + // of output — the latter reports thinking while the request is merely in flight. + const thinking = useMemo( + () => (journalItems ? isStructuredAgentSessionThinking(journalItems) : false), + [journalItems] + ) const turnStatuses = useNativeChatTurnStatus({ messages, latestUserIndex, isWorking: showTurnStatus && isWorking, workingStartedAt: showTurnStatus ? workingStartedAt : null, - settledTurns: showTurnStatus ? settledTurns : null + settledTurns: showTurnStatus ? settledTurns : null, + thinking }) const lifecycleWorking = session.transcriptLifecycle?.state === 'working' const slots = useMemo( @@ -169,7 +176,6 @@ export function NativeChatMessageList({ turnStatuses, turnDiffs, showTurnStatus, - showTypingIndicator, isWorking, lifecycleWorking }), @@ -181,7 +187,6 @@ export function NativeChatMessageList({ messages, receipts, showTurnStatus, - showTypingIndicator, turnDiffs, turnKeys, turnStatuses @@ -280,18 +285,11 @@ export function NativeChatMessageList({ context={rowContext} window={transcriptWindow} /> - {showTurnStatus && - latestUserIndex === -1 && - turnStatuses.active && - showTypingIndicator ? ( - - ) : null} {showTurnStatus && isWorking ? ( - + ) : null} {!showTurnStatus && showTypingIndicator ? : null} diff --git a/src/renderer/src/components/native-chat/NativeChatMessageList.turn-indicator.test.tsx b/src/renderer/src/components/native-chat/NativeChatMessageList.turn-indicator.test.tsx new file mode 100644 index 00000000000..ab706436f91 --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatMessageList.turn-indicator.test.tsx @@ -0,0 +1,563 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' + +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import type { NativeChatLiveSession } from './use-native-chat-live-session' +import { NativeChatMessageList } from './NativeChatMessageList' +import { installNativeChatMessageListTestViewport } from './native-chat-message-list-test-viewport' +import type { + AgentJournalItemBody, + AgentJournalRenderItem +} from '../../../../shared/agent-session-journal-types' + +// The turn record this host writes, and the legacy status row an older host sends. +const turnItem: AgentJournalItemBody = { kind: 'turn', turnId: 'turn-1', state: 'running' } +const legacyTurnRow: AgentJournalItemBody = { + kind: 'status', + text: 'Codex is working…', + turnLifecycle: { turnId: 'turn-1', state: 'running' } +} +const reasoningRow: AgentJournalItemBody = { + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: '' }] +} + +function journalItem(sequence: number, body: AgentJournalItemBody): AgentJournalRenderItem { + return { itemId: `item-${sequence}`, revision: 1, sequence, observedAt: sequence, body } +} + +let restoreViewport = (): void => {} +beforeAll(() => { + restoreViewport = installNativeChatMessageListTestViewport() +}) +afterAll(() => restoreViewport()) +afterEach(cleanup) + +const session: NativeChatLiveSession = { + messages: [ + { + id: 'assistant-1', + role: 'assistant', + blocks: [{ type: 'text', text: 'Selectable agent response.' }], + timestamp: 1, + source: 'transcript' + } + ], + status: 'ready', + sessionId: 'session-1', + agent: 'codex', + hasMore: false, + loadingEarlier: false, + loadEarlier: vi.fn(), + readPhase: 'ready' +} + +// The live turn renders exactly one indicator row; a settled turn keeps its own. +describe('NativeChatMessageList turn indicator', () => { + it('keeps a reduced-motion-safe spinner on the live row of a no-tool Codex turn', () => { + render( + + ) + + const activity = screen.getByText('Working for 0s') + const row = activity.closest('[data-native-chat-turn-activity]') + const spinner = row?.querySelector('svg') + expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') + expect(spinner).toHaveClass('size-4', 'animate-spin', 'motion-reduce:animate-none') + expect(row).toHaveAttribute('aria-live', 'polite') + expect(screen.getByText('The answer is still streaming.').compareDocumentPosition(row!)).toBe( + Node.DOCUMENT_POSITION_FOLLOWING + ) + }) + + it('keeps the live row distinct from the running tool row', () => { + render( + + ) + + const toolLabel = screen.getByText('Running pnpm test') + expect(toolLabel).toHaveClass('animate-pulse') + expect(screen.getAllByText('Running pnpm test')).toHaveLength(1) + const activity = screen.getByText('Working for 0s') + expect(activity.textContent).not.toBe(toolLabel.textContent) + expect(activity).not.toHaveTextContent('shell') + expect(activity).not.toHaveTextContent('pnpm test') + const spinner = activity.closest('[data-native-chat-turn-activity]')?.querySelector('svg') + expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') + expect(spinner).toHaveClass('animate-spin', 'motion-reduce:animate-none') + }) + + it('keeps the live row up after a tool settles', () => { + render( + + ) + + const settledTool = screen.getByText('shell') + const activity = screen.getByText('Working for 0s') + expect(activity.textContent).not.toBe(settledTool.textContent) + expect(activity).not.toHaveTextContent('shell') + expect(activity).not.toHaveTextContent('pnpm test') + expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') + expect(activity.closest('[data-native-chat-turn-activity]')?.querySelector('svg')).toHaveClass( + 'animate-spin' + ) + }) + + it('keeps a completed tool row static while the turn tail spins, then removes the tail', () => { + const workingSession: NativeChatLiveSession = { + ...session, + status: 'working', + messages: [ + { + id: 'assistant-settled-tool', + role: 'assistant', + blocks: [ + { + type: 'tool-call', + name: 'shell', + input: { command: 'pnpm test' }, + state: 'completed' + }, + { type: 'tool-result', output: 'passed' } + ], + timestamp: 1, + source: 'transcript' + } + ] + } + const { container, rerender } = render( + + ) + + const settledTool = screen.getByText('shell') + expect(settledTool).toHaveTextContent('shell pnpm test') + expect(settledTool.closest('button')?.querySelector('.animate-pulse')).toBeNull() + expect(settledTool.closest('button')?.querySelector('.lucide-check')).toBeInTheDocument() + const activity = screen.getByText('Preparing the answer') + expect(activity).not.toHaveClass('animate-pulse', 'animate-spin') + expect(activity.closest('[data-native-chat-turn-activity]')?.querySelector('svg')).toHaveClass( + 'animate-spin' + ) + + rerender( + + ) + + expect(container.querySelector('[data-native-chat-turn-activity]')).toBeNull() + expect(container.querySelector('.animate-pulse')).toBeNull() + expect(container.querySelector('.animate-spin')).toBeNull() + }) + + it('keeps bridge chats on the legacy activity chrome', () => { + render( + + ) + + expect(screen.queryByText('Thinking')).toBeNull() + expect(screen.queryByRole('button', { name: 'Toggle turn details' })).toBeNull() + expect(screen.queryByText('Running sleep 5')).toBeNull() + expect(document.querySelectorAll('.animate-bounce')).toHaveLength(3) + }) + + it('reads "Thinking" on the one live row while the turn is reasoning', () => { + const { container } = render( + + ) + + const user = screen.getByText('Start the task') + const thinking = screen.getByText('Thinking') + expect(user.compareDocumentPosition(thinking)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) + // One indicator, not a "Thinking" row stacked above a spinning "Working…" row. + expect(container.querySelectorAll('[data-native-chat-turn-status]')).toHaveLength(1) + expect(thinking.closest('[data-native-chat-turn-activity]')?.querySelector('svg')).toHaveClass( + 'animate-spin' + ) + expect(container.querySelector('.animate-bounce')).toBeNull() + }) + + it('does not reuse completed-turn reasoning while the next dispatch is pending', () => { + render( + + ) + + expect(screen.queryByText('Thinking')).toBeNull() + expect(screen.getByText('Working for 0s')).toBeInTheDocument() + }) + + it('lets provider activity text beat the reasoning label on the same single row', () => { + const { container } = render( + + ) + + expect(screen.getByText('Exploring the repo layout')).toBeInTheDocument() + expect(screen.queryByText('Thinking')).toBeNull() + expect(container.querySelectorAll('[data-native-chat-turn-status]')).toHaveLength(1) + }) + + it('places the one live row after the newest content in the turn', () => { + render( + + ) + + const status = screen.getByText('Working for 0s') + const assistant = screen.getByText('I am checking now.') + // The live row trails the newest content instead of sitting under the prompt. + expect(assistant.compareDocumentPosition(status)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) + expect(document.querySelectorAll('[data-native-chat-turn-status]')).toHaveLength(1) + }) + + it('shows elapsed working time once tool activity starts', () => { + render( + + ) + + expect(screen.getByText('Working for 3s')).toBeInTheDocument() + }) + + it('keeps the completed duration below the user message', () => { + const startedAt = Date.now() - 3000 + const turnSession: NativeChatLiveSession = { + ...session, + status: 'working', + messages: [ + { + id: 'user-complete', + role: 'user', + blocks: [{ type: 'text', text: 'Complete this task' }], + timestamp: startedAt, + source: 'transcript' + }, + { + id: 'assistant-complete', + role: 'assistant', + blocks: [{ type: 'text', text: 'Task complete.' }], + timestamp: Date.now(), + source: 'transcript' + } + ] + } + const { rerender } = render( + + ) + + rerender( + + ) + + const user = screen.getByText('Complete this task') + const status = screen.getByText('Worked for 3s') + const assistant = screen.getByText('Task complete.') + expect(user.compareDocumentPosition(status)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) + expect(status.compareDocumentPosition(assistant)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) + + rerender( + + ) + + expect(screen.getByText('Worked for 3s')).toBeInTheDocument() + expect(screen.getByText('Working for 0s')).toBeInTheDocument() + }) + + it("uses the completed caret to expand that turn's tool details", () => { + const startedAt = Date.now() - 3000 + render( + + ) + + const status = screen.getByRole('button', { name: 'Toggle turn details' }) + expect(status).toHaveAttribute('aria-expanded', 'false') + expect(screen.queryByRole('button', { name: /1× shell/ })).toBeNull() + fireEvent.click(status) + expect(status).toHaveAttribute('aria-expanded', 'true') + const tool = screen.getByRole('button', { name: /1× shell/ }) + expect(tool).toHaveAttribute('aria-expanded', 'true') + expect(screen.getAllByRole('button', { name: /shell pwd/ })[1]).toHaveAttribute( + 'aria-expanded', + 'false' + ) + }) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatTurnActivityLine.tsx b/src/renderer/src/components/native-chat/NativeChatTurnActivityLine.tsx index da11773105d..72e11ad25a8 100644 --- a/src/renderer/src/components/native-chat/NativeChatTurnActivityLine.tsx +++ b/src/renderer/src/components/native-chat/NativeChatTurnActivityLine.tsx @@ -1,18 +1,55 @@ import { Loader2 } from 'lucide-react' import { translate } from '@/i18n/i18n' -import type { NativeChatTurnActivity } from './native-chat-turn-activity' +import type { NativeChatTurnActivity } from '../../../../shared/native-chat-turn-activity' +import { + describeNativeChatActiveTurnLabel, + NATIVE_CHAT_TURN_STATUS_COPY, + type NativeChatTurnStatus +} from '../../../../shared/native-chat-turn-status' +import { useNativeChatElapsedSeconds } from './use-native-chat-elapsed-seconds' +/** The live turn's one indicator: a spinner plus whatever the turn can say about + * itself — the provider's activity text, else that it is reasoning, else how + * long it has been working. A settled turn keeps its own `NativeChatWorkingStatus` + * row; this one is only ever rendered while the turn is in flight. */ export function NativeChatTurnActivityLine({ - activity + activity, + status }: { activity?: NativeChatTurnActivity | null + status?: NativeChatTurnStatus | null }): React.JSX.Element { - const label = activity?.text ?? translate('components.native-chat.status.working', 'Working…') + const thinking = status?.thinking === true + // The clock only ticks when its number is the label; activity text and + // "Thinking" carry no duration. + const counting = status != null && !thinking && !activity?.text + const elapsedSeconds = useNativeChatElapsedSeconds(status?.startedAt ?? null, counting) + const resolved = describeNativeChatActiveTurnLabel({ + activityText: activity?.text, + thinking, + elapsedSeconds + }) + const label = + resolved.source === 'activity' + ? resolved.text + : status == null + ? translate('components.native-chat.status.working', 'Working…') + : resolved.key === 'thinking' + ? translate( + 'components.native-chat.status.thinking', + NATIVE_CHAT_TURN_STATUS_COPY.thinking + ) + : translate( + 'components.native-chat.status.workingFor', + NATIVE_CHAT_TURN_STATUS_COPY.workingFor, + { value0: resolved.duration } + ) return (
diff --git a/src/renderer/src/components/native-chat/NativeChatWorkingStatus.tsx b/src/renderer/src/components/native-chat/NativeChatWorkingStatus.tsx index e6c38de83f5..8b910daa936 100644 --- a/src/renderer/src/components/native-chat/NativeChatWorkingStatus.tsx +++ b/src/renderer/src/components/native-chat/NativeChatWorkingStatus.tsx @@ -1,13 +1,11 @@ -import { useState } from 'react' import { ChevronRight } from 'lucide-react' import { translate } from '@/i18n/i18n' -import { useNow } from '@/hooks/use-now' import { describeNativeChatTurnStatus, formatNativeChatDuration, - NATIVE_CHAT_TURN_STATUS_COPY, - nativeChatElapsedSeconds + NATIVE_CHAT_TURN_STATUS_COPY } from '../../../../shared/native-chat-turn-status' +import { useNativeChatElapsedSeconds } from './use-native-chat-elapsed-seconds' export { formatNativeChatDuration } @@ -24,15 +22,8 @@ export function NativeChatWorkingStatus({ expanded?: boolean onToggleExpanded?: () => void }): React.JSX.Element { - // Why: elapsed seconds is ordinary render dataflow, not an external system. - // The shared 1s clock is visibility-gated and collapses every in-flight turn - // onto one tick, instead of one interval plus one commit per turn. const counting = !thinking && workedSeconds == null - const now = useNow(1_000, counting) - // Why: preserves the old effect's `startedAt ?? Date.now()` epoch for the - // single frame before the turn's startedAt lands. - const [mountedAt] = useState(() => Date.now()) - const elapsedSeconds = counting ? nativeChatElapsedSeconds(startedAt, mountedAt, now) : 0 + const elapsedSeconds = useNativeChatElapsedSeconds(startedAt, counting) const { key, duration } = describeNativeChatTurnStatus({ thinking, @@ -67,6 +58,7 @@ export function NativeChatWorkingStatus({ return ( | 0:-@html 17:delimiter.curly.svelte@svelte 18:-@typescript 27:delimiter.curly.svelte@svelte 28:-@html 29:delimiter.curly.svelte@svelte 30:-@typescript 35:delimiter.curly.svelte@svelte 36:-@html | embed=html", + "{@html 'raw'} | 0:keyword.control.svelte@svelte 6:-@typescript 21:delimiter.curly.svelte@svelte | embed=none", + " | | embed=html", + " | 0:tag.svelte@svelte | embed=none", ] `) }) -}) -describe('svelte tokenizer regressions', () => { - // Regression: when a Svelte file starts with `{#if}`, `{name}`, or `{@html}`, - // no html embed is active yet. Earlier drafts unconditionally emitted - // `nextEmbedded: '@pop'` from root, which Monaco rejects with - // "cannot pop embedded language if not inside one". The fix splits the - // entry-only `root` state from the html-embedded `markup` state. - it('does not pop a non-existent embed when a file starts with a Svelte block', () => { - const action = findRuleAction('root', '{#if foo}') - expect(action).toMatchObject({ switchTo: '@svelteBlockExpressionEnter' }) - expect(action?.nextEmbedded).toBeUndefined() + // Regression (the field failure): the first interpolation of a file threw + // "cannot pop embedded language if not inside one" — every Svelte file with a + // `{}` in it, which is essentially all of them. + it('highlights every interpolation of a markup line', () => { + const [line] = tokenizeSvelte('

a {first} b {second} c

') + + expect(tokenLanguages(line)).toEqual([ + 'html', + 'svelte', + 'typescript', + 'svelte', + 'html', + 'svelte', + 'typescript', + 'svelte', + 'html' + ]) }) - it('starts the html embed and switches to markup when markup begins', () => { - expect(findRuleAction('root', '

Counter

')).toMatchObject({ - switchTo: '@markup', - nextEmbedded: 'html' - }) + it('opens a file on a Svelte block without popping a missing embed', () => { + // No html embed exists yet at file start, so the block's entry rule must not + // pop one — Monarch throws outright if it does. + const [line] = tokenizeSvelte('{#if count > 0}') + + expect(tokenTypeAt(line, 0)).toBe('keyword.control') + expect(tokenLanguages(line)).toEqual(['svelte', 'typescript', 'svelte']) }) - // Regression: while the html embed is active, only parent rules whose action - // pops the embed are consulted before delegating to html. The first draft - // omitted `nextEmbedded: '@pop'` from ``)).toEqual([ + ['html'], + ['svelte'], + [embeddedLanguageId], + ['svelte'] + ]) + }) + + it.each([ + ['`)).toEqual([ + ['html'], + ['svelte'], + [embeddedLanguageId], + ['svelte'] + ]) + }) +}) + +describe('svelte root state invariant', () => { + // Structural on purpose: behaviour can only reach the root rules some fixture + // happens to exercise, and a root rule that pops an embed throws on the very + // first character of a file. Guard every root rule, exercised or not. + it('has no root rule that pops an embedded language', () => { + const rootRules = (svelteMonarchLanguage.tokenizer as Record).root + const popRules = rootRules.filter( + (rule) => + Array.isArray(rule) && (rule[1] as { nextEmbedded?: string })?.nextEmbedded === '@pop' + ) + + expect(popRules).toEqual([]) }) }) diff --git a/src/renderer/src/lib/monaco-languages/register-svelte.ts b/src/renderer/src/lib/monaco-languages/register-svelte.ts index 8fd197ea112..7e4b0cc0292 100644 --- a/src/renderer/src/lib/monaco-languages/register-svelte.ts +++ b/src/renderer/src/lib/monaco-languages/register-svelte.ts @@ -1,4 +1,8 @@ import type * as Monaco from 'monaco-editor' +import { + restOfLineWithinEmbedBudget, + tagCloseWithinEmbedBudget +} from './monarch-embed-entry-budget' type MonacoModule = typeof Monaco @@ -38,7 +42,18 @@ export const svelteMonarchLanguage: Monaco.languages.IMonarchLanguage = { { token: 'keyword.control', switchTo: '@svelteExpressionEnter' } ], [/\{(?=[^#:/@])/, { token: 'delimiter.curly', switchTo: '@svelteExpressionEnter' }], - [/(?=.)/, { token: '', switchTo: '@markup', nextEmbedded: 'html' }] + // `@rematch` is required: on a zero-width match Monarch's progress check + // `continue`s and silently drops a pending `nextEmbedded` for any other + // token, leaving `markup` without the html embed its pop rules assume. + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@markup', nextEmbedded: 'html' } + ], + // Past the budget: same structure, no embed, so the rest of the line + // cannot deepen the recursion. + [/<\/?[A-Za-z][^>]*>/, 'tag'], + [/[^<{]+/, ''], + [/./, ''] ], // html-embedded markup state. INVARIANT: whenever we are in `markup`, the // html embed is active. Every state that pops back to markup routes @@ -88,7 +103,14 @@ export const svelteMonarchLanguage: Monaco.languages.IMonarchLanguage = { // switches to `markup`. `@rematch` short-circuits Monarch's progress // check — a zero-width match that stays in the same state and stack // depth otherwise throws "no progress in tokenizer". - markupReenter: [[/(?=.)/, { token: '@rematch', switchTo: '@markup', nextEmbedded: 'html' }]], + markupReenter: [ + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@markup', nextEmbedded: 'html' } + ], + // Over budget: `root` carries the same structural rules without embeds. + [/(?=.)/, { token: '@rematch', switchTo: '@root' }] + ], comment: [ [/-->/, { token: 'comment', switchTo: '@markupReenter' }], [/[^-]+/, 'comment'], @@ -105,23 +127,44 @@ export const svelteMonarchLanguage: Monaco.languages.IMonarchLanguage = { // Empty expression `{}`: entry popped the html embed, but we never // entered the typescript embed, so only the state needs to unwind. [/\}/, { token: 'delimiter.curly', switchTo: '@markupReenter' }], - [/(?=.)/, { token: '', switchTo: '@svelteExpression', nextEmbedded: 'typescript' }] + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@svelteExpression', nextEmbedded: 'typescript' } + ], + [/(?=.)/, { token: '@rematch', switchTo: '@svelteExpressionPlain' }] ], svelteExpression: [ [/\}/, { token: 'delimiter.curly', switchTo: '@markupReenter', nextEmbedded: '@pop' }] ], + // Same expression, no typescript embed: reached only past the budget. + svelteExpressionPlain: [ + [/\}/, { token: 'delimiter.curly', switchTo: '@markupReenter' }], + [/[^}]+/, ''] + ], svelteBlockExpressionEnter: [ [/\}/, { token: 'keyword.control', switchTo: '@markupReenter' }], - [/(?=.)/, { token: '', switchTo: '@svelteBlockExpression', nextEmbedded: 'typescript' }] + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@svelteBlockExpression', nextEmbedded: 'typescript' } + ], + [/(?=.)/, { token: '@rematch', switchTo: '@svelteBlockExpressionPlain' }] ], svelteBlockExpression: [ [/\}/, { token: 'keyword.control', switchTo: '@markupReenter', nextEmbedded: '@pop' }] ], + svelteBlockExpressionPlain: [ + [/\}/, { token: 'keyword.control', switchTo: '@markupReenter' }], + [/[^}]+/, ''] + ], scriptOpen: [ // Self-closing `' }, - { line: 9, state: 'root', pattern: '' }, - { line: 11, state: 'styleBody.css', pattern: '' } - ] - - checks.forEach((check) => { - const line = lines.at(check.line - 1) ?? '' - const stateRules = tokenizer[check.state] ?? tokenizer[check.state.split('.')[0]] - const matchedRule = stateRules.find((rule) => { - if (!isRuleEntry(rule)) { - return false - } - const [regexp] = rule - regexp.lastIndex = 0 - const match = regexp.exec(line) - return match !== null && match[0] === check.pattern - }) - if (!matchedRule || !isRuleEntry(matchedRule)) { - return - } - - const actionObject = getRuleAction(matchedRule) - - ruleActions.push({ - line: check.line, - state: check.state, - matched: check.pattern, - nextState: actionObject?.next ? normalizeState(actionObject.next) : undefined, - nextEmbedded: actionObject?.nextEmbedded, - switchTo: actionObject?.switchTo ? normalizeState(actionObject.switchTo) : undefined - }) - }) - - return ruleActions -} - -describe('registerVueLanguage', () => { +describe('registerVueLanguage registration', () => { + // Structural by necessity: covers the registration call itself (ids, + // extensions, idempotence), which tokenizing cannot observe. it('registers the vue language, Monarch tokenizer, and configuration once', () => { const languages: { id: string }[] = [{ id: 'typescript' }] const register = vi.fn((entry: { id: string }) => { @@ -136,8 +54,10 @@ describe('registerVueLanguage', () => { expect(setLanguageConfiguration).toHaveBeenCalledTimes(1) expect(setLanguageConfiguration).toHaveBeenCalledWith('vue', vueLanguageConfiguration) }) +}) - it('captures Vue tokenizer transitions for a representative SFC fixture', () => { +describe('vue tokenization', () => { + it('tokenizes a representative SFC', () => { const fixture = ` @@ -150,121 +70,110 @@ const message = 'hello' p { color: rebeccapurple; } ` - const ruleActions = collectFixtureRuleActions(fixture) - - expect(ruleActions).toMatchInlineSnapshot(` + expect(formatTokenizedLines(tokenizeVue(fixture))).toMatchInlineSnapshot(` [ - { - "line": 1, - "matched": "", - "nextEmbedded": "html", - "nextState": undefined, - "state": "templateOpen", - "switchTo": "templateBody", - }, - { - "line": 2, - "matched": "{{", - "nextEmbedded": "@pop", - "nextState": "templateExpressionEnter", - "state": "templateBody", - "switchTo": undefined, - }, - { - "line": 2, - "matched": "}}", - "nextEmbedded": "@pop", - "nextState": "pop", - "state": "templateExpression", - "switchTo": undefined, - }, - { - "line": 3, - "matched": "", - "nextEmbedded": "@pop", - "nextState": "pop", - "state": "templateBody", - "switchTo": undefined, - }, - { - "line": 5, - "matched": "", - "nextEmbedded": "$S2", - "nextState": undefined, - "state": "scriptOpen.typescript", - "switchTo": "scriptBody.$S2", - }, - { - "line": 7, - "matched": "", - "nextEmbedded": "@pop", - "nextState": "pop", - "state": "scriptBody.typescript", - "switchTo": undefined, - }, - { - "line": 9, - "matched": "", - "nextEmbedded": "$S2", - "nextState": undefined, - "state": "styleOpen.css", - "switchTo": "styleBody.$S2", - }, - { - "line": 11, - "matched": "", - "nextEmbedded": "@pop", - "nextState": "pop", - "state": "styleBody.css", - "switchTo": undefined, - }, + " | 0:tag.vue@vue | embed=none", + " | | embed=none", + " | 0:tag.vue@vue | embed=none", + " | | embed=none", + " | 0:tag.vue@vue | embed=none", ] `) }) - it('tracks embedded languages from Vue block attributes', () => { - expect(findRuleAction('templateExpressionEnter', 'message }}')).toMatchObject({ - nextEmbedded: 'typescript', - switchTo: '@templateExpression' - }) - expect(findRuleAction('scriptLangValue.typescript', '"js"')).toMatchObject({ - switchTo: '@scriptOpen.javascript' - }) - expect(findRuleAction('scriptLangValue.javascript', '"ts"')).toMatchObject({ - switchTo: '@scriptOpen.typescript' - }) - expect(findRuleAction('scriptLangValue.typescript', 'js')).toMatchObject({ - switchTo: '@scriptOpen.javascript' - }) - expect(findRuleAction('styleLangValue.css', '"scss"')).toMatchObject({ - switchTo: '@styleOpen.scss' - }) - expect(findRuleAction('styleLangValue.css', 'less')).toMatchObject({ - switchTo: '@styleOpen.less' - }) + // Regression: every `{{ }}` threw "cannot pop embedded language if not inside + // one" once the template body lost its html embed. + it('highlights every interpolation in a template line', () => { + const [, line] = tokenizeVue('') + + expect(tokenLanguages(line)).toEqual([ + 'html', + 'vue', + 'typescript', + 'vue', + 'html', + 'vue', + 'typescript', + 'vue', + 'html' + ]) + }) + + it('embeds the template body as html', () => { + expect(endEmbeddedLanguages(tokenizeVue(''))).toEqual([ + 'html', + 'html', + null + ]) + }) + + it('keeps the template embedded across a comment before it', () => { + expect(languagesPerLine('\n')).toEqual([ + ['vue'], + ['vue'], + ['html'], + ['vue'] + ]) + }) + + it('does not enter typescript for an empty interpolation', () => { + // `{{}}` pops html on entry but never pushes typescript; the close must + // unwind only the state, or it pops an embed that is not there. + const [, line] = tokenizeVue('') + + expect(tokenLanguages(line)).toEqual(['html', 'vue', 'html']) + }) +}) + +describe('vue embedded language attributes', () => { + it.each([ + ['`)).toEqual([ + ['vue'], + [embeddedLanguageId], + ['vue'] + ]) + }) + + it.each([ + ['`)).toEqual([ + ['vue'], + [embeddedLanguageId], + ['vue'] + ]) + }) +}) + +describe('vue root state invariant', () => { + // Structural on purpose: behaviour can only reach the root rules some fixture + // happens to exercise, and a root rule that pops an embed throws on the very + // first character of a file. Guard every root rule, exercised or not. + it('has no root rule that pops an embedded language', () => { + const rootRules = (vueMonarchLanguage.tokenizer as Record).root + const popRules = rootRules.filter( + (rule) => + Array.isArray(rule) && (rule[1] as { nextEmbedded?: string })?.nextEmbedded === '@pop' + ) + + expect(popRules).toEqual([]) }) }) diff --git a/src/renderer/src/lib/monaco-languages/register-vue.ts b/src/renderer/src/lib/monaco-languages/register-vue.ts index b3531f8f16c..a12f1557f7e 100644 --- a/src/renderer/src/lib/monaco-languages/register-vue.ts +++ b/src/renderer/src/lib/monaco-languages/register-vue.ts @@ -1,4 +1,8 @@ import type * as Monaco from 'monaco-editor' +import { + restOfLineWithinEmbedBudget, + tagCloseWithinEmbedBudget +} from './monarch-embed-entry-budget' type MonacoModule = typeof Monaco @@ -28,31 +32,72 @@ export const vueMonarchLanguage: Monaco.languages.IMonarchLanguage = { ], templateOpen: [ [/\/>/, 'tag', '@pop'], - [/>/, { token: 'tag', switchTo: '@templateBody', nextEmbedded: 'html' }], + [ + tagCloseWithinEmbedBudget, + { token: 'tag', switchTo: '@templateBody', nextEmbedded: 'html' } + ], + [/>/, { token: 'tag', switchTo: '@templateBodyPlain' }], { include: '@tagAttributes' } ], + // INVARIANT: the html embed is active whenever this state is. Leaving an + // interpolation routes back through `templateBodyReenter`, never straight + // here, or the next `{{` would pop an embed that is no longer on the stack. + // Transitions are flat (`switchTo`) so the monarch stack stays at the depth + // `` still pops back to `root`. templateBody: [ [ /\{\{/, - { token: 'delimiter.curly', next: '@templateExpressionEnter', nextEmbedded: '@pop' } + { token: 'delimiter.curly', switchTo: '@templateExpressionEnter', nextEmbedded: '@pop' } ], - [/<\/template\s*>/, { token: 'tag', next: '@pop', nextEmbedded: '@pop' }], - // After a `{{ ... }}` interpolation returns here, the html embed - // has been popped alongside the typescript expression embed. Re-enter - // html so the remaining template markup is tokenized by Monaco's html - // tokenizer instead of falling back to the empty default token. - [/(?=.)/, { token: '', nextEmbedded: 'html' }] + [/<\/template\s*>/, { token: 'tag', next: '@pop', nextEmbedded: '@pop' }] + ], + // Re-entry shim. `@rematch` is required: on a zero-width match Monarch's + // progress check rejects any other token and drops the pending embed with + // it, leaving `templateBody` without the html embed its pop rules assume. + templateBodyReenter: [ + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@templateBody', nextEmbedded: 'html' } + ], + [/(?=.)/, { token: '@rematch', switchTo: '@templateBodyPlain' }] + ], + // Same body with no embeds, so the rest of an over-budget line cannot + // deepen the recursion. Its rules must not touch the embed stack. + templateBodyPlain: [ + [/\{\{/, { token: 'delimiter.curly', switchTo: '@templateExpressionEnter' }], + [/<\/template\s*>/, { token: 'tag', next: '@pop' }], + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@templateBody', nextEmbedded: 'html' } + ], + [/<\/?[A-Za-z][^>]*>/, 'tag'], + [/[^<{]+/, ''], + [/./, ''] ], templateExpressionEnter: [ - [/\}\}/, { token: 'delimiter.curly', next: '@pop' }], - [/(?=.)/, { token: '', switchTo: '@templateExpression', nextEmbedded: 'typescript' }] + [/\}\}/, { token: 'delimiter.curly', switchTo: '@templateBodyReenter' }], + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@templateExpression', nextEmbedded: 'typescript' } + ], + [/(?=.)/, { token: '@rematch', switchTo: '@templateExpressionPlain' }] ], templateExpression: [ - [/\}\}/, { token: 'delimiter.curly', next: '@pop', nextEmbedded: '@pop' }] + [/\}\}/, { token: 'delimiter.curly', switchTo: '@templateBodyReenter', nextEmbedded: '@pop' }] + ], + // Same expression, no typescript embed: reached only past the budget. + templateExpressionPlain: [ + [/\}\}/, { token: 'delimiter.curly', switchTo: '@templateBodyReenter' }], + [/[^}]+/, ''], + [/./, ''] ], scriptOpen: [ [/\/>/, 'tag', '@pop'], - [/>/, { token: 'tag', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' }], + [ + tagCloseWithinEmbedBudget, + { token: 'tag', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' } + ], + [/>/, { token: 'tag', switchTo: '@scriptBodyPlain.$S2' }], [/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@scriptLangBeforeEquals.$S2' }], { include: '@tagAttributes' } ], @@ -80,9 +125,24 @@ export const vueMonarchLanguage: Monaco.languages.IMonarchLanguage = { [/\s+/, 'white'] ], scriptBody: [[/<\/script\s*>/, { token: 'tag', next: '@pop', nextEmbedded: '@pop' }]], + // Over-budget mirror of the body: re-enters `$S2` as soon as the rest of + // the line fits, so a long opening line does not grey out the whole block. + scriptBodyPlain: [ + [/<\/script\s*>/, { token: 'tag', next: '@pop' }], + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' } + ], + [/[^<]+/, ''], + [/./, ''] + ], styleOpen: [ [/\/>/, 'tag', '@pop'], - [/>/, { token: 'tag', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' }], + [ + tagCloseWithinEmbedBudget, + { token: 'tag', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' } + ], + [/>/, { token: 'tag', switchTo: '@styleBodyPlain.$S2' }], [/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@styleLangBeforeEquals.$S2' }], { include: '@tagAttributes' } ], @@ -110,6 +170,15 @@ export const vueMonarchLanguage: Monaco.languages.IMonarchLanguage = { [/\s+/, 'white'] ], styleBody: [[/<\/style\s*>/, { token: 'tag', next: '@pop', nextEmbedded: '@pop' }]], + styleBodyPlain: [ + [/<\/style\s*>/, { token: 'tag', next: '@pop' }], + [ + restOfLineWithinEmbedBudget, + { token: '@rematch', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' } + ], + [/[^<]+/, ''], + [/./, ''] + ], tagAttributes: [ [/[^\s/>=]+/, 'attribute.name'], [/=/, 'delimiter'], diff --git a/src/renderer/src/lib/session-write-subscriber.test.ts b/src/renderer/src/lib/session-write-subscriber.test.ts index 80111781534..9753d8303b7 100644 --- a/src/renderer/src/lib/session-write-subscriber.test.ts +++ b/src/renderer/src/lib/session-write-subscriber.test.ts @@ -389,6 +389,43 @@ describe('createSessionWriteSubscriber', () => { cleanup() }) + it('ignores recovery-ledger-only changes', () => { + // Why: the ledger is stripped from the persisted session, so churning it + // must not rebuild and rewrite the durable payload on every remount. + const persist = vi.fn<(payload: WorkspaceSessionWrite) => void>() + const cleanup = createSessionWriteSubscriber({ store: useAppStore, persist }) + + useAppStore.setState({ + workspaceSessionReady: true, + hydrationSucceeded: true, + ...makeTerminalSessionState('bash') + }) + vi.advanceTimersByTime(200) + persist.mockClear() + + useAppStore.setState({ + tabsByWorktree: { + 'wt-1': [ + { + ...useAppStore.getState().tabsByWorktree['wt-1'][0], + recovery: { + attemptedAt: [1], + generation: 1, + outcome: 'pending', + startedAt: 1, + reason: 'reattach-unverifiable', + tabGeneration: 0 + } + } + ] + } + }) + vi.advanceTimersByTime(200) + + expect(persist).not.toHaveBeenCalled() + cleanup() + }) + it('ignores decorative unified terminal label churn', () => { const persist = vi.fn<(payload: WorkspaceSessionWrite) => void>() const cleanup = createSessionWriteSubscriber({ store: useAppStore, persist }) diff --git a/src/renderer/src/lib/session-write-subscriber.ts b/src/renderer/src/lib/session-write-subscriber.ts index d54675e15ab..685a8e56e3f 100644 --- a/src/renderer/src/lib/session-write-subscriber.ts +++ b/src/renderer/src/lib/session-write-subscriber.ts @@ -14,7 +14,10 @@ type UnifiedTab = UnifiedTabsByWorktree[string][number] const TERMINAL_TAB_LIVE_TITLE_KEYS = new Set(['title']) // Why: this handoff flag is stripped from workspace sessions, so toggling it // alone should not rebuild and rewrite the durable session payload. -const TERMINAL_TAB_TRANSIENT_SESSION_KEYS = new Set(['pendingActivationSpawn']) +const TERMINAL_TAB_TRANSIENT_SESSION_KEYS = new Set([ + 'pendingActivationSpawn', + 'recovery' +]) function terminalTabChangedForSession(prev: TerminalTab, next: TerminalTab): boolean { if (prev === next) { diff --git a/src/renderer/src/lib/workspace-session-patch.test.ts b/src/renderer/src/lib/workspace-session-patch.test.ts index 2f604fb67bb..d2084d6fd03 100644 --- a/src/renderer/src/lib/workspace-session-patch.test.ts +++ b/src/renderer/src/lib/workspace-session-patch.test.ts @@ -182,7 +182,15 @@ describe('buildWorkspaceSessionPatch', () => { title: 'shell', ptyId: 'pty-1', worktreeId: localWorktreeId, - pendingActivationSpawn: true + pendingActivationSpawn: true, + recovery: { + attemptedAt: [1], + generation: 1, + outcome: 'pending', + startedAt: 1, + reason: 'reattach-unverifiable', + tabGeneration: 1 + } } as never ] }, @@ -217,6 +225,9 @@ describe('buildWorkspaceSessionPatch', () => { ].sort() ) expect('pendingActivationSpawn' in patch.tabsByWorktree![localWorktreeId][0]).toBe(false) + // Why: the recovery ledger describes a mounted pane's in-flight heal; a + // persisted one would refuse the first legitimate recovery after restart. + expect('recovery' in patch.tabsByWorktree![localWorktreeId][0]).toBe(false) expect(patch.terminalLayoutsByTabId?.['tab-local'].buffersByLeafId).toBeUndefined() expect(patch.terminalLayoutsByTabId?.['tab-local'].scrollbackRefsByLeafId).toBeUndefined() }) diff --git a/src/renderer/src/lib/workspace-session.ts b/src/renderer/src/lib/workspace-session.ts index 330a150b636..affa2b36ca4 100644 --- a/src/renderer/src/lib/workspace-session.ts +++ b/src/renderer/src/lib/workspace-session.ts @@ -204,12 +204,14 @@ export function buildSanitizedTabsByWorktree( tabsByWorktree: WorkspaceSessionSnapshot['tabsByWorktree'] ): WorkspaceSessionState['tabsByWorktree'] { // Why: strip transient pendingActivationSpawn — session:set persists without Zod re-parse, so a stale flag would drop the first PTY spawn on restart. + // Same for the recovery ledger: it describes a mounted pane's in-flight heal, so a persisted one would refuse the first recovery after restart. return Object.fromEntries( Object.entries(tabsByWorktree).map(([worktreeId, tabs]) => [ worktreeId, tabs.map((tab) => { - const { pendingActivationSpawn: _unused, ...rest } = tab + const { pendingActivationSpawn: _unused, recovery: _recovery, ...rest } = tab void _unused + void _recovery return rest }) ]) diff --git a/src/renderer/src/lib/worktree-sleep-intent.ts b/src/renderer/src/lib/worktree-sleep-intent.ts index 7beac240803..6512abec692 100644 --- a/src/renderer/src/lib/worktree-sleep-intent.ts +++ b/src/renderer/src/lib/worktree-sleep-intent.ts @@ -1,13 +1,69 @@ +// Why: a slept workspace keeps its panes mounted with only dead PTYs behind them. +// Any pane connect that runs while the marker is set waits here, and the clear +// that marks the workspace awake resumes every waiting connect. const sleepingWorktreeIds = new Set() +const tearingDownWorktreeIds = new Set() +const wakeListenersByWorktreeId = new Map void>>() export function markWorktreeSleepIntent(worktreeId: string): void { sleepingWorktreeIds.add(worktreeId) } -export function clearWorktreeSleepIntent(worktreeId: string): void { +/** + * Why: a spawn that resolves while the sleep teardown is still awaiting its host + * would bind a PTY and clear the marker, waking every waiting pane mid-sleep. + * Binds during the teardown window are not wakes. + */ +export async function withWorktreeSleepTeardown( + worktreeId: string, + teardown: () => Promise +): Promise { + tearingDownWorktreeIds.add(worktreeId) + try { + return await teardown() + } finally { + tearingDownWorktreeIds.delete(worktreeId) + } +} + +export function clearWorktreeSleepIntent(worktreeId: string | null): void { + if (!worktreeId || tearingDownWorktreeIds.has(worktreeId)) { + return + } + if (!sleepingWorktreeIds.delete(worktreeId)) { + return + } + const listeners = wakeListenersByWorktreeId.get(worktreeId) + wakeListenersByWorktreeId.delete(worktreeId) + for (const listener of listeners ?? []) { + try { + listener() + } catch (error) { + // Why: one pane's connect failure must not strand its siblings or throw out of a store action. + console.error('[sleep-intent] wake listener failed', { worktreeId, error }) + } + } +} + +// Why: a purged worktree must not wake its panes; they are being unmounted. +export function forgetWorktreeSleepIntent(worktreeId: string): void { sleepingWorktreeIds.delete(worktreeId) + tearingDownWorktreeIds.delete(worktreeId) + wakeListenersByWorktreeId.delete(worktreeId) } export function hasWorktreeSleepIntent(worktreeId: string | null): boolean { return worktreeId !== null && sleepingWorktreeIds.has(worktreeId) } + +export function onWorktreeSleepIntentCleared(worktreeId: string, listener: () => void): () => void { + const listeners = wakeListenersByWorktreeId.get(worktreeId) ?? new Set<() => void>() + listeners.add(listener) + wakeListenersByWorktreeId.set(worktreeId, listeners) + return () => { + listeners.delete(listener) + if (listeners.size === 0 && wakeListenersByWorktreeId.get(worktreeId) === listeners) { + wakeListenersByWorktreeId.delete(worktreeId) + } + } +} diff --git a/src/renderer/src/runtime/runtime-host-connection-state.test.ts b/src/renderer/src/runtime/runtime-host-connection-state.test.ts index 8db0c0eb010..30ee06a5e98 100644 --- a/src/renderer/src/runtime/runtime-host-connection-state.test.ts +++ b/src/renderer/src/runtime/runtime-host-connection-state.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from 'vitest' import type { RuntimeStatus } from '../../../shared/runtime-types' import { isConnectedRuntimeHostState, + isDisconnectedRuntimeHostState, runtimeHostConnectionState, + runtimeHostConnectionStateForEntry, runtimeStatusForOverall } from './runtime-host-connection-state' @@ -177,3 +179,72 @@ describe('runtime host connection state', () => { ).toBe('disconnected') }) }) + +describe('runtime host connection state for a recorded status entry', () => { + it('separates a host that was never probed from one a probe found unreachable', () => { + // The sidebar read raw truthiness, which collapsed these two into the same red glyph. + expect(runtimeHostConnectionStateForEntry(undefined)).toBe('checking') + expect(runtimeHostConnectionStateForEntry({ status: null })).toBe('disconnected') + }) + + it('reads the remote-control diagnostics recorded beside a failed probe', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: null, + remoteControl: remoteControl('reconnecting') + }) + ).toBe('reconnecting') + }) + + it('agrees with the status bar that a closed control channel is disconnected', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: makeStatus({ remoteControl: remoteControl('closed') }) + }) + ).toBe('disconnected') + }) + + it('names only the disconnected verdict as disconnected', () => { + expect(isDisconnectedRuntimeHostState('disconnected')).toBe(true) + for (const state of [ + 'connected', + 'checking', + 'reconnecting', + 'runtime-unavailable', + 'workspace-window-closed' + ] as const) { + expect(isDisconnectedRuntimeHostState(state)).toBe(false) + } + }) +}) + +function remoteControl( + state: NonNullable['state'] +): NonNullable { + return { + state, + pendingRequestCount: 0, + subscriptionCount: 0, + reconnectAttempt: 1, + lastConnectedAt: null, + lastClose: null, + lastError: null + } +} + +it('does not report reconnecting after verification is terminally blocked', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: null, + snapshot: { + environmentId: 'browser', + pairingRevision: 1, + sequence: 1, + checkedAt: 1, + status: null, + verification: 'blocked', + transport: 'disconnected' + } + }) + ).toBe('disconnected') +}) diff --git a/src/renderer/src/runtime/runtime-host-connection-state.ts b/src/renderer/src/runtime/runtime-host-connection-state.ts index 6094c995804..c10477415a1 100644 --- a/src/renderer/src/runtime/runtime-host-connection-state.ts +++ b/src/renderer/src/runtime/runtime-host-connection-state.ts @@ -1,3 +1,4 @@ +import type { RuntimeHostStatusSnapshot } from '../../../shared/runtime-host-status' import type { RuntimeStatus } from '../../../shared/runtime-types' import { isRuntimeWorkspaceWindowClosed } from '../../../shared/runtime-workspace-window-availability' @@ -97,3 +98,44 @@ export function isConnectedRuntimeHostState(state: RuntimeHostConnectionState): state === 'connected' || state === 'runtime-unavailable' || state === 'workspace-window-closed' ) } + +/** + * Only this verdict earns the destructive glyph. 'checking' and 'reconnecting' are + * unverifiable, not down, per docs/reference/ssh-execution-boundary.md. + */ +export function isDisconnectedRuntimeHostState(state: RuntimeHostConnectionState): boolean { + return state === 'disconnected' +} + +/** The same derivation, read straight off a recorded status entry. */ +export function runtimeHostConnectionStateForEntry( + entry: + | { + status: RuntimeStatus | null + remoteControl?: RuntimeStatus['remoteControl'] | null + snapshot?: RuntimeHostStatusSnapshot + } + | null + | undefined +): RuntimeHostConnectionState { + if (entry?.snapshot) { + const snapshot = entry.snapshot + if (snapshot.retired || snapshot.verification === 'blocked') { + return 'disconnected' + } + if (snapshot.transport === 'disconnected') { + return 'reconnecting' + } + if (snapshot.verification === 'checking' && !entry.status) { + return 'checking' + } + if (snapshot.transport === 'ready' && snapshot.verification !== 'verified') { + return 'runtime-unavailable' + } + } + return runtimeHostConnectionState({ + hasStatusEntry: Boolean(entry), + status: entry?.status ?? null, + remoteControl: entry?.remoteControl ?? entry?.status?.remoteControl ?? null + }) +} diff --git a/src/renderer/src/runtime/web-session-tabs-sync/mirrored-terminal-recovery-ledger.test.ts b/src/renderer/src/runtime/web-session-tabs-sync/mirrored-terminal-recovery-ledger.test.ts new file mode 100644 index 00000000000..d1992a27030 --- /dev/null +++ b/src/renderer/src/runtime/web-session-tabs-sync/mirrored-terminal-recovery-ledger.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest' +import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types' +import type { TerminalTab, TerminalTabRecoveryLedger } from '../../../../shared/terminal-tab-types' +import { buildMirroredTerminalTabs } from './terminal-build' +import { toWebTerminalSurfaceTabId } from '../web-terminal-surface-id' + +/** + * The recovery ledger is client-local. The host publishes no such field, so a + * rebuild that does not carry the existing one restores this tab's remount + * allowance on EVERY snapshot — which is the remount storm (b5cfc6ca) the + * ledger exists to end, re-armed on the host's publication cadence. + * + * `generation` is deliberately not asserted here: the host carries none and the + * rebuild emits none, which is why `isSupersededLedger` compares strictly + * forward (`>`) rather than `!==`. See terminal-tab-recovery-ledger.ts. + */ +const WORKTREE = 'repo-1::worktree-1' +const ENVIRONMENT = 'env-1' +const HOST_TAB = 'host-tab-1' + +const LEDGER: TerminalTabRecoveryLedger = { + attemptedAt: [1_000], + generation: 1, + outcome: 'failed', + startedAt: 1_000, + reason: 'reattach-unverifiable', + tabGeneration: 1 +} + +function snapshot(): RuntimeMobileSessionTabsResult { + return { + worktree: WORKTREE, + publicationEpoch: 'epoch-1', + snapshotVersion: 1, + activeGroupId: 'group-1', + activeTabId: null, + activeTabType: null, + tabs: [ + { + type: 'terminal', + id: 'surface-1', + parentTabId: HOST_TAB, + leafId: 'leaf-1', + title: 'Terminal', + status: 'ready', + terminal: 'handle-1', + isActive: true + } + ] + } as RuntimeMobileSessionTabsResult +} + +function rebuild(existing?: Partial): TerminalTab { + const localTabId = toWebTerminalSurfaceTabId(HOST_TAB) + const existingById = new Map( + existing ? [[localTabId, { id: localTabId, ...existing } as TerminalTab]] : [] + ) + const [mirrored] = buildMirroredTerminalTabs(snapshot(), ENVIRONMENT, existingById, {}, 0, 1_000) + return mirrored!.tab +} + +describe('buildMirroredTerminalTabs recovery ledger', () => { + it('carries the client-local ledger across a host snapshot rebuild', () => { + expect(rebuild({ recovery: LEDGER }).recovery).toEqual(LEDGER) + }) + + it('emits none for a tab that never recovered', () => { + expect(rebuild({}).recovery).toBeUndefined() + }) + + it('emits none for a tab the client has never seen', () => { + expect(rebuild().recovery).toBeUndefined() + }) +}) diff --git a/src/renderer/src/runtime/web-session-tabs-sync/terminal-build.ts b/src/renderer/src/runtime/web-session-tabs-sync/terminal-build.ts index dfc3759009d..aacb445bbe3 100644 --- a/src/renderer/src/runtime/web-session-tabs-sync/terminal-build.ts +++ b/src/renderer/src/runtime/web-session-tabs-sync/terminal-build.ts @@ -171,6 +171,10 @@ export function buildMirroredTerminalTabs( // without this dropped the client's agent-prompt label on every snapshot. ...(existing?.generatedTitle ? { generatedTitle: existing.generatedTitle } : {}), ...(existing?.aiVaultTitle ? { aiVaultTitle: existing.aiVaultTitle } : {}), + // Why: the recovery ledger is client-local and the host carries none, so + // rebuilding without it would restore this tab's remount allowance on + // every snapshot — the counting loop recovery is meant to end (b5cfc6ca). + ...(existing?.recovery ? { recovery: existing.recovery } : {}), ...(quickCommandLabel ? { quickCommandLabel } : {}), ...(startupCwd ? { startupCwd } : {}), customTitle: existing?.customTitle ?? null, diff --git a/src/renderer/src/store/slices/runtime-status-diagnostics-generation.ts b/src/renderer/src/store/slices/runtime-status-diagnostics-generation.ts deleted file mode 100644 index 22ba23cbbb1..00000000000 --- a/src/renderer/src/store/slices/runtime-status-diagnostics-generation.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types' -import type { RuntimeEnvironmentStatus } from './runtime-status' - -const diagnosticsGenerationByEnvironment = new Map() - -export function updateRuntimeEnvironmentStatusOverlay( - state: Map, - environmentId: string, - status: RuntimeEnvironmentStatus -): Map { - const current = state.get(environmentId) - if (!current || current.status?.runtimeId !== status.status?.runtimeId) { - return state - } - return new Map(state).set(environmentId, status) -} - -export function acceptRuntimeEnvironmentDiagnosticsGeneration( - environmentId: string, - transportGeneration: number -): boolean { - const previous = diagnosticsGenerationByEnvironment.get(environmentId) - if (previous !== undefined && transportGeneration < previous) { - return false - } - diagnosticsGenerationByEnvironment.set(environmentId, transportGeneration) - return true -} - -export function clearRuntimeEnvironmentDiagnosticsGenerationsForTests(): void { - diagnosticsGenerationByEnvironment.clear() -} - -export function mergePushedRuntimeEnvironmentDiagnostics(args: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics - current: RuntimeEnvironmentStatus | undefined - publish: (status: RuntimeEnvironmentStatus) => void -}): void { - if ( - !args.current?.status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) || - !acceptRuntimeEnvironmentDiagnosticsGeneration(args.environmentId, args.transportGeneration) - ) { - return - } - args.publish({ - ...args.current, - status: { ...args.current.status, remoteControl: args.diagnostics } - }) -} diff --git a/src/renderer/src/store/slices/runtime-status-diagnostics-publish.ts b/src/renderer/src/store/slices/runtime-status-diagnostics-publish.ts deleted file mode 100644 index e51c8621bae..00000000000 --- a/src/renderer/src/store/slices/runtime-status-diagnostics-publish.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types' -import type { AppState } from '../types' -import type { RuntimeEnvironmentStatus } from './runtime-status' -import * as diagnosticsGeneration from './runtime-status-diagnostics-generation' -import * as runtimeStatusRecheck from './runtime-status-recheck' - -export function updateRuntimeStatusStore( - state: AppState, - updater: (state: Map) => Map -): AppState | Pick { - const next = updater(state.runtimeStatusByEnvironmentId) - return next === state.runtimeStatusByEnvironmentId - ? state - : { runtimeStatusByEnvironmentId: next } -} - -export function publishRuntimeEnvironmentDiagnostics(args: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics - getCurrent: () => RuntimeEnvironmentStatus | undefined - updateState: (status: RuntimeEnvironmentStatus) => boolean - afterPublish?: (status: RuntimeEnvironmentStatus) => void -}): void { - diagnosticsGeneration.mergePushedRuntimeEnvironmentDiagnostics({ - environmentId: args.environmentId, - transportGeneration: args.transportGeneration, - diagnostics: args.diagnostics, - current: args.getCurrent(), - publish: (status) => { - if (args.updateState(status)) { - args.afterPublish?.(status) - } - } - }) -} - -export function applyRuntimeEnvironmentStatusOverlay(args: { - environmentId: string - status: RuntimeEnvironmentStatus - setState: ( - updater: (state: Map) => Map - ) => void -}): boolean { - let updated = false - args.setState((state) => { - const next = diagnosticsGeneration.updateRuntimeEnvironmentStatusOverlay( - state, - args.environmentId, - args.status - ) - updated = next !== state - return next - }) - return updated -} - -export function createRuntimeEnvironmentDiagnosticsPublisher(args: { - getCurrent: (environmentId: string) => RuntimeEnvironmentStatus | undefined - setState: ( - updater: (state: Map) => Map - ) => void - afterPublish: (environmentId: string, status: RuntimeEnvironmentStatus) => void -}): (event: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics -}) => void { - return (event) => - publishRuntimeEnvironmentDiagnostics({ - ...event, - getCurrent: () => args.getCurrent(event.environmentId), - updateState: (status) => - applyRuntimeEnvironmentStatusOverlay({ - environmentId: event.environmentId, - status, - setState: args.setState - }), - afterPublish: (status) => args.afterPublish(event.environmentId, status) - }) -} - -export function createRuntimeEnvironmentDiagnosticsSlicePublisher(args: { - getCurrent: (environmentId: string) => RuntimeEnvironmentStatus | undefined - setState: ( - updater: (state: Map) => Map - ) => void - getStore: () => AppState - getConnectionGeneration: (environmentId: string) => number -}): (event: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics -}) => void { - return createRuntimeEnvironmentDiagnosticsPublisher({ - getCurrent: args.getCurrent, - setState: args.setState, - afterPublish: (environmentId, status) => - runtimeStatusRecheck.reconcileRuntimeStatusForSlice( - environmentId, - status.status, - args.getStore, - () => args.getConnectionGeneration(environmentId) - ) - }) -} diff --git a/src/renderer/src/store/slices/runtime-status-diagnostics.test.ts b/src/renderer/src/store/slices/runtime-status-diagnostics.test.ts deleted file mode 100644 index 9faecdcabf1..00000000000 --- a/src/renderer/src/store/slices/runtime-status-diagnostics.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { create } from 'zustand' -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { RuntimeStatus } from '../../../../shared/runtime-types' -import { createRuntimeStatusSlice, type RuntimeStatusSlice } from './runtime-status' - -function makeStatus(overrides: Partial = {}): RuntimeStatus { - return { - runtimeId: 'runtime-a', - rendererGraphEpoch: 0, - graphStatus: 'ready', - authoritativeWindowId: null, - liveTabCount: 3, - liveLeafCount: 0, - runtimeProtocolVersion: 3, - minCompatibleRuntimeClientVersion: 3, - capabilities: ['browser.screencast.v1'], - ...overrides - } as RuntimeStatus -} - -function createSliceStore() { - return create()((...a) => ({ - ...createRuntimeStatusSlice(...(a as unknown as Parameters)) - })) -} - -describe('runtime-status diagnostics', () => { - it('merges transport diagnostics into the complete status and fences stale pushes', () => { - const store = createSliceStore() - const status = makeStatus({ - capabilities: ['browser.screencast.v1', REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] - }) - store.getState().setRuntimeEnvironmentStatus('env-a', { status, checkedAt: 1 }) - const closed = { - state: 'closed' as const, - pendingRequestCount: 0, - subscriptionCount: 1, - reconnectAttempt: 2, - lastConnectedAt: 1, - lastClose: { code: 1006, reason: 'network' }, - lastError: 'connection lost' - } - store.getState().publishRuntimeEnvironmentDiagnostics({ - environmentId: 'env-a', - transportGeneration: 3, - diagnostics: closed - }) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toMatchObject({ - runtimeId: 'runtime-a', - capabilities: expect.arrayContaining([ - 'browser.screencast.v1', - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY - ]), - liveTabCount: 3, - remoteControl: closed - }) - store.getState().publishRuntimeEnvironmentDiagnostics({ - environmentId: 'env-a', - transportGeneration: 2, - diagnostics: { ...closed, state: 'ready' } - }) - expect( - store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl?.state - ).toBe('closed') - }) - - it('ignores diagnostics after the latest status drops shared-control support', () => { - const store = createSliceStore() - const status = makeStatus({ capabilities: [] }) - store.getState().setRuntimeEnvironmentStatus('env-a', { status, checkedAt: 1 }) - - store.getState().publishRuntimeEnvironmentDiagnostics({ - environmentId: 'env-a', - transportGeneration: 3, - diagnostics: { - state: 'reconnecting', - pendingRequestCount: 0, - subscriptionCount: 1, - reconnectAttempt: 2, - lastConnectedAt: 1, - lastClose: { code: 1006, reason: 'network' }, - lastError: 'connection lost' - } - }) - - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(status) - }) -}) diff --git a/src/renderer/src/store/slices/runtime-status-recheck.test.ts b/src/renderer/src/store/slices/runtime-status-recheck.test.ts deleted file mode 100644 index 9f09580c390..00000000000 --- a/src/renderer/src/store/slices/runtime-status-recheck.test.ts +++ /dev/null @@ -1,238 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { create } from 'zustand' -import { toast } from 'sonner' -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments' -import type { RuntimeStatus } from '../../../../shared/runtime-types' -import { - clearRuntimeEnvironmentConnectionGenerationsForTests, - createRuntimeStatusSlice, - setRuntimeEnvironmentConnectionGenerationForTests, - type RuntimeStatusSlice -} from './runtime-status' -import { clearRuntimeStatusRechecksForTests } from './runtime-status-recheck' - -vi.mock('sonner', () => ({ toast: { warning: vi.fn(), dismiss: vi.fn() } })) - -beforeEach(() => { - vi.useFakeTimers() - clearRuntimeStatusRechecksForTests() - clearRuntimeEnvironmentConnectionGenerationsForTests() - vi.mocked(toast.warning).mockReset() -}) - -afterEach(() => { - clearRuntimeStatusRechecksForTests() - vi.useRealTimers() - vi.unstubAllGlobals() -}) - -describe('runtime status recheck', () => { - it('publishes an observe-only ready result through the setter', async () => { - const getStatus = vi.fn().mockResolvedValue(response(status('ready'))) - const store = createStore(getStatus) - - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - await vi.advanceTimersByTimeAsync(3_000) - - expect(getStatus).toHaveBeenCalledWith({ - selector: 'env-a', - timeoutMs: 10_000, - observeOnly: true - }) - expect( - store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl - ).toMatchObject({ - state: 'ready' - }) - await vi.advanceTimersByTimeAsync(120_000) - expect(getStatus).toHaveBeenCalledOnce() - }) - - it('continues indefinitely on the capped ladder, including unchanged publishes', async () => { - const getStatus = vi.fn().mockResolvedValue(response(status('reconnecting'))) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('reconnecting'), - checkedAt: 1 - }) - - await vi.advanceTimersByTimeAsync(3_000 + 6_000 + 12_000 + 30_000 + 60_000 + 60_000) - - expect(getStatus).toHaveBeenCalledTimes(6) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.checkedAt).toBe(1) - }) - - it('cancels on removal, capability loss, and null without probing again', async () => { - const getStatus = vi.fn() - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_authenticated'), - checkedAt: 1 - }) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: { ...status('awaiting_authenticated'), capabilities: [] }, - checkedAt: 2 - }) - await vi.advanceTimersByTimeAsync(60_000) - expect(getStatus).not.toHaveBeenCalled() - - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 3 - }) - store.getState().setRuntimeEnvironments([]) - await vi.advanceTimersByTimeAsync(60_000) - expect(getStatus).not.toHaveBeenCalled() - }) - - it('cancels an armed probe when the connection generation changes', async () => { - const getStatus = vi.fn() - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - - setRuntimeEnvironmentConnectionGenerationForTests('env-a', 2) - await vi.advanceTimersByTimeAsync(3_000) - - expect(getStatus).not.toHaveBeenCalled() - }) - - it('restarts the ladder for a newly published connection generation', async () => { - const getStatus = vi.fn().mockResolvedValue(response(status('ready', 'rt-next'))) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready', 'rt-next'), - checkedAt: 2 - }) - - await vi.advanceTimersByTimeAsync(3_000) - - expect(getStatus).toHaveBeenCalledOnce() - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.runtimeId).toBe( - 'rt-next' - ) - }) - - it('discards an in-flight result after a ready publish bumps the epoch', async () => { - const pending = deferred>() - const getStatus = vi.fn().mockReturnValue(pending.promise) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - await vi.advanceTimersByTimeAsync(3_000) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('ready'), - checkedAt: 2 - }) - - pending.resolve(response(status('reconnecting'))) - await Promise.resolve() - await Promise.resolve() - - expect( - store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl - ).toMatchObject({ - state: 'ready' - }) - }) - - it('keeps setter side effects when a recheck discovers disconnection', async () => { - const getStatus = vi.fn().mockResolvedValue({ - id: 'status.get', - ok: false, - error: { - code: 'runtime_unavailable', - message: 'offline', - data: { remoteControl: status('reconnecting').remoteControl } - }, - _meta: { runtimeId: 'rt' } - }) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - - await vi.advanceTimersByTimeAsync(3_000) - - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBeNull() - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.remoteControl).toMatchObject( - { - state: 'reconnecting' - } - ) - expect(toast.warning).toHaveBeenCalledOnce() - }) -}) - -function createStore(getStatus: ReturnType) { - vi.stubGlobal('window', { - api: { runtimeEnvironments: { getStatus, list: vi.fn() } } - }) - const store = create()((...args) => ({ - ...createRuntimeStatusSlice(...(args as unknown as Parameters)) - })) - store.getState().setRuntimeEnvironments([environment()]) - return store -} - -function status( - controlState: NonNullable['state'], - runtimeId = 'rt' -): RuntimeStatus { - return { - runtimeId, - rendererGraphEpoch: 1, - graphStatus: 'ready', - authoritativeWindowId: null, - liveTabCount: 0, - liveLeafCount: 0, - capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY], - remoteControl: { - state: controlState, - pendingRequestCount: 0, - subscriptionCount: 0, - reconnectAttempt: 1, - lastConnectedAt: null, - lastClose: null, - lastError: null - } - } as RuntimeStatus -} - -function response(result: RuntimeStatus) { - return { id: 'status.get', ok: true as const, result, _meta: { runtimeId: result.runtimeId } } -} - -function environment(): PublicKnownRuntimeEnvironment { - return { - id: 'env-a', - name: 'Dev Box', - createdAt: 1, - updatedAt: 1, - lastUsedAt: null, - runtimeId: 'rt', - endpoints: [{ id: 'ws', kind: 'websocket', label: 'WebSocket', endpoint: 'ws://x' }], - preferredEndpointId: 'ws' - } -} - -function deferred() { - let resolve: (value: T) => void = () => {} - const promise = new Promise((done) => { - resolve = done - }) - return { promise, resolve } -} diff --git a/src/renderer/src/store/slices/runtime-status-recheck.ts b/src/renderer/src/store/slices/runtime-status-recheck.ts deleted file mode 100644 index 303e5b4318d..00000000000 --- a/src/renderer/src/store/slices/runtime-status-recheck.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { RuntimeStatus } from '../../../../shared/runtime-types' -import { unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client' -import { extractRuntimeTransportDiagnostics } from '@/runtime/runtime-status-probe-diagnostics' -import type { RuntimeEnvironmentStatus } from './runtime-status' - -const RECHECK_DELAYS_MS = [3_000, 6_000, 12_000, 30_000, 60_000] - -type RecheckState = { - epoch: number - attempt: number - timer: ReturnType | null - inFlight: boolean - connectionGeneration: number - environmentExists: () => boolean - getConnectionGeneration: () => number - publish: (status: RuntimeEnvironmentStatus) => void -} - -type RuntimeStatusStore = { - runtimeEnvironments: readonly { id: string }[] - setRuntimeEnvironmentStatus: (environmentId: string, status: RuntimeEnvironmentStatus) => void -} - -const rechecks = new Map() - -export function reconcileRuntimeStatusRecheck(args: { - environmentId: string - status: RuntimeStatus | null - connectionGeneration: number - environmentExists: () => boolean - getConnectionGeneration: () => number - publish: (status: RuntimeEnvironmentStatus) => void -}): void { - if (!shouldRecheck(args.status)) { - cancelRuntimeStatusRecheck(args.environmentId) - return - } - let state = rechecks.get(args.environmentId) - if (state && state.connectionGeneration !== args.connectionGeneration) { - cancelRuntimeStatusRecheck(args.environmentId) - state = undefined - } - if (!state) { - state = { - epoch: 0, - attempt: 0, - timer: null, - inFlight: false, - connectionGeneration: args.connectionGeneration, - environmentExists: args.environmentExists, - getConnectionGeneration: args.getConnectionGeneration, - publish: args.publish - } - rechecks.set(args.environmentId, state) - } else { - state.connectionGeneration = args.connectionGeneration - state.environmentExists = args.environmentExists - state.getConnectionGeneration = args.getConnectionGeneration - state.publish = args.publish - } - armRuntimeStatusRecheck(args.environmentId, state) -} - -export function reconcileRuntimeStatusForSlice( - environmentId: string, - status: RuntimeStatus | null, - get: () => RuntimeStatusStore, - getConnectionGeneration: () => number -): void { - reconcileRuntimeStatusRecheck({ - environmentId, - status, - connectionGeneration: getConnectionGeneration(), - environmentExists: () => - get().runtimeEnvironments.some((environment) => environment.id === environmentId), - getConnectionGeneration, - publish: (nextStatus) => get().setRuntimeEnvironmentStatus(environmentId, nextStatus) - }) -} - -export function cancelRuntimeStatusRecheck(environmentId: string): void { - const state = rechecks.get(environmentId) - if (!state) { - return - } - state.epoch += 1 - if (state.timer) { - clearTimeout(state.timer) - } - rechecks.delete(environmentId) -} - -export function cancelRuntimeStatusRechecks(environmentIds: Iterable): void { - for (const environmentId of environmentIds) { - cancelRuntimeStatusRecheck(environmentId) - } -} - -export function clearRuntimeStatusRechecksForTests(): void { - cancelRuntimeStatusRechecks([...rechecks.keys()]) -} - -function shouldRecheck(status: RuntimeStatus | null): boolean { - return Boolean( - status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) && - status.remoteControl && - status.remoteControl.state !== 'ready' - ) -} - -function armRuntimeStatusRecheck(environmentId: string, state: RecheckState): void { - if (state.timer || state.inFlight) { - return - } - const delay = RECHECK_DELAYS_MS[Math.min(state.attempt, RECHECK_DELAYS_MS.length - 1)] - const generation = state.connectionGeneration - state.attempt += 1 - state.timer = setTimeout( - () => void fireRuntimeStatusRecheck(environmentId, state, generation), - delay - ) -} - -async function fireRuntimeStatusRecheck( - environmentId: string, - state: RecheckState, - generation: number -): Promise { - state.timer = null - const epoch = state.epoch - if ( - rechecks.get(environmentId) !== state || - !state.environmentExists() || - state.getConnectionGeneration() !== generation - ) { - cancelRuntimeStatusRecheck(environmentId) - return - } - state.inFlight = true - let nextEntry: RuntimeEnvironmentStatus - try { - const response = await window.api.runtimeEnvironments.getStatus({ - selector: environmentId, - timeoutMs: 10_000, - observeOnly: true - }) - nextEntry = { status: unwrapRuntimeRpcResult(response), checkedAt: Date.now() } - } catch (error: unknown) { - const remoteControl = extractRuntimeTransportDiagnostics(error) - nextEntry = { - status: null, - ...(remoteControl ? { remoteControl } : {}), - checkedAt: Date.now() - } - } - state.inFlight = false - if ( - rechecks.get(environmentId) !== state || - state.epoch !== epoch || - !state.environmentExists() || - state.getConnectionGeneration() !== generation - ) { - return - } - state.publish(nextEntry) -} diff --git a/src/renderer/src/store/slices/runtime-status-refresh.ts b/src/renderer/src/store/slices/runtime-status-refresh.ts index 638a3fae636..fb27eec6a43 100644 --- a/src/renderer/src/store/slices/runtime-status-refresh.ts +++ b/src/renderer/src/store/slices/runtime-status-refresh.ts @@ -15,6 +15,22 @@ export async function refreshRuntimeEnvironmentStatus( selector: environmentId, timeoutMs }) + if (window.api.runtimeEnvironments.getStatusSnapshots) { + try { + const snapshots = await window.api.runtimeEnvironments.getStatusSnapshots() + const snapshot = snapshots.find((entry) => entry.environmentId === environmentId) + if (snapshot) { + publish({ + snapshot, + status: snapshot.verification === 'verified' ? snapshot.status : null, + checkedAt: snapshot.checkedAt + }) + } + } catch (error) { + console.error('Failed to read runtime host status snapshot:', error) + } + return response.ok + } const status = unwrapRuntimeRpcResult(response) if (getRuntimeEnvironmentRevision(environmentId) !== expectedEnvironmentRevision) { return false diff --git a/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts b/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts index 0ba4d645026..d4b788cd49a 100644 --- a/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts +++ b/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts @@ -73,14 +73,10 @@ describe('restored client-hosted browser host attach on reachability', () => { }) }) - // The reconnect policy suppresses the *failure* publish only. A probe that answered still owes - // both recovery follow-ups, or a restored client-hosted page never comes back after the gap. - it('runs both recovery follow-ups on a success when the caller opted out of publishing failures', async () => { + it('runs both recovery follow-ups after a successful refresh', async () => { stubApi(vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a'))) - await storeWithRestoredHandles(true) - .getState() - .refreshRuntimeEnvironmentStatus('env-a', undefined, { publishUnreachable: false }) + await storeWithRestoredHandles(true).getState().refreshRuntimeEnvironmentStatus('env-a') expect(prepareBrowserClientHostPlacement).toHaveBeenCalledWith({ selector: 'env-a', @@ -89,24 +85,12 @@ describe('restored client-hosted browser host attach on reachability', () => { expect(replayClientHostedBrowserCloseIntents).toHaveBeenCalledWith('env-a', expect.anything()) }) - // Under either policy a failed probe owes *no* follow-ups: it verified nothing, so there is no - // recovered host to reattach restored pages to and no one to replay closes at. - it.each([ - { name: 'the default policy', options: undefined }, - { name: 'a caller that opted out of publishing', options: { publishUnreachable: false } } - ])( - 'starts no browser client host when the environment is unreachable: $name', - async (scenario) => { - stubApi(vi.fn().mockRejectedValue(new Error('unreachable'))) - - await storeWithRestoredHandles(true) - .getState() - .refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options) - - expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() - expect(replayClientHostedBrowserCloseIntents).not.toHaveBeenCalled() - } - ) + it('runs no recovery follow-ups when the environment is unreachable', async () => { + stubApi(vi.fn().mockRejectedValue(new Error('unreachable'))) + await storeWithRestoredHandles(true).getState().refreshRuntimeEnvironmentStatus('env-a') + expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() + expect(replayClientHostedBrowserCloseIntents).not.toHaveBeenCalled() + }) it('starts no browser client host for restored pages the server hosts', async () => { stubApi(vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a'))) diff --git a/src/renderer/src/store/slices/runtime-status-snapshot.test.ts b/src/renderer/src/store/slices/runtime-status-snapshot.test.ts new file mode 100644 index 00000000000..fa437065b28 --- /dev/null +++ b/src/renderer/src/store/slices/runtime-status-snapshot.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, expect, it, vi } from 'vitest' +import { create } from 'zustand' +import { toast } from 'sonner' +import { + createRuntimeStatusSlice, + clearRuntimeEnvironmentConnectionGenerationsForTests, + type RuntimeStatusSlice +} from './runtime-status' +import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status' +import type { RuntimeStatus } from '../../../../shared/runtime-types' +import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments' +import { runtimeHostConnectionStateForEntry } from '@/runtime/runtime-host-connection-state' + +vi.mock('sonner', () => ({ toast: { warning: vi.fn(), dismiss: vi.fn() } })) +vi.mock('@/runtime/restored-client-hosted-browser-host-attach', () => ({ + ensureBrowserClientHostsForRestoredPages: vi.fn(), + ensureBrowserClientHostForRestartedRuntime: vi.fn() +})) +vi.mock('@/runtime/client-hosted-browser-close-intent-replay', () => ({ + replayClientHostedBrowserCloseIntents: vi.fn() +})) + +beforeEach(() => { + clearRuntimeEnvironmentConnectionGenerationsForTests() + vi.clearAllMocks() +}) +const environment = { + id: 'env-a', + name: 'Host', + createdAt: 1, + pairingRevision: 1, + endpoints: [], + preferredEndpointId: '' +} as unknown as PublicKnownRuntimeEnvironment +function store() { + const value = create()((...args) => + createRuntimeStatusSlice(...(args as unknown as Parameters)) + ) + value.getState().setRuntimeEnvironments([environment]) + return value +} +function snapshot( + sequence: number, + patch: Partial = {} +): RuntimeHostStatusSnapshot { + return { + environmentId: 'env-a', + pairingRevision: 1, + sequence, + checkedAt: sequence, + transport: 'ready', + verification: 'verified', + status: { runtimeId: 'rt-1' } as RuntimeStatus, + ...patch + } +} + +it('hydrates both viewers and rejects an older read after a newer publication', () => { + for (const viewer of [store(), store()]) { + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2)) + viewer + .getState() + .applyRuntimeHostStatusSnapshot(snapshot(1, { status: null, verification: 'unavailable' })) + expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.runtimeId).toBe( + 'rt-1' + ) + } +}) + +it('represents failed verification honestly without manufacturing a session restart or toast', () => { + const viewer = store() + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(1)) + const generation = viewer + .getState() + .runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2, { verification: 'unavailable' })) + expect( + runtimeHostConnectionStateForEntry(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')) + ).toBe('runtime-unavailable') + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(3)) + expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration).toBe( + generation + ) + expect(toast.warning).not.toHaveBeenCalled() + viewer + .getState() + .applyRuntimeHostStatusSnapshot(snapshot(4, { status: { runtimeId: 'rt-2' } as RuntimeStatus })) + expect( + viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration + ).toBeGreaterThan(generation ?? 0) +}) + +it('retains disconnect ordering and rejects publications for removed or replaced pairings', () => { + const viewer = store() + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(1)) + viewer + .getState() + .applyRuntimeHostStatusSnapshot( + snapshot(3, { retired: true, verification: 'blocked', transport: 'disconnected' }) + ) + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2)) + expect( + runtimeHostConnectionStateForEntry(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')) + ).toBe('disconnected') + viewer.getState().setRuntimeEnvironments([{ ...environment, pairingRevision: 2 }]) + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(4)) + expect(viewer.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false) + viewer.getState().setRuntimeEnvironments([]) + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(5, { pairingRevision: 2 })) + expect(viewer.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false) +}) diff --git a/src/renderer/src/store/slices/runtime-status-snapshot.ts b/src/renderer/src/store/slices/runtime-status-snapshot.ts new file mode 100644 index 00000000000..b3543471d29 --- /dev/null +++ b/src/renderer/src/store/slices/runtime-status-snapshot.ts @@ -0,0 +1,43 @@ +import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status' +import type { AppState } from '../types' +import type { RuntimeEnvironmentStatus } from './runtime-status-types' +import { ensureBrowserClientHostsForRestoredPages } from '@/runtime/restored-client-hosted-browser-host-attach' +import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay' + +export function applyRuntimeHostStatusSnapshot( + snapshot: RuntimeHostStatusSnapshot, + state: AppState, + publishEvidence: (entry: RuntimeEnvironmentStatus) => void +): void { + const environment = state.runtimeEnvironments.find((entry) => entry.id === snapshot.environmentId) + if ( + !environment || + (environment.pairingRevision ?? environment.createdAt) !== snapshot.pairingRevision + ) { + return + } + const previous = state.runtimeStatusByEnvironmentId.get(snapshot.environmentId) + if (previous?.snapshot && previous.snapshot.sequence >= snapshot.sequence) { + return + } + const entry: RuntimeEnvironmentStatus = { + snapshot, + checkedAt: snapshot.checkedAt, + connectionGeneration: previous?.connectionGeneration, + status: snapshot.verification === 'verified' && !snapshot.retired ? snapshot.status : null, + remoteControl: snapshot.remoteControl + } + if (entry.status) { + if (snapshot.remoteControl) { + entry.status = { ...entry.status, remoteControl: snapshot.remoteControl } + } + state.setRuntimeEnvironmentStatus(snapshot.environmentId, entry) + if (previous?.status == null) { + void ensureBrowserClientHostsForRestoredPages(state) + void replayClientHostedBrowserCloseIntents(snapshot.environmentId, state) + } + } else { + // Lost contact or a failed method observes no runtime session ending. + publishEvidence(entry) + } +} diff --git a/src/renderer/src/store/slices/runtime-status-types.ts b/src/renderer/src/store/slices/runtime-status-types.ts index c34feaf07bf..78e47123cf5 100644 --- a/src/renderer/src/store/slices/runtime-status-types.ts +++ b/src/renderer/src/store/slices/runtime-status-types.ts @@ -1,8 +1,9 @@ +import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status' import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments' import type { RuntimeStatus } from '../../../../shared/runtime-types' -import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types' export type RuntimeEnvironmentStatus = { + snapshot?: RuntimeHostStatusSnapshot status: RuntimeStatus | null remoteControl?: RuntimeStatus['remoteControl'] | null appVersion?: string | null @@ -10,11 +11,9 @@ export type RuntimeEnvironmentStatus = { connectionGeneration?: number } -export type RuntimeStatusRefreshOptions = { - publishUnreachable?: boolean -} - export type RuntimeStatusSlice = { + readRuntimeHostStatusSnapshots: () => Promise + applyRuntimeHostStatusSnapshot: (snapshot: RuntimeHostStatusSnapshot) => void runtimeEnvironments: readonly PublicKnownRuntimeEnvironment[] runtimeEnvironmentCatalogHydrated: boolean runtimeEnvironmentCatalogSettled: boolean @@ -26,17 +25,8 @@ export type RuntimeStatusSlice = { status: RuntimeEnvironmentStatus, options?: { suppressDisconnectToast?: boolean } ) => void - publishRuntimeEnvironmentDiagnostics: (args: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics - }) => void clearRuntimeEnvironmentStatus: (environmentId: string) => void retainRuntimeEnvironmentStatuses: (environmentIds: Iterable) => void - refreshRuntimeEnvironmentStatus: ( - environmentId: string, - timeoutMs?: number, - options?: RuntimeStatusRefreshOptions - ) => Promise + refreshRuntimeEnvironmentStatus: (environmentId: string, timeoutMs?: number) => Promise hydrateRuntimeEnvironmentStatuses: () => Promise } diff --git a/src/renderer/src/store/slices/runtime-status.test.ts b/src/renderer/src/store/slices/runtime-status.test.ts index 8e3260957bf..fdc843c748a 100644 --- a/src/renderer/src/store/slices/runtime-status.test.ts +++ b/src/renderer/src/store/slices/runtime-status.test.ts @@ -710,33 +710,37 @@ describe('runtime-status slice', () => { clearRuntimeCompatibilityCacheForTests() }) - // Both directions of the failure-publication policy, from one failing probe. A user-initiated - // check publishes the outage it just observed; a caller holding live transport evidence must - // not, because status.get dials its own socket and its failure is unverifiable, not exited. - it.each([ - { name: 'a user-initiated check', options: undefined, publishes: true }, - { name: 'publishUnreachable defaulted', options: {}, publishes: true }, - { - name: 'a caller that opted out of publishing', - options: { publishUnreachable: false }, - publishes: false - } - ])('records null and returns false when a runtime refresh fails: $name', async (scenario) => { + it('records null and returns false when a runtime refresh fails', async () => { const getStatus = vi.fn().mockRejectedValue(new Error('closed')) stubRuntimeEnvironmentApi({ getStatus }) const store = createSliceStore() const cached = makeStatus() store.getState().setRuntimeEnvironmentStatus('env-a', { status: cached, checkedAt: 1 }) - const reachable = await store - .getState() - .refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options) + const reachable = await store.getState().refreshRuntimeEnvironmentStatus('env-a') - // The dial-answered contract the bridge's bounded retry chain reads is policy-independent. expect(reachable).toBe(false) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe( - scenario.publishes ? null : cached - ) + expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(null) + }) + + it('preserves successful reachability when reading its snapshot fails', async () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.stubGlobal('window', { + api: { + runtimeEnvironments: { + getStatus: vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a')), + getStatusSnapshots: vi.fn().mockRejectedValue(new Error('IPC read failed')) + } + } + }) + try { + const store = createSliceStore() + expect(await store.getState().refreshRuntimeEnvironmentStatus('env-a')).toBe(true) + expect(store.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false) + expect(log).toHaveBeenCalled() + } finally { + log.mockRestore() + } }) it('hydrates saved environments through the single-environment refresh path', async () => { diff --git a/src/renderer/src/store/slices/runtime-status.ts b/src/renderer/src/store/slices/runtime-status.ts index 10985f389d8..c11fba3308a 100644 --- a/src/renderer/src/store/slices/runtime-status.ts +++ b/src/renderer/src/store/slices/runtime-status.ts @@ -1,11 +1,7 @@ import type { StateCreator } from 'zustand' import type { AppState } from '../types' import type { RuntimeStatusSlice } from './runtime-status-types' -export type { - RuntimeEnvironmentStatus, - RuntimeStatusRefreshOptions, - RuntimeStatusSlice -} from './runtime-status-types' +export type { RuntimeEnvironmentStatus, RuntimeStatusSlice } from './runtime-status-types' import { runtimeEnvironmentStatusesEqual } from './runtime-environment-status-equality' import { clearRecentRuntimeCompatibilityFailure, @@ -20,21 +16,16 @@ import { import { reconcileCatalogRows } from './repo-identity-reconcile' import { createRuntimeStatusHydration } from './runtime-status-hydration' import { refreshRuntimeEnvironmentStatus } from './runtime-status-refresh' -import * as runtimeStatusDiagnostics from './runtime-status-diagnostics-generation' import * as runtimeStatusConnectionGeneration from './runtime-status-connection-generation' import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay' import { ensureBrowserClientHostForRestartedRuntime, ensureBrowserClientHostsForRestoredPages } from '@/runtime/restored-client-hosted-browser-host-attach' -import * as runtimeStatusRecheck from './runtime-status-recheck' -import * as runtimeStatusDiagnosticsPublish from './runtime-status-diagnostics-publish' +import { applyRuntimeHostStatusSnapshot } from './runtime-status-snapshot' export const clearRuntimeEnvironmentConnectionGenerationsForTests = (): void => { - runtimeStatusRecheck.cancelRuntimeStatusRechecks( - runtimeStatusConnectionGeneration.clearRuntimeEnvironmentConnectionGenerations() - ) - runtimeStatusDiagnostics.clearRuntimeEnvironmentDiagnosticsGenerationsForTests() + runtimeStatusConnectionGeneration.clearRuntimeEnvironmentConnectionGenerations() } export { @@ -52,6 +43,15 @@ export const createRuntimeStatusSlice: StateCreator { + try { + const snapshots = await window.api.runtimeEnvironments.getStatusSnapshots() + snapshots.forEach((snapshot) => get().applyRuntimeHostStatusSnapshot(snapshot)) + } catch (error) { + console.error('Failed to read runtime host status:', error) + } + }, + setRuntimeEnvironments: (environments) => { const previousRevisionById = new Map( get().runtimeEnvironments.map((environment) => [ @@ -76,7 +76,6 @@ export const createRuntimeStatusSlice: StateCreator environment.id) .filter((id) => !nextIds.has(id)) - runtimeStatusRecheck.cancelRuntimeStatusRechecks([...removedIds, ...replacedEnvironmentIds]) set((s) => { const keep = new Set(environments.map((environment) => environment.id)) const nextStatuses = new Map(s.runtimeStatusByEnvironmentId) @@ -155,15 +154,29 @@ export const createRuntimeStatusSlice: StateCreator + applyRuntimeHostStatusSnapshot(snapshot, get(), (entry) => { + set((s) => ({ + runtimeStatusByEnvironmentId: new Map(s.runtimeStatusByEnvironmentId).set( + snapshot.environmentId, + entry + ) + })) + }), + setRuntimeEnvironmentStatus: (environmentId, status, options) => { const previous = get().runtimeStatusByEnvironmentId.get(environmentId) + if (previous?.snapshot && !status.snapshot) { + return + } + const previousVerifiedStatus = previous?.snapshot?.status ?? previous?.status const pairedDeviceId = status.status?.pairedDeviceId?.trim() // A new runtime id under a known previous one is a restart, not a first connect: the guests are // still ours to host, but only a fresh attach hands them back to the replacement runtime. const runtimeRestarted = Boolean( status.status !== null && - previous?.status != null && - previous.status.runtimeId !== status.status.runtimeId + previousVerifiedStatus != null && + previousVerifiedStatus.runtimeId !== status.status.runtimeId ) // Why: a non-null status proves the runtime just answered, so drop any stale // "offline" compat failure before this online transition fires the @@ -177,7 +190,8 @@ export const createRuntimeStatusSlice: StateCreator - get().runtimeEnvironments.some((environment) => environment.id === environmentId), - getConnectionGeneration: () => - runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration(environmentId), - publish: (entry) => get().setRuntimeEnvironmentStatus(environmentId, entry) - }) if (runtimeRestarted) { void ensureBrowserClientHostForRestartedRuntime(get(), environmentId) } @@ -250,18 +253,7 @@ export const createRuntimeStatusSlice: StateCreator get().runtimeStatusByEnvironmentId.get(environmentId), - setState: (updater) => - set((s) => runtimeStatusDiagnosticsPublish.updateRuntimeStatusStore(s, updater)), - getStore: get, - getConnectionGeneration: - runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration - }), - clearRuntimeEnvironmentStatus: (environmentId) => { - runtimeStatusRecheck.cancelRuntimeStatusRecheck(environmentId) dismissRuntimeDisconnectedToast(environmentId) set((s) => { runtimeStatusConnectionGeneration.advanceRuntimeEnvironmentConnectionGeneration(environmentId) @@ -278,7 +270,6 @@ export const createRuntimeStatusSlice: StateCreator + refreshRuntimeEnvironmentStatus: (environmentId, timeoutMs = 10_000) => refreshRuntimeEnvironmentStatus(environmentId, timeoutMs, (entry) => { - if (entry.status === null && options?.publishUnreachable === false) { - // Unverifiable, not exited: leave the cached verdict for the caller's retry to settle. + if (entry.snapshot) { + get().applyRuntimeHostStatusSnapshot(entry.snapshot) return } // Why: setRuntimeEnvironmentStatus drops any stale compat failure on a non-null diff --git a/src/renderer/src/store/slices/tab-view-mode.test.ts b/src/renderer/src/store/slices/tab-view-mode.test.ts index aa892756b23..56b99c13ba4 100644 --- a/src/renderer/src/store/slices/tab-view-mode.test.ts +++ b/src/renderer/src/store/slices/tab-view-mode.test.ts @@ -81,4 +81,41 @@ describe('tab view mode', () => { store.getState().toggleTabViewMode('missing-tab') expect(store.getState().unifiedTabsByWorktree[WT]).toBe(before) }) + + // Why: terminal-pane recovery asks the terminal row who owns the surface. + // Host sync already writes viewMode there; only these local toggles skipped + // it, which is why the guard had to OR two indices to get a safe answer. + describe('mirrors onto the terminal row', () => { + function terminalRow(tabId: string) { + return store.getState().tabsByWorktree[WT]?.find((tab) => tab.id === tabId) + } + + beforeEach(() => { + const tabId = store.getState().createTab(WT).id + store.setState({ + unifiedTabsByWorktree: { + [WT]: [ + ...store.getState().unifiedTabsByWorktree[WT].filter((tab) => tab.id !== tabId), + makeUnifiedTab({ id: tabId, entityId: tabId, worktreeId: WT, groupId: 'g-left' }) + ] + } + } as Partial) + rowTabId = tabId + }) + + let rowTabId = '' + + it('toggleTabViewMode patches the row in the same write', () => { + store.getState().toggleTabViewMode(rowTabId) + expect(terminalRow(rowTabId)?.viewMode).toBe('chat') + + store.getState().toggleTabViewMode(rowTabId) + expect(terminalRow(rowTabId)?.viewMode).toBe('terminal') + }) + + it('setTabViewMode patches the row in the same write', () => { + store.getState().setTabViewMode(rowTabId, 'chat') + expect(terminalRow(rowTabId)?.viewMode).toBe('chat') + }) + }) }) diff --git a/src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts b/src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts index e87a34f81c2..66f09743e37 100644 --- a/src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts +++ b/src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts @@ -2,23 +2,27 @@ import type { AppState } from '../../types' import type { TerminalTab } from '../../../../../shared/terminal-tab-types' import { findTabAndWorktree } from '../tab-group-state' import { getRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner' +import { locateTerminalTab } from '../../terminals/terminal-tab-location' -export function patchTerminalTabPinned( +/** + * Mirror a host-tracked unified-tab field onto its terminal row, in whichever + * bucket actually holds the row. Reconcile derives these fields from the + * TerminalTab, so a local toggle that only patched the unified tab would be + * recomputed away by the next host snapshot — and recovery's chat-ownership + * guard reads the row, so a lagging row lets a hidden chat surface remount. + */ +export function patchTerminalTabRow( tabsByWorktree: Record, - worktreeId: string, tabId: string, - isPinned: boolean + patch: Partial> ): Partial> { - const tabs = tabsByWorktree[worktreeId] - if (!tabs?.some((tab) => tab.id === tabId)) { + const location = locateTerminalTab(tabsByWorktree, tabId) + if (!location) { return {} } - return { - tabsByWorktree: { - ...tabsByWorktree, - [worktreeId]: tabs.map((tab) => (tab.id === tabId ? { ...tab, isPinned } : tab)) - } - } + const nextTabs = tabsByWorktree[location.worktreeId].slice() + nextTabs[location.index] = { ...location.tab, ...patch } + return { tabsByWorktree: { ...tabsByWorktree, [location.worktreeId]: nextTabs } } } // Why: pin is host-authoritative for remote-server tabs, so mirror it (like setTabColor) or it's lost on reconnect/other clients. diff --git a/src/renderer/src/store/slices/tabs/tabs-label-actions.ts b/src/renderer/src/store/slices/tabs/tabs-label-actions.ts index 8acb925bfae..42b8e3d7163 100644 --- a/src/renderer/src/store/slices/tabs/tabs-label-actions.ts +++ b/src/renderer/src/store/slices/tabs/tabs-label-actions.ts @@ -6,7 +6,7 @@ import { applyTabOrderSortValues, partitionPinnedTabOrder } from './tabs-tab-ord import { mirrorTabPinnedToHost, mirrorTabViewModeToHost, - patchTerminalTabPinned + patchTerminalTabRow } from './tabs-host-mirroring' export function createTabsLabelActions( @@ -62,7 +62,13 @@ export function createTabsLabelActions( }, setTabViewMode: (tabId, mode) => { - set((state) => patchTab(state.unifiedTabsByWorktree, tabId, { viewMode: mode }) ?? {}) + set((state) => ({ + ...patchTab(state.unifiedTabsByWorktree, tabId, { viewMode: mode }), + // Why the row too: viewMode is declared on both types and host-sync + // already writes it to the row. Only these local toggles skipped it, so + // readers had to OR the two indices to find out who owns the surface. + ...patchTerminalTabRow(state.tabsByWorktree, tabId, { viewMode: mode }) + })) mirrorTabViewModeToHost(get(), tabId, mode) }, @@ -86,7 +92,10 @@ export function createTabsLabelActions( (terminal) => terminal.id === found.tab.entityId )?.launchAgent ?? null toggled = { from: fromMode, to: nextMode, agent } - return patchTab(state.unifiedTabsByWorktree, tabId, { viewMode: nextMode }) ?? {} + return { + ...patchTab(state.unifiedTabsByWorktree, tabId, { viewMode: nextMode }), + ...patchTerminalTabRow(state.tabsByWorktree, tabId, { viewMode: nextMode }) + } }) // Why: emit after the state write so the event reflects the committed mode. const committed = toggled as { @@ -141,7 +150,7 @@ export function createTabsLabelActions( [worktreeId]: applyTabOrderSortValues(tabs, tabOrder) }, // Why: reconcile derives pin from the TerminalTab, so mirror it there too or a host snapshot recomputes isPinned:false and un-pins during the echo window. - ...patchTerminalTabPinned(state.tabsByWorktree, worktreeId, tabId, true), + ...patchTerminalTabRow(state.tabsByWorktree, tabId, { isPinned: true }), groupsByWorktree: { ...state.groupsByWorktree, [worktreeId]: updateGroup(groups, { ...group, tabOrder }) @@ -178,7 +187,7 @@ export function createTabsLabelActions( ...state.unifiedTabsByWorktree, [worktreeId]: applyTabOrderSortValues(tabs, tabOrder) }, - ...patchTerminalTabPinned(state.tabsByWorktree, worktreeId, tabId, false), + ...patchTerminalTabRow(state.tabsByWorktree, tabId, { isPinned: false }), groupsByWorktree: { ...state.groupsByWorktree, [worktreeId]: updateGroup(groups, { ...group, tabOrder }) diff --git a/src/renderer/src/store/slices/terminal-tab-recovery-remount.test.ts b/src/renderer/src/store/slices/terminal-tab-recovery-remount.test.ts index f4b28129251..b1db16ae413 100644 --- a/src/renderer/src/store/slices/terminal-tab-recovery-remount.test.ts +++ b/src/renderer/src/store/slices/terminal-tab-recovery-remount.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' import { createTestStore, makeWorktree, seedStore } from './store-test-helpers' +import { isTerminalTabPresent } from './terminal-tab-retirement' +import { FLOATING_TERMINAL_WORKTREE_ID } from '../../../../shared/constants' +import { folderWorkspaceKey } from '../../../../shared/workspace-scope' const WORKTREE_ID = 'repo1::/path/wt1' @@ -20,7 +23,7 @@ describe('remountTerminalTabForRecovery', () => { const remounted = store.getState().remountTerminalTabForRecovery(tabId) - expect(remounted).toBe(true) + expect(remounted.remounted).toBe(true) const after = store.getState().tabsByWorktree[WORKTREE_ID].find((tab) => tab.id === tabId) expect(after?.generation ?? 0).toBe((before?.generation ?? 0) + 1) // Recovery is not user interaction — the remount's PTY updates must not @@ -35,7 +38,7 @@ describe('remountTerminalTabForRecovery', () => { store.getState().queueTabStartupCommand(tabId, startup) const before = store.getState().pendingStartupByTabId[tabId] - expect(store.getState().remountTerminalTabForRecovery(tabId)).toBe(true) + expect(store.getState().remountTerminalTabForRecovery(tabId).remounted).toBe(true) const after = store.getState().pendingStartupByTabId[tabId] expect(after).toEqual(before) @@ -58,6 +61,119 @@ describe('remountTerminalTabForRecovery', () => { const store = createTestStore() seedWorktreeWithTab(store) - expect(store.getState().remountTerminalTabForRecovery('missing-tab')).toBe(false) + expect(store.getState().remountTerminalTabForRecovery('missing-tab')).toEqual({ + remounted: false, + declinedBy: 'tab-missing' + }) + }) +}) + +// Crash b5cfc6ca: recovery released its per-tab remount budget from getTab, which +// reads unifiedTabsByWorktree. That index can drop a tab this one still holds, and +// the release then erased the budget each remount had just consumed. +describe('isTerminalTabPresent as the recovery existence check', () => { + it('answers true for a tab remountTerminalTabForRecovery can still remount', () => { + const store = createTestStore() + const tabId = seedWorktreeWithTab(store) + + expect(isTerminalTabPresent(store.getState(), tabId)).toBe(true) + expect(store.getState().remountTerminalTabForRecovery(tabId).remounted).toBe(true) + }) + + it('stays true when the tab is missing from the unified tab index', () => { + const store = createTestStore() + const tabId = seedWorktreeWithTab(store) + store.setState({ unifiedTabsByWorktree: {} }) + + expect(store.getState().getTab(tabId)).toBeNull() + expect(isTerminalTabPresent(store.getState(), tabId)).toBe(true) + }) + + it('answers false once the tab leaves the remount index', () => { + const store = createTestStore() + const tabId = seedWorktreeWithTab(store) + store.setState({ tabsByWorktree: { [WORKTREE_ID]: [] } }) + + expect(isTerminalTabPresent(store.getState(), tabId)).toBe(false) + expect(store.getState().remountTerminalTabForRecovery(tabId).remounted).toBe(false) + }) + + // The budget release still has to fire for a real close, or a closed tab's + // timestamps and pending retry outlive it. + it('answers false after a genuine closeTab', () => { + const store = createTestStore() + const tabId = seedWorktreeWithTab(store) + + store.getState().closeTab(tabId) + + expect(isTerminalTabPresent(store.getState(), tabId)).toBe(false) + }) +}) + +// Not every workspace is a repository checkout. The ledger is keyed to the tab +// ROW and resolved through locateTerminalTab, which scans every bucket in +// tabsByWorktree — so a folder workspace and the floating-terminal bucket must +// behave identically without a single branch for them. The predecessor kept the +// budget in a module map keyed by tabId, and its row patcher made the caller +// name the bucket, which is where a non-worktree key could go wrong. +describe.each([ + ['a repository worktree', WORKTREE_ID], + ['a folder workspace', folderWorkspaceKey('fw-1')], + ['the floating terminal bucket', FLOATING_TERMINAL_WORKTREE_ID] +])('the recovery ledger on %s', (_label, bucketId) => { + function seedBucket(store: ReturnType): string { + seedStore(store, { + worktreesByRepo: { + repo1: [makeWorktree({ id: WORKTREE_ID, repoId: 'repo1', path: '/path/wt1' })] + } + }) + return store.getState().createTab(bucketId).id + } + + const AUTOMATIC = { reason: 'write-stalled', trigger: 'automatic', now: 0 } as const + + it('admits, observes and then refuses the same reason until a new trigger', () => { + const store = createTestStore() + const tabId = seedBucket(store) + const row = (): { recovery?: unknown } | undefined => + store.getState().tabsByWorktree[bucketId]?.find((tab) => tab.id === tabId) + + const first = store.getState().remountTerminalTabForRecovery(tabId, AUTOMATIC) + expect(first.remounted).toBe(true) + // The ledger landed on the row in this bucket, not in a worktree-keyed map. + expect(row()?.recovery).toMatchObject({ outcome: 'pending', reason: 'write-stalled' }) + + // Unsettled blocks the next automatic ask, even past the cooldown. + expect( + store.getState().remountTerminalTabForRecovery(tabId, { ...AUTOMATIC, now: 16_000 }) + ).toEqual({ remounted: false, declinedBy: 'unsettled', retryInMs: 15_000 }) + + if (!first.remounted) { + throw new Error('unreachable: the first remount was admitted') + } + store.getState().settleTerminalTabRecovery(tabId, first.generation, 'failed') + expect(row()?.recovery).toMatchObject({ outcome: 'failed' }) + expect( + store.getState().remountTerminalTabForRecovery(tabId, { ...AUTOMATIC, now: 600_000 }) + ).toEqual({ remounted: false, declinedBy: 'settled-failure' }) + + // The user asking again is the new trigger the refusal waits for. + expect( + store + .getState() + .remountTerminalTabForRecovery(tabId, { ...AUTOMATIC, trigger: 'user', now: 600_000 }) + .remounted + ).toBe(true) + }) + + it('drops the ledger with the row when the tab closes', () => { + const store = createTestStore() + const tabId = seedBucket(store) + store.getState().remountTerminalTabForRecovery(tabId, AUTOMATIC) + + store.getState().closeTab(tabId) + + expect(isTerminalTabPresent(store.getState(), tabId)).toBe(false) + expect(store.getState().tabsByWorktree[bucketId]?.some((tab) => tab.id === tabId)).toBeFalsy() }) }) diff --git a/src/renderer/src/store/slices/terminal-tab-retirement.ts b/src/renderer/src/store/slices/terminal-tab-retirement.ts index b57ac4946fc..80e67824699 100644 --- a/src/renderer/src/store/slices/terminal-tab-retirement.ts +++ b/src/renderer/src/store/slices/terminal-tab-retirement.ts @@ -9,6 +9,7 @@ import { resolveTerminalHostOwnership } from '@/lib/terminal-worktree-route' import { FLOATING_TERMINAL_WORKTREE_ID } from '../../../../shared/constants' import { isEphemeralSetupTerminalWorktreeId } from '../../../../shared/ephemeral-setup-terminal-worktree-id' import { parseWorkspaceKey } from '../../../../shared/workspace-scope' +import { locateTerminalTab } from '../terminals/terminal-tab-location' export type TerminalTabCloseReason = 'user' | 'cleanup' | 'pty-exit' @@ -131,7 +132,7 @@ export function isTerminalTabPresent( state: Pick, tabId: string ): boolean { - return Object.values(state.tabsByWorktree).some((tabs) => tabs.some((tab) => tab.id === tabId)) + return locateTerminalTab(state.tabsByWorktree, tabId) !== null } export function buildTerminalTabRetirementPlan( diff --git a/src/renderer/src/store/slices/worktree-helpers.ts b/src/renderer/src/store/slices/worktree-helpers.ts index 3fa3219fa32..d6d9e9de74f 100644 --- a/src/renderer/src/store/slices/worktree-helpers.ts +++ b/src/renderer/src/store/slices/worktree-helpers.ts @@ -25,6 +25,11 @@ import type { import type { WorktreeRemovalTarget } from '../../../../shared/worktree/removal' import type { TerminalGitHubPRLink } from '../../../../shared/terminal-github-pr-link-detector' import type { ExecutionHostId } from '../../../../shared/execution-host' +import type { TerminalPaneRecoveryOutcome } from '../../../../shared/terminal-tab-types' +import type { + TerminalRecoveryRemountRequest, + TerminalRecoveryRemountResult +} from '../terminals/terminal-tab-recovery-ledger' import type { RemoveWorktreeOptions } from './worktree-removal-options' import type { HostQualifiedDetectedWorktreeResult, @@ -310,9 +315,24 @@ export type WorktreeSlice = { * TerminalPane unmounts, detaches (preserving a live PTY), and remounts with * a fresh xterm that reattaches and replays. Used by terminal-pane-recovery * when a pane's write pipeline is certified dead or its input is - * undeliverable while the PTY is alive. Returns false when the tab is gone. + * undeliverable while the PTY is alive. + * + * The generation bump and the tab's recovery ledger are written together, so + * the budget cannot outlive — or be released independently of — the row it + * belongs to. Omitting the request marks an external lifecycle remount: it + * skips admission and writes no ledger. */ - remountTerminalTabForRecovery: (tabId: string) => boolean + remountTerminalTabForRecovery: ( + tabId: string, + request?: TerminalRecoveryRemountRequest + ) => TerminalRecoveryRemountResult + /** Record what a mounted pane observed for its recovery attempt. Ignored + * unless `generation` is the row's current, still-pending ledger epoch. */ + settleTerminalTabRecovery: ( + tabId: string, + generation: number, + outcome: Exclude + ) => void setActiveFolderWorkspace: (folderWorkspaceId: string, executionHostId?: ExecutionHostId) => void setRenamingWorktreeId: (request: string | WorktreeRenameRequest | null) => void allWorktrees: () => Worktree[] diff --git a/src/renderer/src/store/slices/worktree-sleep-intent-lifecycle.test.ts b/src/renderer/src/store/slices/worktree-sleep-intent-lifecycle.test.ts new file mode 100644 index 00000000000..8b3c73e2ff1 --- /dev/null +++ b/src/renderer/src/store/slices/worktree-sleep-intent-lifecycle.test.ts @@ -0,0 +1,167 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as intent from '@/lib/worktree-sleep-intent' +import { buildWorktreePurgeState } from './worktrees/teardown/worktree-purge-state' +import { createTestStore, makeWorktree, seedStore } from './store-test-helpers' +import { createStoreCascadesMockApi } from './store-cascades-test-harness' + +const { clearWorktreeSleepIntent, hasWorktreeSleepIntent, markWorktreeSleepIntent } = intent +const WORKTREE_ID = 'repo1::/path/wt1' +const FOLDER_KEY = 'folder:folder-1' + +createStoreCascadesMockApi() + +function seedWorktree(store: ReturnType): void { + seedStore(store, { + worktreesByRepo: { + repo1: [makeWorktree({ id: WORKTREE_ID, repoId: 'repo1', path: '/path/wt1' })] + }, + refreshGitHubForWorktree: vi.fn(), + refreshGitHubForWorktreeIfStale: vi.fn() + }) +} + +// Why this suite exists: the sleep marker outlives teardown so mounted panes stay cold +// (#10205). Every route that makes a workspace awake again must release it, or the +// workspace is stuck cold and its PTY exits stop counting as activity. +describe('worktree sleep intent lifecycle', () => { + beforeEach(() => { + clearWorktreeSleepIntent(WORKTREE_ID) + clearWorktreeSleepIntent(FOLDER_KEY) + }) + + it('is released by activating the worktree', () => { + const store = createTestStore() + seedWorktree(store) + markWorktreeSleepIntent(WORKTREE_ID) + + store.getState().setActiveWorktree(WORKTREE_ID) + + expect(hasWorktreeSleepIntent(WORKTREE_ID)).toBe(false) + }) + + it('survives the sleep flow clearing the active selection', () => { + const store = createTestStore() + seedWorktree(store) + markWorktreeSleepIntent(WORKTREE_ID) + + store.getState().setActiveWorktree(null) + + expect(hasWorktreeSleepIntent(WORKTREE_ID)).toBe(true) + }) + + it('is released by activating a folder workspace', () => { + const store = createTestStore() + store.setState({ + folderWorkspaces: [ + { + id: 'folder-1', + projectGroupId: 'group-1', + name: 'Folder', + folderPath: '/folder', + executionHostId: 'local', + linkedTask: null, + comment: '', + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 0, + lastActivityAt: 0, + createdAt: 1, + updatedAt: 1 + } + ] + }) + markWorktreeSleepIntent(FOLDER_KEY) + + store.getState().setActiveFolderWorkspace('folder-1') + + expect(hasWorktreeSleepIntent(FOLDER_KEY)).toBe(false) + }) + + it('is released when any PTY binds to a tab in the worktree', () => { + const store = createTestStore() + seedWorktree(store) + const tab = store.getState().createTab(WORKTREE_ID, undefined, undefined, { activate: false }) + markWorktreeSleepIntent(WORKTREE_ID) + + store.getState().updateTabPtyId(tab.id, 'pty-cli-created') + + expect(hasWorktreeSleepIntent(WORKTREE_ID)).toBe(false) + }) + + it('is released when a tab is created with a live PTY', () => { + const store = createTestStore() + seedWorktree(store) + markWorktreeSleepIntent(WORKTREE_ID) + + store.getState().createTab(WORKTREE_ID, undefined, undefined, { + activate: false, + initialPtyId: 'pty-cli-created' + }) + + expect(hasWorktreeSleepIntent(WORKTREE_ID)).toBe(false) + }) + + it('notifies wake listeners once and only on a real clear', () => { + const { onWorktreeSleepIntentCleared } = intent + const woke = vi.fn() + markWorktreeSleepIntent(WORKTREE_ID) + const unsubscribe = onWorktreeSleepIntentCleared(WORKTREE_ID, woke) + + clearWorktreeSleepIntent('repo1::/path/other') + expect(woke).not.toHaveBeenCalled() + clearWorktreeSleepIntent(WORKTREE_ID) + clearWorktreeSleepIntent(WORKTREE_ID) + expect(woke).toHaveBeenCalledTimes(1) + + markWorktreeSleepIntent(WORKTREE_ID) + clearWorktreeSleepIntent(WORKTREE_ID) + expect(woke).toHaveBeenCalledTimes(1) + unsubscribe() + }) + + it('ignores a PTY bind that lands while the sleep teardown is in flight', async () => { + const store = createTestStore() + seedWorktree(store) + const tab = store.getState().createTab(WORKTREE_ID, undefined, undefined, { activate: false }) + markWorktreeSleepIntent(WORKTREE_ID) + const woke = vi.fn() + intent.onWorktreeSleepIntentCleared(WORKTREE_ID, woke) + + await intent.withWorktreeSleepTeardown(WORKTREE_ID, async () => { + store.getState().updateTabPtyId(tab.id, 'pty-late-spawn') + }) + + expect(hasWorktreeSleepIntent(WORKTREE_ID)).toBe(true) + expect(woke).not.toHaveBeenCalled() + store.getState().updateTabPtyId(tab.id, 'pty-after-teardown') + expect(hasWorktreeSleepIntent(WORKTREE_ID)).toBe(false) + }) + + it('keeps notifying siblings when one wake listener throws', () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + const woke = vi.fn() + markWorktreeSleepIntent(WORKTREE_ID) + intent.onWorktreeSleepIntentCleared(WORKTREE_ID, () => { + throw new Error('boom') + }) + intent.onWorktreeSleepIntentCleared(WORKTREE_ID, woke) + + expect(() => clearWorktreeSleepIntent(WORKTREE_ID)).not.toThrow() + expect(woke).toHaveBeenCalledTimes(1) + errorSpy.mockRestore() + }) + + it('is forgotten without waking panes when the worktree is purged', () => { + const store = createTestStore() + seedWorktree(store) + markWorktreeSleepIntent(WORKTREE_ID) + const woke = vi.fn() + intent.onWorktreeSleepIntentCleared(WORKTREE_ID, woke) + + store.setState(buildWorktreePurgeState(store.getState(), [WORKTREE_ID])) + + expect(hasWorktreeSleepIntent(WORKTREE_ID)).toBe(false) + expect(woke).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/store/slices/worktrees.ts b/src/renderer/src/store/slices/worktrees.ts index 606a5a26857..330d27e7bb2 100644 --- a/src/renderer/src/store/slices/worktrees.ts +++ b/src/renderer/src/store/slices/worktrees.ts @@ -52,6 +52,7 @@ import { createGetKnownWorktreeById, createPurgeWorktreeTerminalState, createRemountTerminalTabForRecovery, + createSettleTerminalTabRecovery, createSetRenamingWorktreeId } from './worktrees/session/worktree-slice-lookups' import { createPurgeStaleRuntimeHostState } from './worktrees/teardown/purge-stale-runtime-host-state' @@ -108,6 +109,7 @@ export const createWorktreeSlice: StateCreator seedActiveWorktreeLastVisitedIfMissing: createSeedActiveWorktreeLastVisitedIfMissing(set, get), setRenamingWorktreeId: createSetRenamingWorktreeId(set, get), remountTerminalTabForRecovery: createRemountTerminalTabForRecovery(set, get), + settleTerminalTabRecovery: createSettleTerminalTabRecovery(set, get), setActiveWorktree: createSetActiveWorktree(set, get), setActiveFolderWorkspace: createSetActiveFolderWorkspace(set, get), allWorktrees: createAllWorktrees(set, get), diff --git a/src/renderer/src/store/slices/worktrees/session/set-active-folder-workspace.ts b/src/renderer/src/store/slices/worktrees/session/set-active-folder-workspace.ts index d2b5af79114..b89da1ace44 100644 --- a/src/renderer/src/store/slices/worktrees/session/set-active-folder-workspace.ts +++ b/src/renderer/src/store/slices/worktrees/session/set-active-folder-workspace.ts @@ -9,6 +9,7 @@ import { } from '../listing/detected-worktree-meta' import { shouldDeferActivationTerminalPrep } from './activation-terminal-prep' import { deriveActiveSurfaceForWorktree } from '../../tabs/tabs-surface' +import { clearWorktreeSleepIntent } from '@/lib/worktree-sleep-intent' export function createSetActiveFolderWorkspace( set: WorktreeSliceSet, @@ -62,6 +63,8 @@ export function createSetActiveFolderWorkspace( : s.folderWorkspaces } }) + // Why: cleared after the set() so a waiting pane connects against the activated state. + clearWorktreeSleepIntent(workspaceKey) if (workspace.isUnread) { void get().updateFolderWorkspace( folderWorkspaceId, diff --git a/src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts b/src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts index b4ec0b0f99a..31c7e8bbb37 100644 --- a/src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts +++ b/src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts @@ -24,6 +24,7 @@ import { } from '../listing/detected-worktree-meta' import { persistPassiveWorktreeMetaForOwner } from '../listing/worktree-owner-settings' import { resolveActivatedWorktreeSurface } from './active-worktree-surface' +import { clearWorktreeSleepIntent } from '@/lib/worktree-sleep-intent' import { pendingActivationTerminalPrepCancels, shouldDeferActivationTerminalPrep @@ -206,6 +207,11 @@ export function createSetActiveWorktree( } }) + // Why: any activation is an explicit wake (null is the sleep flow clearing selection). + // Cleared after the set() above so a pane still waiting on the marker connects once, + // in the remounted generation, instead of connecting and then being remounted. + clearWorktreeSleepIntent(worktreeId) + if (worktreeId && shouldPrepareTerminalTabs) { const prepareTerminalTabs = (): void => { pendingActivationTerminalPrepCancels.delete(worktreeId) diff --git a/src/renderer/src/store/slices/worktrees/session/worktree-slice-lookups.ts b/src/renderer/src/store/slices/worktrees/session/worktree-slice-lookups.ts index c6fd3143b17..78e0e397bca 100644 --- a/src/renderer/src/store/slices/worktrees/session/worktree-slice-lookups.ts +++ b/src/renderer/src/store/slices/worktrees/session/worktree-slice-lookups.ts @@ -4,6 +4,16 @@ import { FLOATING_TERMINAL_WORKTREE_ID } from '../../../../../../shared/constant import { getTerminalActivationSpawnSuppression } from '../../terminal-activation-spawn-suppression' import { findKnownWorktreeById } from '../listing/detected-worktree-meta' import { buildWorktreePurgeState } from '../teardown/worktree-purge-state' +import { locateTerminalTab } from '../../../terminals/terminal-tab-location' +import { + admitTerminalRecoveryRemount, + nextTerminalRecoveryLedger, + settledTerminalRecoveryLedger +} from '../../../terminals/terminal-tab-recovery-ledger' +import type { + TerminalRecoveryRemountRequest, + TerminalRecoveryRemountResult +} from '../../../terminals/terminal-tab-recovery-ledger' export function createSetRenamingWorktreeId( set: WorktreeSliceSet, @@ -20,47 +30,97 @@ export function createRemountTerminalTabForRecovery( set: WorktreeSliceSet, _get: WorktreeSliceGet ): WorktreeSlice['remountTerminalTabForRecovery'] { - return (tabId) => { - let remounted = false + return (tabId, request) => { + const remountRequest: TerminalRecoveryRemountRequest = request ?? { + // The lifetime bridge's host-hydration remount is an external trigger: it + // is not a heal attempt, so it neither consumes nor consults the ledger. + reason: 'reattach-unverifiable', + trigger: 'external', + now: Date.now() + } + let result: TerminalRecoveryRemountResult = { + remounted: false, + declinedBy: 'tab-missing' + } set((s) => { - for (const [worktreeId, tabs] of Object.entries(s.tabsByWorktree)) { - const index = tabs.findIndex((tab) => tab.id === tabId) - if (index === -1) { - continue - } - const tab = tabs[index] - const nextTabs = tabs.slice() - const pendingStartup = s.pendingStartupByTabId[tabId] - nextTabs[index] = { - ...tab, - // Why: bump generation to remount a pane whose renderer died while its PTY stayed alive, so it reattaches, not spawns. - generation: (tab.generation ?? 0) + 1, - // Why: recovery isn't a user interaction — suppress its PTY updates from reshuffling Recent, like activation remounts. - pendingActivationSpawn: getTerminalActivationSpawnSuppression( - s.terminalLayoutsByTabId[tab.id] - ) - } - remounted = true - return { - tabsByWorktree: { - ...s.tabsByWorktree, - [worktreeId]: nextTabs - }, - ...(pendingStartup - ? { - // Why: a remounted pane must own a distinct one-shot startup record so a stale - // pane cannot consume the successor's command during teardown. - pendingStartupByTabId: { - ...s.pendingStartupByTabId, - [tabId]: { ...pendingStartup } - } - } - : {}) + const location = locateTerminalTab(s.tabsByWorktree, tabId) + // Why re-admit inside the write: the caller's read happened before an + // async liveness probe, and a concurrent detector may have consumed the + // budget across it. Locating the row and spending its budget is one step. + const admission = admitTerminalRecoveryRemount(location?.tab, remountRequest) + if (!location || !admission.admitted) { + if (admission.admitted) { + result = { remounted: false, declinedBy: 'tab-missing' } + } else { + const { admitted: _admitted, ...decline } = admission + result = { remounted: false, ...decline } } + return {} + } + const { worktreeId, index, tab } = location + const nextTabs = s.tabsByWorktree[worktreeId].slice() + const pendingStartup = s.pendingStartupByTabId[tabId] + // Why: bump generation to remount a pane whose renderer died while its PTY stayed alive, so it reattaches, not spawns. + const nextTabGeneration = (tab.generation ?? 0) + 1 + // An external remount is not a heal attempt, so it writes no ledger. The + // generation bump alone supersedes any ledger already on the row, which + // is exactly right: an external remount IS a new trigger. + const recovery = + remountRequest.trigger === 'external' + ? tab.recovery + : nextTerminalRecoveryLedger(tab, remountRequest, nextTabGeneration) + nextTabs[index] = { + ...tab, + generation: nextTabGeneration, + // Why: recovery isn't a user interaction — suppress its PTY updates from reshuffling Recent, like activation remounts. + pendingActivationSpawn: getTerminalActivationSpawnSuppression( + s.terminalLayoutsByTabId[tab.id] + ), + // The remount and the budget it spends are one write, so no disposal, + // release path or index drift can undo half of it (crash b5cfc6ca). + ...(recovery ? { recovery } : {}) + } + result = { remounted: true, generation: recovery?.generation ?? 0 } + return { + tabsByWorktree: { + ...s.tabsByWorktree, + [worktreeId]: nextTabs + }, + ...(pendingStartup + ? { + // Why: a remounted pane must own a distinct one-shot startup record so a stale + // pane cannot consume the successor's command during teardown. + pendingStartupByTabId: { + ...s.pendingStartupByTabId, + [tabId]: { ...pendingStartup } + } + } + : {}) } - return {} }) - return remounted + return result + } +} + +export function createSettleTerminalTabRecovery( + set: WorktreeSliceSet, + _get: WorktreeSliceGet +): WorktreeSlice['settleTerminalTabRecovery'] { + return (tabId, generation, outcome) => { + set((s) => { + const location = locateTerminalTab(s.tabsByWorktree, tabId) + if (!location) { + return {} + } + const { worktreeId, index, tab } = location + const recovery = settledTerminalRecoveryLedger(tab, generation, outcome) + if (!recovery) { + return {} + } + const nextTabs = s.tabsByWorktree[worktreeId].slice() + nextTabs[index] = { ...tab, recovery } + return { tabsByWorktree: { ...s.tabsByWorktree, [worktreeId]: nextTabs } } + }) } } diff --git a/src/renderer/src/store/slices/worktrees/teardown/remove-worktree.ts b/src/renderer/src/store/slices/worktrees/teardown/remove-worktree.ts index 075172ddce1..a6a9c87d838 100644 --- a/src/renderer/src/store/slices/worktrees/teardown/remove-worktree.ts +++ b/src/renderer/src/store/slices/worktrees/teardown/remove-worktree.ts @@ -6,6 +6,7 @@ import { parseExecutionHostId } from '../../../../../../shared/execution-host' import { ensureHooksConfirmed } from '@/lib/ensure-hooks-confirmed' import { getActiveRuntimeTarget } from '../../../../runtime/runtime-rpc-client' import { forgetHugeRepoWarningDismissalsForWorktrees } from '@/lib/source-control-huge-repo-warning-dismissals' +import { forgetWorktreeSleepIntent } from '@/lib/worktree-sleep-intent' import { showPreservedBranchToast } from '@/components/sidebar/preserved-branch-toast' import { resolveWorktreeOperationRouteResult, @@ -222,6 +223,7 @@ export function createRemoveWorktree( // Why: invalidate stale probes once deletion is authoritative, so an old toast can't mutate a same-path replacement. forgetHugeRepoWarningDismissalsForWorktrees([worktreeId]) + forgetWorktreeSleepIntent(worktreeId) // Why: forget-local is legal while the host is unreachable, so record the removal here too — otherwise an // in-flight metadata read that snapshotted this row re-appends it, and disconnected polls never drop it. if (hostId && parseExecutionHostId(hostId)?.kind === 'ssh') { diff --git a/src/renderer/src/store/slices/worktrees/teardown/worktree-purge-state.ts b/src/renderer/src/store/slices/worktrees/teardown/worktree-purge-state.ts index 78a57da7955..e2f6e5945fe 100644 --- a/src/renderer/src/store/slices/worktrees/teardown/worktree-purge-state.ts +++ b/src/renderer/src/store/slices/worktrees/teardown/worktree-purge-state.ts @@ -8,6 +8,7 @@ import { createWorktreePurgeOmitters } from './worktree-purge-omitters' import { removeDeleteStatesForWorktreeIds } from './worktree-delete-state' import { removeWorktreeVisitEntriesForTargets } from '@/lib/worktree-visit-recency' import { forgetAmbiguousOwnerWarnings } from '../listing/worktree-owner-settings' +import { forgetWorktreeSleepIntent } from '@/lib/worktree-sleep-intent' export function buildWorktreePurgeState( s: AppState, @@ -18,6 +19,10 @@ export function buildWorktreePurgeState( ) const worktreeIdSet = new Set(normalizedTargets.map((target) => target.id)) pruneHostedReviewLinkMutationGenerations(worktreeIdSet) + // Why: ids are repo::path, so a worktree recreated at the same path must not inherit a stale sleep. + for (const id of worktreeIdSet) { + forgetWorktreeSleepIntent(id) + } // Why: every authoritative and explicit purge converges here, so a deleted path can't inherit stale UI state. forgetHugeRepoWarningDismissalsForWorktrees(worktreeIdSet) forgetAmbiguousOwnerWarnings(worktreeIdSet) diff --git a/src/renderer/src/store/terminals/terminal-pty-bindings.ts b/src/renderer/src/store/terminals/terminal-pty-bindings.ts index 2e0397aff45..499ea63e8de 100644 --- a/src/renderer/src/store/terminals/terminal-pty-bindings.ts +++ b/src/renderer/src/store/terminals/terminal-pty-bindings.ts @@ -9,6 +9,7 @@ import { isRemoteRuntimePtyId } from './terminal-pty-identities' import { omitUnverifiedPtyLossTabIds } from './terminal-unverified-pty-loss' +import { clearWorktreeSleepIntent } from '@/lib/worktree-sleep-intent' import { omitDisownedPtyIds } from './terminal-disowned-pty-sources' export function createTerminalPtyBindingActions( @@ -275,6 +276,8 @@ export function createTerminalPtyBindingActions( ...(shouldBumpSortEpoch ? { sortEpoch: s.sortEpoch + 1 } : {}) } }) + // Why: a bound PTY means the workspace is awake by any route (CLI, automation, client wake), not only activation. + clearWorktreeSleepIntent(worktreeId) // Why: activation spawns come from clicking a worktree, not work in it — skip the lastActivityAt stamp and sortEpoch bump; other spawn reasons still bump. if (worktreeId && !wasActivationSpawn && !isRemoteRuntimeMirror) { get().bumpWorktreeActivity(worktreeId) diff --git a/src/renderer/src/store/terminals/terminal-tab-creation.ts b/src/renderer/src/store/terminals/terminal-tab-creation.ts index 11f9d1d2a59..83310850475 100644 --- a/src/renderer/src/store/terminals/terminal-tab-creation.ts +++ b/src/renderer/src/store/terminals/terminal-tab-creation.ts @@ -1,3 +1,4 @@ +import { clearWorktreeSleepIntent } from '@/lib/worktree-sleep-intent' import type { TerminalTab } from '../../../../shared/terminal-tab-types' import { isValidHostTerminalTabId } from '../../../../shared/terminal-tab-id' import { emptyLayoutSnapshot, singlePaneLayoutSnapshot } from '../slices/terminal-helpers' @@ -271,6 +272,10 @@ export function createTerminalTabCreationActions( } } }) + if (options?.initialPtyId) { + // Why: a tab born with a live PTY (CLI/runtime create) wakes the workspace like any other bind. + clearWorktreeSleepIntent(worktreeId) + } const shouldRecordInteraction = options?.recordInteraction ?? (!options?.pendingActivationSpawn && !options?.initialPtyId) if (shouldRecordInteraction) { diff --git a/src/renderer/src/store/terminals/terminal-tab-location.ts b/src/renderer/src/store/terminals/terminal-tab-location.ts new file mode 100644 index 00000000000..893d52f3ef0 --- /dev/null +++ b/src/renderer/src/store/terminals/terminal-tab-location.ts @@ -0,0 +1,20 @@ +import type { TerminalTab } from '../../../../shared/terminal-tab-types' + +/** + * The one scan over `tabsByWorktree`. Recovery's remount, its budget release + * and its native-chat guard must all resolve a tab through this: answering + * from a different index (getTab's `unifiedTabsByWorktree`) made every remount + * erase the budget it had just consumed, and the cap never held (crash b5cfc6ca). + */ +export function locateTerminalTab( + tabsByWorktree: Readonly>, + tabId: string +): { worktreeId: string; index: number; tab: TerminalTab } | null { + for (const [worktreeId, tabs] of Object.entries(tabsByWorktree)) { + const index = tabs.findIndex((candidate) => candidate.id === tabId) + if (index !== -1) { + return { worktreeId, index, tab: tabs[index] } + } + } + return null +} diff --git a/src/renderer/src/store/terminals/terminal-tab-recovery-ledger.ts b/src/renderer/src/store/terminals/terminal-tab-recovery-ledger.ts new file mode 100644 index 00000000000..b06b6de5d30 --- /dev/null +++ b/src/renderer/src/store/terminals/terminal-tab-recovery-ledger.ts @@ -0,0 +1,218 @@ +import { DIRECT_SSH_PANE_RETRY_SETTLEMENT_TIMEOUT_MS } from '@/components/terminal-pane/pty-connection/pty-connect-limits' +import type { + TerminalPaneRecoveryOutcome, + TerminalPaneRecoveryReason, + TerminalTab, + TerminalTabRecoveryLedger +} from '../../../../shared/terminal-tab-types' + +// Why this module exists: recovery's budget used to live in module-level Maps +// keyed by tabId. Anything keyed outside the row needs a release path, and the +// release fired on every remount-driven pane disposal — so each remount erased +// the budget it had just consumed and the cap never held (crash b5cfc6ca). +// The ledger now lives on the row, so "the budget released itself" has no +// expression: reading the budget IS reading the tab. +// +// The control is not the count. A remount that mounts a pane which fails the +// same way is not evidence that anything changed, so recovery gates on an +// OBSERVED outcome, borrowing the direct-SSH pane retry vocabulary +// (DirectSshPaneRetryResult): an attempt that has not settled blocks the next +// one, and a settled failure refuses the same reason until a new trigger. + +// Backstop only — a breadcrumb-emitting ceiling for a loop the outcome gate +// somehow failed to catch. The outcome gate is what stops a storm. +export const MAX_RECOVERIES_PER_WINDOW = 3 +export const RECOVERY_WINDOW_MS = 5 * 60_000 +// Why a cooldown exists: one incident can trip several detectors (stall watch, +// replay guard, input path) within seconds; the first remount fixes all of +// them, the rest must coalesce instead of re-remounting mid-reattach. +export const RECOVERY_COOLDOWN_MS = 15_000 +// Why reuse the direct-SSH settlement timeout: the same 31s bound already +// decides when a pane's attach attempt has stopped being in flight. A 'pending' +// ledger older than that describes a pane that never reported, not one still +// working, so it must stop blocking rather than wedge recovery forever. +export const RECOVERY_SETTLEMENT_TIMEOUT_MS = DIRECT_SSH_PANE_RETRY_SETTLEMENT_TIMEOUT_MS + +/** Why a request exists at all. Only 'automatic' is subject to the + * settled-failure refusal: a user pressing Retry, or an external lifecycle + * remount, IS the new trigger the refusal is waiting for. */ +export type TerminalRecoveryTrigger = 'automatic' | 'user' | 'external' + +export type TerminalRecoveryRemountRequest = { + reason: TerminalPaneRecoveryReason + trigger: TerminalRecoveryTrigger + /** The recovery epoch the requesting pane captured, when it has one. */ + generation?: number + now: number +} + +export type TerminalRecoveryDecline = + | { declinedBy: 'tab-missing' } + | { declinedBy: 'stale-generation' } + | { declinedBy: 'settled-failure' } + | { declinedBy: 'window-cap'; retryInMs: number } + | { declinedBy: 'unsettled'; retryInMs: number } + | { declinedBy: 'cooldown'; retryInMs: number } + +export type TerminalRecoveryAdmission = + | { admitted: true } + | ({ admitted: false } & TerminalRecoveryDecline) + +export type TerminalRecoveryRemountResult = + /** `generation` is the ledger epoch the remounted pane will capture. */ + { remounted: true; generation: number } | ({ remounted: false } & TerminalRecoveryDecline) + +const ADMITTED: TerminalRecoveryAdmission = { admitted: true } + +function recentAttempts(ledger: TerminalTabRecoveryLedger, now: number): number[] { + return ledger.attemptedAt.filter((at) => now - at < RECOVERY_WINDOW_MS) +} + +/** True once the ledger describes an attempt nothing can still settle: the row + * moved to a generation this ledger never saw (authority change, SSH pane + * retry, activation respawn, external remount). Derived, so no writer can + * forget to mark it — and none can mark it wrongly either. */ +function isSupersededLedger(tab: TerminalTab, ledger: TerminalTabRecoveryLedger): boolean { + // Strictly forward: generation only ever increments, so a row that reads + // LOWER is a host-snapshot rebuild that dropped the field, not a new trigger. + // Treating that as one would hand the tab a fresh allowance per snapshot. + return (tab.generation ?? 0) > ledger.tabGeneration +} + +export function readTerminalRecoveryOutcome( + tab: TerminalTab, + now: number +): TerminalPaneRecoveryOutcome | null { + const ledger = tab.recovery + if (!ledger) { + return null + } + if (isSupersededLedger(tab, ledger)) { + return 'superseded' + } + if (ledger.outcome === 'pending' && now - ledger.startedAt >= RECOVERY_SETTLEMENT_TIMEOUT_MS) { + return 'timed-out' + } + return ledger.outcome +} + +/** Narrowed to the one field it reads, so the connect path can pass the row it + * already resolved rather than looking the full TerminalTab up a second time. */ +export function captureTabRecoveryGeneration( + tab: Pick | null | undefined +): number { + return tab?.recovery?.generation ?? 0 +} + +/** + * The single admission decision. Runs read-only to fail a request fast, and + * again inside the store write so a probe's await cannot open a window for two + * panes to both consume the budget. + */ +export function admitTerminalRecoveryRemount( + tab: TerminalTab | null | undefined, + request: TerminalRecoveryRemountRequest +): TerminalRecoveryAdmission { + if (!tab) { + return { admitted: false, declinedBy: 'tab-missing' } + } + const ledger = tab.recovery + if ( + request.generation !== undefined && + request.generation !== captureTabRecoveryGeneration(tab) + ) { + return { admitted: false, declinedBy: 'stale-generation' } + } + if (request.trigger === 'external' || !ledger) { + return ADMITTED + } + const recent = recentAttempts(ledger, request.now) + if (recent.length >= MAX_RECOVERIES_PER_WINDOW) { + // Unconditional: the backstop must survive supersession, or anything that + // bumps tab.generation each cycle would lift the ceiling along with it. + return { + admitted: false, + declinedBy: 'window-cap', + retryInMs: recent[0] + RECOVERY_WINDOW_MS - request.now + } + } + if (request.trigger === 'user') { + // The user asking again IS the new evidence. Only the window cap — the + // backstop against a loop neither side can see — survives it. + return ADMITTED + } + const outcome = readTerminalRecoveryOutcome(tab, request.now) + if (outcome !== 'superseded') { + if (ledger.outcome === 'pending') { + if (outcome === 'pending') { + // Re-requesting under an unsettled attempt is the storm: the remounted + // pane fails the same way and asks again with a freshly captured epoch, + // so an epoch check can never refuse it. Nothing has been observed yet. + return { + admitted: false, + declinedBy: 'unsettled', + retryInMs: ledger.startedAt + RECOVERY_SETTLEMENT_TIMEOUT_MS - request.now + } + } + // Aged past the settlement bound with nobody reporting. Deliberately NOT + // read as an observed failure: a pane kind with no settle path would + // otherwise wedge its tab's recovery forever. The cooldown and the window + // cap bound it instead. + } else if ( + (ledger.outcome === 'failed' || ledger.outcome === 'timed-out') && + ledger.reason === request.reason + ) { + // A pane OBSERVED this reason fail after the last remount. Repeating it + // re-requests exactly the action that just failed with no evidence + // anything changed — wait for a real trigger (generation move, or user). + return { admitted: false, declinedBy: 'settled-failure' } + } + } + const last = recent.at(-1) + if (last !== undefined && request.now - last < RECOVERY_COOLDOWN_MS) { + return { + admitted: false, + declinedBy: 'cooldown', + retryInMs: last + RECOVERY_COOLDOWN_MS - request.now + } + } + return ADMITTED +} + +/** The ledger a remount writes, in the same object as the generation bump. */ +export function nextTerminalRecoveryLedger( + tab: TerminalTab, + request: TerminalRecoveryRemountRequest, + nextTabGeneration: number +): TerminalTabRecoveryLedger { + const previous = tab.recovery + // Carried across supersession on purpose — see the window-cap note above. + const carriedAttempts = previous ? recentAttempts(previous, request.now) : [] + return { + attemptedAt: [...carriedAttempts, request.now], + generation: captureTabRecoveryGeneration(tab) + 1, + outcome: 'pending', + startedAt: request.now, + reason: request.reason, + tabGeneration: nextTabGeneration + } +} + +/** Record what the mounted pane observed. Returns null when this settlement is + * not the current attempt's, so the caller can leave the store untouched. */ +export function settledTerminalRecoveryLedger( + tab: TerminalTab, + generation: number, + outcome: Exclude +): TerminalTabRecoveryLedger | null { + const ledger = tab.recovery + if ( + !ledger || + ledger.generation !== generation || + ledger.outcome !== 'pending' || + isSupersededLedger(tab, ledger) + ) { + return null + } + return { ...ledger, outcome } +} diff --git a/src/renderer/src/web/preload-api/web-runtime-environments-api.ts b/src/renderer/src/web/preload-api/web-runtime-environments-api.ts index e6ce2fd5a5d..8e9b0ecc245 100644 --- a/src/renderer/src/web/preload-api/web-runtime-environments-api.ts +++ b/src/renderer/src/web/preload-api/web-runtime-environments-api.ts @@ -16,6 +16,9 @@ import { translateHostAccessLinkError } from '@/lib/remote-pairing-copy' import { callEnvironmentEnvelope } from './web-runtime-calls' import { closeActiveRuntimeClients, + subscribeWebRuntimeStatus, + readWebRuntimeStatusSnapshots, + observeWebRuntimeStatus, disconnectActiveRuntimeEnvironment, getClientForEnvironment, manuallyDisconnectedEnvironmentIds, @@ -29,6 +32,8 @@ export function createRuntimeEnvironmentsApi(): NonNullable< Partial['runtimeEnvironments'] > { return { + onStatusChanged: subscribeWebRuntimeStatus, + getStatusSnapshots: async () => readWebRuntimeStatusSnapshots(), list: async () => { const environment = requireActiveEnvironmentOrNull() return environment ? [redactStoredWebRuntimeEnvironment(environment)] : [] @@ -146,6 +151,12 @@ export function createRuntimeEnvironmentsApi(): NonNullable< manuallyDisconnectedEnvironmentIds.clear() closeActiveRuntimeClients() webRuntimeState.activeEnvironment = nextEnvironment + getClientForEnvironment(nextEnvironment).statusOwner?.acceptVerified({ + id: 'status.get', + ok: true, + result: runtimeStatus, + _meta: { runtimeId: runtimeStatus.runtimeId } + }) return { ok: true, environment: redactStoredWebRuntimeEnvironment(nextEnvironment), @@ -173,6 +184,7 @@ export function createRuntimeEnvironmentsApi(): NonNullable< connect: ({ selector, timeoutMs }) => { const environment = resolveEnvironment(selector) manuallyDisconnectedEnvironmentIds.delete(environment.id) + closeActiveRuntimeClients() return callEnvironmentEnvelope( environment.id, 'status.get', @@ -180,8 +192,10 @@ export function createRuntimeEnvironmentsApi(): NonNullable< timeoutMs ) }, - getStatus: ({ selector, timeoutMs }) => - callEnvironmentEnvelope(selector, 'status.get', undefined, timeoutMs), + getStatus: ({ selector, timeoutMs, observeOnly }) => + observeOnly + ? observeWebRuntimeStatus(selector, timeoutMs) + : callEnvironmentEnvelope(selector, 'status.get', undefined, timeoutMs), retryControlConnection: () => Promise.resolve(), prepareBrowserClientHostPlacement: async () => ({ kind: 'server' }), call: ({ selector, method, params, timeoutMs }) => diff --git a/src/renderer/src/web/preload-api/web-runtime-session.ts b/src/renderer/src/web/preload-api/web-runtime-session.ts index 17a19136002..15b6cb63ee7 100644 --- a/src/renderer/src/web/preload-api/web-runtime-session.ts +++ b/src/renderer/src/web/preload-api/web-runtime-session.ts @@ -1,3 +1,7 @@ +import type { + RuntimeHostStatusSnapshot, + RuntimeHostStatusResponse +} from '../../../../shared/runtime-host-status' import type { WorktreeVisibilityDefaults } from '../../../../shared/global-settings-types' import { RuntimeRpcCallQueuePool } from '../../../../shared/runtime-rpc-call-queue' import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope' @@ -30,6 +34,43 @@ export const webRuntimeState: { cachedDetectedWorktrees: null } +const statusListeners = new Set<(snapshot: RuntimeHostStatusSnapshot) => void>() +export function subscribeWebRuntimeStatus( + callback: (snapshot: RuntimeHostStatusSnapshot) => void +): () => void { + statusListeners.add(callback) + return () => { + statusListeners.delete(callback) + } +} +export function readWebRuntimeStatusSnapshots(): RuntimeHostStatusSnapshot[] { + const snapshot = webRuntimeState.activeClient?.statusOwner?.read() + return snapshot ? [snapshot] : [] +} +export async function observeWebRuntimeStatus( + selector: string, + timeoutMs?: number +): Promise { + const environment = resolveEnvironment(selector) + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return manuallyDisconnectedResponse(environment) + } + const existing = webRuntimeState.activeClient?.statusOwner + if (existing) { + return existing.refresh({ timeoutMs, observeOnly: true }) + } + const transient = new WebRuntimeClient(getPreferredWebPairingOffer(environment), { + reconnect: false + }) + try { + return (await transient.call('status.get', undefined, { + timeoutMs + })) as RuntimeHostStatusResponse + } finally { + transient.close() + } +} + export const manuallyDisconnectedEnvironmentIds = new Set() export const runtimeCallQueuePool = new RuntimeRpcCallQueuePool() @@ -50,7 +91,18 @@ export function getClientForEnvironment( webRuntimeState.activeClientEnvironmentId !== environment.id ) { webRuntimeState.activeClient?.close() - webRuntimeState.activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment)) + webRuntimeState.activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment), { + status: { + environmentId: environment.id, + pairingRevision: environment.pairingRevision ?? environment.createdAt, + publish: (snapshot) => { + for (const listener of statusListeners) { + listener(snapshot) + } + }, + verified: (response) => updateEnvironmentFromResponse(environment, response) + } + }) webRuntimeState.activeClientEnvironmentId = environment.id } return webRuntimeState.activeClient diff --git a/src/renderer/src/web/web-runtime-client-export-parity.test.ts b/src/renderer/src/web/web-runtime-client-export-parity.test.ts index 27dce452549..6ba87eaad36 100644 --- a/src/renderer/src/web/web-runtime-client-export-parity.test.ts +++ b/src/renderer/src/web/web-runtime-client-export-parity.test.ts @@ -6,8 +6,13 @@ it('keeps the paired-web client public export surface exact', () => { expectTypeOf().toEqualTypeOf() expectTypeOf().toEqualTypeOf() expectTypeOf>().toEqualTypeOf< - [pairing: WebPairingOffer] + [ + pairing: WebPairingOffer, + options?: ConstructorParameters[1] + ] + >() + expectTypeOf().toEqualTypeOf< + 'call' | 'close' | 'subscribe' | 'statusOwner' >() - expectTypeOf().toEqualTypeOf<'call' | 'close' | 'subscribe'>() expect(Object.keys(WebClient)).toEqual(['WebRuntimeClient']) }) diff --git a/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts b/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts index 71568adc4da..95417748271 100644 --- a/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts +++ b/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts @@ -70,7 +70,7 @@ describe('WebRuntimeClient timeout budget', () => { await vi.advanceTimersByTimeAsync(60_000) expect(settled).toBe(false) - expect(waitForConnected).toHaveBeenCalledWith(25) + expect(waitForConnected).toHaveBeenCalledWith(25, undefined) resolveConnection() await Promise.resolve() diff --git a/src/renderer/src/web/web-runtime-client.ts b/src/renderer/src/web/web-runtime-client.ts index 7bc756ca0c0..8a3b944f701 100644 --- a/src/renderer/src/web/web-runtime-client.ts +++ b/src/renderer/src/web/web-runtime-client.ts @@ -1,3 +1,8 @@ +import { RuntimeHostStatusOwner } from '../../../shared/runtime-host-status-owner' +import type { + RuntimeHostStatusSnapshot, + RuntimeHostStatusResponse +} from '../../../shared/runtime-host-status' import type { RuntimeRpcResponse } from '../../../shared/runtime-rpc-envelope' import { WebRuntimeConnectionTransport } from './web-runtime-connection-transport' import { subscribeWebRuntimeFileWatch } from './web-runtime-file-watch-subscription' @@ -24,11 +29,62 @@ export class WebRuntimeClient { private readonly fileWatchTeardownRetries = new Map Promise>>() private readonly childClients = new Set() - constructor(private readonly pairing: WebPairingOffer) { - this.transport = new WebRuntimeConnectionTransport(pairing, { - now: () => this.now(), - isDocumentVisible: () => this.isDocumentVisible() - }) + readonly statusOwner?: RuntimeHostStatusOwner + + constructor( + private readonly pairing: WebPairingOffer, + options: { + reconnect?: boolean + status?: { + environmentId: string + pairingRevision: number + publish: (snapshot: RuntimeHostStatusSnapshot) => void + verified: (response: RuntimeHostStatusResponse) => void + } + } = {} + ) { + this.transport = new WebRuntimeConnectionTransport( + pairing, + { + now: () => this.now(), + isDocumentVisible: () => this.isDocumentVisible() + }, + { + reconnect: options.reconnect, + onStateChanged: (state) => { + if (state === 'auth-failed') { + this.statusOwner?.authenticationRejected() + } + this.statusOwner?.connectionChanged( + state === 'connected' + ? 'ready' + : state === 'disconnected' || state === 'auth-failed' + ? 'disconnected' + : 'connecting' + ) + } + } + ) + if (options.status) { + const status = options.status + this.statusOwner = new RuntimeHostStatusOwner({ + ...status, + persistent: true, + request: (signal) => + this.transport.call('status.get', undefined, { + timeoutMs: 15_000, + signal + }) as Promise, + verified: (response) => { + status.verified(response) + return true + } + }) + this.statusOwner.connectionChanged( + this.transport.state === 'connected' ? 'ready' : 'connecting' + ) + this.statusOwner.activate() + } } call( @@ -36,7 +92,9 @@ export class WebRuntimeClient { params?: unknown, options?: { timeoutMs?: number } ): Promise> { - return this.transport.call(method, params, options) + return method === 'status.get' && this.statusOwner + ? this.statusOwner.refresh(options) + : this.transport.call(method, params, options) } async subscribe( @@ -94,6 +152,7 @@ export class WebRuntimeClient { } close(options: { notifySubscriptions?: boolean } = {}): void { + this.statusOwner?.dispose() const shouldNotifySubscriptions = options.notifySubscriptions ?? true for (const child of Array.from(this.childClients)) { child.close({ notifySubscriptions: shouldNotifySubscriptions }) diff --git a/src/renderer/src/web/web-runtime-connection-transport.ts b/src/renderer/src/web/web-runtime-connection-transport.ts index d62cf3b48ba..1ead257bbc5 100644 --- a/src/renderer/src/web/web-runtime-connection-transport.ts +++ b/src/renderer/src/web/web-runtime-connection-transport.ts @@ -43,7 +43,11 @@ export class WebRuntimeConnectionTransport { constructor( private readonly pairing: WebPairingOffer, - clock: { now: () => number; isDocumentVisible: () => boolean } + clock: { now: () => number; isDocumentVisible: () => boolean }, + private readonly lifecycle: { + onStateChanged?: (state: WebRuntimeConnectionState) => void + reconnect?: boolean + } = {} ) { this.serverPublicKey = publicKeyFromBase64(pairing.publicKeyB64) this.connectionWaiters = new WebRuntimeConnectionWaiters({ @@ -60,7 +64,7 @@ export class WebRuntimeConnectionTransport { this.requestRegistry = new WebRuntimeRequestRegistry({ deviceToken: pairing.deviceToken, nextId: () => this.nextId(), - waitForConnected: (timeoutMs) => this.connectionWaiters.wait(timeoutMs), + waitForConnected: (timeoutMs, signal) => this.connectionWaiters.wait(timeoutMs, signal), sendEncrypted: (message) => this.sendEncrypted(message) }) this.heartbeat = new WebRuntimeConnectionHeartbeat({ @@ -82,7 +86,7 @@ export class WebRuntimeConnectionTransport { async call( method: string, params?: unknown, - options?: { timeoutMs?: number } + options?: { timeoutMs?: number; signal?: AbortSignal } ): Promise> { return this.requestRegistry.call(method, params, options) } @@ -153,6 +157,7 @@ export class WebRuntimeConnectionTransport { } else if (next === 'auth-failed') { this.connectionWaiters.rejectAll(createWebRuntimeUnauthorizedError()) } + this.lifecycle.onStateChanged?.(next) } private openConnection(): void { @@ -232,7 +237,7 @@ export class WebRuntimeConnectionTransport { } private scheduleReconnect(): void { - if (this.reconnectTimer || this.intentionallyClosed) { + if (this.reconnectTimer || this.intentionallyClosed || this.lifecycle.reconnect === false) { return } const delay = withReconnectJitter( diff --git a/src/renderer/src/web/web-runtime-connection-waiters.ts b/src/renderer/src/web/web-runtime-connection-waiters.ts index c16e30cd779..a8f3d081e86 100644 --- a/src/renderer/src/web/web-runtime-connection-waiters.ts +++ b/src/renderer/src/web/web-runtime-connection-waiters.ts @@ -13,7 +13,10 @@ export class WebRuntimeConnectionWaiters { constructor(private readonly options: WebRuntimeConnectionWaiterOptions) {} - wait(timeoutMs = 30_000): Promise { + wait(timeoutMs = 30_000, signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(signal.reason) + } if (this.options.getState() === 'connected') { return Promise.resolve() } @@ -24,11 +27,20 @@ export class WebRuntimeConnectionWaiters { return Promise.reject(new Error('Remote Orca runtime connection closed.')) } return new Promise((resolve, reject) => { - const timeout = window.setTimeout(() => { - const index = this.waiters.findIndex((waiter) => waiter.resolve === resolve) + const cleanup = (): void => { + window.clearTimeout(timeout) + signal?.removeEventListener('abort', abort) + const index = this.waiters.indexOf(waiter) if (index !== -1) { this.waiters.splice(index, 1) } + } + const abort = (): void => { + cleanup() + reject(signal?.reason) + } + const timeout = window.setTimeout(() => { + cleanup() reject( new Error( withRemoteRuntimeTailscaleHint( @@ -38,16 +50,18 @@ export class WebRuntimeConnectionWaiters { ) ) }, timeoutMs) - this.waiters.push({ + const waiter = { resolve: () => { - window.clearTimeout(timeout) + cleanup() resolve() }, - reject: (error) => { - window.clearTimeout(timeout) + reject: (error: Error) => { + cleanup() reject(error) } - }) + } + this.waiters.push(waiter) + signal?.addEventListener('abort', abort, { once: true }) }) } diff --git a/src/renderer/src/web/web-runtime-request-registry.ts b/src/renderer/src/web/web-runtime-request-registry.ts index 1347d580f00..0329e208ba3 100644 --- a/src/renderer/src/web/web-runtime-request-registry.ts +++ b/src/renderer/src/web/web-runtime-request-registry.ts @@ -6,7 +6,7 @@ const REQUEST_TIMEOUT_MS = 30_000 type WebRuntimeRequestRegistryOptions = { deviceToken: string nextId: () => string - waitForConnected: (timeoutMs?: number) => Promise + waitForConnected: (timeoutMs?: number, signal?: AbortSignal) => Promise sendEncrypted: (message: unknown) => boolean } @@ -18,17 +18,41 @@ export class WebRuntimeRequestRegistry { async call( method: string, params?: unknown, - callOptions?: { timeoutMs?: number } + callOptions?: { timeoutMs?: number; signal?: AbortSignal } ): Promise> { - await this.options.waitForConnected(callOptions?.timeoutMs) + const signal = callOptions?.signal + await this.options.waitForConnected(callOptions?.timeoutMs, signal) + signal?.throwIfAborted() return new Promise((resolve, reject) => { const id = this.options.nextId() const timeoutMs = callOptions?.timeoutMs ?? REQUEST_TIMEOUT_MS const timeout = window.setTimeout(() => { this.pending.delete(id) + cleanup() reject(new Error(`Request timed out: ${method}`)) }, timeoutMs) - this.pending.set(id, { method, resolve, reject, timeout }) + const cleanup = (): void => { + signal?.removeEventListener('abort', abort) + } + const abort = (): void => { + this.pending.delete(id) + window.clearTimeout(timeout) + cleanup() + reject(signal?.reason) + } + signal?.addEventListener('abort', abort, { once: true }) + this.pending.set(id, { + method, + resolve: (value) => { + cleanup() + resolve(value) + }, + reject: (error) => { + cleanup() + reject(error) + }, + timeout + }) if ( !this.options.sendEncrypted({ id, @@ -39,6 +63,7 @@ export class WebRuntimeRequestRegistry { ) { this.pending.delete(id) window.clearTimeout(timeout) + cleanup() reject(new Error('Remote Orca runtime is not connected.')) } }) diff --git a/src/renderer/src/web/web-runtime-status-owner.test.ts b/src/renderer/src/web/web-runtime-status-owner.test.ts new file mode 100644 index 00000000000..7abfe6f19f4 --- /dev/null +++ b/src/renderer/src/web/web-runtime-status-owner.test.ts @@ -0,0 +1,49 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import WebSocket from 'ws' +import { + createSharedControlTestServer, + closeSharedControlTestServers +} from '../../../shared/remote-runtime-shared-control-test-server' +import { WebRuntimeClient } from './web-runtime-client' + +const clients: WebRuntimeClient[] = [] +beforeEach(() => { + vi.stubGlobal('WebSocket', WebSocket) + vi.stubGlobal('window', { + setTimeout, + clearTimeout, + setInterval, + clearInterval, + atob: (value: string) => Buffer.from(value, 'base64').toString('binary'), + btoa: (value: string) => Buffer.from(value, 'binary').toString('base64') + }) +}) +afterEach(async () => { + clients.splice(0).forEach((client) => client.close()) + await closeSharedControlTestServers() + vi.unstubAllGlobals() +}) + +it('primary browser status follows the authenticated socket and closing it retires the owner', async () => { + let runtimeId = 'before' + const server = await createSharedControlTestServer({ + resultForRequest: () => ({ runtimeId, capabilities: [] }) + }) + const publish = vi.fn() + const client = new WebRuntimeClient(server.pairing, { + status: { environmentId: 'browser', pairingRevision: 1, publish, verified: vi.fn() } + }) + clients.push(client) + await expect + .poll(() => client.statusOwner?.read().verification, { timeout: 3_000 }) + .toBe('verified') + expect(client.statusOwner?.read().status?.runtimeId).toBe('before') + runtimeId = 'after' + server.closeClients() + await expect + .poll(() => client.statusOwner?.read().status?.runtimeId, { timeout: 3_000 }) + .toBe('after') + expect(client.statusOwner?.read().transport).toBe('ready') + client.close() + expect(publish.mock.lastCall?.[0]).toMatchObject({ retired: true, verification: 'blocked' }) +}) diff --git a/src/shared/commit-message-agent-specs-primary.ts b/src/shared/commit-message-agent-specs-primary.ts index e6ba42f775b..3e42a4c5865 100644 --- a/src/shared/commit-message-agent-specs-primary.ts +++ b/src/shared/commit-message-agent-specs-primary.ts @@ -197,7 +197,6 @@ export function buildPrimaryCommitMessageAgentSpecs({ '--print', '--no-session', '--no-tools', - '--no-extensions', '--no-skills', '--no-context-files', '--mode', diff --git a/src/shared/commit-message-plan.test.ts b/src/shared/commit-message-plan.test.ts index 0b728307bf7..3c2fa62aea5 100644 --- a/src/shared/commit-message-plan.test.ts +++ b/src/shared/commit-message-plan.test.ts @@ -2,6 +2,29 @@ import { describe, expect, it } from 'vitest' import { planCommitMessageGeneration, planAgentBinary } from './commit-message-plan' describe('planCommitMessageGeneration', () => { + it('keeps extension-provided Pi models available in generated Git text plans', () => { + const result = planCommitMessageGeneration( + { agentId: 'pi', model: 'local-extension/model' }, + 'Write a commit message' + ) + expect(result.ok).toBe(true) + if (!result.ok) { + throw new Error(result.error) + } + expect(result.plan.args).not.toContain('--no-extensions') + expect(result.plan.args).toEqual( + expect.arrayContaining([ + '--no-session', + '--no-tools', + '--no-skills', + '--no-context-files', + '--model', + 'local-extension/model' + ]) + ) + expect(result.plan.stdinPayload).toBe('Write a commit message') + }) + it('plans Claude non-interactive generation with the prompt on stdin only', () => { const result = planCommitMessageGeneration( { diff --git a/src/shared/execution-host-registry.test.ts b/src/shared/execution-host-registry.test.ts index e509fe05bfc..e397e8ed5bf 100644 --- a/src/shared/execution-host-registry.test.ts +++ b/src/shared/execution-host-registry.test.ts @@ -320,17 +320,15 @@ describe('execution host registry', () => { ]) }) - it('includes runtime hosts from repo ownership but marks them disconnected without live status', () => { + it('keeps runtime hosts checking before their first status result', () => { const hosts = buildExecutionHostRegistry({ repos: [{ connectionId: null, executionHostId: 'runtime:env-2' }], settings: { activeRuntimeEnvironmentId: null } }) - // No live status means no evidence the Orca server is reachable, so it must - // read 'disconnected' rather than defaulting to 'available'/"Connected". expect(hosts).toMatchObject([ { id: 'local', health: 'local' }, - { id: 'runtime:env-2', kind: 'runtime', label: 'env-2', health: 'disconnected' } + { id: 'runtime:env-2', kind: 'runtime', label: 'env-2', health: 'connecting' } ]) }) @@ -373,3 +371,29 @@ describe('execution host registry', () => { ]) }) }) + +it('keeps an initial unknown-transport verification connecting', () => { + const hosts = buildExecutionHostRegistry({ + repos: [], + settings: null, + runtimeEnvironments: [{ id: 'host', name: 'Host' }], + runtimeStatusByEnvironmentId: new Map([ + [ + 'host', + { + status: null, + snapshot: { + environmentId: 'host', + pairingRevision: 1, + sequence: 1, + checkedAt: 0, + status: null, + verification: 'checking', + transport: 'unknown' + } + } + ] + ]) + }) + expect(hosts.find((host) => host.id === 'runtime:host')?.health).toBe('connecting') +}) diff --git a/src/shared/execution-host-registry.ts b/src/shared/execution-host-registry.ts index a970f9da45c..bad6b40f257 100644 --- a/src/shared/execution-host-registry.ts +++ b/src/shared/execution-host-registry.ts @@ -1,3 +1,4 @@ +import type { RuntimeHostStatusSnapshot } from './runtime-host-status' import { LOCAL_EXECUTION_HOST_ID, getLocalExecutionHostLabel, @@ -49,6 +50,7 @@ type RuntimeEnvironmentSummary = { } type RuntimeHostStatus = { + snapshot?: RuntimeHostStatusSnapshot status?: RuntimeStatus | null remoteControl?: RuntimeStatus['remoteControl'] | null appVersion?: string | null @@ -158,9 +160,24 @@ function addRuntimeHost( const hostId = toRuntimeExecutionHostId(environmentId) const runtimeStatus = statusByEnvironmentId?.get(environmentId) const status = runtimeStatus?.status - const compatibility = runtimeCompatibility(status) + const snapshot = runtimeStatus?.snapshot + const metadata = status ?? snapshot?.status + const compatibility = runtimeCompatibility(metadata) const remoteControl = runtimeStatus?.remoteControl ?? status?.remoteControl - const controlHealth = runtimeControlHealth(remoteControl) + const controlHealth = snapshot?.retired + ? 'disconnected' + : snapshot?.verification === 'blocked' + ? 'blocked' + : !runtimeStatus || + snapshot?.verification === 'checking' || + snapshot?.transport === 'disconnected' || + snapshot?.transport === 'connecting' + ? 'connecting' + : snapshot?.transport === 'ready' + ? compatibility?.kind === 'blocked' + ? 'blocked' + : 'available' + : runtimeControlHealth(remoteControl) setHost(hosts, { id: hostId, kind: 'runtime', @@ -168,12 +185,12 @@ function addRuntimeHost( detail: 'Orca server', health: controlHealth ?? runtimeHealth(status, compatibility, remoteControl), compatibility: compatibility ?? undefined, - capabilities: status?.capabilities, - appVersion: runtimeStatus?.appVersion ?? status?.appVersion ?? null, - protocolVersion: status?.runtimeProtocolVersion ?? status?.protocolVersion ?? null, + capabilities: metadata?.capabilities, + appVersion: runtimeStatus?.appVersion ?? metadata?.appVersion ?? null, + protocolVersion: metadata?.runtimeProtocolVersion ?? metadata?.protocolVersion ?? null, minCompatibleClientVersion: - status?.minCompatibleRuntimeClientVersion ?? status?.minCompatibleMobileVersion ?? null, - platform: status?.hostPlatform ?? null, + metadata?.minCompatibleRuntimeClientVersion ?? metadata?.minCompatibleMobileVersion ?? null, + platform: metadata?.hostPlatform ?? null, remoteControlState: remoteControl ?? null, ...(source ? { source } : {}) }) diff --git a/src/renderer/src/components/native-chat/native-chat-turn-activity.test.ts b/src/shared/native-chat-turn-activity.test.ts similarity index 72% rename from src/renderer/src/components/native-chat/native-chat-turn-activity.test.ts rename to src/shared/native-chat-turn-activity.test.ts index a819e3054a2..3cc64f4e106 100644 --- a/src/renderer/src/components/native-chat/native-chat-turn-activity.test.ts +++ b/src/shared/native-chat-turn-activity.test.ts @@ -1,8 +1,5 @@ import { describe, expect, it } from 'vitest' -import type { - AgentJournalItemBody, - AgentJournalRenderItem -} from '../../../../shared/agent-session-journal-types' +import type { AgentJournalItemBody, AgentJournalRenderItem } from './agent-session-journal-types' import { selectStructuredAgentTurnActivity } from './native-chat-turn-activity' function item(sequence: number, body: AgentJournalItemBody): AgentJournalRenderItem { @@ -34,6 +31,49 @@ describe('selectStructuredAgentTurnActivity', () => { expect(activity).toEqual({ kind: 'description', text: 'Preparing the answer' }) }) + it("never puts the model's reasoning on the indicator line", () => { + const reasoning = item(2, { + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'Let me check whether the journal already records this' }] + }) + + // Reasoning is the turn's content; the row says the turn is thinking instead. + expect(selectStructuredAgentTurnActivity([turnStart, reasoning], 'turn-1')).toBeNull() + // An ordinary status row is still a description of what the turn is doing. + expect( + selectStructuredAgentTurnActivity( + [turnStart, reasoning, item(3, { kind: 'status', text: 'Updating the plan' })], + 'turn-1' + ) + ).toEqual({ kind: 'description', text: 'Updating the plan' }) + // Provider-authored copy is unaffected, so Codex keeps its line. + expect( + selectStructuredAgentTurnActivity([turnStart, reasoning], 'turn-1', { + turnId: 'turn-1', + text: 'Running a command' + }) + ).toEqual({ kind: 'description', text: 'Running a command' }) + }) + + it('skips reasoning behind a typed turn item too', () => { + const typedTurnStart = item(1, { kind: 'turn', turnId: 'turn-1', state: 'running' }) + + expect( + selectStructuredAgentTurnActivity( + [ + typedTurnStart, + item(2, { + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'Weighing two approaches' }] + }) + ], + 'turn-1' + ) + ).toBeNull() + }) + it('prefers matching ephemeral provider activity over journal-derived status', () => { const activity = selectStructuredAgentTurnActivity( [turnStart, item(2, { kind: 'status', text: 'Older journal status' })], diff --git a/src/renderer/src/components/native-chat/native-chat-turn-activity.ts b/src/shared/native-chat-turn-activity.ts similarity index 86% rename from src/renderer/src/components/native-chat/native-chat-turn-activity.ts rename to src/shared/native-chat-turn-activity.ts index e42abea2bf6..fbc1756529b 100644 --- a/src/renderer/src/components/native-chat/native-chat-turn-activity.ts +++ b/src/shared/native-chat-turn-activity.ts @@ -1,11 +1,8 @@ -import { readAgentJournalTurn } from '../../../../shared/agent-session-turn-record' -import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' -import type { AgentSessionTurnActivity } from '../../../../shared/agent-session-wire' -import { normalizePromptField } from '../../../../shared/agent-status-field-normalization' -import { - describeActiveToolCall, - formatActiveToolLabel -} from '../../../../shared/native-chat-tool-activity' +import { readAgentJournalTurn } from './agent-session-turn-record' +import type { AgentJournalRenderItem } from './agent-session-journal-types' +import type { AgentSessionTurnActivity } from './agent-session-wire' +import { normalizePromptField } from './agent-status-field-normalization' +import { describeActiveToolCall, formatActiveToolLabel } from './native-chat-tool-activity' export type NativeChatTurnActivity = { kind: 'description'; text: string } diff --git a/src/shared/native-chat-turn-status.test.ts b/src/shared/native-chat-turn-status.test.ts index 3fdcccb7bf3..c98d4f79a73 100644 --- a/src/shared/native-chat-turn-status.test.ts +++ b/src/shared/native-chat-turn-status.test.ts @@ -1,24 +1,16 @@ import { describe, expect, it } from 'vitest' -import type { NativeChatMessage } from './native-chat-types' import { + describeNativeChatActiveTurnLabel, describeNativeChatTurnStatus, + formatNativeChatActiveTurnLabel, formatNativeChatDuration, formatNativeChatTurnStatusLabel, nativeChatElapsedSeconds, - nativeChatTurnHasResponse, reduceNativeChatTurnTiming, selectNativeChatTurnStatuses, type NativeChatTurnTimingByTurn } from './native-chat-turn-status' -function message( - id: string, - role: NativeChatMessage['role'], - blocks: NativeChatMessage['blocks'] -): NativeChatMessage { - return { id, role, blocks, timestamp: null, source: 'transcript' } -} - describe('formatNativeChatDuration', () => { it.each([ [0, '0s'], @@ -59,6 +51,51 @@ describe('describeNativeChatTurnStatus', () => { }) }) +describe('describeNativeChatActiveTurnLabel', () => { + it('lets provider activity beat both fallbacks', () => { + expect( + describeNativeChatActiveTurnLabel({ + activityText: 'Reading src/main.ts', + thinking: true, + elapsedSeconds: 12 + }) + ).toEqual({ source: 'activity', text: 'Reading src/main.ts' }) + }) + + it('falls back to reasoning when the provider says nothing usable', () => { + expect( + describeNativeChatActiveTurnLabel({ activityText: ' ', thinking: true, elapsedSeconds: 12 }) + ).toEqual({ source: 'status', key: 'thinking', duration: null }) + expect( + describeNativeChatActiveTurnLabel({ activityText: null, thinking: true, elapsedSeconds: 12 }) + ).toEqual({ source: 'status', key: 'thinking', duration: null }) + }) + + it('falls back to the running clock when the turn is neither talking nor reasoning', () => { + expect(describeNativeChatActiveTurnLabel({ thinking: false, elapsedSeconds: 184 })).toEqual({ + source: 'status', + key: 'workingFor', + duration: '3m 4s' + }) + }) +}) + +describe('formatNativeChatActiveTurnLabel', () => { + it('renders the one live row in English for platforms without i18n', () => { + expect( + formatNativeChatActiveTurnLabel({ + activityText: 'Running pnpm test', + thinking: false, + elapsedSeconds: 4 + }) + ).toBe('Running pnpm test') + expect(formatNativeChatActiveTurnLabel({ thinking: true, elapsedSeconds: 4 })).toBe('Thinking') + expect(formatNativeChatActiveTurnLabel({ thinking: false, elapsedSeconds: 12 })).toBe( + 'Working for 12s' + ) + }) +}) + describe('formatNativeChatTurnStatusLabel', () => { it('renders each state in English for platforms without i18n', () => { expect( @@ -73,45 +110,6 @@ describe('formatNativeChatTurnStatusLabel', () => { }) }) -describe('nativeChatTurnHasResponse', () => { - const user = message('u1', 'user', [{ type: 'text', text: 'go' }]) - - it('is false while the turn has produced nothing', () => { - expect(nativeChatTurnHasResponse([user], 0)).toBe(false) - }) - - it('ignores a whitespace-only assistant block', () => { - const blank = message('a1', 'assistant', [{ type: 'text', text: ' \n ' }]) - expect(nativeChatTurnHasResponse([user, blank], 0)).toBe(false) - }) - - it('is true on the first real text, tool call, or tool result', () => { - expect( - nativeChatTurnHasResponse( - [user, message('a1', 'assistant', [{ type: 'text', text: 'hi' }])], - 0 - ) - ).toBe(true) - expect( - nativeChatTurnHasResponse( - [user, message('t1', 'tool', [{ type: 'tool-call', name: 'Read', input: {} }])], - 0 - ) - ).toBe(true) - expect( - nativeChatTurnHasResponse( - [user, message('t1', 'tool', [{ type: 'tool-result', output: 'ok' }])], - 0 - ) - ).toBe(true) - }) - - it('does not count output that preceded the latest user turn', () => { - const earlier = message('a0', 'assistant', [{ type: 'text', text: 'old' }]) - expect(nativeChatTurnHasResponse([earlier, user], 1)).toBe(false) - }) -}) - describe('reduceNativeChatTurnTiming', () => { const validTurnKeys = new Set(['u1']) @@ -290,18 +288,18 @@ describe('reduceNativeChatTurnTiming', () => { }) describe('selectNativeChatTurnStatuses', () => { - it('reports the working turn as thinking until it produces output', () => { + it('carries the reasoning verdict it is given onto the working turn', () => { const { active } = selectNativeChatTurnStatuses( { u1: { startedAt: 1_000, workedSeconds: null } }, - { activeTurnKey: 'u1', isWorking: true, hasCurrentTurnResponse: false } + { activeTurnKey: 'u1', isWorking: true, thinking: true } ) expect(active).toEqual({ startedAt: 1_000, thinking: true, workedSeconds: null }) }) - it('stops thinking once the turn has output', () => { + it('reports a working turn that is not reasoning as counting', () => { const { active } = selectNativeChatTurnStatuses( { u1: { startedAt: 1_000, workedSeconds: null } }, - { activeTurnKey: 'u1', isWorking: true, hasCurrentTurnResponse: true } + { activeTurnKey: 'u1', isWorking: true, thinking: false } ) expect(active?.thinking).toBe(false) }) @@ -309,7 +307,7 @@ describe('selectNativeChatTurnStatuses', () => { it('exposes settled turns and resolves the active one from them when idle', () => { const { active, completedByTurn } = selectNativeChatTurnStatuses( { u1: { startedAt: 1_000, workedSeconds: 12 } }, - { activeTurnKey: 'u1', isWorking: false, hasCurrentTurnResponse: true } + { activeTurnKey: 'u1', isWorking: false, thinking: false } ) expect(completedByTurn.u1).toEqual({ startedAt: 1_000, thinking: false, workedSeconds: 12 }) expect(active).toEqual(completedByTurn.u1) @@ -318,7 +316,7 @@ describe('selectNativeChatTurnStatuses', () => { it('omits an in-flight turn from the completed map', () => { const { completedByTurn } = selectNativeChatTurnStatuses( { u1: { startedAt: 1_000, workedSeconds: null } }, - { activeTurnKey: 'u1', isWorking: true, hasCurrentTurnResponse: true } + { activeTurnKey: 'u1', isWorking: true, thinking: false } ) expect(completedByTurn).toEqual({}) }) diff --git a/src/shared/native-chat-turn-status.ts b/src/shared/native-chat-turn-status.ts index aaecbc05ce5..b34b199e9d0 100644 --- a/src/shared/native-chat-turn-status.ts +++ b/src/shared/native-chat-turn-status.ts @@ -3,8 +3,6 @@ // and the mobile app (used directly — mobile ships English only) so the two // surfaces never drift. Everything here is pure; each platform owns its own clock. -import type { NativeChatMessage } from './native-chat-types' - export const NATIVE_CHAT_TURN_STATUS_COPY = { thinking: 'Thinking', workingFor: 'Working for {{value0}}', @@ -48,6 +46,52 @@ export function describeNativeChatTurnStatus({ return { key: 'workingFor', duration: formatNativeChatDuration(elapsedSeconds) } } +/** The two readings that label a live turn's one indicator row, carried together + * so a surface cannot pick up one without the other. */ +export type NativeChatLiveTurnIndicator = { + thinking: boolean + activityText: string | null +} + +export type NativeChatActiveTurnLabel = + | { source: 'activity'; text: string } + | { source: 'status'; key: 'thinking' | 'workingFor'; duration: string | null } + +/** The live turn's single indicator label. Provider activity wins because it is the + * only text that says what the turn is actually doing; reasoning is next; the + * running clock is the floor. Shared so desktop and mobile cannot disagree. */ +export function describeNativeChatActiveTurnLabel({ + activityText, + thinking, + elapsedSeconds +}: { + activityText?: string | null + thinking: boolean + elapsedSeconds: number +}): NativeChatActiveTurnLabel { + const text = activityText?.trim() + if (text) { + return { source: 'activity', text } + } + return thinking + ? { source: 'status', key: 'thinking', duration: null } + : { source: 'status', key: 'workingFor', duration: formatNativeChatDuration(elapsedSeconds) } +} + +/** The live turn's label in English. For platforms without i18n (mobile). */ +export function formatNativeChatActiveTurnLabel(input: { + activityText?: string | null + thinking: boolean + elapsedSeconds: number +}): string { + const label = describeNativeChatActiveTurnLabel(input) + if (label.source === 'activity') { + return label.text + } + const copy = NATIVE_CHAT_TURN_STATUS_COPY[label.key] + return label.duration == null ? copy : copy.replaceAll('{{value0}}', label.duration) +} + /** Resolve the turn-status label in English. For platforms without i18n (mobile). */ export function formatNativeChatTurnStatusLabel(input: { thinking: boolean @@ -59,26 +103,6 @@ export function formatNativeChatTurnStatusLabel(input: { return duration == null ? copy : copy.replaceAll('{{value0}}', duration) } -/** True once the current turn has produced anything renderable — the boundary - * between the "Thinking" label and the counting "Working for N" label. */ -export function nativeChatTurnHasResponse( - messages: readonly NativeChatMessage[], - latestUserIndex: number -): boolean { - return messages - .slice(latestUserIndex + 1) - .some( - (message) => - (message.role === 'assistant' || message.role === 'tool') && - message.blocks.some( - (block) => - block.type === 'tool-call' || - block.type === 'tool-result' || - (block.type === 'text' && block.text.trim().length > 0) - ) - ) -} - export type NativeChatTurnTiming = { startedAt: number workedSeconds: number | null @@ -183,13 +207,14 @@ export function selectNativeChatTurnStatuses( activeTurnKey, isWorking, workingStartedAt, - hasCurrentTurnResponse, + thinking, settledByTurn }: { activeTurnKey: string isWorking: boolean workingStartedAt?: number | null - hasCurrentTurnResponse: boolean + /** Whether the active turn is reasoning right now, from its journal content. */ + thinking: boolean settledByTurn?: NativeChatSettledTurns } ): { active: NativeChatTurnStatus | null; completedByTurn: Record } { @@ -216,7 +241,7 @@ export function selectNativeChatTurnStatuses( active: isWorking ? { startedAt: workingStartedAt ?? timingByTurn[activeTurnKey]?.startedAt ?? null, - thinking: !hasCurrentTurnResponse, + thinking, workedSeconds: null } : (completedByTurn[activeTurnKey] ?? null), diff --git a/src/shared/native-chat-unverifiable-turn-status.test.ts b/src/shared/native-chat-unverifiable-turn-status.test.ts index 7e93f9cecc4..592ca699183 100644 --- a/src/shared/native-chat-unverifiable-turn-status.test.ts +++ b/src/shared/native-chat-unverifiable-turn-status.test.ts @@ -57,7 +57,7 @@ describe('authoritative unknown turn duration at the shared status consumer', () const options = { activeTurnKey: user.itemId, isWorking: false, - hasCurrentTurnResponse: true, + thinking: false, settledByTurn: selectStructuredAgentSettledTurns([user, recoveredTurn]) } @@ -87,7 +87,7 @@ describe('authoritative unknown turn duration at the shared status consumer', () { activeTurnKey: 'next', isWorking: true, - hasCurrentTurnResponse: true, + thinking: false, settledByTurn: selectStructuredAgentSettledTurns([userItem('turn-1'), item]) } ) @@ -113,7 +113,7 @@ describe('authoritative unknown turn duration at the shared status consumer', () { activeTurnKey: 'turn-1', isWorking: false, - hasCurrentTurnResponse: true, + thinking: false, settledByTurn: selectStructuredAgentSettledTurns([ userItem('turn-1'), userItem('old'), diff --git a/src/shared/remote-runtime-shared-control-test-server.ts b/src/shared/remote-runtime-shared-control-test-server.ts index 0e4adb1a69c..701ea0e5893 100644 --- a/src/shared/remote-runtime-shared-control-test-server.ts +++ b/src/shared/remote-runtime-shared-control-test-server.ts @@ -20,6 +20,7 @@ export type SharedControlTestServer = { } type ServerOptions = { + resultForRequest?: (method: string) => unknown delaySubscriptionReady?: boolean sendKeepaliveBeforeResponse?: boolean keepaliveDelayMs?: number @@ -174,7 +175,7 @@ function handleRequest( const streaming = isStreamingMethod(request.method) const result = streaming ? { type: 'ready', subscriptionId: `${request.method}:subscription` } - : { method: request.method } + : (options.resultForRequest?.(request.method) ?? { method: request.method }) const sendResponse = (): void => { if (options.sendUnknownResponseBeforeResponse) { sendEncrypted(ws, sharedKey, { diff --git a/src/shared/remote-workspace-session-projection.test.ts b/src/shared/remote-workspace-session-projection.test.ts index fdccd75b8e9..a11026e3581 100644 --- a/src/shared/remote-workspace-session-projection.test.ts +++ b/src/shared/remote-workspace-session-projection.test.ts @@ -6,6 +6,52 @@ import { import { getDefaultWorkspaceSession } from './constants' describe('remote workspace session projection', () => { + // The transient set this boundary mirrors. `recovery` is the tab's in-flight + // heal, timestamped with THIS machine's clock, and `pendingActivationSpawn` is + // a one-shot mount handoff — neither means anything on another client's row, + // and a foreign `startedAt` would be compared against the reader's Date.now(). + it('strips client-local transient tab fields on the way out', () => { + const session = { + ...getDefaultWorkspaceSession(), + activeRepoId: 'repo-a', + activeWorktreeId: 'repo-a::/srv/app', + activeTabId: 'tab-1', + tabsByWorktree: { + 'repo-a::/srv/app': [ + { + id: 'tab-1', + ptyId: 'pty-1', + worktreeId: 'repo-a::/srv/app', + title: 'Remote', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1, + pendingActivationSpawn: true, + recovery: { + attemptedAt: [1_000], + generation: 1, + outcome: 'failed' as const, + startedAt: 1_000, + reason: 'reattach-unverifiable' as const, + tabGeneration: 1 + } + } + ] + }, + terminalLayoutsByTabId: {} + } + + const projected = exportRemoteWorkspaceSession(session, { + isTargetWorktree: (worktreeId) => worktreeId.startsWith('repo-a::') + }) + + const exported = projected.tabsByWorktreePath['/srv/app'][0] as Record + expect(exported.recovery).toBeUndefined() + expect(exported.pendingActivationSpawn).toBeUndefined() + expect(exported.id).toBe('tab-1') + }) + it('exports terminal state using remote worktree paths instead of local repo ids', () => { const session = { ...getDefaultWorkspaceSession(), diff --git a/src/shared/remote-workspace-session-projection.ts b/src/shared/remote-workspace-session-projection.ts index 7f923050d36..29b3e15cd1d 100644 --- a/src/shared/remote-workspace-session-projection.ts +++ b/src/shared/remote-workspace-session-projection.ts @@ -32,9 +32,20 @@ function worktreePathFromId(worktreeId: string): string | null { } function tabToRemote(tab: TerminalTab, worktreePath: string): RemoteWorkspaceTerminalTab { - const { worktreeId: _worktreeId, pendingActivationSpawn: _pendingActivationSpawn, ...rest } = tab + // `recovery` joins the transient set for the same reason as + // pendingActivationSpawn: it describes THIS client's in-flight heal, and its + // timestamps are this machine's clock. On another client's row they would be + // compared against a foreign `Date.now()`. Nothing hands an unsanitized + // session to this boundary today; stripping here keeps that from mattering. + const { + worktreeId: _worktreeId, + pendingActivationSpawn: _pendingActivationSpawn, + recovery: _recovery, + ...rest + } = tab void _worktreeId void _pendingActivationSpawn + void _recovery return { ...rest, worktreePath } } diff --git a/src/shared/remote-workspace-types.ts b/src/shared/remote-workspace-types.ts index 4d6eec6021e..9544b709f9c 100644 --- a/src/shared/remote-workspace-types.ts +++ b/src/shared/remote-workspace-types.ts @@ -1,6 +1,12 @@ import type { TerminalLayoutSnapshot, TerminalTab } from './terminal-tab-types' -export type RemoteWorkspaceTerminalTab = Omit & { +// Transient client-local fields are omitted, not merely unset: `recovery` is +// this client's in-flight heal, stamped with this machine's clock, so the type +// must not let a future producer put one on the wire. +export type RemoteWorkspaceTerminalTab = Omit< + TerminalTab, + 'worktreeId' | 'pendingActivationSpawn' | 'recovery' +> & { worktreePath: string } diff --git a/src/shared/runtime-host-status-owner.test.ts b/src/shared/runtime-host-status-owner.test.ts new file mode 100644 index 00000000000..32331991988 --- /dev/null +++ b/src/shared/runtime-host-status-owner.test.ts @@ -0,0 +1,207 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { RuntimeHostStatusOwner } from './runtime-host-status-owner' +import { runtimeHostStatusFailure, type RuntimeHostStatusResponse } from './runtime-host-status' +import type { RuntimeStatus } from './runtime-types' + +const owners: RuntimeHostStatusOwner[] = [] +beforeEach(() => vi.useFakeTimers()) +afterEach(() => { + owners.splice(0).forEach((owner) => owner.dispose()) + vi.useRealTimers() +}) +function success(runtimeId = 'host-1'): RuntimeHostStatusResponse & { ok: true } { + return { + id: 'status', + ok: true, + result: { runtimeId, capabilities: [] } as unknown as RuntimeStatus, + _meta: { runtimeId } + } +} +function deferred() { + let resolve!: (response: RuntimeHostStatusResponse) => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} +function createOwner(persistent = false) { + const request = vi + .fn<(signal: AbortSignal) => Promise>() + .mockResolvedValue(success()) + const publish = vi.fn() + const verified = vi.fn((_response: RuntimeHostStatusResponse, _active: boolean) => persistent) + const owner = new RuntimeHostStatusOwner({ + environmentId: 'env-a', + pairingRevision: 1, + persistent, + request, + publish, + verified + }) + owners.push(owner) + return { owner, request, publish, verified } +} + +it('shares one verification between viewers with independent deadlines', async () => { + const { owner, request } = createOwner() + const pending = deferred() + request.mockReturnValue(pending.promise) + const impatient = owner.refresh({ timeoutMs: 100 }) + const patient = owner.refresh({ timeoutMs: 1_000 }) + await vi.advanceTimersByTimeAsync(100) + expect((await impatient).ok).toBe(false) + expect(request).toHaveBeenCalledOnce() + expect(request.mock.calls[0][0].aborted).toBe(false) + pending.resolve(success()) + expect((await patient).ok).toBe(true) +}) + +it('uses ready transitions, not diagnostic updates or a healthy polling timer', async () => { + const { owner, request } = createOwner(true) + await owner.refresh() + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(0) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledTimes(2) + owner.connectionChanged('disconnected') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledTimes(2) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(3) +}) + +it('retries a failed status operation while retaining healthy transport and last good metadata', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + await owner.refresh() + request.mockResolvedValueOnce(runtimeHostStatusFailure('runtime_unavailable', 'status timed out')) + await owner.refresh() + expect(owner.read()).toMatchObject({ + transport: 'ready', + verification: 'unavailable', + status: { runtimeId: 'host-1' } + }) + await vi.advanceTimersByTimeAsync(3_000) + expect(owner.read().verification).toBe('verified') + expect(request).toHaveBeenCalledTimes(3) +}) + +it('retires a lost-socket request before explicit fallback and rejects its late result', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + const old = deferred() + request.mockReturnValueOnce(old.promise) + const waiting = owner.refresh() + owner.connectionChanged('disconnected') + expect(request.mock.calls[0][0].aborted).toBe(true) + request.mockResolvedValueOnce(success('fallback-host')) + expect((await owner.refresh()).ok).toBe(true) + expect((await waiting).ok).toBe(true) + old.resolve(success('obsolete-host')) + await vi.advanceTimersByTimeAsync(0) + expect(owner.read().status?.runtimeId).toBe('fallback-host') + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(3) +}) + +it('a reconnect transfers waiting readers to a fresh verification', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + const old = deferred() + request.mockReturnValueOnce(old.promise) + const waiting = owner.refresh() + owner.connectionChanged('disconnected') + owner.connectionChanged('ready') + expect((await waiting).ok).toBe(true) + old.resolve(success('old')) + await vi.advanceTimersByTimeAsync(0) + expect(owner.read().status?.runtimeId).toBe('host-1') +}) + +it('disconnect settles readers and prevents late results and retry resurrection', async () => { + const { owner, request, publish } = createOwner() + const old = deferred() + request.mockReturnValue(old.promise) + const waiting = owner.refresh() + owner.dispose() + expect((await waiting).ok).toBe(false) + const sequence = owner.read().sequence + old.resolve(success()) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(owner.read()).toMatchObject({ retired: true, sequence }) + expect(publish.mock.lastCall?.[0].retired).toBe(true) + expect(request).toHaveBeenCalledOnce() +}) + +it('passive reads create neither standing retries nor connection intent', async () => { + const { owner, request, verified } = createOwner() + request.mockResolvedValueOnce(runtimeHostStatusFailure('runtime_unavailable', 'offline')) + await owner.refresh({ observeOnly: true }) + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledOnce() + await owner.refresh({ observeOnly: true }) + expect(verified.mock.lastCall?.[1]).toBe(false) +}) + +it('authentication rejection blocks automatic verification until explicit reconnect', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + request.mockResolvedValueOnce(runtimeHostStatusFailure('unauthorized', 're-pair')) + await owner.refresh() + owner.connectionChanged('disconnected') + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledOnce() + expect((await owner.refresh({ reconnect: true })).ok).toBe(true) +}) + +it('blocks a rejected reconnect even without an outstanding status request', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + await owner.refresh() + owner.connectionChanged('disconnected') + owner.authenticationRejected() + expect(owner.read()).toMatchObject({ verification: 'blocked', status: { runtimeId: 'host-1' } }) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledOnce() +}) + +it('cancelling one reader leaves the shared request available to other readers', async () => { + const { owner, request } = createOwner() + const pending = deferred() + request.mockReturnValue(pending.promise) + const controller = new AbortController() + const cancelled = owner.refresh({ signal: controller.signal }) + const remaining = owner.refresh() + const rejection = expect(cancelled).rejects.toThrow('cancelled') + controller.abort(new Error('cancelled')) + await rejection + expect(request.mock.calls[0][0].aborted).toBe(false) + pending.resolve(success()) + expect((await remaining).ok).toBe(true) +}) + +it.each(['unknown', 'ready'] as const)( + 'distinguishes the caller deadline with %s transport', + async (transport) => { + const { owner, request } = createOwner() + owner.connectionChanged(transport) + request.mockReturnValue(deferred().promise) + const response = owner.refresh({ timeoutMs: 100 }) + await vi.advanceTimersByTimeAsync(100) + expect(await response).toMatchObject({ + ok: false, + error: { + message: + transport === 'ready' + ? 'Status request timed out.' + : 'Timed out waiting for the remote Orca runtime.' + } + }) + } +) diff --git a/src/shared/runtime-host-status-owner.ts b/src/shared/runtime-host-status-owner.ts new file mode 100644 index 00000000000..d9c52395c89 --- /dev/null +++ b/src/shared/runtime-host-status-owner.ts @@ -0,0 +1,274 @@ +import { + isRuntimeHostStatusBlocked, + runtimeHostStatusError, + runtimeHostStatusFailure, + type RuntimeHostStatusResponse, + type RuntimeHostStatusSnapshot +} from './runtime-host-status' + +const RETRY_DELAYS_MS = [3_000, 6_000, 12_000, 30_000, 60_000] +const REQUEST_TIMEOUT_MS = 15_000 +let publicationSequence = 0 + +type Waiter = { + resolve: (response: RuntimeHostStatusResponse) => void + cleanup: () => void +} + +type StatusOwnerOptions = { + environmentId: string + pairingRevision: number + persistent?: boolean + request: (signal: AbortSignal) => Promise + verified: (response: Extract, active: boolean) => boolean + publish: (snapshot: RuntimeHostStatusSnapshot) => void +} + +/** One verification and one retry slot, shared by all readers of this connection. */ +export class RuntimeHostStatusOwner { + private active = false + private disposed = false + private persistent: boolean + private attempt = 0 + private retry: ReturnType | null = null + private request: AbortController | null = null + private readonly waiters = new Set() + private response: RuntimeHostStatusResponse = runtimeHostStatusFailure( + 'runtime_unavailable', + 'Status has not been checked.' + ) + private snapshot: RuntimeHostStatusSnapshot + + constructor(private readonly options: StatusOwnerOptions) { + this.persistent = options.persistent ?? false + this.snapshot = { + environmentId: options.environmentId, + pairingRevision: options.pairingRevision, + sequence: ++publicationSequence, + checkedAt: 0, + status: null, + verification: 'checking', + transport: 'unknown' + } + } + + read(): RuntimeHostStatusSnapshot { + return this.snapshot + } + + activate(): void { + if (this.active || this.disposed) { + return + } + this.active = true + this.startRequest() + } + + acceptVerified(response: Extract): void { + if (this.disposed) { + return + } + this.active = true + this.retireRequest() + this.clearRetry() + this.complete(response) + } + + refresh( + options: { timeoutMs?: number; observeOnly?: true; reconnect?: true; signal?: AbortSignal } = {} + ): Promise { + if (options.signal?.aborted) { + return Promise.reject(options.signal.reason) + } + if (this.disposed) { + return Promise.resolve(this.response) + } + if (!options.observeOnly) { + this.active = true + } + if (options.reconnect) { + this.attempt = 0 + this.update({ verification: 'checking' }) + } + if (this.snapshot.verification === 'blocked') { + return Promise.resolve(this.response) + } + const result = new Promise((resolve, reject) => { + const release = (): void => { + waiter.cleanup() + this.waiters.delete(waiter) + if (!this.active && this.waiters.size === 0) { + this.retireRequest() + } + } + const abort = (): void => { + release() + reject(options.signal?.reason) + } + const timer = setTimeout(() => { + release() + resolve( + runtimeHostStatusFailure( + 'runtime_unavailable', + this.snapshot.transport === 'ready' + ? 'Status request timed out.' + : 'Timed out waiting for the remote Orca runtime.' + ) + ) + }, options.timeoutMs ?? REQUEST_TIMEOUT_MS) + const waiter: Waiter = { + resolve, + cleanup: () => { + clearTimeout(timer) + options.signal?.removeEventListener('abort', abort) + } + } + this.waiters.add(waiter) + options.signal?.addEventListener('abort', abort, { once: true }) + }) + this.startRequest() + return result + } + + connectionChanged( + transport: RuntimeHostStatusSnapshot['transport'], + remoteControl?: RuntimeHostStatusSnapshot['remoteControl'] + ): void { + if (this.disposed) { + return + } + const previous = this.snapshot.transport + this.update({ transport, ...(remoteControl !== undefined ? { remoteControl } : {}) }) + if (transport === previous) { + return + } + if (previous === 'ready') { + this.retireRequest() + this.clearRetry() + if (this.snapshot.verification !== 'blocked') { + this.update({ verification: 'unavailable' }) + } + } + if (transport === 'ready' && this.snapshot.verification !== 'blocked') { + // A pre-reconnect answer cannot verify the new socket's runtime. + this.retireRequest() + if (this.active || this.waiters.size > 0) { + this.startRequest() + } + } + } + + authenticationRejected(): void { + if (this.disposed) { + return + } + this.retireRequest() + this.clearRetry() + this.complete(runtimeHostStatusFailure('unauthorized', 'Pair this client again.')) + } + + dispose(): void { + if (this.disposed) { + return + } + this.disposed = true + this.active = false + this.retireRequest() + this.clearRetry() + this.response = runtimeHostStatusFailure( + 'runtime_manually_disconnected', + 'Runtime environment was disconnected or replaced.' + ) + this.update({ retired: true, transport: 'disconnected', verification: 'blocked' }) + this.settleWaiters() + } + + private startRequest(): void { + if (this.disposed || this.request || this.snapshot.verification === 'blocked') { + return + } + this.clearRetry() + const controller = new AbortController() + this.request = controller + if (this.snapshot.verification !== 'verified') { + this.update({ verification: 'checking' }) + } + void this.verify(controller) + } + + private async verify(controller: AbortController): Promise { + let response: RuntimeHostStatusResponse + try { + response = await this.options.request(controller.signal) + } catch (error) { + response = runtimeHostStatusError(error) + if (error instanceof TypeError || error instanceof SyntaxError) { + console.error('Runtime status verification failed:', error) + response = runtimeHostStatusFailure('invalid_runtime_response', error.message) + } + } + if (this.request !== controller || this.disposed) { + return + } + this.request = null + this.complete(response) + } + + private complete(response: RuntimeHostStatusResponse): void { + this.response = response + if (response.ok) { + this.attempt = 0 + this.update({ status: response.result, checkedAt: Date.now(), verification: 'verified' }) + this.persistent = this.options.verified(response, this.active) + } else { + this.update({ + checkedAt: Date.now(), + verification: isRuntimeHostStatusBlocked(response) ? 'blocked' : 'unavailable' + }) + this.scheduleRetry() + } + this.settleWaiters() + } + + private scheduleRetry(): void { + if ( + !this.active || + this.disposed || + this.snapshot.verification === 'blocked' || + (this.persistent && this.snapshot.transport !== 'ready') + ) { + return + } + const delay = RETRY_DELAYS_MS[Math.min(this.attempt++, RETRY_DELAYS_MS.length - 1)] + this.retry = setTimeout(() => { + this.retry = null + this.startRequest() + }, delay) + } + + private settleWaiters(): void { + for (const waiter of this.waiters) { + waiter.cleanup() + waiter.resolve(this.response) + } + this.waiters.clear() + } + + private retireRequest(): void { + const request = this.request + this.request = null + request?.abort() + } + + private clearRetry(): void { + if (this.retry) { + clearTimeout(this.retry) + } + this.retry = null + } + + private update(patch: Partial): void { + this.snapshot = { ...this.snapshot, ...patch, sequence: ++publicationSequence } + this.options.publish(this.snapshot) + } +} diff --git a/src/shared/runtime-host-status.ts b/src/shared/runtime-host-status.ts new file mode 100644 index 00000000000..4dd7ff7cc22 --- /dev/null +++ b/src/shared/runtime-host-status.ts @@ -0,0 +1,44 @@ +import type { RemoteRuntimeSharedConnectionDiagnostics } from './remote-runtime-shared-control-types' +import type { RuntimeRpcFailure, RuntimeRpcResponse } from './runtime-rpc-envelope' +import type { RuntimeStatus } from './runtime-types' + +export const RUNTIME_HOST_STATUS_CHANNEL = 'runtimeEnvironments:statusChanged' + +/** Local client state; never exchanged with the paired host. */ +export type RuntimeHostStatusSnapshot = { + environmentId: string + pairingRevision: number + sequence: number + checkedAt: number + status: RuntimeStatus | null + verification: 'checking' | 'verified' | 'unavailable' | 'blocked' + transport: 'unknown' | 'connecting' | 'ready' | 'disconnected' + remoteControl?: RemoteRuntimeSharedConnectionDiagnostics | null + retired?: true +} + +export type RuntimeHostStatusResponse = RuntimeRpcResponse + +export function runtimeHostStatusFailure(code: string, message: string): RuntimeRpcFailure { + return { id: 'status.get', ok: false, error: { code, message } } +} + +export function runtimeHostStatusError(error: unknown): RuntimeRpcFailure { + const code = + error instanceof Error && 'code' in error && typeof error.code === 'string' + ? error.code + : 'runtime_unavailable' + return runtimeHostStatusFailure(code, error instanceof Error ? error.message : String(error)) +} + +export function isRuntimeHostStatusBlocked(response: RuntimeRpcFailure): boolean { + return [ + 'unauthorized', + 'forbidden', + 'invalid_argument', + 'invalid_runtime_response', + 'protocol_version_mismatch', + 'method_not_found', + 'unsupported_method' + ].includes(response.error.code) +} diff --git a/src/shared/runtime-terminal-contracts.ts b/src/shared/runtime-terminal-contracts.ts index db1c3751ba8..ad392a2b9a0 100644 --- a/src/shared/runtime-terminal-contracts.ts +++ b/src/shared/runtime-terminal-contracts.ts @@ -330,6 +330,10 @@ export type RuntimeTerminalClose = { export type RuntimeTerminalWaitCondition = 'exit' | 'tui-idle' +// Why both spellings: the codex-* members were published by every host before the agent-neutral +// rename, so they are permanent — a client still has to read them off an older host. This build +// keeps a codex-* reason only where the matched wording is plausibly Codex's own; every matcher +// that inspects no agent publishes the agent-* spelling. export type RuntimeTerminalWaitBlockedReason = | 'codex-update-prompt' | 'codex-trust-workspace' @@ -337,6 +341,11 @@ export type RuntimeTerminalWaitBlockedReason = | 'codex-model-migration-prompt' | 'codex-hooks-review-prompt' | 'codex-interactive-prompt' + | 'agent-update-prompt' + | 'agent-trust-workspace' + | 'agent-cwd-prompt' + | 'agent-hooks-review-prompt' + | 'agent-interactive-prompt' | 'agent-approval-prompt' export type RuntimeTerminalWait = { diff --git a/src/shared/structured-agent-session-live-turn.test.ts b/src/shared/structured-agent-session-live-turn.test.ts new file mode 100644 index 00000000000..b9ce5f2c022 --- /dev/null +++ b/src/shared/structured-agent-session-live-turn.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from 'vitest' +import type { AgentJournalRenderItem } from './agent-session-journal-types' +import { isStructuredAgentSessionThinking } from './structured-agent-session-live-turn' + +function item( + itemId: string, + sequence: number, + body: AgentJournalRenderItem['body'] +): AgentJournalRenderItem { + return { itemId, sequence, revision: 1, observedAt: sequence, body } +} + +describe('isStructuredAgentSessionThinking', () => { + const turnStart = item('turn-start', 1, { + kind: 'status', + text: 'Working', + turnLifecycle: { turnId: 'turn-1', state: 'running' } + }) + const reasoning = (sequence: number): AgentJournalRenderItem => + item(`reasoning-${sequence}`, sequence, { + kind: 'message', + role: 'reasoning', + blocks: [{ type: 'text', text: 'Weighing two approaches' }] + }) + + it('is true while reasoning is the newest thing the turn produced', () => { + expect(isStructuredAgentSessionThinking([turnStart, reasoning(2)])).toBe(true) + }) + + it('is false once a tool call, a message or a diff lands after the reasoning', () => { + const after = (body: AgentJournalRenderItem['body']): boolean => + isStructuredAgentSessionThinking([turnStart, reasoning(2), item('after', 3, body)]) + expect(after({ kind: 'tool-call', name: 'shell', input: null, state: 'running' })).toBe(false) + expect( + after({ kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'Here you go' }] }) + ).toBe(false) + expect( + after({ + kind: 'diff', + path: 'src/a.ts', + patch: { head: '@@', byteLength: 2, digest: 'd', truncated: false } + }) + ).toBe(false) + }) + + it('is false when a turn produced no reasoning at all', () => { + expect(isStructuredAgentSessionThinking([turnStart])).toBe(false) + expect(isStructuredAgentSessionThinking([])).toBe(false) + }) + + it('does not read an earlier turn as this one reasoning', () => { + // The scan stops at this turn's own record, so the previous turn's reasoning + // cannot leak forward into a turn that has produced nothing yet. + const newTurn = item('turn-2-start', 2, { + kind: 'status', + text: 'Working', + turnLifecycle: { turnId: 'turn-2', state: 'running' } + }) + expect(isStructuredAgentSessionThinking([reasoning(1), newTurn])).toBe(false) + }) + + it('does not read a completed turn as reasoning during the next pending dispatch', () => { + const completedTurn = item('turn-1', 1, { + kind: 'turn', + turnId: 'turn-1', + state: 'completed' + }) + expect(isStructuredAgentSessionThinking([completedTurn, reasoning(2)])).toBe(false) + }) + + it('stops at a typed turn item, the carrier this host writes', () => { + const typedTurn = (sequence: number, turnId: string): AgentJournalRenderItem => + item(`turn-${turnId}`, sequence, { kind: 'turn', turnId, state: 'running' }) + expect(isStructuredAgentSessionThinking([typedTurn(1, 'turn-1'), reasoning(2)])).toBe(true) + expect(isStructuredAgentSessionThinking([reasoning(1), typedTurn(2, 'turn-2')])).toBe(false) + }) + + it('lets an unmarked status stay transparent to the latest reasoning state', () => { + const plan = item('plan', 3, { kind: 'status', text: 'Step 1. Read the file' }) + expect(isStructuredAgentSessionThinking([turnStart, plan])).toBe(false) + expect(isStructuredAgentSessionThinking([turnStart, reasoning(2), plan])).toBe(true) + }) + + it.each([ + { + kind: 'approval' as const, + title: 'Run the command?', + detail: null, + options: [], + resolution: { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null + } + }, + { + kind: 'question' as const, + question: 'Which path?', + options: [], + resolution: { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null + } + } + ])('stops thinking when the turn is waiting on a $kind', (prompt) => { + expect( + isStructuredAgentSessionThinking([turnStart, reasoning(2), item('prompt', 3, prompt)]) + ).toBe(false) + }) +}) diff --git a/src/shared/structured-agent-session-live-turn.ts b/src/shared/structured-agent-session-live-turn.ts new file mode 100644 index 00000000000..319a6d7d579 --- /dev/null +++ b/src/shared/structured-agent-session-live-turn.ts @@ -0,0 +1,76 @@ +// What the newest turn in a structured journal is doing right now, read off the +// tail of the item list. Every scan here stops at the turn's own record — the +// typed `turn` item, or the legacy status row that carries one — because state +// from an earlier turn is never this turn's state. + +import type { + AgentJournalRenderItem, + AgentJournalToolCallItem +} from './agent-session-journal-types' +import { readAgentJournalTurn } from './agent-session-turn-record' + +export function activeStructuredAgentSessionTurnId( + items: readonly AgentJournalRenderItem[] +): string | null { + for (let index = items.length - 1; index >= 0; index -= 1) { + const turn = readAgentJournalTurn(items[index]?.body) + if (turn) { + return turn.state === 'running' ? turn.turnId : null + } + } + return null +} + +/** + * Whether the newest thing the active turn produced is the model's own reasoning. + * + * This is what "thinking" has to mean for the indicator to be honest: the turn is reasoning + * *right now*. The older rule — "the turn has produced no renderable output yet" — reports + * thinking while the request is merely in flight, and stops reporting it the moment a tool call + * lands, which is usually when reasoning actually starts. + */ +export function isStructuredAgentSessionThinking( + items: readonly AgentJournalRenderItem[] +): boolean { + let newestContentIsReasoning: boolean | null = null + for (let index = items.length - 1; index >= 0; index -= 1) { + const body = items[index]?.body + const turn = readAgentJournalTurn(body) + if (turn) { + return turn.state === 'running' && newestContentIsReasoning === true + } + if (newestContentIsReasoning !== null) { + continue + } + if (body?.kind === 'message') { + newestContentIsReasoning = body.role === 'reasoning' + } else if ( + body?.kind === 'tool-call' || + body?.kind === 'diff' || + body?.kind === 'approval' || + body?.kind === 'question' + ) { + newestContentIsReasoning = false + } + // Plain status copy is activity chrome, not newer transcript content. + } + return false +} + +/** The tool call the newest turn is still inside, or null when nothing is running. + * An abandoned `running` call from an earlier crashed turn can never be reported + * as live work. */ +export function activeStructuredAgentSessionToolCall( + items: readonly AgentJournalRenderItem[] +): AgentJournalToolCallItem | null { + for (let index = items.length - 1; index >= 0; index -= 1) { + const body = items[index]?.body + if (readAgentJournalTurn(body)) { + return null + } + if (body?.kind === 'tool-call' && body.state === 'running') { + return body + } + } + return null +} diff --git a/src/shared/structured-agent-session-projection.ts b/src/shared/structured-agent-session-projection.ts index bd13d600a34..e0189ed7124 100644 --- a/src/shared/structured-agent-session-projection.ts +++ b/src/shared/structured-agent-session-projection.ts @@ -3,20 +3,26 @@ import { normalizeOptionalField, normalizePromptField } from './agent-status-field-normalization' -import type { - AgentJournalRenderItem, - AgentJournalSubmission, - AgentJournalToolCallItem -} from './agent-session-journal-types' +import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' import { AGENT_STATUS_TOOL_INPUT_MAX_LENGTH, AGENT_STATUS_TOOL_NAME_MAX_LENGTH } from './agent-status-types' import { describeToolInput } from './native-chat-tool-summary' -import { readAgentJournalTurn } from './agent-session-turn-record' +import { + activeStructuredAgentSessionToolCall, + activeStructuredAgentSessionTurnId +} from './structured-agent-session-live-turn' + import type { NativeChatBlock, NativeChatMessage } from './native-chat-types' import { sha256 } from './sha256' +// Re-exported so the live-turn readers' existing consumers keep one import site. +export { + activeStructuredAgentSessionToolCall, + activeStructuredAgentSessionTurnId +} from './structured-agent-session-live-turn' + function boundedText(payload: { head: string; truncated: boolean; byteLength: number }): string { return payload.truncated ? `${payload.head}\n… (${payload.byteLength} bytes)` : payload.head } @@ -159,18 +165,6 @@ export function projectStructuredItemToNativeChat( return message } -export function activeStructuredAgentSessionTurnId( - items: readonly AgentJournalRenderItem[] -): string | null { - for (let index = items.length - 1; index >= 0; index -= 1) { - const turn = readAgentJournalTurn(items[index]?.body) - if (turn) { - return turn.state === 'running' ? turn.turnId : null - } - } - return null -} - export function hasPersistedStructuredAgentSessionTurn( items: readonly AgentJournalRenderItem[] ): boolean { @@ -270,24 +264,6 @@ export function latestStructuredAgentSessionAssistantMessage( return '' } -/** The tool call the newest turn is still inside, or null when nothing is running. - * Scanning stops at the turn's own lifecycle row so an abandoned `running` call - * from an earlier crashed turn can never be reported as live work. */ -export function activeStructuredAgentSessionToolCall( - items: readonly AgentJournalRenderItem[] -): AgentJournalToolCallItem | null { - for (let index = items.length - 1; index >= 0; index -= 1) { - const body = items[index]?.body - if (readAgentJournalTurn(body)) { - return null - } - if (body?.kind === 'tool-call' && body.state === 'running') { - return body - } - } - return null -} - /** The activity fields a sidebar row shows beside the prompt, named as the agent-status * entry names them so the client can hand them straight to a row. */ export type StructuredAgentSessionStatusProjection = { diff --git a/src/shared/structured-agent-session-read-refusal.ts b/src/shared/structured-agent-session-read-refusal.ts new file mode 100644 index 00000000000..18487f12be3 --- /dev/null +++ b/src/shared/structured-agent-session-read-refusal.ts @@ -0,0 +1,45 @@ +/** + * The one refusal a structured-session READ can raise that is not a failure to read. + * + * `agentSession.history` and `agentSession.subscribe` both resolve the session through the host's + * `requireSession`, which raises this code when the host holds no session object by that id. That + * is never a transcript Orca could not read — it is a session this host has not attached YET (the + * surface's hold is what attaches one) or one it has just closed. Both windows end on their own: + * the first when the hold lands, the second when the chat tab retires. + * + * The genuinely latched lease — "Orca cannot prove the previous owner exited" — reaches the client + * through the ACQUISITION path instead, so narrowing on the code costs a read no real diagnosis. + * See `agent-session-lease-adjudication`. + */ + +/** Raised by a host that holds no attached session by that id. */ +export const AGENT_SESSION_UNATTACHED_REFUSAL_CODE = 'agent_session_ownership_unknown' + +/** + * How long a read may keep refusing this way before the pane is allowed to call it a failure. + * + * Both windows this code covers are sub-second in practice, so an unattached read that outlives + * this one is no longer transitional and the user is owed the error rather than a spinner that + * never resolves. + */ +export const AGENT_SESSION_UNATTACHED_READ_GRACE_MS = 5_000 + +/** + * Whether a read failure is that refusal. + * + * Takes both shapes the client sees: the thrown RPC error, whose `code` and `message` are each the + * bare refusal code, and the raw failure payload a stream delivers to its error callback. + */ +export function isUnattachedAgentSessionReadRefusal(error: unknown): boolean { + if (typeof error === 'string') { + return error === AGENT_SESSION_UNATTACHED_REFUSAL_CODE + } + if (typeof error !== 'object' || error === null) { + return false + } + const { code, message } = error as { code?: unknown; message?: unknown } + return ( + code === AGENT_SESSION_UNATTACHED_REFUSAL_CODE || + message === AGENT_SESSION_UNATTACHED_REFUSAL_CODE + ) +} diff --git a/src/shared/structured-agent-session-turn-timing.test.ts b/src/shared/structured-agent-session-turn-timing.test.ts index f1487f0ef42..769d30fab10 100644 --- a/src/shared/structured-agent-session-turn-timing.test.ts +++ b/src/shared/structured-agent-session-turn-timing.test.ts @@ -158,7 +158,7 @@ describe('host-settled turns override local observation', () => { { activeTurnKey: 'u1', isWorking: false, - hasCurrentTurnResponse: true, + thinking: false, settledByTurn: settled } ) diff --git a/src/shared/terminal-tab-types.ts b/src/shared/terminal-tab-types.ts index 1c455333ea9..d99472e2fde 100644 --- a/src/shared/terminal-tab-types.ts +++ b/src/shared/terminal-tab-types.ts @@ -1,6 +1,58 @@ import type { AiVaultSessionTitle } from './ai-vault-session-title' import type { TuiAgent } from './tui-agent' +/** Why recovery reasons live in the shared row type: the tab row carries the + * recovery ledger, and the ledger records which reason it last acted on. */ +export type TerminalPaneRecoveryReason = + | 'write-stalled' + | 'replay-wedged' + | 'input-undeliverable' + // The paired runtime that owns the PTY refused this write and said so on the + // wire. Distinct from 'input-undeliverable' because it skips the liveness + // probe: main's registry holds no entry for a `remote:` id, so `pty:hasPty` + // routes it to the local provider and answers a fabricated "dead". The + // rejection frame is the evidence instead — it came from the process that + // owns the PTY, over a connection that is by construction still up. + | 'input-rejected-by-host' + | 'reattach-unverifiable' + // A restore was requested for a certified-dead pipeline (reveal path). + | 'restore-blocked' + // A spawn resolved without a PTY id, so the pane is mounted with no transport + // binding. pty:data for the old id then lands in the pre-handler buffer, which + // ACKs it — main's delivery health stays green while the pane shows nothing. + | 'spawn-left-pane-unbound' + +/** Same vocabulary the direct-SSH pane retry ledger settles with + * (DirectSshPaneRetryResult), so a pane reports both through one call. */ +export type TerminalPaneRecoveryOutcome = + | 'pending' + | 'success' + | 'failed' + | 'timed-out' + | 'superseded' + +/** The tab's recovery ledger. Lives on the row — not in a module- or + * store-level map keyed by tabId — so a tab's existence and its recovery + * budget are the same object: nothing can release the budget while keeping + * the row, and closing the tab drops both together (crash b5cfc6ca). */ +export type TerminalTabRecoveryLedger = { + /** Remount timestamps inside the rolling window. Backstop, not the control. */ + attemptedAt: number[] + /** Recovery epoch. A mounted pane captures it and stale requests are refused. */ + generation: number + /** What the mounted pane observed for the attempt this ledger describes. */ + outcome: TerminalPaneRecoveryOutcome + /** When that attempt was requested. Bounds how long 'pending' may block. */ + startedAt: number + /** The reason this attempt acted on. A settled failure refuses the SAME + * reason again until a new trigger arrives. */ + reason: TerminalPaneRecoveryReason + /** `tab.generation` right after the remount. Any later bump — authority + * change, SSH pane retry, activation respawn — is a new trigger, so the + * mismatch alone supersedes this ledger. No writer required. */ + tabGeneration: number +} + // ─── Terminal Tab (legacy — used by persistence and TerminalContentSlice) ─ export type TerminalTab = { id: string @@ -53,6 +105,11 @@ export type TerminalTab = { * `sortEpoch` increments. Split layouts use a numeric count because one tab * can remount several panes. Never persisted — it is a transient handoff. */ pendingActivationSpawn?: boolean | number + /** Transient recovery ledger for this tab. Never persisted — it describes a + * mounted pane's in-flight heal, and a stale one would refuse the first + * legitimate recovery after restart. Stripped exactly like + * `pendingActivationSpawn` (buildSanitizedTabsByWorktree). */ + recovery?: TerminalTabRecoveryLedger } export type TerminalPaneSplitDirection = 'vertical' | 'horizontal' diff --git a/src/shared/terminal-wait-blocked-reason-legacy-alias.test.ts b/src/shared/terminal-wait-blocked-reason-legacy-alias.test.ts new file mode 100644 index 00000000000..2e3e7e4687c --- /dev/null +++ b/src/shared/terminal-wait-blocked-reason-legacy-alias.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from 'vitest' +import { + agentNeutralTerminalWaitBlockedReason, + describeTerminalWaitBlockedReason +} from './terminal-wait-blocked-reason-legacy-alias' +import type { RuntimeTerminalWaitBlockedReason } from './runtime-terminal-contracts' + +describe('agentNeutralTerminalWaitBlockedReason', () => { + it.each([ + ['codex-update-prompt', 'agent-update-prompt'], + ['codex-trust-workspace', 'agent-trust-workspace'], + ['codex-cwd-prompt', 'agent-cwd-prompt'], + ['codex-hooks-review-prompt', 'agent-hooks-review-prompt'], + ['codex-interactive-prompt', 'agent-interactive-prompt'] + ] as const)('renames %s published by an older host to %s', (legacy, neutral) => { + expect(agentNeutralTerminalWaitBlockedReason(legacy)).toBe(neutral) + }) + + // Why no alias: this build still publishes it, so aliasing it would rename a live reason rather + // than reinterpret an older host's -- and 'codex just got an upgrade' does name Codex. + it('leaves the agent-specific codex-model-migration-prompt alone', () => { + expect(agentNeutralTerminalWaitBlockedReason('codex-model-migration-prompt')).toBeNull() + }) + + it.each(['agent-approval-prompt', 'agent-trust-workspace', 'agent-hooks-review-prompt'] as const)( + 'reports no alias for the already-neutral %s', + (reason) => { + expect(agentNeutralTerminalWaitBlockedReason(reason)).toBeNull() + } + ) + + // Why: the reason is JSON off the wire with no enum to validate it, and an object-literal lookup + // would answer these from Object.prototype -- the CLI would then print a function to the user. + it.each(['constructor', 'toString', 'valueOf', '__proto__', 'hasOwnProperty'])( + 'reports no alias for the prototype key %s', + (reason) => { + expect( + agentNeutralTerminalWaitBlockedReason(reason as RuntimeTerminalWaitBlockedReason) + ).toBeNull() + } + ) +}) + +// Why one formatter: the CLI's wait/show output and the worker and federation "Agent startup +// blocked:" receipts all render this token, and only the CLI used to alias it. +describe('describeTerminalWaitBlockedReason', () => { + it('names the neutral spelling beside a legacy token', () => { + expect(describeTerminalWaitBlockedReason('codex-trust-workspace')).toBe( + 'codex-trust-workspace (agent-trust-workspace)' + ) + }) + + it.each(['agent-trust-workspace', 'codex-model-migration-prompt'] as const)( + 'renders %s unannotated', + (reason) => { + expect(describeTerminalWaitBlockedReason(reason)).toBe(reason) + } + ) +}) diff --git a/src/shared/terminal-wait-blocked-reason-legacy-alias.ts b/src/shared/terminal-wait-blocked-reason-legacy-alias.ts new file mode 100644 index 00000000000..50ce940e608 --- /dev/null +++ b/src/shared/terminal-wait-blocked-reason-legacy-alias.ts @@ -0,0 +1,34 @@ +import type { RuntimeTerminalWaitBlockedReason } from './runtime-terminal-contracts' + +// Why: hosts older than the agent-neutral spellings still publish the codex-* tokens for dialogs +// their matcher never proved were Codex's, so a paired client renders the neutral equivalent +// instead of showing a Codex label to a Gemini/Cursor/Antigravity user. +// Why a Map: the reason arrives off the wire unvalidated, and a plain object would answer +// 'constructor' or 'toString' from Object.prototype and print a function to the user. +// Why one-directional: nothing consumes an agent-* -> codex-* mapping. A new host's agent-* token +// reaching an old client is rendered by that client's shipped code, which this build cannot change. +const LEGACY_CODEX_REASON_ALIASES = new Map([ + ['codex-update-prompt', 'agent-update-prompt'], + ['codex-trust-workspace', 'agent-trust-workspace'], + ['codex-cwd-prompt', 'agent-cwd-prompt'], + ['codex-hooks-review-prompt', 'agent-hooks-review-prompt'], + ['codex-interactive-prompt', 'agent-interactive-prompt'] +]) + +/** Neutral spelling for a reason an older host published, or null when it is already neutral or agent-specific. */ +export function agentNeutralTerminalWaitBlockedReason( + reason: RuntimeTerminalWaitBlockedReason +): RuntimeTerminalWaitBlockedReason | null { + return LEGACY_CODEX_REASON_ALIASES.get(reason) ?? null +} + +/** + * A blocked reason as shown to a user: the token the host published, plus the neutral spelling when + * the host predates it. Why append and not replace: the raw token is what scripts parse. + */ +export function describeTerminalWaitBlockedReason( + reason: RuntimeTerminalWaitBlockedReason +): string { + const neutral = agentNeutralTerminalWaitBlockedReason(reason) + return neutral ? `${reason} (${neutral})` : reason +} diff --git a/src/shared/workspace-session-schema.ts b/src/shared/workspace-session-schema.ts index 48fe00a7f4d..0a24551381e 100644 --- a/src/shared/workspace-session-schema.ts +++ b/src/shared/workspace-session-schema.ts @@ -99,6 +99,12 @@ const terminalTabSchema = z.object({ customTitle: z.string().nullable(), color: z.string().nullable(), isPinned: z.boolean().optional(), + // Why: recovery asks the terminal row who owns the surface, so a row that + // loses viewMode on reload reads as "not chat-owned" and lets a hidden chat + // surface remount itself. Declared here so the row survives the parse, with + // the same `.catch('terminal')` degradation the unified tab uses below. + // Legacy rows that predate this stay undefined → 'terminal' in the renderer. + viewMode: z.enum(['terminal', 'chat']).catch('terminal').optional(), sortOrder: z.number(), createdAt: z.number(), generation: z.number().optional(), diff --git a/src/shared/workspace-session-terminal-schema.test.ts b/src/shared/workspace-session-terminal-schema.test.ts index 5878b70a1b9..18a930fa492 100644 --- a/src/shared/workspace-session-terminal-schema.test.ts +++ b/src/shared/workspace-session-terminal-schema.test.ts @@ -72,4 +72,53 @@ describe('parseWorkspaceSession terminal fields', () => { expect(result.value.tabsByWorktree.wt).toEqual([]) } }) + + // Why this matters beyond persistence hygiene: terminal-pane recovery asks + // the terminal ROW who owns the surface. While the row lost viewMode on load, + // a chat-owned tab read as "not chat-owned" after every restart and recovery + // would remount its hidden surface — the race #19745's guard exists to stop. + describe('terminal row viewMode', () => { + function parseRow(row: Record): Record | undefined { + const result = parseWorkspaceSession({ + activeRepoId: null, + activeWorktreeId: 'wt', + activeTabId: 'tab1', + tabsByWorktree: { + wt: [ + { + id: 'tab1', + ptyId: null, + worktreeId: 'wt', + title: 'Terminal 1', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 0, + ...row + } + ] + }, + terminalLayoutsByTabId: {} + }) + expect(result.ok).toBe(true) + return result.ok ? result.value.tabsByWorktree.wt[0] : undefined + } + + it('survives the load boundary so a restored row still reads chat-owned', () => { + expect(parseRow({ viewMode: 'chat' })?.viewMode).toBe('chat') + }) + + it('keeps an explicit terminal mode', () => { + expect(parseRow({ viewMode: 'terminal' })?.viewMode).toBe('terminal') + }) + + it('leaves a row persisted by an older build undefined rather than failing', () => { + expect(parseRow({})?.viewMode).toBeUndefined() + }) + + it('degrades an unknown mode from a newer build instead of dropping the tab', () => { + // .catch('terminal') — the safe default, never a whole-session parse failure. + expect(parseRow({ viewMode: 'holographic' })?.viewMode).toBe('terminal') + }) + }) }) diff --git a/src/shared/worktree/removal.ts b/src/shared/worktree/removal.ts index 59e5803eddb..237d73630df 100644 --- a/src/shared/worktree/removal.ts +++ b/src/shared/worktree/removal.ts @@ -22,11 +22,12 @@ export type WorktreeForceDeleteReason = // rather than scanning the whole message and letting a path spell out a verdict. export const UNSTOPPED_PTY_DETAIL_SEPARATOR = ' — ' -// Why: verification distinguishes a PTY it watched stay alive from one it could not reach, +// Why: verification distinguishes a process it watched stay alive from one it could not reach, // and the delete toast must not flatten the two — a user waiving "we could not confirm" is -// making a different decision than one killing a terminal Orca just saw running. The marker -// and its matcher stay together for the same reason the force hint does. -export const UNSTOPPED_PTY_LIVE_DETAIL_PREFIX = 'still live:' +// making a different decision than one killing something Orca just saw running. The marker +// and its matcher stay together for the same reason the force hint does. Shared by the PTY +// sweep and the structured-session sweep, which both re-observe after their stop. +export const STILL_LIVE_DETAIL_PREFIX = 'still live:' // Why (#11960): a sweep that never answers wedges removal exactly like a stop that could not // be proven, and the waiver clears both — but this error carries different words, so without @@ -54,7 +55,15 @@ export function isUnstoppedPtyRemovalError(error: string): boolean { export function isProvenLivePtyRemovalError(error: string): boolean { return ( isUnstoppedPtyRemovalError(error) && - error.includes(`${UNSTOPPED_PTY_DETAIL_SEPARATOR}${UNSTOPPED_PTY_LIVE_DETAIL_PREFIX}`) + error.includes(`${UNSTOPPED_PTY_DETAIL_SEPARATOR}${STILL_LIVE_DETAIL_PREFIX}`) + ) +} + +/** True only when the observation AFTER the close found the session still attached. */ +export function isProvenLiveStructuredSessionRemovalError(error: string): boolean { + return ( + isRunningAgentSessionRemovalError(error) && + error.includes(`${UNSTOPPED_PTY_DETAIL_SEPARATOR}${STILL_LIVE_DETAIL_PREFIX}`) ) } diff --git a/tests/e2e/helpers/relay-execution-process.ts b/tests/e2e/helpers/relay-execution-process.ts new file mode 100644 index 00000000000..c9247bffca8 --- /dev/null +++ b/tests/e2e/helpers/relay-execution-process.ts @@ -0,0 +1,125 @@ +import { randomUUID } from 'node:crypto' +import { mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises' +import path from 'node:path' +import { createInterface } from 'node:readline' +import { spawnProcess } from '../../../src/shared/child-process/run-process' + +const executionProgram = ` +const fs = require('node:fs'); +const readline = require('node:readline'); +const marker = process.argv[1]; +let sequence = 0; +readline.createInterface({ input: process.stdin }).on('line', line => { + const { id, value } = JSON.parse(line); + if (value === 'mutation-1') fs.appendFileSync('mutations.log', marker + '\\n'); + process.stdout.write(JSON.stringify({ id, pid: process.pid, cwd: fs.realpathSync('.'), + marker, sequence: ++sequence, value }) + '\\n'); +}); +` + +export async function createRelayExecutionProcess() { + await mkdir(path.join(process.cwd(), '.tmp'), { recursive: true }) + const folder = await mkdtemp(path.join(process.cwd(), '.tmp', 'relay-execution-')) + const executionCwd = await realpath(folder) + const marker = randomUUID() + const pending = new Map< + number, + { + resolve: (value: string) => void + reject: (error: Error) => void + timer: ReturnType + } + >() + let sequence = 0 + let nextId = 0 + let failure: Error | null = null + const child = spawnProcess({ + program: process.execPath, + args: ['-e', executionProgram, marker], + cwd: folder, + env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' } + }) + const fail = (error: Error) => { + failure = error + for (const item of pending.values()) { + clearTimeout(item.timer) + item.reject(error) + } + pending.clear() + } + child.on('error', fail) + child.stdin.on('error', fail) + child.stdout.on('error', fail) + child.stderr.on('error', fail) + child.stderr.resume() + const closed = new Promise((resolve) => + child.once('close', () => { + fail(new Error('execution process exited')) + resolve() + }) + ) + const lines = createInterface({ input: child.stdout }) + lines.on('line', (line) => { + try { + const output = JSON.parse(line) + const item = pending.get(output.id) + if ( + !item || + output.pid !== child.pid || + output.cwd !== executionCwd || + output.marker !== marker || + output.sequence !== sequence + 1 + ) { + throw new Error('execution ownership or output sequence changed') + } + sequence = output.sequence + clearTimeout(item.timer) + pending.delete(output.id) + item.resolve(output.value) + } catch (error) { + fail(error as Error) + } + }) + return { + pid: child.pid, + sequence: () => sequence, + execute: (value: string) => + new Promise((resolve, reject) => { + if (failure) { + reject(failure) + return + } + const id = ++nextId + const timer = setTimeout(() => fail(new Error('execution response timed out')), 5_000) + pending.set(id, { resolve, reject, timer }) + child.stdin.write(`${JSON.stringify({ id, value })}\n`) + }), + mutations: async () => { + try { + const entries = (await readFile(path.join(folder, 'mutations.log'), 'utf8')) + .trim() + .split('\n') + if (entries.some((entry) => entry !== marker)) { + throw new Error('unexpected execution artifact') + } + return entries.length + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return 0 + } + throw error + } + }, + close: async () => { + child.stdin.end() + const timer = setTimeout(() => child.kill('SIGKILL'), 5_000) + try { + await closed + } finally { + clearTimeout(timer) + lines.close() + await rm(folder, { recursive: true, force: true }) + } + } + } +} diff --git a/tests/e2e/helpers/slept-workspace-probe.ts b/tests/e2e/helpers/slept-workspace-probe.ts new file mode 100644 index 00000000000..b2b5635b526 --- /dev/null +++ b/tests/e2e/helpers/slept-workspace-probe.ts @@ -0,0 +1,133 @@ +/** + * Shared probes for GH #10205: a deliberately slept workspace must stay cold. + * Drives the shipping sleep path (sidebar context menu) and reads both the + * renderer's live PTY model and host truth. + */ +import type { Locator, Page } from '@stablyai/playwright-test' +import { expect } from '@stablyai/playwright-test' +import { ensureTerminalVisible } from './store' +import { waitForActivePanePtyId, waitForActiveTerminalManager } from './terminal' + +export type WorkspaceSample = { + livePtyCount: number + tabCount: number + tabIds: string[] + mountedTabIds: string[] + tabPtyHints: (string | null)[] +} + +export function rowLocator(page: Page, worktreeId: string): Locator { + return page + .locator( + `[data-worktree-sidebar] [role="option"][data-worktree-id=${JSON.stringify(worktreeId)}]` + ) + .first() +} + +export async function readWorkspaceSample( + page: Page, + worktreeId: string +): Promise { + return page.evaluate((id) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('window.__store is not available') + } + const tabs = state.tabsByWorktree[id] ?? [] + const tabIds = new Set(tabs.map((tab) => tab.id)) + const managers = window.__paneManagers + return { + livePtyCount: tabs.reduce( + (count, tab) => count + (state.ptyIdsByTabId[tab.id]?.length ?? 0), + 0 + ), + tabCount: tabs.length, + tabIds: tabs.map((tab) => tab.id), + mountedTabIds: managers + ? Array.from(managers.keys()).filter((tabId) => tabIds.has(tabId)) + : [], + tabPtyHints: tabs.map((tab) => tab.ptyId ?? null) + } + }, worktreeId) +} + +/** Host-side truth: a revived workspace shows a freshly created live session here. */ +export async function readHostLiveTerminalCount(page: Page, worktreeId: string): Promise { + return (await page.evaluate(async (id) => { + const result = await window.api.runtime.call({ + method: 'terminal.list', + params: { worktree: `id:${id}`, requireFreshPtyLiveness: true } + }) + if (!result.ok) { + throw new Error(result.error.message) + } + return (result.result as { totalCount: number }).totalCount + }, worktreeId)) as number +} + +/** Connect-verdict lines (REATTACH / ATTACH / FRESH SPAWN / SKIP SPAWN) for one workspace. */ +export async function readConnectDiagnostics(page: Page, worktreeId: string): Promise { + return page.evaluate((id) => { + const state = window.__store?.getState() + const target = globalThis as unknown as Record + const diag = (target.__ptyConnectDiag as string[] | undefined) ?? [] + const tabIds = new Set((state?.tabsByWorktree[id] ?? []).map((tab) => tab.id)) + // Pane ids restart at 1 per worktree, so a verdict line is attributed to the + // tab named by the most recent connect line for that same pane id. + const tabByPaneId = new Map() + const owned: string[] = [] + for (const line of diag) { + const connect = /^pane=(\d+) tab=(\S+) /.exec(line) + if (connect) { + tabByPaneId.set(connect[1], connect[2]) + if (tabIds.has(connect[2])) { + owned.push(line) + } + continue + } + const verdict = /^pane=(\d+) ->/.exec(line) + if (verdict) { + const tabId = tabByPaneId.get(verdict[1]) + if (tabId && tabIds.has(tabId)) { + owned.push(line) + } + } + } + return owned + }, worktreeId) +} + +export async function giveWorkspaceALivePty(page: Page, worktreeId: string): Promise { + await page.evaluate((id) => { + window.__store?.getState().setActiveWorktree(id) + }, worktreeId) + await ensureTerminalVisible(page) + await waitForActiveTerminalManager(page, 30_000) + return waitForActivePanePtyId(page, 30_000) +} + +/** The shipping sleep path: right-click the sidebar row, click "Sleep". */ +export async function sleepWorkspaceViaSidebar(page: Page, worktreeId: string): Promise { + const row = rowLocator(page, worktreeId) + await expect(row).toBeVisible() + await row.scrollIntoViewIfNeeded() + const scope = row.locator('[data-worktree-context-menu-scope="worktree"]').first() + const target = (await scope.count()) > 0 ? scope : row + await target.click({ button: 'right' }) + const sleepItem = page.getByRole('menuitem', { name: 'Sleep', exact: true }).first() + await expect(sleepItem).toBeVisible() + await sleepItem.click() +} + +export async function activateWorkspaceByClick(page: Page, worktreeId: string): Promise { + const row = rowLocator(page, worktreeId) + await expect(row).toBeVisible() + await row.scrollIntoViewIfNeeded() + await row.click() + await expect + .poll(() => page.evaluate(() => window.__store?.getState().activeWorktreeId ?? null), { + timeout: 10_000, + message: `sidebar click did not activate ${worktreeId}` + }) + .toBe(worktreeId) +} diff --git a/tests/e2e/paired-runtime-rejected-input-remount.unit.test.ts b/tests/e2e/paired-runtime-rejected-input-remount.unit.test.ts index fae82b45d44..3967f7934eb 100644 --- a/tests/e2e/paired-runtime-rejected-input-remount.unit.test.ts +++ b/tests/e2e/paired-runtime-rejected-input-remount.unit.test.ts @@ -27,7 +27,11 @@ type StoreState = Record let mockStoreState: StoreState let storeSubscribers: ((state: StoreState) => void)[] = [] -const remountTerminalTabForRecovery = vi.fn<(tabId: string) => boolean>(() => true) +/** The store action reports admission now, not a bare boolean. */ +const REMOUNTED = { remounted: true as const, generation: 1 } +const remountTerminalTabForRecovery = vi.fn<(tabId: string, request?: unknown) => typeof REMOUNTED>( + () => REMOUNTED +) vi.mock('@/store', () => ({ useAppStore: { @@ -278,7 +282,7 @@ describe('host-rejected paired-runtime input reaches a pane remount', () => { vi.resetModules() vi.clearAllMocks() storeSubscribers = [] - remountTerminalTabForRecovery.mockReturnValue(true) + remountTerminalTabForRecovery.mockReturnValue(REMOUNTED) mockStoreState = { activeWorktreeId: 'wt-1', activeWorkspaceExecutionHostId: `runtime:${ENVIRONMENT_ID}`, @@ -417,7 +421,12 @@ describe('host-rejected paired-runtime input reaches a pane remount', () => { // Hop 1: the host turned the refusal into the negotiated frame. await vi.waitFor(() => expect(hostOpcodes).toContain(TerminalStreamOpcode.WriteUnavailable)) // Hop 2 (the one that was missing): it survives pane recovery as a remount. - await vi.waitFor(() => expect(remountTerminalTabForRecovery).toHaveBeenCalledWith('tab-1')) + await vi.waitFor(() => + expect(remountTerminalTabForRecovery).toHaveBeenCalledWith( + 'tab-1', + expect.objectContaining({ reason: 'input-rejected-by-host', trigger: 'automatic' }) + ) + ) binding.dispose() _resetTerminalPaneRecoveryForTests() diff --git a/tests/e2e/relay-region-compatibility.unit.test.ts b/tests/e2e/relay-region-compatibility.unit.test.ts new file mode 100644 index 00000000000..9019498ada7 --- /dev/null +++ b/tests/e2e/relay-region-compatibility.unit.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it, vi } from 'vitest' +import { + AssignmentRequestSchema as BaselineRequest, + AssignmentResponseSchema as BaselineResponse +} from '../../cloud/apps/relay/src/test-fixtures/relay-contract-baseline/director-messages' +import { + DrainSchema as BaselineDrain, + HostHelloSchema as BaselineHello +} from '../../cloud/apps/relay/src/test-fixtures/relay-contract-baseline/control-messages' +import { AssignmentRequestSchema } from '../../cloud/packages/relay-contract/src/director-messages' +import { HostHelloSchema } from '../../cloud/packages/relay-contract/src/control-messages' +import { requestRelayAssignment } from '../../src/main/runtime/relay/relay-http-client' +import { RelayAssignRateGate } from '../../src/main/runtime/relay/relay-assign-rate-gate' + +const assignment = { + v: 1, + cellUrl: 'https://asia.example.test', + assignmentEpoch: 3, + lease: 'synthetic-assignment' +} +const window = { + generation: 1, + assignmentEpoch: 3, + incumbentRegion: 'asia-east2', + expiresAt: 100_000_000, + policyVersion: 1 +} +function request(fetch: typeof globalThis.fetch) { + return requestRelayAssignment({ + directorUrl: 'https://director.example.test', + relayHostId: 'abcdefghijklmnop', + relayToken: 'synthetic-authorization', + preferredRegion: 'asia-east2', + reconnect: true, + regionCorrection: { v: 1, action: 'issue-window' }, + fetch, + assignRateGate: new RelayAssignRateGate() + }) +} + +describe('relay correction mixed-version wire contracts', () => { + it('new desktop falls back against the actual pinned old director parser', async () => { + const bodies: unknown[] = [] + const fetch = vi.fn(async (_url, init) => { + const body: unknown = JSON.parse(String(init?.body)) + bodies.push(body) + return BaselineRequest.safeParse(body).success + ? Response.json(BaselineResponse.parse(assignment)) + : new Response(null, { status: 400 }) + }) + expect(await request(fetch)).toEqual(assignment) + expect(bodies).toHaveLength(2) + expect(AssignmentRequestSchema.safeParse(bodies[0]).success).toBe(true) + expect(BaselineRequest.safeParse(bodies[0]).success).toBe(false) + expect(bodies[1]).toEqual({ + v: 1, + relayHostId: 'abcdefghijklmnop', + preferredRegion: 'asia-east2', + reconnect: true + }) + }) + + it('the old desktop assignment shape remains accepted by the new director', () => { + const request = BaselineRequest.parse({ v: 1, relayHostId: 'abcdefghijklmnop' }) + expect(AssignmentRequestSchema.parse(request)).toEqual(request) + expect(BaselineResponse.parse(assignment)).toEqual(assignment) + }) + + it('the negotiated capability requires no change to the strict old host hello', () => { + const hello = { + v: 1, + relayHostId: 'abcdefghijklmnop', + assignmentEpoch: 3, + hostPublicKeyB64: Buffer.alloc(32).toString('base64'), + appVersion: 'test' + } + expect(BaselineHello.parse(HostHelloSchema.parse(hello))).toEqual(hello) + expect(BaselineHello.safeParse({ ...hello, idleRegionalRehome: true }).success).toBe(false) + }) + + it('the idle cutover uses a drain frame understood by the pinned old desktop', () => { + const drain = { recovery: 'resolve-director', graceMs: 0 } + expect(BaselineDrain.parse(drain)).toEqual(drain) + }) + + it.each([ + { v: 1, window: { ...window, policyVersion: 2 } }, + { v: 2, window }, + { v: 1, window: { ...window, expiresAt: -1 } }, + { v: 1, window: { ...window, unexpectedField: true } } + ])( + 'defers unsupported or malformed optional correction without losing placement: %j', + async (regionCorrection) => { + const result = await request(async () => Response.json({ ...assignment, regionCorrection })) + expect(result).toMatchObject(assignment) + expect(result.regionCorrection).toBeUndefined() + } + ) + + it('still accepts supported correction metadata', async () => { + const regionCorrection = { v: 1, window } + expect(await request(async () => Response.json({ ...assignment, regionCorrection }))).toEqual({ + ...assignment, + regionCorrection + }) + }) + + it.each([ + { cellUrl: 'http://untrusted.example.test' }, + { assignmentEpoch: -1 }, + { lease: '' }, + { unexpectedField: true } + ])('keeps the core assignment strict: %j', async (invalid) => { + await expect(request(async () => Response.json({ ...assignment, ...invalid }))).rejects.toThrow( + 'relay_assignment_failed_502' + ) + }) +}) diff --git a/tests/e2e/relay-region-correction.unit.test.ts b/tests/e2e/relay-region-correction.unit.test.ts new file mode 100644 index 00000000000..614e5c08c1d --- /dev/null +++ b/tests/e2e/relay-region-correction.unit.test.ts @@ -0,0 +1,519 @@ +import { createHash, randomUUID } from 'node:crypto' +import { once } from 'node:events' +import { afterEach, describe, expect, it, vi } from 'vitest' +import nacl from 'tweetnacl' +import WebSocket from 'ws' +import type { IdleRegionalRehomeRequest } from '../../cloud/packages/relay-contract/src/idle-regional-rehome' +import { + openInMemoryRelayDatabase, + readRelayDatabasePoolPressure +} from '../../cloud/apps/relay/src/database' +import { createRelayServer } from '../../cloud/apps/relay/src/relay-server' +import type { RelayConfig } from '../../cloud/apps/relay/src/config' +import type * as AdminTokenVerifier from '../../cloud/apps/relay/src/admin-token-verifier' +import { RelayOriginPool } from '../../src/main/runtime/relay/relay-origin-pool' +import { RELAY_HOST_CAPABILITY_HEADERS } from '../../src/main/runtime/relay/relay-control-protocol' +import type { MobileSocketTransport } from '../../src/main/runtime/rpc/mobile-socket-wiring' +import { createRelayExecutionProcess } from './helpers/relay-execution-process' + +vi.mock('../../cloud/apps/relay/src/relay-token-verifier', () => ({ + createRelayTokenVerifier: () => async (hostId: string) => ({ + sub: 'transport-test-user', + prof: 'profile-1', + org: 'org-1', + relayHostId: hostId, + purpose: 'host-control', + exp: 4_102_444_800 + }), + readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null +})) + +vi.mock('../../cloud/apps/relay/src/admin-token-verifier', async (importOriginal) => ({ + ...(await importOriginal()), + createRegionalRehomeTokenVerifier: () => async (token: string) => token === 'test-director-token' +})) + +const cleanups: (() => Promise)[] = [] +afterEach(async () => { + const failures: unknown[] = [] + for (const cleanup of cleanups.splice(0).toReversed()) { + try { + await cleanup() + } catch (error) { + failures.push(error) + } + } + vi.restoreAllMocks() + if (failures.length > 0) { + throw new AggregateError(failures, 'relay topology cleanup failed') + } +}) + +async function topology() { + const execution = await createRelayExecutionProcess() + cleanups.push(() => execution.close()) + let clock = Date.now() + vi.spyOn(Date, 'now').mockImplementation(() => clock) + const database = await openInMemoryRelayDatabase() + cleanups.push(() => database.close()) + const keypair = nacl.box.keyPair() + const hostId = createHash('sha256').update(keypair.publicKey).digest('base64url').slice(0, 16) + const identity = { userId: 'transport-test-user', relayHostId: hostId } + const cells = [ + { + id: 'transport-us', + url: 'https://transport-us.example.test', + region: 'us-central1' as const, + capacityRequests: 100 + }, + { + id: 'transport-asia', + url: 'https://transport-asia.example.test', + region: 'asia-east2' as const, + capacityRequests: 100 + } + ] + const incarnations = [ + '11111111-1111-4111-8111-111111111111', + '22222222-2222-4222-8222-222222222222' + ] + const endpoints = new Map() + const sockets = new Set() + const servers = cells.map((cell, index) => + createRelayServer( + { + port: 0, + publicUrl: cell.url, + cellUrl: cell.url, + role: 'cell', + cellId: cell.id, + region: cell.region, + cells, + dataDir: '', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + adminJwksUrl: 'https://auth.example.test/jwks', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new Uint8Array(32), + adminAudience: 'https://director.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + rehomeAudience: 'https://director.example.test/v1/admin/host-drain', + rehomeDirectorServiceAccount: 'director@example.test', + databasePoolMax: 1, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + regionCorrectionCohortPercent: 100 + } as RelayConfig, + database, + { now: () => clock, random: () => 0.5, cellIncarnation: incarnations[index] } + ) + ) + cleanups.push(async () => { + for (const socket of sockets) { + socket.terminate() + } + for (const relay of servers) { + relay.sessions.drain(0) + await new Promise((resolve) => relay.server.close(() => resolve())) + } + }) + const source = servers[0]! + const target = servers[1]! + await source.assignments.inspectRegionalRehomeControl() + clock += 86_400_000 + await source.assignments.applyRegionalRehomeControl({ + expectedGeneration: 0, + enabled: true, + notBefore: clock, + ratePerMinute: 10, + preferenceMaxAgeMs: 86_400_000, + hostCooldownMs: 604_800_000, + drainGraceMs: 60_000 + }) + await source.assignments.reconcileCells(cells) + const startedAt = clock - 1_000 + const safety = () => ({ + observedAt: clock, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + }) + const heartbeat = async () => { + for (const [index, cell] of cells.entries()) { + const relay = servers[index]! + relay.observability.flush({ + ...relay.runtimeCounts(), + ...readRelayDatabasePoolPressure(database) + }) + await source.assignments.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + region: cell.region, + cellIncarnation: incarnations[index]!, + startedAt, + ready: true, + observedRequests: 0 + }) + await source.assignments.recordCellRegionalRehomeStatus({ + cellId: cell.id, + cellIncarnation: incarnations[index]!, + regionalRehomeProtocol: 3, + safety: { + observedAt: clock, + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + }) + } + } + await heartbeat() + for (const [index, relay] of servers.entries()) { + relay.server.listen(0, '127.0.0.1') + await once(relay.server, 'listening') + const address = relay.server.address() + if (!address || typeof address === 'string') { + throw new Error('missing local address') + } + endpoints.set(new URL(cells[index]!.url).host, `ws://127.0.0.1:${address.port}`) + } + const connect = (url: string, headers?: Record) => { + const parsed = new URL(url) + const socket = new WebSocket(`${endpoints.get(parsed.host)}${parsed.pathname}`, { headers }) + sockets.add(socket) + return socket + } + let failCorroboration = 0 + let pauseCorroboration = false + let corroborationFailures = 0 + let rejectTargetControls = false + let targetControlFailures = 0 + const executionErrors: unknown[] = [] + let delayedReply: (() => void) | null = null + const received: string[] = [] + const pool = new RelayOriginPool({ + directorUrl: 'https://director.example.test', + relayHostId: hostId, + identity: { userId: identity.userId, profileId: 'profile-1', organizationId: 'org-1' }, + keypair: { ...keypair, publicKeyB64: Buffer.from(keypair.publicKey).toString('base64') }, + appVersion: 'transport-test', + isCurrent: () => true, + onStatus: () => {}, + now: () => clock, + mobileSocketWiring: { + attachTransport: (transport: MobileSocketTransport) => { + transport.onMessage((raw, reply) => { + const value = raw.toString() + received.push(value) + void execution + .execute(value) + .then((output) => { + if (output === 'mutation-1') { + delayedReply = () => reply('mutation-1-ack') + } else { + reply(`host:${output}`) + } + }) + .catch((error) => executionErrors.push(error)) + }) + return () => {} + } + } as never, + createControlSocket: (url, token) => { + if (rejectTargetControls && new URL(url).host === new URL(cells[1]!.url).host) { + targetControlFailures++ + throw new Error('simulated_target_unavailable') + } + const socket = connect(url, { + authorization: `Bearer ${token}`, + ...RELAY_HOST_CAPABILITY_HEADERS + }) + if (process.env.ORCA_RELAY_TRANSPORT_DIAGNOSTICS === '1') { + const cell = new URL(url).host + console.info('transport-control-created', { + cell, + stack: new Error('transport control created').stack + }) + socket.on('message', (raw) => { + const message = JSON.parse(raw.toString()) + if (['region-restored', 'host-hello-ack', 'drain'].includes(message.type)) { + console.info('transport-control-message', { + cell, + type: message.type, + assignmentEpoch: message.assignmentEpoch, + generation: message.generation + }) + } + }) + socket.on('close', (code) => console.info('transport-control-close', { cell, code })) + } + return socket + }, + createDataSocket: (url) => connect(url), + fetch: (async () => { + if (failCorroboration > 0 || pauseCorroboration) { + failCorroboration = Math.max(0, failCorroboration - 1) + corroborationFailures++ + return Response.json({ error: 'temporary_director_failure' }, { status: 503 }) + } + const assignment = await source.assignments.resolve(identity) + if (!assignment) { + return Response.json({ error: 'assignment_not_found' }, { status: 409 }) + } + return Response.json({ + v: 1, + cellUrl: assignment.cellUrl, + assignmentEpoch: assignment.assignmentEpoch, + lease: 'synthetic-assignment-lease' + }) + }) as typeof fetch + }) + cleanups.push(async () => { + pool.closeNow() + }) + const assignment = await source.assignments.assign(identity, 'us-central1') + await pool.openInitial( + { + v: 1, + cellUrl: assignment.cellUrl, + assignmentEpoch: assignment.assignmentEpoch, + lease: 'synthetic-assignment-lease' + }, + hostId + ) + const attachPhone = async (cellIndex: number, device: string) => { + const invite = await source.store.createInvite(identity, device) + const socket = connect(`${cells[cellIndex]!.url}/v1/connect/${hostId}`) + await once(socket, 'open') + const hello = once(socket, 'message') + socket.send( + JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) + ) + const [raw] = await hello + expect(JSON.parse(raw.toString())).toMatchObject({ type: 'relay-hello', ok: true }) + return socket + } + let candidate: (IdleRegionalRehomeRequest & { sourceCellUrl: string }) | undefined + const prepareMove = async () => { + const issued = await source.assignments.exchangeRegionCorrection( + identity, + { v: 1, action: 'issue-window' }, + assignment.assignmentEpoch + ) + await source.assignments.exchangeRegionCorrection( + identity, + { + v: 1, + action: 'report', + generation: issued.window!.generation, + assignmentEpoch: assignment.assignmentEpoch, + policyVersion: 1, + outcome: 'conclusive', + measurements: { 'us-central1': 180, 'asia-east2': 40 } + }, + assignment.assignmentEpoch + ) + candidate = (await source.assignments.selectIdleRegionalRehomeCandidates(safety()))[0] + expect(candidate).toBeDefined() + return candidate! + } + const move = async () => { + if (!candidate) { + await prepareMove() + } + const { sourceCellUrl, ...request } = candidate! + const address = endpoints.get(new URL(sourceCellUrl).host)!.replace('ws:', 'http:') + const response = await fetch(`${address}/v1/admin/host-idle-rehome`, { + method: 'POST', + headers: { authorization: 'Bearer test-director-token', 'content-type': 'application/json' }, + body: JSON.stringify({ ...request, cohortPercent: 100, directorSafety: safety() }) + }) + const body = (await response.json()) as { v: number; outcome: string } + expect(response.status, JSON.stringify(body)).toBe(200) + return { outcome: body.outcome } + } + return { + source, + target, + pool, + identity, + database, + cells, + attachPhone, + connectDevice: () => connect(`${cells[0]!.url}/v1/connect/${hostId}`), + move, + prepareMove, + heartbeat, + now: () => clock, + advance: (ms: number) => { + clock += ms + }, + received, + failNextCorroboration: () => { + failCorroboration = 1 + }, + pauseCorroboration: (paused: boolean) => { + pauseCorroboration = paused + }, + corroborationFailures: () => corroborationFailures, + targetControlFailures: () => targetControlFailures, + failTarget: () => { + rejectTargetControls = true + const session = target.sessions.get(identity) + if (session?.socket) { + session.socket.terminate() + } + }, + execution, + executionErrors, + mutations: () => execution.mutations(), + reply: () => { + if (!delayedReply) { + throw new Error('no delayed mutation') + } + delayedReply() + } + } +} + +async function echo(socket: WebSocket, value: string) { + const marker = `${value}:${randomUUID()}` + const response = once(socket, 'message') + socket.send(marker) + const [raw] = await response + expect(raw.toString()).toBe(`host:${marker}`) +} + +describe('idle region correction across real relay and desktop WebSockets', () => { + it('releases the empty source and recovers normally when the target never registers', async () => { + const context = await topology() + await context.prepareMove() + context.failTarget() + expect(await context.move()).toEqual({ outcome: 'committed' }) + await expect.poll(() => context.source.sessions.get(context.identity)).toBeNull() + await expect + .poll(async () => + context.database.query( + `SELECT activity_id FROM relay_assignment_activity_leases + WHERE user_id = ? AND relay_host_id = ? AND cell_id = ?`, + [context.identity.userId, context.identity.relayHostId, context.cells[0]!.id] + ) + ) + .toEqual([]) + await expect.poll(context.targetControlFailures).toBeGreaterThan(0) + context.advance(15 * 60_000 + 1) + await context.heartbeat() + expect(await context.source.assignments.abortExpiredEvacuations()).toBe(1) + expect(await context.source.assignments.resolve(context.identity)).toMatchObject({ + cellId: context.cells[0]!.id, + assignmentEpoch: 3 + }) + await expect + .poll(() => context.pool.activeAssignment?.cellUrl, { timeout: 15_000 }) + .toBe(context.cells[0]!.url) + await expect + .poll(() => context.source.sessions.get(context.identity)?.state, { timeout: 15_000 }) + .toBe('active') + const returning = await context.attachPhone(0, 'phone-after-target-failure') + await echo(returning, 'after-target-failure') + expect(await context.mutations()).toBe(0) + expect(context.executionErrors).toEqual([]) + }, 30_000) + + it('rejects an arrival during cutover and restores admissions after a definite failed commit', async () => { + const context = await topology() + await context.prepareMove() + const original = context.source.sessions.get(context.identity)! + let entered!: () => void + let release!: () => void + const committing = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + release = resolve + }) + vi.spyOn(context.source.assignments, 'commitIdleRegionalRehome').mockImplementationOnce( + async () => { + entered() + await gate + throw new Error('simulated_database_unavailable_before_commit') + } + ) + const move = context.move() + await committing + try { + const invite = await context.source.store.createInvite(context.identity, 'racing-phone') + const arriving = context.connectDevice() + const rejected = once(arriving, 'close') + await once(arriving, 'open') + arriving.send( + JSON.stringify({ + type: 'relay-auth', + v: 1, + mode: 'connect', + credential: invite.inviteToken + }) + ) + expect((await rejected)[0]).toBe(4409) + expect(context.source.sessions.get(context.identity)).toBe(original) + } finally { + release() + await move + } + expect(await move).toEqual({ outcome: 'deferred' }) + expect(context.source.sessions.get(context.identity)).toBe(original) + const returning = await context.attachPhone(0, 'retrying-phone') + await echo(returning, 'after-definite-abort') + expect(await context.mutations()).toBe(0) + expect(context.executionErrors).toEqual([]) + }, 30_000) + + it('defers for either connected device, then moves after both disconnect without replaying work', async () => { + const context = await topology() + const phone = await context.attachPhone(0, 'phone') + const tablet = await context.attachPhone(0, 'tablet') + const sourceSession = context.source.sessions.get(context.identity)! + await echo(phone, 'before-cutover') + phone.send('mutation-1') + await expect.poll(context.mutations).toBe(1) + expect(await context.move()).toEqual({ outcome: 'busy' }) + expect(context.source.sessions.get(context.identity)).toBe(sourceSession) + expect((await context.source.assignments.resolve(context.identity))?.cellId).toBe( + context.cells[0]!.id + ) + const acknowledged = once(phone, 'message') + context.reply() + expect((await acknowledged)[0].toString()).toBe('mutation-1-ack') + const phoneClosed = once(phone, 'close') + phone.close() + await phoneClosed + await expect.poll(() => sourceSession.activeSplices.size).toBe(1) + expect(await context.move()).toEqual({ outcome: 'busy' }) + await echo(tablet, 'quiet-tablet-still-connected') + const tabletClosed = once(tablet, 'close') + tablet.close() + await tabletClosed + await expect.poll(() => sourceSession.activeSplices.size).toBe(0) + expect(await context.move()).toEqual({ outcome: 'committed' }) + await expect + .poll(() => context.pool.activeAssignment?.cellUrl, { timeout: 15_000 }) + .toBe(context.cells[1]!.url) + await expect.poll(() => context.source.sessions.get(context.identity)).toBeNull() + const returning = await context.attachPhone(1, 'returning-phone') + await echo(returning, 'after-idle-cutover') + expect(await context.mutations()).toBe(1) + expect(context.executionErrors).toEqual([]) + expect(context.execution.sequence()).toBe(4) + }, 30_000) +}) diff --git a/tests/e2e/runtime-host-status-recovery.spec.ts b/tests/e2e/runtime-host-status-recovery.spec.ts new file mode 100644 index 00000000000..09707606da1 --- /dev/null +++ b/tests/e2e/runtime-host-status-recovery.spec.ts @@ -0,0 +1,221 @@ +import { createConnection, createServer, type Socket, type AddressInfo } from 'node:net' +import type { Page } from '@stablyai/playwright-test' +import { decodePairingOffer, encodePairingOffer } from '../../src/shared/pairing' +import { expect, test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient, + launchPairedWebClient, + type RuntimeDesktopPairingOffer +} from './helpers/paired-electron-client' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' + +async function interruptibleHost(offer: RuntimeDesktopPairingOffer) { + const pairing = decodePairingOffer(offer.pairingUrl) + const endpoint = new URL(pairing.endpoint) + const sockets = new Set() + let online = true + const server = createServer((client) => { + if (!online) { + client.destroy() + return + } + const host = createConnection({ host: endpoint.hostname, port: Number(endpoint.port) }) + for (const socket of [client, host]) { + sockets.add(socket) + socket.on('error', () => { + client.destroy() + host.destroy() + }) + socket.on('close', () => { + sockets.delete(socket) + client.destroy() + host.destroy() + }) + } + client.pipe(host).pipe(client) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() as AddressInfo + const pairingUrl = encodePairingOffer({ ...pairing, endpoint: `ws://127.0.0.1:${address.port}` }) + let webClientUrl: string | undefined + if (offer.webClientUrl) { + const url = new URL(offer.webClientUrl) + url.search = '' + url.hash = new URLSearchParams({ pairing: pairingUrl }).toString() + webClientUrl = url.href + } + return { + offer: { pairingUrl, webClientUrl }, + setOnline(value: boolean) { + online = value + if (!online) { + sockets.forEach((socket) => socket.destroy()) + } + }, + async close() { + sockets.forEach((socket) => socket.destroy()) + await new Promise((resolve) => server.close(() => resolve())) + } + } +} + +async function statusEvidence(page: Page, environmentId?: string) { + return page.evaluate((id) => { + const entries = window.__store?.getState().runtimeStatusByEnvironmentId + const entry = id ? entries?.get(id) : entries?.values().next().value + return entry?.snapshot + ? { + verification: entry.snapshot.verification, + transport: entry.snapshot.transport, + runtimeId: entry.status?.runtimeId, + sequence: entry.snapshot.sequence + } + : null + }, environmentId) +} + +async function expectWorkspaceHostAppearance( + page: Page, + disconnected: boolean, + hostLabel?: string +) { + const cards = page.locator('[data-worktree-card-surface="true"]') + const card = ( + hostLabel ? cards.filter({ has: page.getByText(hostLabel, { exact: true }) }) : cards + ).first() + await expect(card).toBeVisible() + await expect(card).toHaveCSS('opacity', disconnected ? '0.6' : '1') + const icon = card.locator(disconnected ? 'svg.lucide-server-off' : 'svg.lucide-server').first() + await expect(icon).toBeVisible() + await expect( + card.locator(disconnected ? 'svg.lucide-server' : 'svg.lucide-server-off') + ).toHaveCount(0) + await expect(icon).toHaveClass(disconnected ? /text-destructive/ : /text-muted-foreground/) + await icon.hover() + await expect( + page.getByRole('tooltip', { name: disconnected ? /disconnected/i : /Project on/ }) + ).toBeVisible() + await page.mouse.move(900, 600) +} + +for (const topology of ['desktop', 'headless'] as const) { + test(`connection-owned status recovers with a ${topology} host and independent viewers`, async ({ + electronApp, + orcaPage: page, + testRepoPath + }, testInfo) => { + test.setTimeout(180_000) + let headless: Awaited> | null = null + let proxy: Awaited> | undefined + let client: Awaited> | undefined + let browser: Awaited> | undefined + try { + headless = + topology === 'headless' + ? await launchHeadlessPairedRuntimeHost({ pinnedServePort: true }) + : null + const offer = headless?.offer ?? (await createRuntimeDesktopPairingOffer(page)) + await (headless + ? headless.client.call('repo.add', { path: testRepoPath }) + : page.evaluate(async (path) => { + await window.api.repos.add({ path }) + await window.__store?.getState().fetchRepos() + }, testRepoPath)) + proxy = await interruptibleHost(offer) + client = await launchPairedElectronClient(offer, testInfo, 'Direct host') + proxy.setOnline(false) + const offlineId = await client.page.evaluate(async (pairingCode) => { + const { environment } = await window.api.runtimeEnvironments.addFromPairingCode({ + name: 'Recovering host', + pairingCode + }) + const store = window.__store!.getState() + store.setRuntimeEnvironments(await window.api.runtimeEnvironments.list()) + await store.refreshRuntimeEnvironmentStatus(environment.id, 1_000) + return environment.id + }, proxy.offer.pairingUrl) + await expect + .poll(() => statusEvidence(client!.page, offlineId)) + .toMatchObject({ verification: 'unavailable' }) + expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({ + verification: 'verified' + }) + proxy.setOnline(true) + await expect + .poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + const initial = await statusEvidence(client!.page, offlineId) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expect(client.page.getByText('Recovering host', { exact: true }).first()).toBeVisible() + await client.page.screenshot({ path: testInfo.outputPath(`${topology}-recovered.png`) }) + browser = await launchPairedWebClient(electronApp, proxy.offer) + await expect + .poll(() => statusEvidence(browser!.page), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + await expectWorkspaceHostAppearance(browser.page, false) + proxy.setOnline(false) + await expect + .poll(() => statusEvidence(client!.page, offlineId)) + .toMatchObject({ transport: 'disconnected' }) + await expect + .poll(() => statusEvidence(browser!.page), { timeout: 30_000 }) + .toMatchObject({ transport: 'disconnected' }) + expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({ + verification: 'verified', + transport: 'ready' + }) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expectWorkspaceHostAppearance(client.page, false, 'Direct host') + await expectWorkspaceHostAppearance(browser.page, false) + await client.page.screenshot({ + path: testInfo.outputPath(`${topology}-sidebar-reconnecting.png`) + }) + await browser.page.screenshot({ + path: testInfo.outputPath(`${topology}-browser-reconnecting.png`) + }) + proxy.setOnline(true) + await expect + .poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + await expect + .poll(() => statusEvidence(browser!.page), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + expect((await statusEvidence(client!.page, offlineId))!.sequence).toBeGreaterThan( + initial!.sequence + ) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expectWorkspaceHostAppearance(browser.page, false) + await browser.page.screenshot({ + path: testInfo.outputPath(`${topology}-browser-recovered.png`) + }) + await client.page.evaluate(async (selector) => { + await window.api.runtimeEnvironments.disconnect({ selector }) + }, offlineId) + await expect + .poll(() => statusEvidence(client!.page, offlineId)) + .toMatchObject({ verification: 'blocked', transport: 'disconnected' }) + await expectWorkspaceHostAppearance(client.page, true, 'Recovering host') + await expectWorkspaceHostAppearance(client.page, false, 'Direct host') + await client.page.screenshot({ + path: testInfo.outputPath(`${topology}-sidebar-disconnected.png`) + }) + await client.page.evaluate(async (selector) => { + await window.api.runtimeEnvironments.connect({ selector }) + }, offlineId) + await expect + .poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expectWorkspaceHostAppearance(browser.page, false) + await client.page.screenshot({ + path: testInfo.outputPath(`${topology}-sidebar-restored.png`) + }) + } finally { + await browser?.dispose() + await client?.dispose() + await proxy?.close() + await headless?.dispose() + } + }) +} diff --git a/tests/e2e/slept-workspace-remount-wake.spec.ts b/tests/e2e/slept-workspace-remount-wake.spec.ts new file mode 100644 index 00000000000..f820a7c7f69 --- /dev/null +++ b/tests/e2e/slept-workspace-remount-wake.spec.ts @@ -0,0 +1,87 @@ +/** + * GH #10205: a manual sleep keeps the tab's session id as a wake hint, so a later + * remount of its still-mounted pane reattaches that dead id and the daemon spawns + * a fresh shell. Production parking timings are deliberate: a shrunk park delay + * unmounts the slept panes and hides the behavior. + */ +import type { Page } from '@stablyai/playwright-test' +import { expect, test } from './helpers/orca-app' +import { getAllWorktreeIds, waitForSessionReady } from './helpers/store' +import { + activateWorkspaceByClick, + giveWorkspaceALivePty, + readConnectDiagnostics, + readHostLiveTerminalCount, + readWorkspaceSample, + sleepWorkspaceViaSidebar +} from './helpers/slept-workspace-probe' + +const OBSERVATION_MS = 8_000 +const SAMPLE_INTERVAL_MS = 200 + +async function assertStaysCold(page: Page, worktreeId: string): Promise { + let peakLivePty = 0 + let peakTabs = 0 + const deadline = Date.now() + OBSERVATION_MS + while (Date.now() < deadline) { + const sample = await readWorkspaceSample(page, worktreeId) + peakLivePty = Math.max(peakLivePty, sample.livePtyCount) + peakTabs = Math.max(peakTabs, sample.tabCount) + await page.waitForTimeout(SAMPLE_INTERVAL_MS) + } + const hostLive = await readHostLiveTerminalCount(page, worktreeId) + const diag = await readConnectDiagnostics(page, worktreeId) + console.error(`[#10205] ${JSON.stringify({ peakLivePty, peakTabs, hostLive, diag })}`) + expect(peakLivePty, 'slept workspace grew a live PTY').toBe(0) + expect(peakTabs, 'slept workspace grew a tab').toBe(1) + expect(hostLive, 'host created a session for the slept workspace').toBe(0) + // Why: proves the gate held rather than the pane having quietly unmounted. + expect(diag.at(-1), 'remounted pane did not wait for the wake').toContain('WAIT FOR WAKE') +} + +test('remounting a slept hidden pane does not respawn its PTY', async ({ orcaPage }) => { + await waitForSessionReady(orcaPage) + const [slept, other] = await getAllWorktreeIds(orcaPage) + expect(other, 'seeded repo must expose two worktrees').toBeTruthy() + await giveWorkspaceALivePty(orcaPage, slept) + await giveWorkspaceALivePty(orcaPage, other) + await activateWorkspaceByClick(orcaPage, slept) + expect((await readWorkspaceSample(orcaPage, slept)).livePtyCount).toBeGreaterThan(0) + + await sleepWorkspaceViaSidebar(orcaPage, slept) + await expect + .poll(async () => (await readWorkspaceSample(orcaPage, slept)).livePtyCount, { + timeout: 20_000, + message: 'sleep did not release the workspace PTYs' + }) + .toBe(0) + await activateWorkspaceByClick(orcaPage, other) + + const sample = await readWorkspaceSample(orcaPage, slept) + const sleptTabId = sample.tabIds[0] + expect(sleptTabId, 'slept workspace must retain a tab').toBeTruthy() + // Presence preconditions: the pane is still mounted and still carries its wake hint, + // otherwise a remount has nothing to reattach and the oracle passes vacuously. + expect(sample.mountedTabIds, 'slept pane was parked before the remount').toContain(sleptTabId) + expect(sample.tabPtyHints[0], 'sleep must keep the session id as a wake hint').toBeTruthy() + + const remounted = await orcaPage.evaluate( + (tabId) => window.__store?.getState().remountTerminalTabForRecovery(tabId) ?? false, + sleptTabId + ) + expect(remounted, 'remountTerminalTabForRecovery did not find the slept tab').toBe(true) + await assertStaysCold(orcaPage, slept) + + // Non-vacuity: a deliberate click must still wake it, and exactly once — the + // waiting pane and its remounted successor must not both reattach. + await activateWorkspaceByClick(orcaPage, slept) + await expect + .poll(async () => (await readWorkspaceSample(orcaPage, slept)).livePtyCount, { + timeout: 40_000, + message: 'the slept workspace never wakes even on deliberate activation' + }) + .toBeGreaterThan(0) + await orcaPage.waitForTimeout(3_000) + expect((await readWorkspaceSample(orcaPage, slept)).livePtyCount).toBe(1) + expect(await readHostLiveTerminalCount(orcaPage, slept)).toBe(1) +}) diff --git a/tests/e2e/terminal-quick-command-pre-bind-recovery.spec.ts b/tests/e2e/terminal-quick-command-pre-bind-recovery.spec.ts index c734301d33f..fd4915d0ce3 100644 --- a/tests/e2e/terminal-quick-command-pre-bind-recovery.spec.ts +++ b/tests/e2e/terminal-quick-command-pre-bind-recovery.spec.ts @@ -160,14 +160,16 @@ process.stdout.write(${JSON.stringify(`${marker}\n`)}) observe() }, blocked.tabId) - const remounted = await orcaPage.evaluate((tabId) => { + // No request argument: an external lifecycle remount, which skips the + // recovery ledger entirely and so reports generation 0. + const remountResult = await orcaPage.evaluate((tabId) => { const state = window.__store?.getState() if (!state) { throw new Error('Renderer store unavailable') } return state.remountTerminalTabForRecovery(tabId) }, blocked.tabId) - expect(remounted).toBe(true) + expect(remountResult).toMatchObject({ remounted: true }) // Keep the original pre-spawn attempt gated until React has committed the // successor pane. Releasing earlier lets a loaded CI renderer finish the diff --git a/tests/e2e/worktree-switch-first-paint.spec.ts b/tests/e2e/worktree-switch-first-paint.spec.ts new file mode 100644 index 00000000000..f062b6e246a --- /dev/null +++ b/tests/e2e/worktree-switch-first-paint.spec.ts @@ -0,0 +1,481 @@ +import { execFileSync } from 'node:child_process' +import { mkdtempSync, realpathSync, rmSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { Page, TestInfo } from '@stablyai/playwright-test' +import { expect, test } from './helpers/orca-app' +import { loadWorktreesUntilPathsPresent } from './helpers/worktree-registration' +import { + ensureTerminalVisible, + getAllWorktreeIds, + switchToWorktree, + waitForActiveWorktree, + waitForSessionReady +} from './helpers/store' +import { + execInTerminal, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForTerminalOutput +} from './helpers/terminal' + +/** + * Worktree-switch first-paint budget. + * + * Why this exists: worktree-switch-responsiveness.spec.ts proves the click task + * stays short, and the reveal-convergence spec proves the buffer eventually + * matches. Neither covers the symptom users report — the revealed terminal is + * BLANK for a beat after the switch. This measures the phase that owns that + * beat: switch click -> revealed pane has painted its restored content. + * + * The scenario is the one that dominates at many-worktree scale: a switch to a + * worktree whose tabs are in the persisted session but have never been mounted + * in this renderer. Hot-retain only keeps 4 worktrees warm, so with hundreds of + * worktrees essentially every switch is this one. Reloading the renderer between + * rounds reproduces it exactly, at production parking timings. + */ + +// Why 3: the field profile that motivated this budget has 449 worktrees whose +// median tab count is 2-3, so a 3-tab worktree is the switch users actually pay for. +const TABS_PER_WORKTREE = Number(process.env.ORCA_SWITCH_TABS ?? '3') +const SCROLLBACK_LINES = 1_500 +// Budget: a switch has to look instant. Anything over this reads as a stall. +const FIRST_PAINT_BUDGET_MS = Number(process.env.ORCA_SWITCH_BUDGET_MS ?? '250') +// Why repeat: a single cold reveal on a loaded dev machine swings by tens of ms, +// which is the same order as the effect under test. +const SWITCH_SAMPLE_COUNT = Number(process.env.ORCA_SWITCH_ROUNDS ?? '5') + +type SwitchSample = { + activationMs: number | null + paneMountedMs: number | null + contentRestoredMs: number | null + maxFrameGapMs: number + longTaskTotalMs: number + worstLongTaskMs: number + mountedAtActivation: number + settledPaneManagers: number + settledPanes: number + settledWebglContexts: number +} + +type SwitchPaintProbe = { + t0: number + activationMs: number | null + paneMountedMs: number | null + contentRestoredMs: number | null + frames: number[] + longTasks: number[] + mountedAtActivation: number + stop: () => void +} + +declare global { + var __switchPaintProbe: SwitchPaintProbe | undefined +} + +async function ensureTabs(page: Page, worktreeId: string, marker: string): Promise { + await switchToWorktree(page, worktreeId) + await ensureTerminalVisible(page) + const tabIds: string[] = [] + for (let index = 0; index < TABS_PER_WORKTREE; index += 1) { + const tabId = await page.evaluate( + ({ id, wanted }) => { + const state = window.__store!.getState() + const existing = state.tabsByWorktree[id] ?? [] + const reuse = existing[wanted] + const tab = reuse ?? state.createTab(id, undefined, undefined, { activate: true }) + state.setActiveTab(tab.id) + state.setActiveTabType('terminal') + return tab.id + }, + { id: worktreeId, wanted: index } + ) + await waitForActiveTerminalManager(page, 30_000) + const ptyId = await waitForActivePanePtyId(page, 30_000) + const label = `${marker}_T${index}` + await execInTerminal( + page, + ptyId, + `for i in $(seq 1 ${SCROLLBACK_LINES}); do echo "${label}_$i ${'y'.repeat(48)}"; done; echo ${label}_READY` + ) + await waitForTerminalOutput(page, `${label}_READY`, 60_000) + tabIds.push(tabId) + } + return tabIds +} + +async function waitForUnmountedTabs(page: Page, tabIds: readonly string[]): Promise { + return expect + .poll( + () => + page.evaluate((ids) => ids.every((id) => window.__paneManagers?.has(id) !== true), tabIds), + { timeout: 20_000, message: 'switch target still had mounted panes' } + ) + .toBe(true) + .then( + () => true, + () => false + ) +} + +/** Tabs with a mounted pane, once the post-reveal warm-up has settled. */ +async function waitForMountedTabs(page: Page, tabIds: readonly string[]): Promise { + const read = () => + page.evaluate( + (ids) => ids.filter((id) => window.__paneManagers?.has(id) === true).sort(), + [...tabIds] + ) + await expect + .poll(async () => (await read()).length, { + timeout: 20_000, + message: 'activation-deferred tabs never mounted after the reveal' + }) + .toBe(tabIds.length) + .catch(() => undefined) + return read() +} + +async function measureSwitch( + page: Page, + targetWorktreeId: string, + targetTabIds: readonly string[] +): Promise { + await page.evaluate( + ({ worktreeId, tabIds }) => { + const probe = { + t0: performance.now(), + activationMs: null as number | null, + paneMountedMs: null as number | null, + contentRestoredMs: null as number | null, + frames: [] as number[], + longTasks: [] as number[], + mountedAtActivation: 0, + stop: () => {} + } + globalThis.__switchPaintProbe = probe + let observer: PerformanceObserver | null = null + try { + observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + probe.longTasks.push(entry.duration) + } + }) + observer.observe({ entryTypes: ['longtask'] }) + } catch { + /* longtask unsupported */ + } + let running = true + const visibleTabId = () => { + const state = window.__store!.getState() + return state.activeWorktreeId === worktreeId && state.activeTabType === 'terminal' + ? state.activeTabId + : (state.activeTabIdByWorktree?.[worktreeId] ?? null) + } + const tick = () => { + if (!running) { + return + } + const now = performance.now() - probe.t0 + probe.frames.push(now) + const state = window.__store!.getState() + if (probe.activationMs === null && state.activeWorktreeId === worktreeId) { + probe.activationMs = now + // Why here and not at paint: this is the switch's own frame, before any + // idle admission can run, so it measures what the SWITCH mounted. + probe.mountedAtActivation = tabIds.filter( + (id) => window.__paneManagers?.has(id) === true + ).length + } + const tabId = visibleTabId() + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + if (probe.paneMountedMs === null && pane?.container?.isConnected) { + probe.paneMountedMs = now + } + if (probe.contentRestoredMs === null && pane) { + // Restored = the revealed viewport carries real text rather than an + // empty grid. Read on a frame callback, so this is the frame the + // content became renderable — one frame ahead of the pixels, and not + // a pixel assertion. Both arms are measured identically. + const buffer = pane.terminal.buffer.active + let filledRows = 0 + for (let row = 0; row < pane.terminal.rows; row += 1) { + const line = buffer.getLine(buffer.viewportY + row) + if (line && line.translateToString(true).trim().length > 0) { + filledRows += 1 + } + } + if (filledRows >= Math.min(5, pane.terminal.rows)) { + probe.contentRestoredMs = now + } + } + requestAnimationFrame(tick) + } + requestAnimationFrame(tick) + probe.stop = () => { + running = false + try { + observer?.disconnect() + } catch { + /* ignore */ + } + } + window.__store!.getState().setActiveWorktree(worktreeId) + }, + { worktreeId: targetWorktreeId, tabIds: [...targetTabIds] } + ) + + // Why poll rather than sample a fixed window: the measurement is "how long did + // the restore take", so the harness must outlast the slowest runner rather than + // give up at a deadline and report the reveal as never restoring. + await expect + .poll(() => page.evaluate(() => globalThis.__switchPaintProbe?.contentRestoredMs ?? null), { + timeout: 30_000, + message: 'revealed terminal never restored its content' + }) + .not.toBeNull() + // Let the idle admission drain so the settled-resource readings are steady. + await page.waitForTimeout(2_000) + + return page.evaluate(() => { + const probe = globalThis.__switchPaintProbe! + probe.stop() + let maxGap = 0 + let previous = 0 + for (const frame of probe.frames) { + maxGap = Math.max(maxGap, frame - previous) + previous = frame + } + let settledPanes = 0 + let settledWebglContexts = 0 + const managers = window.__paneManagers + for (const manager of managers?.values() ?? []) { + settledPanes += (manager.getPanes?.() ?? []).length + // Why diagnostics and not `pane.webglAddon`: getPanes() hands back a public + // projection that has no webglAddon field, so reading it is always falsy. + const diagnostics = + ( + manager as { getRenderingDiagnostics?: () => { hasWebgl?: boolean }[] } + ).getRenderingDiagnostics?.() ?? [] + settledWebglContexts += diagnostics.filter((entry) => entry.hasWebgl === true).length + } + return { + settledPaneManagers: managers?.size ?? 0, + settledPanes, + settledWebglContexts, + activationMs: probe.activationMs, + paneMountedMs: probe.paneMountedMs, + contentRestoredMs: probe.contentRestoredMs, + maxFrameGapMs: +maxGap.toFixed(1), + longTaskTotalMs: +probe.longTasks.reduce((total, value) => total + value, 0).toFixed(1), + worstLongTaskMs: +probe.longTasks + .reduce((worst, value) => Math.max(worst, value), 0) + .toFixed(1), + mountedAtActivation: probe.mountedAtActivation + } + }) +} + +function report(label: string, sample: SwitchSample): string { + return [ + `${label}:`, + ` activation ${sample.activationMs?.toFixed(1) ?? 'n/a'}ms`, + ` pane mounted ${sample.paneMountedMs?.toFixed(1) ?? 'n/a'}ms`, + ` content restored ${sample.contentRestoredMs?.toFixed(1) ?? 'never'}ms`, + ` max frame gap ${sample.maxFrameGapMs}ms`, + ` long tasks total=${sample.longTaskTotalMs}ms worst=${sample.worstLongTaskMs}ms`, + ` panes at switch ${sample.mountedAtActivation}/${TABS_PER_WORKTREE}`, + ` settled resources managers=${sample.settledPaneManagers} panes=${sample.settledPanes} webgl=${sample.settledWebglContexts}` + ].join('\n') +} + +async function publish(testInfo: TestInfo, name: string, body: string): Promise { + console.log(body) + await testInfo.attach(name, { body, contentType: 'text/plain' }) +} + +// Why 8 extra: hot-retain keeps the 4 most recently hidden worktrees mounted and +// exempts the last-active one, so a target only cold-parks once enough other +// worktrees have been visited after it. That is the steady state at field scale. +const FILLER_WORKTREE_COUNT = Number(process.env.ORCA_SWITCH_FILLER_WORKTREES ?? '8') + +async function addFillerWorktrees( + page: Page, + testRepoPath: string +): Promise<{ ids: string[]; cleanup: () => void }> { + const parent = realpathSync(mkdtempSync(path.join(os.tmpdir(), 'orca-switch-paint-'))) + const paths = Array.from({ length: FILLER_WORKTREE_COUNT }, (_, index) => + path.join(parent, `filler-${index}`) + ) + const removeAll = (): void => { + for (const worktreePath of paths) { + try { + execFileSync('git', ['worktree', 'remove', '--force', worktreePath], { + cwd: testRepoPath, + stdio: 'ignore' + }) + } catch { + /* best effort */ + } + } + rmSync(parent, { recursive: true, force: true }) + } + // Why clean up before rethrowing: testRepoPath is worker-scoped and reused by + // later specs, so a half-built fixture would leak worktrees into them. + try { + for (const worktreePath of paths) { + execFileSync('git', ['worktree', 'add', '--detach', worktreePath, 'HEAD'], { + cwd: testRepoPath, + stdio: 'ignore' + }) + } + } catch (error) { + removeAll() + throw error + } + try { + return await registerFillerWorktrees(page, testRepoPath, paths, removeAll) + } catch (error) { + removeAll() + throw error + } +} + +async function registerFillerWorktrees( + page: Page, + testRepoPath: string, + paths: readonly string[], + cleanup: () => void +): Promise<{ ids: string[]; cleanup: () => void }> { + const repoId = await page.evaluate( + (repoPath) => + window.__store!.getState().repos.find((repo) => repo.path === repoPath)?.id ?? null, + testRepoPath + ) + if (!repoId) { + throw new Error(`seeded repo not registered: ${testRepoPath}`) + } + await loadWorktreesUntilPathsPresent(page, repoId, [...paths]) + const ids = await page.evaluate( + ({ id, wanted }) => + (window.__store!.getState().worktreesByRepo[id] ?? []) + .filter((worktree) => wanted.includes(worktree.path)) + .map((worktree) => worktree.id), + { id: repoId, wanted: paths } + ) + return { ids, cleanup } +} + +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle] +} + +test.describe('Worktree switch first paint', () => { + test('repaints an unmounted worktree within the switch budget', async ({ + orcaPage, + testRepoPath + }, testInfo) => { + test.setTimeout(900_000) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + await ensureTerminalVisible(orcaPage) + + const worktreeIds = await getAllWorktreeIds(orcaPage) + expect(worktreeIds.length).toBeGreaterThanOrEqual(2) + const [primaryId, targetId] = worktreeIds + const filler = await addFillerWorktrees(orcaPage, testRepoPath) + + const samples: SwitchSample[] = [] + const lines: string[] = [] + try { + const targetTabIds = await ensureTabs(orcaPage, targetId, 'WTB') + + // Give the filler worktrees persisted tabs without mounting them, so the + // store carries a field-scale tab population (the profile that motivated + // this budget has 846 tabs across 449 worktrees). + await orcaPage.evaluate( + ({ ids, perWorktree }) => { + const state = window.__store!.getState() + for (const id of ids) { + const existing = state.tabsByWorktree[id] ?? [] + for (let index = existing.length; index < perWorktree; index += 1) { + state.createTab(id) + } + } + }, + { ids: filler.ids, perWorktree: 2 } + ) + + for (let round = 0; round < SWITCH_SAMPLE_COUNT; round += 1) { + // Leave the primary active and let the session persist before reloading: + // startup restores the persisted active worktree, so this is what makes + // the target come back with tabs in the session and no pane ever mounted + // — the state every switch lands in once the worktree count exceeds the + // hot-retain working set. + await switchToWorktree(orcaPage, primaryId) + await ensureTerminalVisible(orcaPage) + await orcaPage.waitForTimeout(2_500) + await orcaPage.reload() + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + await ensureTerminalVisible(orcaPage) + await orcaPage.waitForTimeout(2_500) + const unmounted = await waitForUnmountedTabs(orcaPage, targetTabIds) + expect(unmounted, 'target worktree was already mounted before the switch').toBe(true) + + const sample = await measureSwitch(orcaPage, targetId, targetTabIds) + samples.push(sample) + lines.push(report(`round ${round + 1} (target unmounted=${unmounted})`, sample)) + + // The half of the contract that keeps the speed-up free: the hidden tabs + // the switch skipped still end up mounted, so the next tab switch is as + // warm as it was before the reveal stopped mounting them up front. + const warmedTabIds = await waitForMountedTabs(orcaPage, targetTabIds) + expect(warmedTabIds, 'deferred tabs never joined the warm working set').toEqual( + [...targetTabIds].sort() + ) + } + } finally { + filler.cleanup() + } + + const restored = samples + .map((sample) => sample.contentRestoredMs) + .filter((value): value is number => value !== null) + expect(restored.length, 'revealed terminal never restored its content').toBe(samples.length) + const summary = [ + `first activation -> ${TABS_PER_WORKTREE}-tab worktree, ${samples.length} rounds`, + ` content restored: median=${median(restored).toFixed(1)}ms samples=${restored + .map((value) => value.toFixed(0)) + .join(', ')}ms`, + ` activation: median=${median( + samples.map((sample) => sample.activationMs ?? 0) + ).toFixed(1)}ms`, + ` panes at switch: ${samples.map((sample) => sample.mountedAtActivation).join(', ')}`, + ` settled panes: ${samples.map((sample) => sample.settledPanes).join(', ')}`, + ` settled webgl: ${samples.map((sample) => sample.settledWebglContexts).join(', ')}`, + '', + ...lines + ].join('\n') + await publish(testInfo, 'first-activation-switch.txt', summary) + + for (const sample of samples) { + expect( + sample.mountedAtActivation, + 'the switch mounted more than the pane the user is looking at' + ).toBe(1) + } + // Why CI is exempt from the budget and not from the invariants: shared + // runners cannot hold a latency threshold, but "the switch mounted one pane" + // and "the warm set came back" are exact and are the real regression guards. + if (process.env.CI) { + console.log( + `[switch-budget] CI run, latency budget not enforced (median ${median(restored).toFixed(1)}ms)` + ) + return + } + expect(median(restored)).toBeLessThanOrEqual(FIRST_PAINT_BUDGET_MS) + }) +}) diff --git a/tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs b/tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs index 63b7222a93d..ffac29bddea 100644 --- a/tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs +++ b/tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs @@ -27,19 +27,19 @@ import { pickFreePort, stopDevApp, waitForStoreReady -} from '../../config/scripts/windows-apphang-repro/electron-dev-session.mjs' +} from '../../../config/scripts/windows-apphang-repro/electron-dev-session.mjs' import { createCompletedOnboardingProfile, safeRemoveLocalDirectory -} from '../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs' +} from '../../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs' import { pollUntil, rendererActionTimeoutMs, runWithTimeout, setupTimeoutMs -} from '../../config/scripts/windows-apphang-repro/repro-timing.mjs' +} from '../../../config/scripts/windows-apphang-repro/repro-timing.mjs' -const rootDir = path.resolve(fileURLToPath(new URL('../..', import.meta.url))) +const rootDir = path.resolve(fileURLToPath(new URL('../../..', import.meta.url))) const PARK_DELAY_MS = 1_500 const SETTLE_AFTER_PARK_MS = 4_000 // Short root so the daemon Unix socket fits under the macOS 104-char limit; diff --git a/tests/tools/benchmarks/terminal-cold-park-reveal-bench.mjs b/tests/tools/benchmarks/terminal-cold-park-reveal-bench.mjs index a01cbf2cc3a..70bf1f66944 100644 --- a/tests/tools/benchmarks/terminal-cold-park-reveal-bench.mjs +++ b/tests/tools/benchmarks/terminal-cold-park-reveal-bench.mjs @@ -33,19 +33,19 @@ import { pickFreePort, stopDevApp, waitForStoreReady -} from '../../config/scripts/windows-apphang-repro/electron-dev-session.mjs' +} from '../../../config/scripts/windows-apphang-repro/electron-dev-session.mjs' import { createCompletedOnboardingProfile, safeRemoveLocalDirectory -} from '../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs' +} from '../../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs' import { pollUntil, rendererActionTimeoutMs, runWithTimeout, setupTimeoutMs -} from '../../config/scripts/windows-apphang-repro/repro-timing.mjs' +} from '../../../config/scripts/windows-apphang-repro/repro-timing.mjs' -const rootDir = path.resolve(fileURLToPath(new URL('../..', import.meta.url))) +const rootDir = path.resolve(fileURLToPath(new URL('../../..', import.meta.url))) const scenarioTimeoutMs = 300_000 // Short enough that a tab parks within a few seconds of being hidden, long diff --git a/tests/tools/benchmarks/terminal-perf-bench.mjs b/tests/tools/benchmarks/terminal-perf-bench.mjs index a73ffae6bc9..551f33d0433 100644 --- a/tests/tools/benchmarks/terminal-perf-bench.mjs +++ b/tests/tools/benchmarks/terminal-perf-bench.mjs @@ -19,16 +19,16 @@ import { pickFreePort, stopDevApp, waitForStoreReady -} from '../../config/scripts/windows-apphang-repro/electron-dev-session.mjs' +} from '../../../config/scripts/windows-apphang-repro/electron-dev-session.mjs' import { pollUntil, rendererActionTimeoutMs, runWithTimeout, setupTimeoutMs -} from '../../config/scripts/windows-apphang-repro/repro-timing.mjs' -import { safeRemoveLocalDirectory } from '../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs' +} from '../../../config/scripts/windows-apphang-repro/repro-timing.mjs' +import { safeRemoveLocalDirectory } from '../../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs' -const rootDir = path.resolve(fileURLToPath(new URL('../..', import.meta.url))) +const rootDir = path.resolve(fileURLToPath(new URL('../../..', import.meta.url))) const scenarioTimeoutMs = 300_000 const defaultIterations = 8 const defaultSwitches = 24 diff --git a/tests/tools/pi-owner-runtime-smoke.mjs b/tests/tools/pi-owner-runtime-smoke.mjs new file mode 100644 index 00000000000..204627340ac --- /dev/null +++ b/tests/tools/pi-owner-runtime-smoke.mjs @@ -0,0 +1,128 @@ +// Run: node tests/tools/pi-owner-runtime-smoke.mjs /path/to/pi-coding-agent +import assert from 'node:assert/strict' +import { once } from 'node:events' +import { mkdtemp, writeFile, rm } from 'node:fs/promises' +import { createServer } from 'node:http' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { build } from 'esbuild' + +const piRoot = resolve(process.argv[2] || '') +assert.ok(process.argv[2], 'Pass an installed pi-coding-agent package directory') +const scratch = await mkdtemp(join(tmpdir(), 'orca-pi-owner-')) +const received = [] +const server = createServer(async (request, response) => { + let body = '' + for await (const chunk of request) { + body += chunk + } + received.push(JSON.parse(body)) + response.end('{}') +}) +try { + const bundle = join(scratch, 'orca.cjs') + await build({ + stdin: { + contents: [ + "export { getPiAgentStatusExtensionSource } from './src/main/pi/agent-status-extension-source';", + "export { runProcess } from './src/shared/child-process/run-process';" + ].join('\n'), + resolveDir: process.cwd() + }, + bundle: true, + platform: 'node', + format: 'cjs', + outfile: bundle, + packages: 'external' + }) + const { getPiAgentStatusExtensionSource, runProcess } = createRequire(import.meta.url)(bundle) + server.listen(0, '127.0.0.1') + await once(server, 'listening') + const dead = await runProcess({ + program: process.execPath, + args: ['-e', 'console.log(process.pid)'] + }) + assert.equal(dead.code, 0) + const deadPid = Number(dead.stdout.trim()) + assert.throws(() => process.kill(deadPid, 0), { code: 'ESRCH' }) + const worker = join(scratch, 'worker.mjs') + const moduleUrl = (file) => JSON.stringify(pathToFileURL(join(piRoot, file)).href) + await writeFile( + worker, + ` + import assert from 'node:assert/strict' + import { loadExtensions } from ${moduleUrl('dist/core/extensions/loader.js')} + import { ExtensionRunner } from ${moduleUrl('dist/core/extensions/runner.js')} + import { SessionManager } from ${moduleUrl('dist/core/session-manager.js')} + const loaded = await loadExtensions([process.argv[2]], process.cwd()) + assert.deepEqual(loaded.errors, []) + const runner = new ExtensionRunner(loaded.extensions, loaded.runtime, process.cwd(), SessionManager.inMemory(process.cwd()), undefined) + const errors = [] + runner.onError(error => errors.push(error)) + await runner.emit({ type: 'agent_start' }) + await new Promise(resolve => setTimeout(resolve, 250)) + assert.deepEqual(errors, []) + console.log(JSON.stringify({pid: process.pid, owner: process.env[process.argv[3]], handlers: loaded.extensions[0].handlers.size})) + ` + ) + const results = [] + for (const kind of ['pi', 'omp', 'prime-agent']) { + const ownerKey = + kind === 'prime-agent' ? 'ORCA_PRIME_AGENT_STATUS_OWNED' : 'ORCA_PI_STATUS_OWNED' + for (const scenario of ['baseline-dead', 'fixed-dead', 'fixed-live']) { + let source = getPiAgentStatusExtensionSource(kind) + if (scenario === 'baseline-dead') { + const guard = 'if (ownerPid && ownerPid !== selfPid && isStatusOwnerAlive(ownerPid)) return' + assert.ok( + source.includes(guard), + 'Baseline mutation must replace the actual ownership guard' + ) + source = source.replace(guard, 'if (ownerPid && ownerPid !== selfPid) return') + } + const extension = join(scratch, `${kind}-${scenario}.ts`) + await writeFile(extension, source) + const before = received.length + const owner = scenario === 'fixed-live' ? process.pid : deadPid + const child = await runProcess({ + program: process.execPath, + args: [worker, extension, ownerKey], + cwd: scratch, + env: { + ...process.env, + ORCA_BACKGROUND_LAUNCH: '1', + ORCA_PANE_KEY: 'owner-proof', + ORCA_AGENT_HOOK_PORT: String(server.address().port), + ORCA_AGENT_HOOK_TOKEN: 'isolated-proof-token', + ORCA_AGENT_HOOK_ENV: 'proof', + ORCA_AGENT_HOOK_ENDPOINT: '', + ORCA_PI_STATUS_OWNED: '', + ORCA_PRIME_AGENT_STATUS_OWNED: '', + PRIME_AGENT_INTERNAL_DAEMON_WORKER: kind === 'prime-agent' ? '1' : '', + [ownerKey]: String(owner) + }, + timeoutMs: 15000 + }) + assert.equal(child.code, 0, child.stderr) + const observation = JSON.parse(child.stdout.trim().split('\n').at(-1)) + const shouldReport = scenario === 'fixed-dead' + assert.equal( + received.length - before, + shouldReport ? 1 : 0, + `${kind}/${scenario}: HTTP delivery` + ) + assert.equal(observation.owner, String(shouldReport ? observation.pid : owner)) + assert.equal(observation.handlers > 0, shouldReport) + if (shouldReport) { + assert.equal(received.at(-1).payload.hook_event_name, 'agent_start') + } + results.push({ kind, scenario, posts: received.length - before, ...observation }) + } + } + console.log(JSON.stringify({ platform: process.platform, results }, null, 2)) +} finally { + server.closeAllConnections() + server.close() + await rm(scratch, { recursive: true, force: true }) +} diff --git a/tests/tools/pi-provider-runtime-smoke.mjs b/tests/tools/pi-provider-runtime-smoke.mjs new file mode 100644 index 00000000000..bda4f6a0f89 --- /dev/null +++ b/tests/tools/pi-provider-runtime-smoke.mjs @@ -0,0 +1,130 @@ +import assert from 'node:assert/strict' +import { once } from 'node:events' +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises' +import { createServer } from 'node:http' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { build } from 'esbuild' +const piCli = process.argv[2] && resolve(process.argv[2]) +assert.ok(piCli, 'Pass the installed Pi CLI entrypoint') +const scratch = await mkdtemp(join(tmpdir(), 'orca-pi-provider-')) +const requests = [] +const server = createServer(async (req, res) => { + let body = '' + for await (const part of req) { + body += part + } + requests.push(JSON.parse(body)) + res.writeHead(200, { 'content-type': 'text/event-stream' }) + for (const chunk of [ + { + id: 'proof', + object: 'chat.completion.chunk', + choices: [ + { + index: 0, + delta: { role: 'assistant', content: 'fixture-generated-commit' }, + finish_reason: null + } + ] + }, + { + id: 'proof', + object: 'chat.completion.chunk', + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } + } + ]) { + res.write(`data: ${JSON.stringify(chunk)}\n\n`) + } + res.end('data: [DONE]\n\n') +}) +try { + const bundle = join(scratch, 'orca.cjs') + await build({ + stdin: { + contents: + "export {planCommitMessageGeneration} from './src/shared/commit-message-plan'; export {runProcess} from './src/shared/child-process/run-process';", + resolveDir: process.cwd() + }, + bundle: true, + platform: 'node', + format: 'cjs', + outfile: bundle, + packages: 'external' + }) + const { planCommitMessageGeneration, runProcess } = createRequire(import.meta.url)(bundle) + server.listen(0, '127.0.0.1') + await once(server, 'listening') + const dir = join(scratch, 'agent') + await mkdir(join(dir, 'extensions'), { recursive: true }) + await writeFile( + join(dir, 'extensions', 'provider.ts'), + `export default function(pi){pi.registerProvider('orca-proof',{name:'Proof',baseUrl:'http://127.0.0.1:${server.address().port}/v1',apiKey:'fixture-only',api:'openai-completions',models:[{id:'local',name:'Proof',reasoning:false,input:['text'],cost:{input:0,output:0,cacheRead:0,cacheWrite:0},contextWindow:8192,maxTokens:256}]})}` + ) + await writeFile( + join(dir, 'settings.json'), + JSON.stringify({ defaultProvider: 'orca-proof', defaultModel: 'local' }) + ) + const planned = planCommitMessageGeneration( + { agentId: 'pi', model: 'orca-proof/local' }, + 'Generate one short commit message.' + ) + assert.equal(planned.ok, true) + const fixedArgs = planned.plan.args + assert.ok(!fixedArgs.includes('--no-extensions')) + const variants = [ + ['baseline', [...fixedArgs, '--no-extensions']], + ['extensions-enabled', fixedArgs] + ] + const results = [] + for (const [variant, args] of variants) { + const n = requests.length + const result = await runProcess({ + program: process.execPath, + args: [piCli, ...args], + cwd: scratch, + env: { + PATH: process.env.PATH, + SystemRoot: process.env.SystemRoot, + WINDIR: process.env.WINDIR, + HOME: scratch, + USERPROFILE: scratch, + ORCA_BACKGROUND_LAUNCH: '1', + PI_CODING_AGENT_DIR: dir + }, + input: planned.plan.stdinPayload, + timeoutMs: 20000 + }) + results.push({ + variant, + args, + code: result.code, + stdout: result.stdout, + stderr: result.stderr, + requests: requests.length - n + }) + } + assert.equal(results[0].requests, 0) + assert.notEqual(results[0].code, 0) + assert.equal(results[1].code, 0, results[1].stderr) + assert.match(results[1].stdout, /fixture-generated-commit/) + assert.equal(results[1].requests, 1) + console.log( + JSON.stringify( + { + scope: + 'Actual Pi CLI and production command planner; isolated extension provider with local OpenAI-compatible fixture.', + platform: process.platform, + results + }, + null, + 2 + ) + ) +} finally { + server.closeAllConnections() + server.close() + await rm(scratch, { recursive: true, force: true }) +} diff --git a/tests/tools/relay-bench/find-cell.mjs b/tests/tools/relay-bench/find-cell.mjs new file mode 100644 index 00000000000..cc48be054c9 --- /dev/null +++ b/tests/tools/relay-bench/find-cell.mjs @@ -0,0 +1,32 @@ +import { createRequire } from 'node:module' +const WebSocket = createRequire(import.meta.url)('ws') +const hostId = process.argv[2] +const bogus = 'A'.repeat(43) +const probe = (cell) => + new Promise((resolve) => { + const ws = new WebSocket(`wss://${cell}.relay.onorca.dev/v1/connect/${hostId}`, { + perMessageDeflate: false + }) + const t0 = performance.now() + const done = (r) => { + try { + ws.terminate() + } catch {} + resolve({ cell, ms: Math.round(performance.now() - t0), ...r }) + } + ws.on('open', () => + ws.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: bogus })) + ) + ws.on('message', (m) => done({ hello: JSON.parse(m.toString()).code })) + ws.on('error', (e) => done({ error: e.code ?? e.message })) + ws.on('close', (c) => done({ close: c })) + setTimeout(() => done({ error: 'timeout' }), 8000) + }) +const cells = Array.from({ length: 30 }, (_, i) => `c${i + 1}`) +const results = await Promise.all(cells.map(probe)) +for (const r of results) { + if (r.hello !== 4409 || process.argv[3]) { + console.log(JSON.stringify(r)) + } +} +console.log('probed', results.length, 'wrong-cell:', results.filter((r) => r.hello === 4409).length)