From d21f63e6fc3f4a79393e3ce718dc71382a8fa54b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:42:54 -0700 Subject: [PATCH 01/10] fix(browser): keep cookie-import warnings readable from newer hosts (#15002) * fix(browser): keep cookie-import warnings readable from newer hosts The import summary is cast, not decoded, coming off the runtime RPC wire, so a newer host can publish a warning code or undecryptable reason this build has never seen. Both switches then matched nothing and returned undefined, which rendered as a blank toast.warning(undefined). #14683 replaced the absorbing default: arm with case 'unknown' to satisfy switch-exhaustiveness-check, which forbids default: on an exhaustive switch. Guard before each switch instead: narrow the wire value against the handled set and route anything else to a generic message. The handled sets are keyed by the unions themselves, so a fourth member still fails typecheck here. * fix(browser): reject non-string cookie-import warning discriminants Object.hasOwn coerces its key, so the membership guards admitted any value whose toString() matched a handled variant. A host that widened reason to an array sends ['unknown'], which passed the guard and then fell straight back out of the switch -- the same undefined-to-toast.warning blank the guards exist to prevent. Take unknown and check typeof first, matching isTopLevelView and isTuiAgent. The Record membership sets still fail typecheck on a new union member. --- src/renderer/src/i18n/locales/en.json | 4 +- .../lib/browser-cookie-import-toast.test.ts | 80 +++++++++++++++++++ .../src/lib/browser-cookie-import-toast.ts | 44 +++++++++- 3 files changed, 126 insertions(+), 2 deletions(-) diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index bacd04d4ba1..a66f2e3b481 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -788,7 +788,9 @@ "undecryptableAppBoundMixed": "Orca cannot decrypt {{value0}} of this browser's cookies because they use app-bound encryption; {{value1}} more could not be decrypted for another reason. You can import cookies from a file using “From File…”.", "undecryptableKeyring": "{{value0}} cookies could not be decrypted because the system keyring was unavailable. Unlock your login keyring (or install a Secret Service provider such as gnome-keyring) and import again.", "undecryptableKeyringMixed": "{{value0}} cookies could not be decrypted because the system keyring was unavailable; {{value1}} more could not be decrypted for another reason. Unlock your login keyring (or install a Secret Service provider such as gnome-keyring) and import again.", - "undecryptableUnknown": "{{value0}} cookies could not be decrypted and were skipped. Close the source browser completely and try the import again." + "undecryptableUnknown": "{{value0}} cookies could not be decrypted and were skipped. Close the source browser completely and try the import again.", + "unrecognizedWarning": "The cookie import finished with a warning this version of Orca does not recognize. Update Orca to see the details, then check this profile before relying on its cookies.", + "undecryptableUnrecognizedReason": "{{value0}} cookies could not be decrypted and were skipped for a reason this version of Orca does not recognize. Update Orca to see the details, then try the import again." } } } diff --git a/src/renderer/src/lib/browser-cookie-import-toast.test.ts b/src/renderer/src/lib/browser-cookie-import-toast.test.ts index 23e04c53034..a090ba5a99f 100644 --- a/src/renderer/src/lib/browser-cookie-import-toast.test.ts +++ b/src/renderer/src/lib/browser-cookie-import-toast.test.ts @@ -101,6 +101,86 @@ describe('emitBrowserCookieImportToast', () => { expect(warningToastMock).not.toHaveBeenCalled() }) + // Why: the summary crosses the runtime RPC wire and is cast, not decoded, so a newer host can + // publish a reason/code this client build has never heard of (#14683 follow-up). + it('still warns when a newer host sends an undeclared undecryptable reason', () => { + emitBrowserCookieImportToast( + { + ...summary, + warning: { + code: 'cookies-undecryptable', + failedCookies: 3, + reason: 'hardware-token-required' + } as unknown as BrowserCookieImportSummary['warning'] + }, + 'Imported 0 cookies.', + 'Remote Linux' + ) + + const message = warningToastMock.mock.calls[0]?.[0] + expect(typeof message).toBe('string') + expect(message).not.toBe('') + expect(message).toContain('3') + }) + + it('still warns when a newer host sends an undeclared warning code', () => { + emitBrowserCookieImportToast( + { + ...summary, + warning: { + code: 'profile-locked', + failedCookies: 3 + } as unknown as BrowserCookieImportSummary['warning'] + }, + 'Imported 0 cookies.', + 'Remote Linux' + ) + + const message = warningToastMock.mock.calls[0]?.[0] + expect(typeof message).toBe('string') + expect(message).not.toBe('') + }) + + // Why: hasOwn coerces its key, so a host that widened `reason` to an array sends ['unknown'], + // which a hasOwn-only guard admits before the switch drops it back out. + it('still warns when a newer host sends the reason as an array', () => { + emitBrowserCookieImportToast( + { + ...summary, + warning: { + code: 'cookies-undecryptable', + failedCookies: 3, + reason: ['unknown'] + } as unknown as BrowserCookieImportSummary['warning'] + }, + 'Imported 0 cookies.', + 'Remote Linux' + ) + + const message = warningToastMock.mock.calls[0]?.[0] + expect(typeof message).toBe('string') + expect(message).not.toBe('') + }) + + it('still warns when a newer host sends the warning code as an array', () => { + emitBrowserCookieImportToast( + { + ...summary, + warning: { + code: ['cookies-undecryptable'], + failedCookies: 3, + reason: 'unknown' + } as unknown as BrowserCookieImportSummary['warning'] + }, + 'Imported 0 cookies.', + 'Remote Linux' + ) + + const message = warningToastMock.mock.calls[0]?.[0] + expect(typeof message).toBe('string') + expect(message).not.toBe('') + }) + it('keeps both applicable warnings when restart fallback is unavailable', () => { emitBrowserCookieImportToast( { diff --git a/src/renderer/src/lib/browser-cookie-import-toast.ts b/src/renderer/src/lib/browser-cookie-import-toast.ts index a8fa4aa6bae..d8afad09db3 100644 --- a/src/renderer/src/lib/browser-cookie-import-toast.ts +++ b/src/renderer/src/lib/browser-cookie-import-toast.ts @@ -3,8 +3,41 @@ import type { BrowserCookieImportSummary } from '../../../shared/browser-workspa import { translate } from '@/i18n/i18n' type CookieImportWarning = NonNullable +type CookieImportWarningCode = CookieImportWarning['code'] +type UndecryptableReason = Extract['reason'] + +// Why: the summary is cast, not decoded, on the way off the runtime RPC wire, so a newer host can +// send a code/reason this build has never heard of. The Record keys are the union itself, so a new +// member fails typecheck here instead of silently falling out of the switch (#14683 follow-up). +const HANDLED_WARNING_CODES: Record = { + 'restart-fallback-unavailable': true, + 'cookies-undecryptable': true +} + +const HANDLED_UNDECRYPTABLE_REASONS: Record = { + 'app-bound-encryption': true, + 'linux-keyring-unavailable': true, + unknown: true +} + +// Why: typeof first — hasOwn coerces its key, so a host that widened `reason` to an array would +// send ['unknown'], pass a hasOwn-only guard, then fall straight back out of the switch. +function isHandledWarningCode(code: unknown): code is CookieImportWarningCode { + return typeof code === 'string' && Object.hasOwn(HANDLED_WARNING_CODES, code) +} + +function isHandledUndecryptableReason(reason: unknown): reason is UndecryptableReason { + return typeof reason === 'string' && Object.hasOwn(HANDLED_UNDECRYPTABLE_REASONS, reason) +} function formatCookieImportWarning(warning: CookieImportWarning): string { + const code: unknown = warning.code + if (!isHandledWarningCode(code)) { + return translate( + 'auto.lib.browser.cookie.import.toast.unrecognizedWarning', + 'The cookie import finished with a warning this version of Orca does not recognize. Update Orca to see the details, then check this profile before relying on its cookies.' + ) + } switch (warning.code) { case 'restart-fallback-unavailable': return warning.loadedCookies === 0 @@ -21,7 +54,15 @@ function formatCookieImportWarning(warning: CookieImportWarning): string { value1: warning.loadedCookies + warning.failedCookies } ) - case 'cookies-undecryptable': + case 'cookies-undecryptable': { + const reason: unknown = warning.reason + if (!isHandledUndecryptableReason(reason)) { + return translate( + 'auto.lib.browser.cookie.import.toast.undecryptableUnrecognizedReason', + '{{value0}} cookies could not be decrypted and were skipped for a reason this version of Orca does not recognize. Update Orca to see the details, then try the import again.', + { value0: warning.failedCookies } + ) + } switch (warning.reason) { case 'app-bound-encryption': return warning.otherFailedCookies @@ -54,6 +95,7 @@ function formatCookieImportWarning(warning: CookieImportWarning): string { { value0: warning.failedCookies } ) } + } } } From 81d7f9b24eb343096af6ad43a12349975f304ad9 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:44:43 -0700 Subject: [PATCH 02/10] refactor: split db.ts under 400 lines (#14979) * refactor: split db.ts under 400 lines * rm plan * fix(orchestration-db): add safety guards to database operations Add status guards to UPDATE statements to prevent late operations from overwriting changes made by concurrent requests. Validate mutation results to surface silent no-ops. Extract circuit-break threshold, add transaction wrapping, and sanitize untrusted input. Bump schema version to v28. * Add transactional safety to dispatch and message operations Wrap dispatch failures and batched message updates with SAVEPOINTs to ensure atomicity and idempotency: - Dispatch failures now check status guards and roll back if the related task update fails, preventing partial state corruption - Message batches (across multiple 500-id chunks) roll back entirely if any batch fails, avoiding partial mutations - Add tests verifying idempotency and atomicity under failure conditions * Handle concurrent writes and improve transactional safety - Remote question answering: add classification check before and after UPDATE to safely detect concurrent modifications. Prevents false success when the UPDATE loses a race. - Transaction rollback: wrap in try-catch to prevent errors from masking the original failure. - Question thread reset: use status update instead of deletion to preserve message references. * test: add answer replay and race condition edge case coverage Add test cases for answer replay idempotency, conflict detection, and a race condition between concurrent answer updates in federation relay. Also verify local question state transitions during orchestration reset. --- config/max-lines-baseline.txt | 1 - src/main/runtime/orchestration/db.ts | 7761 +---------------- .../db/attach-orchestration-db-methods.ts | 121 + .../orchestration/db/contract-constants.ts | 10 + .../coordinator-runs/coordinator-run-store.ts | 91 + .../db/database-file-permissions.ts | 17 + .../db/decision-gates/decision-gate-store.ts | 116 + .../db/dispatch-capability-hash.ts | 5 + .../dispatch-context/dispatch-capability.ts | 105 + .../dispatch-circuit-breaker.ts | 2 + .../dispatch-context/dispatch-completion.ts | 134 + .../dispatch-context-store.ts | 131 + .../db/dispatch-context/dispatch-lookup.ts | 108 + .../worker-report-settlement.ts | 125 + .../federated-worker-report-outcome.test.ts | 18 + .../db/federated-worker-report-outcome.ts | 18 + .../db/federation/federated-dispatch-store.ts | 114 + .../db/federation/federation-relay-ack.ts | 202 + .../db/federation/federation-relay-enqueue.ts | 185 + .../db/federation/federation-relay-import.ts | 140 + .../db/federation/federation-relay-item.ts | 69 + .../remote-dispatch-attachment-authority.ts | 173 + .../remote-dispatch-attachment-create.ts | 177 + .../remote-dispatch-attachment-stop.ts | 106 + .../db/federation/remote-question-store.ts | 139 + .../runtime/orchestration/db/generated-id.ts | 5 + .../db/legacy-question-identity.test.ts | 48 + .../db/legacy-question-identity.ts | 34 + .../db/legacy/legacy-ask-operation.ts | 148 + .../legacy/legacy-compatibility-candidates.ts | 185 + .../legacy/legacy-compatibility-principals.ts | 188 + .../db/legacy/legacy-lifecycle-operation.ts | 196 + .../db/legacy/legacy-mail-acknowledge.ts | 113 + .../db/legacy/legacy-question-acknowledge.ts | 100 + .../db/legacy/legacy-question-lookup.ts | 126 + .../db/legacy/legacy-recovery-cohort.ts | 163 + .../db/legacy/legacy-reply-operation.ts | 191 + .../db/legacy/legacy-worker-completion.ts | 161 + .../db/lifecycle-rejection-marker.test.ts | 23 + .../db/lifecycle-rejection-marker.ts | 38 + .../db/messages/direct-mailbox-routing.ts | 198 + .../foreign-direct-mailbox-routing.ts | 197 + .../db/messages/mailbox-routing-page.ts | 13 + .../db/messages/message-inbox.ts | 293 + .../db/messages/message-insert.ts | 64 + .../mutation-receipt-store.ts | 155 + .../db/orchestration-db-methods.ts | 119 + .../orchestration/db/orchestration-db.ts | 48 + .../orchestration/db/pane-key-match.test.ts | 29 + .../orchestration/db/pane-key-match.ts | 36 + .../db/questions/question-threads.ts | 176 + .../db/reset/orchestration-reset.ts | 100 + .../orchestration/db/run-list-cursor.test.ts | 17 + .../orchestration/db/run-list-cursor.ts | 25 + .../runtime/orchestration/db/run-list-page.ts | 18 + .../runs/legacy-coordinator-mail-takeover.ts | 170 + .../orchestration/db/runs/run-binding.ts | 166 + .../db/runs/run-coordinator-mail-routing.ts | 89 + .../orchestration/db/runs/run-create.ts | 44 + .../orchestration/db/runs/run-delivery.ts | 268 + .../orchestration/db/runs/run-lookup.ts | 140 + .../db/schema/adopt-legacy-run.ts | 142 + .../backfill-legacy-question-threads.ts | 129 + .../db/schema/create-core-tables-sql.ts | 178 + .../db/schema/create-graph-tables-sql.ts | 170 + .../orchestration/db/schema/create-tables.ts | 18 + .../schema/migrate-legacy-contract-storage.ts | 150 + .../db/schema/migrate-v13-v28.ts | 169 + .../orchestration/db/schema/migrate-v2-v12.ts | 233 + .../orchestration/db/schema/migrate.ts | 35 + .../db/schema/schema-column-probes.ts | 70 + .../orchestration/db/tasks/task-store.ts | 255 + .../runtime/orchestration/db/utc-timestamp.ts | 48 + .../federated-worker-start-reconcile.ts | 113 + .../worker-dispatch-abandon.ts | 89 + .../worker-dispatch-authority.ts | 154 + .../worker-dispatch-outcome.ts | 136 + .../worker-dispatch/worker-dispatch-stage.ts | 92 + .../worker-dispatch/worker-dispatch-start.ts | 152 + .../worker-dispatch/worker-dispatch-stop.ts | 238 + .../worker-terminal-recovery.ts | 98 + .../worker-terminal-archive.ts | 204 + .../worker-terminal-listing.ts | 143 + .../worker-terminal-release.ts | 168 + .../worker-terminal-resource-store.ts | 185 + .../worker-terminal-transfer.ts | 93 + .../dispatch-failure-idempotency.test.ts | 60 + ...ederation-acknowledgment-integrity.test.ts | 64 +- .../message-batch-atomicity.test.ts | 94 + .../orchestration-reset-db.test.ts | 21 +- 90 files changed, 10102 insertions(+), 7751 deletions(-) create mode 100644 src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts create mode 100644 src/main/runtime/orchestration/db/contract-constants.ts create mode 100644 src/main/runtime/orchestration/db/coordinator-runs/coordinator-run-store.ts create mode 100644 src/main/runtime/orchestration/db/database-file-permissions.ts create mode 100644 src/main/runtime/orchestration/db/decision-gates/decision-gate-store.ts create mode 100644 src/main/runtime/orchestration/db/dispatch-capability-hash.ts create mode 100644 src/main/runtime/orchestration/db/dispatch-context/dispatch-capability.ts create mode 100644 src/main/runtime/orchestration/db/dispatch-context/dispatch-circuit-breaker.ts create mode 100644 src/main/runtime/orchestration/db/dispatch-context/dispatch-completion.ts create mode 100644 src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts create mode 100644 src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts create mode 100644 src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts create mode 100644 src/main/runtime/orchestration/db/federated-worker-report-outcome.test.ts create mode 100644 src/main/runtime/orchestration/db/federated-worker-report-outcome.ts create mode 100644 src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts create mode 100644 src/main/runtime/orchestration/db/federation/federation-relay-ack.ts create mode 100644 src/main/runtime/orchestration/db/federation/federation-relay-enqueue.ts create mode 100644 src/main/runtime/orchestration/db/federation/federation-relay-import.ts create mode 100644 src/main/runtime/orchestration/db/federation/federation-relay-item.ts create mode 100644 src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-authority.ts create mode 100644 src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-create.ts create mode 100644 src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts create mode 100644 src/main/runtime/orchestration/db/federation/remote-question-store.ts create mode 100644 src/main/runtime/orchestration/db/generated-id.ts create mode 100644 src/main/runtime/orchestration/db/legacy-question-identity.test.ts create mode 100644 src/main/runtime/orchestration/db/legacy-question-identity.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-ask-operation.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-compatibility-candidates.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-lifecycle-operation.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-mail-acknowledge.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-question-acknowledge.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-question-lookup.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-recovery-cohort.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-reply-operation.ts create mode 100644 src/main/runtime/orchestration/db/legacy/legacy-worker-completion.ts create mode 100644 src/main/runtime/orchestration/db/lifecycle-rejection-marker.test.ts create mode 100644 src/main/runtime/orchestration/db/lifecycle-rejection-marker.ts create mode 100644 src/main/runtime/orchestration/db/messages/direct-mailbox-routing.ts create mode 100644 src/main/runtime/orchestration/db/messages/foreign-direct-mailbox-routing.ts create mode 100644 src/main/runtime/orchestration/db/messages/mailbox-routing-page.ts create mode 100644 src/main/runtime/orchestration/db/messages/message-inbox.ts create mode 100644 src/main/runtime/orchestration/db/messages/message-insert.ts create mode 100644 src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts create mode 100644 src/main/runtime/orchestration/db/orchestration-db-methods.ts create mode 100644 src/main/runtime/orchestration/db/orchestration-db.ts create mode 100644 src/main/runtime/orchestration/db/pane-key-match.test.ts create mode 100644 src/main/runtime/orchestration/db/pane-key-match.ts create mode 100644 src/main/runtime/orchestration/db/questions/question-threads.ts create mode 100644 src/main/runtime/orchestration/db/reset/orchestration-reset.ts create mode 100644 src/main/runtime/orchestration/db/run-list-cursor.test.ts create mode 100644 src/main/runtime/orchestration/db/run-list-cursor.ts create mode 100644 src/main/runtime/orchestration/db/run-list-page.ts create mode 100644 src/main/runtime/orchestration/db/runs/legacy-coordinator-mail-takeover.ts create mode 100644 src/main/runtime/orchestration/db/runs/run-binding.ts create mode 100644 src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts create mode 100644 src/main/runtime/orchestration/db/runs/run-create.ts create mode 100644 src/main/runtime/orchestration/db/runs/run-delivery.ts create mode 100644 src/main/runtime/orchestration/db/runs/run-lookup.ts create mode 100644 src/main/runtime/orchestration/db/schema/adopt-legacy-run.ts create mode 100644 src/main/runtime/orchestration/db/schema/backfill-legacy-question-threads.ts create mode 100644 src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts create mode 100644 src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts create mode 100644 src/main/runtime/orchestration/db/schema/create-tables.ts create mode 100644 src/main/runtime/orchestration/db/schema/migrate-legacy-contract-storage.ts create mode 100644 src/main/runtime/orchestration/db/schema/migrate-v13-v28.ts create mode 100644 src/main/runtime/orchestration/db/schema/migrate-v2-v12.ts create mode 100644 src/main/runtime/orchestration/db/schema/migrate.ts create mode 100644 src/main/runtime/orchestration/db/schema/schema-column-probes.ts create mode 100644 src/main/runtime/orchestration/db/tasks/task-store.ts create mode 100644 src/main/runtime/orchestration/db/utc-timestamp.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/federated-worker-start-reconcile.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-abandon.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-authority.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-outcome.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stage.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-recovery.ts create mode 100644 src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts create mode 100644 src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts create mode 100644 src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts create mode 100644 src/main/runtime/orchestration/db/worker-terminal/worker-terminal-resource-store.ts create mode 100644 src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts create mode 100644 src/main/runtime/orchestration/dispatch-failure-idempotency.test.ts create mode 100644 src/main/runtime/orchestration/message-batch-atomicity.test.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index d7af7e7e4d4..2d90785d513 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -65,7 +65,6 @@ inline src/main/runtime/orca-runtime-files.ts inline src/main/runtime/orca-runtime-git.ts inline src/main/runtime/orca-runtime.test.ts inline src/main/runtime/orca-runtime.ts -inline src/main/runtime/orchestration/db.ts inline src/main/runtime/rpc/methods/orchestration.ts inline src/main/runtime/rpc/methods/terminal.ts inline src/main/runtime/runtime-rpc.ts diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index 725d8c5da41..fc5253aa411 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -1,116 +1,15 @@ -/* eslint-disable max-lines -- Why: the orchestration DB keeps schema creation, message CRUD, task DAG resolution, and dispatch context management in one class so transactional invariants (e.g. promoteReadyTasks running inside the same writer as updateTaskStatus) are enforced by locality. */ -import { createHash, randomBytes, timingSafeEqual } from 'node:crypto' -import { chmodSync, existsSync } from 'node:fs' -import Database from '../../sqlite/sync-database' -import type { - MessageType, - MessagePriority, - MessageDeliveryContract, - TaskStatus, - DispatchStatus, - GateStatus, - CoordinatorStatus, - MessageRow, - TaskRow, - DispatchContextRow, - DecisionGateRow, - CoordinatorRun, - WorkerReportOutcome, - WorkerReportSettlement, - RunRow, - DeliveryRow, - DeliveryStatus, - LegacyAdoptionRow, - LegacyCompatibilityPrincipalRow, - LegacyPrincipalRole, - LegacyOperationReceiptRow, - LegacyMailReceiptRow, - QuestionRow, - QuestionStatus, - MutationReceiptRow, - MutationState, - WorkerDispatchRow, - WorkerDispatchState, - LegacyWorkerTerminalRecoveryRow, - FederatedDispatchRow, - RemoteDispatchAttachmentRow, - FederationRelayDirection, - FederationRelayItemRow -} from './types' -import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display' -import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../shared/orchestration-rpc-contract' -import { parsePaneKey } from '../../../shared/stable-pane-id' -import { OrchestrationError } from './orchestration-error' -import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew' -import { - deriveWorkerTerminalListState, - WORKER_SETTLED_STATES, - type WorkerTerminalResourceRow, - type WorkerTerminalArchiveRow, - type WorkerTerminalArchiveStatus, - type WorkerTerminalListState, - type WorkerTerminalOwnershipState, - type WorkerTerminalRetainedReason -} from './worker-terminal-ownership' -import { ORCHESTRATION_RUN_PAGE_LIMIT } from '../../../shared/orchestration-run-pagination' -import { - ORCHESTRATION_CONTRACT_VERSION, - ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION -} from '../../../shared/protocol-version' -import { - releaseContextOnlyDispatch, - type ContextOnlyDispatchReleaseResult -} from './context-only-dispatch-release' -import { - ensureMutationReceiptCapacity, - migrateMutationReceiptCapacity -} from './mutation-receipt-capacity' - -// Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys. -function isEquivalentPaneKey(a: string, b: string): boolean { - if (a === b) { - return true - } - const aLeaf = parsePaneKey(a)?.leafId - const bLeaf = parsePaneKey(b)?.leafId - return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) -} - -function parseWorkerTerminalPriorOwnerIds(value: string): string[] | null { - try { - const parsed: unknown = JSON.parse(value) - return Array.isArray(parsed) && parsed.every((entry) => typeof entry === 'string') - ? parsed - : null - } catch { - return null - } -} - -const MESSAGE_ID_UPDATE_BATCH_SIZE = 500 -export const ORCHESTRATION_DELIVERY_BATCH_LIMIT = 50 - -export type MailboxRoutingPage = { - routedCount: number - hasMore: boolean - types: MessageType[] -} - -export type ForeignDirectMailboxRoutingPage = MailboxRoutingPage & { - mailboxes: { mailboxHandle: string; types: MessageType[] }[] -} -// Why: indexable pre-filter for isEquivalentPaneKey — equal strings and equal leaves both share the -// text after the first ':', so this narrows candidates without deciding equivalence itself. -const RUN_PANE_KEY_MATCH_SUFFIX_SQL = - "substr(coordinator_pane_key, instr(coordinator_pane_key, ':') + 1)" -const DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL = - "substr(assignee_pane_key, instr(assignee_pane_key, ':') + 1)" -const REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL = "substr(pane_key, instr(pane_key, ':') + 1)" - -function paneKeyMatchSuffix(paneKey: string): string { - const colon = paneKey.indexOf(':') - return colon === -1 ? paneKey : paneKey.slice(colon + 1) -} +export { OrchestrationDb } from './db/orchestration-db' +export { + CURRENT_CONTRACT_VERSION, + LEGACY_CONTRACT_VERSION, + LEGACY_RUN_ID +} from './db/contract-constants' +export type { RunListPage, TaskRuntimeLineageRow } from './db/run-list-page' +export { ORCHESTRATION_DELIVERY_BATCH_LIMIT } from './db/messages/mailbox-routing-page' +export type { + ForeignDirectMailboxRoutingPage, + MailboxRoutingPage +} from './db/messages/mailbox-routing-page' export type { MessageType, @@ -141,7638 +40,4 @@ export type { MutationState, WorkerDispatchRow, WorkerDispatchState -} - -function generateId(prefix: string): string { - return `${prefix}_${randomBytes(6).toString('hex')}` -} - -function hashDispatchCapability(capability: string): string { - return createHash('sha256').update(capability).digest('hex') -} - -function addLifecycleRejectionMarker(payload: string | null, code: string, reason: string): string { - let parsed: Record = {} - try { - const value: unknown = payload ? JSON.parse(payload) : {} - if (value && typeof value === 'object' && !Array.isArray(value)) { - parsed = value as Record - } - } catch { - // Authority reconciliation only reaches this path with object payloads. - } - return JSON.stringify({ - ...parsed, - _orcaLifecycleRejection: { code, reason } - }) -} - -function hasLifecycleRejectionMarker(payload: string | null): boolean { - try { - const value: unknown = JSON.parse(payload ?? 'null') - if (!value || typeof value !== 'object' || Array.isArray(value)) { - return false - } - const marker = (value as Record)._orcaLifecycleRejection - return Boolean( - marker && - typeof marker === 'object' && - !Array.isArray(marker) && - typeof (marker as Record).code === 'string' && - typeof (marker as Record).reason === 'string' - ) - } catch { - return false - } -} - -const SQLITE_UTC_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/ - -function exposeUtcTimestamp(timestamp: string | null): string | null { - if (!timestamp || !SQLITE_UTC_TIMESTAMP_RE.test(timestamp)) { - return timestamp - } - return `${timestamp.replace(' ', 'T')}Z` -} - -function exposeMessageTimestamps(message: MessageRow): MessageRow { - // Why: SQLite stores UTC as timezone-less space format for SQL ordering, but RPC/CLI consumers need an explicit offset. - return { - ...message, - created_at: exposeUtcTimestamp(message.created_at) ?? message.created_at, - delivered_at: exposeUtcTimestamp(message.delivered_at) - } -} - -function exposeMessageListTimestamps(messages: MessageRow[]): MessageRow[] { - return messages.map(exposeMessageTimestamps) -} - -function exposeRunTimestamps(run: RunRow): RunRow { - return { - ...run, - created_at: exposeUtcTimestamp(run.created_at) ?? run.created_at, - updated_at: exposeUtcTimestamp(run.updated_at) ?? run.updated_at - } -} - -function encodeRunListCursor(run: RunRow): string { - const cursor: RunListCursor = { createdAt: run.created_at, id: run.id } - return Buffer.from(JSON.stringify(cursor)).toString('base64url') -} - -function decodeRunListCursor(value: string): RunListCursor { - try { - const parsed: unknown = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')) - if ( - !parsed || - typeof parsed !== 'object' || - typeof (parsed as RunListCursor).createdAt !== 'string' || - typeof (parsed as RunListCursor).id !== 'string' - ) { - throw new Error('invalid cursor shape') - } - return parsed as RunListCursor - } catch { - throw new OrchestrationError('cursor_invalid', 'The Run list cursor is invalid.') - } -} - -function exposeDeliveryTimestamps(delivery: DeliveryRow): DeliveryRow { - return { - ...delivery, - created_at: exposeUtcTimestamp(delivery.created_at) ?? delivery.created_at, - acknowledged_at: exposeUtcTimestamp(delivery.acknowledged_at) - } -} - -function exposeQuestionTimestamps(question: QuestionRow): QuestionRow { - return { - ...question, - created_at: exposeUtcTimestamp(question.created_at) ?? question.created_at, - answered_at: exposeUtcTimestamp(question.answered_at), - closed_at: exposeUtcTimestamp(question.closed_at) - } -} - -function normalizeLegacyQuestionText(value: string): string { - return value.replace(/\r\n/g, '\n').trim() -} - -function normalizeLegacyQuestionOptions(options: unknown): string { - if (!Array.isArray(options) || !options.every((option) => typeof option === 'string')) { - return '[]' - } - return JSON.stringify(options.map((option) => option.trim())) -} - -function legacyMessageMatchesQuestion( - message: MessageRow, - question: string, - options: string[], - recipientHandles: readonly string[] -): boolean { - if ( - !recipientHandles.includes(message.to_handle) || - normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question) - ) { - return false - } - try { - const payload = JSON.parse(message.payload ?? '{}') as { options?: unknown } - return ( - normalizeLegacyQuestionOptions(payload.options) === normalizeLegacyQuestionOptions(options) - ) - } catch { - return false - } -} - -export const LEGACY_RUN_ID = ORCHESTRATION_LEGACY_RUN_ID - -export const LEGACY_CONTRACT_VERSION = 0 -export const CURRENT_CONTRACT_VERSION = ORCHESTRATION_CONTRACT_VERSION - -export type RunListPage = { - runs: RunRow[] - nextCursor: string | null -} - -export type TaskRuntimeLineageRow = TaskRow & { - creator_dispatch_id: string | null - creator_dispatch_run_id: string | null - creator_dispatch_pane_key: string | null - creator_dispatch_process_incarnation: string | null -} - -type RunListCursor = { - createdAt: string - id: string -} - -// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity. -const SCHEMA_VERSION = 28 - -function hardenOrchestrationDatabaseFiles(dbPath: (string & {}) | ':memory:'): void { - if (dbPath === ':memory:' || process.platform === 'win32') { - // Why: Windows protects these files through Orca's current-user-only userData DACL; POSIX mode bits are inert there. - return - } - for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) { - if (existsSync(path)) { - chmodSync(path, 0o600) - } - } -} - -export class OrchestrationDb { - private db: Database.Database - - // Why: the orchestration DB is created lazily for ALL users, but only the - // small minority who dispatch work ever have dispatch_contexts rows. The - // renderer graph publish rebuilds orchestration context on every 16ms tick - // (buildAgentOrchestrationByPaneKey), issuing 2 queries per terminal. Cache - // emptiness so the non-orchestration majority short-circuits the whole - // per-terminal fan-out. Only createDispatchContext flips this false→true. - private hasAnyDispatchContextsCache: boolean | undefined - private localMutationCallerFingerprint: string | undefined - - constructor(dbPath: (string & {}) | ':memory:') { - this.db = new Database(dbPath) - this.db.pragma('journal_mode = WAL') - this.db.pragma('synchronous = NORMAL') - this.db.pragma('busy_timeout = 5000') - this.createTables() - this.migrate() - this.createCoordinatorMailRoutingTrigger() - this.rememberCurrentRunCoordinatorHandles() - hardenOrchestrationDatabaseFiles(dbPath) - } - - private createTables(): void { - this.db.exec(` - CREATE TABLE IF NOT EXISTS runs ( - id TEXT PRIMARY KEY, - objective TEXT NOT NULL, - home_database TEXT NOT NULL DEFAULT 'this_database', - coordinator_handle TEXT, - coordinator_pane_key TEXT, - consumer_generation INTEGER NOT NULL DEFAULT 0, - legacy INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - - CREATE TABLE IF NOT EXISTS messages ( - id TEXT NOT NULL, - run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', - delivery_contract TEXT NOT NULL DEFAULT 'current_delivery' - CHECK(delivery_contract IN ('legacy_direct', 'current_delivery', 'audit_only')), - from_handle TEXT NOT NULL, - to_handle TEXT NOT NULL, - subject TEXT NOT NULL, - body TEXT NOT NULL DEFAULT '', - type TEXT NOT NULL DEFAULT 'status' - CHECK(type IN ( - 'status', 'dispatch', 'worker_done', 'merge_ready', - 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' - )), - priority TEXT NOT NULL DEFAULT 'normal' - CHECK(priority IN ('normal', 'high', 'urgent')), - thread_id TEXT, - payload TEXT, - read INTEGER NOT NULL DEFAULT 0, - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - delivered_at TEXT, - sender_pane_key TEXT - ); - - CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_id ON messages(id); - CREATE INDEX IF NOT EXISTS idx_inbox ON messages(to_handle, read); - CREATE INDEX IF NOT EXISTS idx_thread ON messages(thread_id); - - CREATE TABLE IF NOT EXISTS run_coordinator_handles ( - run_id TEXT NOT NULL, - terminal_handle TEXT NOT NULL, - first_bound_at TEXT NOT NULL DEFAULT (datetime('now')), - PRIMARY KEY (run_id, terminal_handle) - ); - - CREATE INDEX IF NOT EXISTS idx_run_coordinator_handles_handle - ON run_coordinator_handles(terminal_handle, run_id); - - CREATE TRIGGER IF NOT EXISTS trg_runs_remember_coordinator_insert - AFTER INSERT ON runs - WHEN NEW.legacy = 0 AND NEW.coordinator_handle IS NOT NULL - BEGIN - INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) - VALUES (NEW.id, NEW.coordinator_handle); - END; - - CREATE TRIGGER IF NOT EXISTS trg_runs_remember_coordinator_update - AFTER UPDATE OF coordinator_handle ON runs - WHEN NEW.legacy = 0 AND NEW.coordinator_handle IS NOT NULL - BEGIN - INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) - VALUES (NEW.id, NEW.coordinator_handle); - END; - - CREATE TRIGGER IF NOT EXISTS trg_runs_forget_coordinator_handles - AFTER DELETE ON runs - BEGIN - DELETE FROM run_coordinator_handles WHERE run_id = OLD.id; - END; - - CREATE TABLE IF NOT EXISTS deliveries ( - id TEXT PRIMARY KEY, - run_id TEXT NOT NULL, - consumer_generation INTEGER NOT NULL, - message_ids TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'outstanding' - CHECK(status IN ('outstanding', 'acknowledged', 'fenced')), - created_at TEXT NOT NULL DEFAULT (datetime('now')), - acknowledged_at TEXT - ); - - CREATE UNIQUE INDEX IF NOT EXISTS idx_deliveries_one_outstanding - ON deliveries(run_id) WHERE status = 'outstanding'; - CREATE INDEX IF NOT EXISTS idx_deliveries_run_created - ON deliveries(run_id, created_at); - - CREATE TABLE IF NOT EXISTS mutation_receipts ( - caller_fingerprint TEXT NOT NULL, - request_id TEXT NOT NULL, - method TEXT NOT NULL, - payload_hash TEXT NOT NULL, - state TEXT NOT NULL DEFAULT 'pending' - CHECK(state IN ('pending', 'completed')), - receipt TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')), - PRIMARY KEY (caller_fingerprint, request_id) - ); - - CREATE TABLE IF NOT EXISTS mutation_caller_identities ( - transport TEXT PRIMARY KEY, - caller_fingerprint TEXT NOT NULL UNIQUE - ); - - CREATE TABLE IF NOT EXISTS worker_dispatches ( - dispatch_id TEXT PRIMARY KEY, - runtime_epoch TEXT, - state TEXT NOT NULL DEFAULT 'starting' - CHECK(state IN ( - 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', - 'stopping', 'stop_unknown', 'stopped', 'abandoned' - )), - stage TEXT NOT NULL DEFAULT 'accepted', - worktree_id TEXT, - agent_terminal_handle TEXT, - setup_state TEXT NOT NULL DEFAULT 'not_applicable', - effects TEXT NOT NULL DEFAULT '[]', - residual_resources TEXT NOT NULL DEFAULT '[]', - start_options TEXT NOT NULL DEFAULT '{}', - last_error TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - - CREATE TABLE IF NOT EXISTS worker_terminal_resources ( - id TEXT PRIMARY KEY, - origin_dispatch_id TEXT NOT NULL, - owner_dispatch_id TEXT NOT NULL, - prior_owner_dispatch_ids TEXT NOT NULL DEFAULT '[]', - worktree_id TEXT, - terminal_handle TEXT NOT NULL, - pane_key TEXT, - process_incarnation TEXT, - host_scope TEXT, - ownership_state TEXT NOT NULL DEFAULT 'owned' - CHECK(ownership_state IN ('owned', 'transferred', 'user_owned', 'external', 'released')), - release_state TEXT NOT NULL DEFAULT 'not_requested' - CHECK(release_state IN ( - 'not_requested', 'retained', 'requested', 'releasing', 'released', 'unknown' - )), - retained_reason TEXT, - release_requested_at TEXT, - release_completed_at TEXT, - release_error TEXT, - archive_source TEXT, - archive_status TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - - CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_terminal_resources_owner - ON worker_terminal_resources(owner_dispatch_id); - CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_handle - ON worker_terminal_resources(terminal_handle); - CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_pane - ON worker_terminal_resources(pane_key); - CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_identity - ON worker_terminal_resources(process_incarnation, host_scope); - CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_release - ON worker_terminal_resources(release_state); - - CREATE TABLE IF NOT EXISTS worker_terminal_archives ( - dispatch_id TEXT PRIMARY KEY, - resource_id TEXT NOT NULL, - kind TEXT NOT NULL CHECK(kind IN ('transcript_pin', 'terminal_tail')), - content TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - - CREATE TABLE IF NOT EXISTS federated_dispatches ( - dispatch_id TEXT PRIMARY KEY, - environment_id TEXT NOT NULL, - environment_name TEXT NOT NULL, - peer_fingerprint TEXT NOT NULL, - remote_runtime_epoch TEXT, - protocol_version INTEGER NOT NULL DEFAULT 1, - remote_worktree_id TEXT, - remote_terminal_handle TEXT, - to_home_imported_sequence INTEGER NOT NULL DEFAULT 0, - to_home_acknowledged_sequence INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - - CREATE TABLE IF NOT EXISTS remote_dispatch_attachments ( - dispatch_id TEXT PRIMARY KEY, - task_id TEXT NOT NULL, - home_peer_fingerprint TEXT NOT NULL, - protocol_version INTEGER NOT NULL DEFAULT 1, - runtime_epoch TEXT NOT NULL, - capability_hash TEXT, - pane_key TEXT, - process_incarnation TEXT, - state TEXT NOT NULL DEFAULT 'starting' - CHECK(state IN ( - 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', - 'stopping', 'stop_unknown', 'stopped', 'abandoned' - )), - stage TEXT NOT NULL DEFAULT 'accepted', - worktree_id TEXT, - terminal_handle TEXT, - setup_state TEXT NOT NULL DEFAULT 'not_applicable', - effects TEXT NOT NULL DEFAULT '[]', - residual_resources TEXT NOT NULL DEFAULT '[]', - to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0, - last_error TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - CREATE INDEX IF NOT EXISTS idx_remote_dispatch_attachments_active_pane - ON remote_dispatch_attachments(pane_key) - WHERE state IN ('starting', 'ready'); - CREATE INDEX IF NOT EXISTS idx_remote_dispatch_attachments_active_pane_suffix - ON remote_dispatch_attachments(${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL}) - WHERE state IN ('starting', 'ready') AND pane_key IS NOT NULL; - - CREATE TABLE IF NOT EXISTS federation_relay_items ( - dispatch_id TEXT NOT NULL, - direction TEXT NOT NULL CHECK(direction IN ('to_home', 'to_worker')), - sequence INTEGER NOT NULL, - message_id TEXT NOT NULL, - kind TEXT NOT NULL, - payload TEXT NOT NULL, - byte_count INTEGER NOT NULL, - acked_at TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - PRIMARY KEY (dispatch_id, direction, sequence), - UNIQUE (dispatch_id, direction, message_id) - ); - - CREATE INDEX IF NOT EXISTS idx_federation_relay_pending - ON federation_relay_items(dispatch_id, direction, acked_at, sequence); - - CREATE TABLE IF NOT EXISTS remote_questions ( - message_id TEXT PRIMARY KEY, - dispatch_id TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending' - CHECK(status IN ('pending', 'answered', 'closed')), - answer_message_id TEXT, - answer_body TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - answered_at TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_remote_questions_dispatch_status - ON remote_questions(dispatch_id, status); - - CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY, - run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', - parent_id TEXT, - created_by_terminal_handle TEXT, - created_by_pane_key TEXT, - created_by_process_incarnation TEXT, - created_by_run_generation INTEGER, - task_title TEXT, - display_name TEXT, - spec TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending' - CHECK(status IN ( - 'pending', 'ready', 'dispatched', - 'completed', 'failed', 'blocked' - )), - deps TEXT NOT NULL DEFAULT '[]', - result TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - completed_at TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status); - CREATE INDEX IF NOT EXISTS idx_tasks_parent ON tasks(parent_id); - - CREATE TABLE IF NOT EXISTS dispatch_contexts ( - id TEXT PRIMARY KEY, - run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', - task_id TEXT NOT NULL, - contract_version INTEGER NOT NULL DEFAULT ${CURRENT_CONTRACT_VERSION}, - launch_token_hash TEXT, - assignee_handle TEXT, - assignee_pane_key TEXT, - capability_hash TEXT, - process_incarnation TEXT, - capability_revoked_at TEXT, - status TEXT NOT NULL DEFAULT 'pending' - CHECK(status IN ('pending', 'dispatched', 'completed', 'failed', 'circuit_broken')), - failure_count INTEGER NOT NULL DEFAULT 0, - last_failure TEXT, - dispatched_at TEXT, - completed_at TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - last_heartbeat_at TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_dispatch_task ON dispatch_contexts(task_id); - CREATE INDEX IF NOT EXISTS idx_dispatch_status ON dispatch_contexts(status); - CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_handle ON dispatch_contexts(assignee_handle); - - CREATE TABLE IF NOT EXISTS decision_gates ( - id TEXT PRIMARY KEY, - run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', - task_id TEXT NOT NULL, - question TEXT NOT NULL, - options TEXT NOT NULL DEFAULT '[]', - status TEXT NOT NULL DEFAULT 'pending' - CHECK(status IN ('pending', 'resolved', 'timeout')), - resolution TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - resolved_at TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_gates_task ON decision_gates(task_id); - CREATE INDEX IF NOT EXISTS idx_gates_status ON decision_gates(status); - - CREATE INDEX IF NOT EXISTS idx_runs_coordinator_pane_leaf - ON runs(${RUN_PANE_KEY_MATCH_SUFFIX_SQL}) - WHERE coordinator_pane_key IS NOT NULL; - - CREATE TABLE IF NOT EXISTS coordinator_runs ( - id TEXT PRIMARY KEY, - spec TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'idle' - CHECK(status IN ('idle', 'running', 'completed', 'failed')), - coordinator_handle TEXT NOT NULL, - poll_interval_ms INTEGER NOT NULL DEFAULT 2000, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - completed_at TEXT, - scheduler_lost_at TEXT - ); - `) - this.createMailboxDeliveryIndexesIfPossible() - } - - // Why: CREATE TABLE IF NOT EXISTS won't alter existing DBs; migrate in a txn that bumps user_version only on success (atomic all-or-nothing). - private migrate(): void { - const storedVersion = this.db.pragma('user_version', { simple: true }) as number - const current = resolveOrchestrationMigrationStartVersion( - this.db, - storedVersion, - SCHEMA_VERSION - ) - if (current >= SCHEMA_VERSION) { - return - } - - this.db.exec('BEGIN IMMEDIATE') - try { - // v1 → v2: SQLite can't ALTER a CHECK, so rebuild messages to allow 'heartbeat'; fold in v3's delivered_at to skip a second rebuild. - if (current < 2) { - if (!this.hasColumn('dispatch_contexts', 'last_heartbeat_at')) { - this.db.exec(`ALTER TABLE dispatch_contexts ADD COLUMN last_heartbeat_at TEXT`) - } - - if (!this.messagesTypeCheckAllowsHeartbeat()) { - // Why: recreate indexes here — DROP TABLE drops them; createTables re-runs only next startup, so skipping full-scans until restart. - this.db.exec(` - CREATE TABLE messages_new ( - id TEXT NOT NULL, - from_handle TEXT NOT NULL, - to_handle TEXT NOT NULL, - subject TEXT NOT NULL, - body TEXT NOT NULL DEFAULT '', - type TEXT NOT NULL DEFAULT 'status' - CHECK(type IN ( - 'status', 'dispatch', 'worker_done', 'merge_ready', - 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' - )), - priority TEXT NOT NULL DEFAULT 'normal' - CHECK(priority IN ('normal', 'high', 'urgent')), - thread_id TEXT, - payload TEXT, - read INTEGER NOT NULL DEFAULT 0, - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - delivered_at TEXT - ); - INSERT INTO messages_new ( - id, from_handle, to_handle, subject, body, type, priority, - thread_id, payload, read, sequence, created_at - ) - SELECT - id, from_handle, to_handle, subject, body, type, priority, - thread_id, payload, read, sequence, created_at - FROM messages; - DROP TABLE messages; - ALTER TABLE messages_new RENAME TO messages; - - CREATE UNIQUE INDEX idx_messages_id ON messages(id); - CREATE INDEX idx_inbox ON messages(to_handle, read); - CREATE INDEX idx_messages_undelivered_inbox - ON messages(to_handle, read, delivered_at, sequence); - CREATE INDEX idx_thread ON messages(thread_id); - `) - } - } - - // v2 → v3: add messages.delivered_at. hasColumn probe skips DBs that already got it via the v1→v2 rebuild (else a dup-column error aborts the txn). - if (current < 3) { - if (!this.hasColumn('messages', 'delivered_at')) { - this.db.exec(`ALTER TABLE messages ADD COLUMN delivered_at TEXT`) - } - } - if (current < 4) { - if (!this.hasColumn('tasks', 'created_by_terminal_handle')) { - this.db.exec(`ALTER TABLE tasks ADD COLUMN created_by_terminal_handle TEXT`) - } - } - if (current < 5) { - if (!this.hasColumn('tasks', 'task_title')) { - this.db.exec(`ALTER TABLE tasks ADD COLUMN task_title TEXT`) - } - if (!this.hasColumn('tasks', 'display_name')) { - this.db.exec(`ALTER TABLE tasks ADD COLUMN display_name TEXT`) - } - } - if (current < 6) { - if (!this.hasColumn('dispatch_contexts', 'assignee_pane_key')) { - this.db.exec(`ALTER TABLE dispatch_contexts ADD COLUMN assignee_pane_key TEXT`) - } - if (!this.hasColumn('messages', 'sender_pane_key')) { - this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`) - } - } - if (current < 7) { - this.db - .prepare( - `INSERT OR IGNORE INTO runs ( - id, objective, home_database, consumer_generation, legacy - ) VALUES (?, ?, 'this_database', 0, 1)` - ) - .run(LEGACY_RUN_ID, 'Legacy orchestration state (inspect only)') - for (const table of ['messages', 'tasks', 'dispatch_contexts', 'decision_gates']) { - if (!this.hasColumn(table, 'run_id')) { - this.db.exec( - `ALTER TABLE ${table} ADD COLUMN run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}'` - ) - } - } - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_messages_run_sequence ON messages(run_id, sequence); - CREATE INDEX IF NOT EXISTS idx_tasks_run_status ON tasks(run_id, status); - CREATE INDEX IF NOT EXISTS idx_dispatch_run_status ON dispatch_contexts(run_id, status); - CREATE INDEX IF NOT EXISTS idx_gates_run_status ON decision_gates(run_id, status); - CREATE INDEX IF NOT EXISTS idx_runs_coordinator_pane ON runs(coordinator_pane_key); - `) - } - if (current < 8) { - this.db.exec(` - CREATE TABLE IF NOT EXISTS deliveries ( - id TEXT PRIMARY KEY, - run_id TEXT NOT NULL, - consumer_generation INTEGER NOT NULL, - message_ids TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'outstanding' - CHECK(status IN ('outstanding', 'acknowledged', 'fenced')), - created_at TEXT NOT NULL DEFAULT (datetime('now')), - acknowledged_at TEXT - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_deliveries_one_outstanding - ON deliveries(run_id) WHERE status = 'outstanding'; - CREATE INDEX IF NOT EXISTS idx_deliveries_run_created - ON deliveries(run_id, created_at); - - CREATE TABLE IF NOT EXISTS question_threads ( - message_id TEXT PRIMARY KEY, - run_id TEXT NOT NULL, - dispatch_id TEXT NOT NULL, - asker_handle TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending' - CHECK(status IN ('pending', 'answered', 'closed')), - answer_message_id TEXT, - answer_body TEXT, - answered_by_generation INTEGER, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - answered_at TEXT, - closed_at TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_questions_dispatch_status - ON question_threads(dispatch_id, status); - `) - } - if (current < 9 && !this.messagesTypeCheckAllowsQuestion()) { - this.db.exec(` - CREATE TABLE messages_new ( - id TEXT NOT NULL, - run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', - from_handle TEXT NOT NULL, - to_handle TEXT NOT NULL, - subject TEXT NOT NULL, - body TEXT NOT NULL DEFAULT '', - type TEXT NOT NULL DEFAULT 'status' - CHECK(type IN ( - 'status', 'dispatch', 'worker_done', 'merge_ready', - 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' - )), - priority TEXT NOT NULL DEFAULT 'normal' - CHECK(priority IN ('normal', 'high', 'urgent')), - thread_id TEXT, - payload TEXT, - read INTEGER NOT NULL DEFAULT 0, - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - delivered_at TEXT, - sender_pane_key TEXT - ); - INSERT INTO messages_new ( - id, run_id, from_handle, to_handle, subject, body, type, priority, - thread_id, payload, read, sequence, created_at, delivered_at, sender_pane_key - ) - SELECT - id, run_id, from_handle, to_handle, subject, body, type, priority, - thread_id, payload, read, sequence, created_at, delivered_at, sender_pane_key - FROM messages; - DROP TABLE messages; - ALTER TABLE messages_new RENAME TO messages; - - CREATE UNIQUE INDEX idx_messages_id ON messages(id); - CREATE INDEX idx_inbox ON messages(to_handle, read); - CREATE INDEX idx_thread ON messages(thread_id); - CREATE INDEX idx_messages_run_sequence ON messages(run_id, sequence); - CREATE INDEX idx_messages_undelivered_inbox - ON messages(to_handle, read, delivered_at, sequence); - `) - } - if (current < 10) { - if (!this.hasColumn('dispatch_contexts', 'capability_hash')) { - this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN capability_hash TEXT') - } - if (!this.hasColumn('dispatch_contexts', 'process_incarnation')) { - this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN process_incarnation TEXT') - } - if (!this.hasColumn('dispatch_contexts', 'capability_revoked_at')) { - this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN capability_revoked_at TEXT') - } - } - if (current < 11) { - this.db.exec(` - CREATE TABLE IF NOT EXISTS mutation_receipts ( - caller_fingerprint TEXT NOT NULL, - request_id TEXT NOT NULL, - method TEXT NOT NULL, - payload_hash TEXT NOT NULL, - state TEXT NOT NULL DEFAULT 'pending' - CHECK(state IN ('pending', 'completed')), - receipt TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')), - PRIMARY KEY (caller_fingerprint, request_id) - ); - `) - } - if (current < 12) { - this.db.exec(` - CREATE TABLE IF NOT EXISTS worker_dispatches ( - dispatch_id TEXT PRIMARY KEY, - runtime_epoch TEXT, - state TEXT NOT NULL DEFAULT 'starting' - CHECK(state IN ( - 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', - 'stopping', 'stop_unknown', 'stopped', 'abandoned' - )), - stage TEXT NOT NULL DEFAULT 'accepted', - worktree_id TEXT, - agent_terminal_handle TEXT, - setup_state TEXT NOT NULL DEFAULT 'not_applicable', - effects TEXT NOT NULL DEFAULT '[]', - residual_resources TEXT NOT NULL DEFAULT '[]', - start_options TEXT NOT NULL DEFAULT '{}', - last_error TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - `) - } - if (current < 13 && !this.hasColumn('worker_dispatches', 'runtime_epoch')) { - this.db.exec('ALTER TABLE worker_dispatches ADD COLUMN runtime_epoch TEXT') - } - if (current < 14) { - this.db.exec(` - CREATE TABLE IF NOT EXISTS federated_dispatches ( - dispatch_id TEXT PRIMARY KEY, - environment_id TEXT NOT NULL, - environment_name TEXT NOT NULL, - peer_fingerprint TEXT NOT NULL, - remote_runtime_epoch TEXT, - protocol_version INTEGER NOT NULL DEFAULT 1, - remote_worktree_id TEXT, - remote_terminal_handle TEXT, - to_home_imported_sequence INTEGER NOT NULL DEFAULT 0, - to_home_acknowledged_sequence INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - CREATE TABLE IF NOT EXISTS remote_dispatch_attachments ( - dispatch_id TEXT PRIMARY KEY, - task_id TEXT NOT NULL, - home_peer_fingerprint TEXT NOT NULL, - protocol_version INTEGER NOT NULL DEFAULT 1, - runtime_epoch TEXT NOT NULL, - capability_hash TEXT, - pane_key TEXT, - process_incarnation TEXT, - state TEXT NOT NULL DEFAULT 'starting' - CHECK(state IN ( - 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', - 'stopping', 'stop_unknown', 'stopped', 'abandoned' - )), - stage TEXT NOT NULL DEFAULT 'accepted', - worktree_id TEXT, - terminal_handle TEXT, - setup_state TEXT NOT NULL DEFAULT 'not_applicable', - effects TEXT NOT NULL DEFAULT '[]', - residual_resources TEXT NOT NULL DEFAULT '[]', - to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0, - last_error TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - `) - } - if (current < 15) { - if (!this.hasColumn('federated_dispatches', 'to_home_imported_sequence')) { - this.db.exec( - 'ALTER TABLE federated_dispatches ADD COLUMN to_home_imported_sequence INTEGER NOT NULL DEFAULT 0' - ) - } - if (!this.hasColumn('remote_dispatch_attachments', 'to_worker_imported_sequence')) { - this.db.exec( - 'ALTER TABLE remote_dispatch_attachments ADD COLUMN to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0' - ) - } - this.db.exec(` - CREATE TABLE IF NOT EXISTS federation_relay_items ( - dispatch_id TEXT NOT NULL, - direction TEXT NOT NULL CHECK(direction IN ('to_home', 'to_worker')), - sequence INTEGER NOT NULL, - message_id TEXT NOT NULL, - kind TEXT NOT NULL, - payload TEXT NOT NULL, - byte_count INTEGER NOT NULL, - acked_at TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - PRIMARY KEY (dispatch_id, direction, sequence), - UNIQUE (dispatch_id, direction, message_id) - ); - CREATE INDEX IF NOT EXISTS idx_federation_relay_pending - ON federation_relay_items(dispatch_id, direction, acked_at, sequence); - `) - } - if (current < 16) { - this.db.exec(` - CREATE TABLE IF NOT EXISTS remote_questions ( - message_id TEXT PRIMARY KEY, - dispatch_id TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending' - CHECK(status IN ('pending', 'answered', 'closed')), - answer_message_id TEXT, - answer_body TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - answered_at TEXT - ); - CREATE INDEX IF NOT EXISTS idx_remote_questions_dispatch_status - ON remote_questions(dispatch_id, status); - `) - } - if (current < 17 && !this.hasColumn('remote_dispatch_attachments', 'protocol_version')) { - this.db.exec( - 'ALTER TABLE remote_dispatch_attachments ADD COLUMN protocol_version INTEGER NOT NULL DEFAULT 1' - ) - } - if (current < 19) { - this.migrateLegacyContractStorage() - } - if (current < 20) { - this.backfillLegacyQuestionThreads() - } - if (current < 21) { - this.migrateLegacySchedulerLossProvenance() - } - if (current < 22) { - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_handle - ON dispatch_contexts(assignee_handle); - `) - } - if (current < 23) { - this.backfillWorkerTerminalResources() - } - if (current < 24) { - if (!this.hasColumn('tasks', 'created_by_pane_key')) { - this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_pane_key TEXT') - } - if (!this.hasColumn('tasks', 'created_by_process_incarnation')) { - this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_process_incarnation TEXT') - } - if (!this.hasColumn('tasks', 'created_by_run_generation')) { - this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_run_generation INTEGER') - } - } - if (current < 25) { - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_dispatch_active_assignee_handle - ON dispatch_contexts(assignee_handle) - WHERE assignee_handle IS NOT NULL AND status IN ('pending', 'dispatched'); - `) - } - if (current < 26) { - migrateMutationReceiptCapacity(this.db) - } - if ( - current < 27 && - !this.hasColumn('federated_dispatches', 'to_home_acknowledged_sequence') - ) { - this.db.exec( - 'ALTER TABLE federated_dispatches ADD COLUMN to_home_acknowledged_sequence INTEGER NOT NULL DEFAULT 0' - ) - } - if (current < 28) { - this.db.exec(` - CREATE TABLE IF NOT EXISTS mutation_caller_identities ( - transport TEXT PRIMARY KEY, - caller_fingerprint TEXT NOT NULL UNIQUE - ); - `) - } - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_pane_leaf - ON dispatch_contexts(${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL}) - WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched'); - CREATE INDEX IF NOT EXISTS idx_dispatch_active_run_assignee_handle - ON dispatch_contexts(run_id, assignee_handle) - WHERE assignee_handle IS NOT NULL AND status IN ('pending', 'dispatched'); - CREATE INDEX IF NOT EXISTS idx_dispatch_active_run_pane_leaf - ON dispatch_contexts(run_id, ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL}) - WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched'); - CREATE INDEX IF NOT EXISTS idx_dispatch_active_assignee_pane_key - ON dispatch_contexts(assignee_pane_key) - WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched'); - `) - this.createMailboxDeliveryIndexesIfPossible() - - this.db.pragma(`user_version = ${SCHEMA_VERSION}`) - this.db.exec('COMMIT') - } catch (err) { - this.db.exec('ROLLBACK') - throw err - } - } - - private migrateLegacyContractStorage(): void { - if (!this.hasColumn('dispatch_contexts', 'contract_version')) { - this.db.exec( - `ALTER TABLE dispatch_contexts - ADD COLUMN contract_version INTEGER NOT NULL DEFAULT ${CURRENT_CONTRACT_VERSION}` - ) - } - if (!this.hasColumn('dispatch_contexts', 'launch_token_hash')) { - this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN launch_token_hash TEXT') - } - if (!this.hasColumn('messages', 'delivery_contract')) { - this.db.exec( - `ALTER TABLE messages - ADD COLUMN delivery_contract TEXT NOT NULL DEFAULT 'current_delivery' - CHECK(delivery_contract IN ('legacy_direct', 'current_delivery', 'audit_only'))` - ) - } - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_messages_delivery_contract - ON messages(run_id, delivery_contract, to_handle, read, sequence); - - CREATE TABLE IF NOT EXISTS legacy_adoptions ( - source_run_id TEXT PRIMARY KEY, - adopted_run_id TEXT UNIQUE NOT NULL, - scheduler_state_lost INTEGER NOT NULL, - adopted_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - - CREATE TABLE IF NOT EXISTS legacy_compatibility_principals ( - id TEXT PRIMARY KEY, - run_id TEXT NOT NULL, - dispatch_id TEXT, - role TEXT NOT NULL CHECK(role IN ('worker', 'coordinator')), - host_scope TEXT NOT NULL, - terminal_handle TEXT NOT NULL, - pane_key TEXT NOT NULL, - launch_token_hash TEXT NOT NULL, - process_incarnation TEXT, - status TEXT NOT NULL - CHECK(status IN ('committed', 'settled', 'revoked')), - CHECK( - (role = 'worker' AND dispatch_id IS NOT NULL) OR - (role = 'coordinator' AND dispatch_id IS NULL) - ), - UNIQUE(role, run_id, dispatch_id) - ); - - CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_principal_coordinator - ON legacy_compatibility_principals(run_id) - WHERE role = 'coordinator'; - CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_principal_dispatch - ON legacy_compatibility_principals(dispatch_id) - WHERE role = 'worker'; - - CREATE TABLE IF NOT EXISTS legacy_operation_receipts ( - principal_id TEXT NOT NULL, - operation_key TEXT NOT NULL, - method TEXT NOT NULL, - payload_hash TEXT NOT NULL, - effect_id TEXT NOT NULL, - response_json TEXT NOT NULL, - completed_at TEXT NOT NULL DEFAULT (datetime('now')), - PRIMARY KEY(principal_id, operation_key) - ); - - CREATE TABLE IF NOT EXISTS legacy_mail_receipts ( - principal_id TEXT NOT NULL, - message_id TEXT NOT NULL, - acknowledged_at TEXT, - PRIMARY KEY(principal_id, message_id) - ); - `) - - this.db - .prepare( - `UPDATE dispatch_contexts - SET contract_version = ? - WHERE run_id = ? AND capability_hash IS NULL` - ) - .run(LEGACY_CONTRACT_VERSION, LEGACY_RUN_ID) - this.classifyLegacyMessageContracts(LEGACY_RUN_ID, false) - this.ensureLegacySchedulerLossColumn() - this.adoptLegacyRunIfNeeded() - } - - private classifyLegacyMessageContracts(runId: string, adoptedOnly: boolean): void { - const contractFilter = adoptedOnly - ? " AND delivery_contract IN ('legacy_direct', 'audit_only')" - : '' - this.db - .prepare( - `UPDATE messages SET delivery_contract = 'legacy_direct' - WHERE run_id = ?${contractFilter}` - ) - .run(runId) - const rows = this.db - .prepare(`SELECT id, payload FROM messages WHERE run_id = ?${contractFilter}`) - .all(runId) as { id: string; payload: string | null }[] - const markAuditOnly = this.db.prepare( - "UPDATE messages SET delivery_contract = 'audit_only' WHERE id = ? AND run_id = ?" - ) - for (const row of rows) { - if (hasLifecycleRejectionMarker(row.payload)) { - markAuditOnly.run(row.id, runId) - } - } - } - - private migrateLegacySchedulerLossProvenance(): void { - this.ensureLegacySchedulerLossColumn() - this.adoptLegacyRunIfNeeded() - const adoption = this.getLegacyAdoption() - if (adoption) { - this.classifyLegacyMessageContracts(adoption.adopted_run_id, true) - } - } - - private ensureLegacySchedulerLossColumn(): void { - if (!this.hasColumn('coordinator_runs', 'scheduler_lost_at')) { - this.db.exec('ALTER TABLE coordinator_runs ADD COLUMN scheduler_lost_at TEXT') - } - } - - private backfillLegacyQuestionThreads(): void { - const messages = this.db - .prepare( - `SELECT id, run_id, from_handle, to_handle, payload, created_at, sequence - FROM messages - WHERE type = 'decision_gate' - AND delivery_contract IN ('legacy_direct', 'current_delivery') - ORDER BY sequence` - ) - .all() as { - id: string - run_id: string - from_handle: string - to_handle: string - payload: string | null - created_at: string - sequence: number - }[] - const getDispatch = this.db.prepare( - 'SELECT id, run_id, task_id FROM dispatch_contexts WHERE id = ? AND contract_version = ?' - ) - const getDispatchesForLegacyQuestion = this.db.prepare( - `SELECT id, run_id, task_id - FROM dispatch_contexts - WHERE contract_version = ? AND assignee_handle = ? - AND (? IS NULL OR task_id = ?) - AND created_at <= ? - AND (completed_at IS NULL OR completed_at >= ?) - ORDER BY rowid - LIMIT 2` - ) - const getAnswer = this.db.prepare( - `SELECT id, body, created_at - FROM messages - WHERE run_id = ? - AND thread_id = ? - AND delivery_contract IN ('legacy_direct', 'current_delivery') - AND from_handle = ? - AND to_handle IN (?, ?) - AND sequence > ? - ORDER BY sequence - LIMIT 1` - ) - const insert = this.db.prepare( - `INSERT OR IGNORE INTO question_threads ( - message_id, run_id, dispatch_id, asker_handle, status, - answer_message_id, answer_body, created_at, answered_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const message of messages) { - let payload: { taskId?: unknown; dispatchId?: unknown } - try { - payload = JSON.parse(message.payload ?? '{}') as { - taskId?: unknown - dispatchId?: unknown - } - } catch { - continue - } - const inferredDispatches = - typeof payload.dispatchId === 'string' - ? [] - : (getDispatchesForLegacyQuestion.all( - LEGACY_CONTRACT_VERSION, - message.from_handle, - typeof payload.taskId === 'string' ? payload.taskId : null, - typeof payload.taskId === 'string' ? payload.taskId : null, - message.created_at, - message.created_at - ) as { id: string; run_id: string; task_id: string }[]) - const dispatch = - typeof payload.dispatchId === 'string' - ? (getDispatch.get(payload.dispatchId, LEGACY_CONTRACT_VERSION) as - | { id: string; run_id: string; task_id: string } - | undefined) - : inferredDispatches.length === 1 - ? inferredDispatches[0] - : undefined - if ( - !dispatch || - (typeof payload.taskId === 'string' && payload.taskId !== dispatch.task_id) || - (message.run_id !== LEGACY_RUN_ID && message.run_id !== dispatch.run_id) - ) { - continue - } - const answer = getAnswer.get( - message.run_id, - message.id, - message.to_handle, - message.from_handle, - `dispatch:${dispatch.id}`, - message.sequence - ) as { id: string; body: string; created_at: string } | undefined - insert.run( - message.id, - dispatch.run_id, - dispatch.id, - message.from_handle, - answer ? 'answered' : 'pending', - answer?.id ?? null, - answer?.body ?? null, - message.created_at, - answer?.created_at ?? null - ) - } - const adoption = this.getLegacyAdoption() - const coordinator = adoption - ? this.getLegacyCoordinatorPrincipal(adoption.adopted_run_id) - : undefined - if (adoption && coordinator?.status === 'revoked') { - this.promoteLegacyCoordinatorMailForTakeover( - adoption.adopted_run_id, - coordinator.terminal_handle - ) - } - } - - private adoptLegacyRunIfNeeded(): void { - const existing = this.db - .prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?') - .get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined - const hasGraph = this.db - .prepare( - `SELECT 1 - WHERE EXISTS(SELECT 1 FROM tasks WHERE run_id = ?) - OR EXISTS(SELECT 1 FROM dispatch_contexts WHERE run_id = ?) - OR EXISTS(SELECT 1 FROM decision_gates WHERE run_id = ?) - OR EXISTS(SELECT 1 FROM messages WHERE run_id = ?) - OR EXISTS(SELECT 1 FROM question_threads WHERE run_id = ?) - OR EXISTS(SELECT 1 FROM deliveries WHERE run_id = ?)` - ) - .get(LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID) - if (!existing && !hasGraph) { - return - } - - const adoptedRunId = existing?.adopted_run_id ?? generateId('run') - this.db - .prepare( - `INSERT OR IGNORE INTO runs ( - id, objective, home_database, consumer_generation, legacy - ) VALUES (?, ?, 'this_database', 0, 0)` - ) - .run(adoptedRunId, 'Recovered orchestration work from a contract update') - this.db - .prepare( - `INSERT OR IGNORE INTO legacy_adoptions ( - source_run_id, adopted_run_id, scheduler_state_lost - ) VALUES (?, ?, 1)` - ) - .run(LEGACY_RUN_ID, adoptedRunId) - this.db - .prepare( - `UPDATE coordinator_runs - SET status = 'failed', - completed_at = COALESCE( - completed_at, - (SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ?) - ), - scheduler_lost_at = ( - SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ? - ) - WHERE status = 'running' - AND julianday(created_at) <= julianday(( - SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ? - ))` - ) - .run(LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID) - - this.db - .prepare( - `UPDATE deliveries SET status = 'fenced' - WHERE run_id = ? AND status = 'outstanding'` - ) - .run(LEGACY_RUN_ID) - for (const table of [ - 'tasks', - 'dispatch_contexts', - 'decision_gates', - 'messages', - 'question_threads', - 'deliveries' - ]) { - this.db - .prepare(`UPDATE ${table} SET run_id = ? WHERE run_id = ?`) - .run(adoptedRunId, LEGACY_RUN_ID) - } - this.db - .prepare( - `UPDATE runs - SET objective = 'Legacy orchestration state (adopted; inspect only)', - coordinator_handle = NULL, coordinator_pane_key = NULL, - updated_at = datetime('now') - WHERE id = ?` - ) - .run(LEGACY_RUN_ID) - - const mismatch = this.db - .prepare( - `WITH migration_runs(run_id) AS (VALUES (?), (?)) - SELECT 1 - WHERE EXISTS( - SELECT 1 FROM dispatch_contexts d - INNER JOIN tasks t ON t.id = d.task_id - WHERE d.run_id <> t.run_id - AND ( - d.run_id IN (SELECT run_id FROM migration_runs) - OR t.run_id IN (SELECT run_id FROM migration_runs) - ) - ) - OR EXISTS( - SELECT 1 FROM decision_gates g - INNER JOIN tasks t ON t.id = g.task_id - WHERE g.run_id <> t.run_id - AND ( - g.run_id IN (SELECT run_id FROM migration_runs) - OR t.run_id IN (SELECT run_id FROM migration_runs) - ) - ) - OR EXISTS( - SELECT 1 FROM question_threads q - INNER JOIN dispatch_contexts d ON d.id = q.dispatch_id - WHERE q.run_id <> d.run_id - AND ( - q.run_id IN (SELECT run_id FROM migration_runs) - OR d.run_id IN (SELECT run_id FROM migration_runs) - ) - ) - OR EXISTS( - SELECT 1 FROM deliveries d - INNER JOIN json_each(d.message_ids) ids - INNER JOIN messages m ON m.id = ids.value - WHERE d.run_id <> m.run_id - AND ( - d.run_id IN (SELECT run_id FROM migration_runs) - OR m.run_id IN (SELECT run_id FROM migration_runs) - ) - )` - ) - .get(LEGACY_RUN_ID, adoptedRunId) - if (mismatch) { - throw new Error('Legacy orchestration adoption produced inconsistent Run ownership.') - } - } - - private hasColumn(table: string, column: string): boolean { - const rows = this.db.pragma(`table_info(${table})`) as { name: string }[] - return rows.some((r) => r.name === column) - } - - private createMailboxDeliveryIndexesIfPossible(): void { - const hasDeliveredAt = this.hasColumn('messages', 'delivered_at') - if (hasDeliveredAt) { - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_messages_undelivered_inbox - ON messages(to_handle, read, delivered_at, sequence) - `) - } - - if ( - !hasDeliveredAt || - !this.hasColumn('messages', 'run_id') || - !this.hasColumn('messages', 'delivery_contract') - ) { - return - } - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_messages_undelivered_direct_run - ON messages(run_id, to_handle, sequence) - WHERE read = 0 AND delivered_at IS NULL - AND delivery_contract = 'current_delivery'; - CREATE INDEX IF NOT EXISTS idx_messages_unread_current_inbox - ON messages(to_handle, sequence) - WHERE read = 0 AND delivery_contract = 'current_delivery'; - CREATE INDEX IF NOT EXISTS idx_messages_unread_current_inbox_type - ON messages(to_handle, type, sequence) - WHERE read = 0 AND delivery_contract = 'current_delivery'; - CREATE INDEX IF NOT EXISTS idx_messages_unread_current_run_type - ON messages(run_id, to_handle, type, sequence) - WHERE read = 0 AND delivery_contract = 'current_delivery'; - `) - } - - // Why: sqlite_master holds the table's CREATE SQL incl. the CHECK — cheapest reliable probe for whether it already allows 'heartbeat'. - private messagesTypeCheckAllowsHeartbeat(): boolean { - const row = this.db - .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages'") - .get() as { sql: string } | undefined - return !!row && row.sql.includes("'heartbeat'") - } - - private messagesTypeCheckAllowsQuestion(): boolean { - const row = this.db - .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages'") - .get() as { sql: string } | undefined - return !!row && row.sql.includes("'question'") - } - - // ── Durable mutation receipts ── - - getOrCreateLocalMutationCallerFingerprint(): string { - if (this.localMutationCallerFingerprint) { - return this.localMutationCallerFingerprint - } - const transport = 'local_authenticated_transport' - const existing = this.db - .prepare('SELECT caller_fingerprint FROM mutation_caller_identities WHERE transport = ?') - .get(transport) as { caller_fingerprint: string } | undefined - if (existing) { - this.localMutationCallerFingerprint = existing.caller_fingerprint - return this.localMutationCallerFingerprint - } - this.db - .prepare( - `INSERT OR IGNORE INTO mutation_caller_identities (transport, caller_fingerprint) - VALUES (?, ?)` - ) - .run(transport, randomBytes(32).toString('hex')) - const created = this.db - .prepare('SELECT caller_fingerprint FROM mutation_caller_identities WHERE transport = ?') - .get(transport) as { caller_fingerprint: string } | undefined - if (!created) { - throw new Error('Failed to create the local orchestration mutation caller identity.') - } - this.localMutationCallerFingerprint = created.caller_fingerprint - return this.localMutationCallerFingerprint - } - - beginMutationReceipt(params: { - callerFingerprint: string - requestId: string - method: string - payloadHash: string - }): - | { disposition: 'started'; row: MutationReceiptRow } - | { disposition: 'pending'; row: MutationReceiptRow } - | { disposition: 'completed'; row: MutationReceiptRow } { - this.db.exec('BEGIN IMMEDIATE') - try { - const existing = this.getMutationReceipt(params.callerFingerprint, params.requestId) - if (existing) { - if (existing.method !== params.method || existing.payload_hash !== params.payloadHash) { - throw new OrchestrationError( - 'request_mismatch', - `Mutation request ${params.requestId} was already used with different input.` - ) - } - this.db.exec('COMMIT') - return { disposition: existing.state, row: existing } - } - ensureMutationReceiptCapacity(this.db) - this.db - .prepare( - `INSERT INTO mutation_receipts ( - caller_fingerprint, request_id, method, payload_hash, state - ) VALUES (?, ?, ?, ?, 'pending')` - ) - .run(params.callerFingerprint, params.requestId, params.method, params.payloadHash) - const row = this.getMutationReceipt(params.callerFingerprint, params.requestId) - this.db.exec('COMMIT') - return { disposition: 'started', row: row as MutationReceiptRow } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - completeMutationReceipt(params: { - callerFingerprint: string - requestId: string - method: string - payloadHash: string - receipt: string - }): MutationReceiptRow { - const result = this.db - .prepare( - `UPDATE mutation_receipts - SET state = 'completed', receipt = ?, updated_at = datetime('now') - WHERE caller_fingerprint = ? AND request_id = ? AND method = ? - AND payload_hash = ?` - ) - .run( - params.receipt, - params.callerFingerprint, - params.requestId, - params.method, - params.payloadHash - ) - const row = this.getMutationReceipt(params.callerFingerprint, params.requestId) - if (result.changes !== 1 || !row) { - throw new OrchestrationError( - 'request_mismatch', - `Mutation request ${params.requestId} no longer matches its pending operation.` - ) - } - return row - } - - discardPendingMutationReceipt(callerFingerprint: string, requestId: string): void { - this.db - .prepare( - `DELETE FROM mutation_receipts - WHERE caller_fingerprint = ? AND request_id = ? AND state = 'pending'` - ) - .run(callerFingerprint, requestId) - } - - getMutationReceipt(callerFingerprint: string, requestId: string): MutationReceiptRow | undefined { - return this.db - .prepare( - `SELECT * FROM mutation_receipts - WHERE caller_fingerprint = ? AND request_id = ?` - ) - .get(callerFingerprint, requestId) as MutationReceiptRow | undefined - } - - // ── Legacy adoption and compatibility principals ── - - getLegacyAdoption(): LegacyAdoptionRow | undefined { - return this.db - .prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?') - .get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined - } - - commitLegacyCompatibilityPrincipal(params: { - runId: string - dispatchId?: string - role: LegacyPrincipalRole - hostScope: string - terminalHandle: string - paneKey: string - launchTokenHash: string - processIncarnation?: string - }): { principal: LegacyCompatibilityPrincipalRow; duplicate: boolean } { - this.db.exec('BEGIN IMMEDIATE') - try { - const adoption = this.getLegacyAdoption() - if (!adoption || adoption.adopted_run_id !== params.runId) { - throw new OrchestrationError( - 'request_mismatch', - `Run ${params.runId} is not the adopted legacy Run.` - ) - } - const dispatchId = params.role === 'worker' ? (params.dispatchId ?? null) : null - let initialStatus: 'committed' | 'settled' = 'committed' - if (params.role === 'worker') { - const dispatch = dispatchId ? this.getDispatchContextById(dispatchId) : undefined - if ( - !dispatch || - dispatch.run_id !== params.runId || - dispatch.contract_version !== LEGACY_CONTRACT_VERSION - ) { - throw new OrchestrationError( - 'request_mismatch', - `Dispatch ${dispatchId ?? '(missing)'} is not a legacy attempt in this Run.` - ) - } - initialStatus = ['pending', 'dispatched'].includes(dispatch.status) - ? 'committed' - : 'settled' - } else if (params.dispatchId) { - throw new OrchestrationError( - 'request_mismatch', - 'A coordinator compatibility principal cannot name a Dispatch.' - ) - } - - const existing = this.db - .prepare( - `SELECT * FROM legacy_compatibility_principals - WHERE role = ? AND run_id = ? AND dispatch_id IS ?` - ) - .get(params.role, params.runId, dispatchId) as LegacyCompatibilityPrincipalRow | undefined - if (existing) { - const same = - existing.host_scope === params.hostScope && - existing.terminal_handle === params.terminalHandle && - existing.pane_key === params.paneKey && - existing.launch_token_hash === params.launchTokenHash && - existing.process_incarnation === (params.processIncarnation ?? null) - if (!same) { - throw new OrchestrationError( - 'request_mismatch', - `The ${params.role} compatibility principal is already committed to different proof.` - ) - } - if (existing.status === 'revoked') { - throw new OrchestrationError( - 'legacy_read_only', - `The ${params.role} compatibility principal has been revoked. No effects were applied.`, - { effectsApplied: false } - ) - } - this.db.exec('COMMIT') - return { principal: existing, duplicate: true } - } - if ( - params.role === 'coordinator' && - !this.resolveLegacyCoordinatorCandidate({ - runId: params.runId, - terminalHandle: params.terminalHandle, - paneKey: params.paneKey - }) - ) { - throw new OrchestrationError( - 'legacy_read_only', - 'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.', - { effectsApplied: false } - ) - } - - const id = generateId('legacy_principal') - this.db - .prepare( - `INSERT INTO legacy_compatibility_principals ( - id, run_id, dispatch_id, role, host_scope, terminal_handle, - pane_key, launch_token_hash, process_incarnation, status - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - .run( - id, - params.runId, - dispatchId, - params.role, - params.hostScope, - params.terminalHandle, - params.paneKey, - params.launchTokenHash, - params.processIncarnation ?? null, - initialStatus - ) - const principal = this.getLegacyCompatibilityPrincipal(id) as LegacyCompatibilityPrincipalRow - if (principal.status === 'committed') { - this.initializeLegacyRecoveryCohort(principal) - } - this.db.exec('COMMIT') - return { principal, duplicate: false } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getLegacyCompatibilityPrincipal(id: string): LegacyCompatibilityPrincipalRow | undefined { - return this.db.prepare('SELECT * FROM legacy_compatibility_principals WHERE id = ?').get(id) as - | LegacyCompatibilityPrincipalRow - | undefined - } - - listLegacyCompatibilityPrincipals(runId: string): LegacyCompatibilityPrincipalRow[] { - return this.db - .prepare( - `SELECT * FROM legacy_compatibility_principals - WHERE run_id = ? ORDER BY rowid` - ) - .all(runId) as LegacyCompatibilityPrincipalRow[] - } - - getLegacyCoordinatorPrincipal(runId: string): LegacyCompatibilityPrincipalRow | undefined { - return this.db - .prepare( - `SELECT * FROM legacy_compatibility_principals - WHERE run_id = ? AND role = 'coordinator'` - ) - .get(runId) as LegacyCompatibilityPrincipalRow | undefined - } - - resolveLegacyCompatibilityPrincipalByIdentity(params: { - runId: string - role: LegacyPrincipalRole - terminalHandle?: string - paneKey?: string - }): LegacyCompatibilityPrincipalRow | undefined { - if (!params.terminalHandle && !params.paneKey) { - return undefined - } - const rows = ( - this.db - .prepare( - `SELECT * FROM legacy_compatibility_principals - WHERE run_id = ? AND role = ? AND status IN ('committed', 'settled') - ORDER BY rowid` - ) - .all(params.runId, params.role) as LegacyCompatibilityPrincipalRow[] - ).filter((principal) => - params.paneKey - ? isEquivalentPaneKey(principal.pane_key, params.paneKey) - : principal.terminal_handle === params.terminalHandle - ) - if (rows.length > 1) { - throw new OrchestrationError( - 'operation_unknown', - 'Multiple legacy principals match this process identity.' - ) - } - return rows[0] - } - - resolveLegacyWorkerCandidate(params: { - runId?: string - terminalHandle?: string - paneKey?: string - dispatchId?: string - taskId?: string - }): { dispatch: DispatchContextRow } | undefined { - if (!params.runId || (!params.terminalHandle && !params.paneKey)) { - return undefined - } - const rows = ( - params.dispatchId - ? [this.getDispatchContextById(params.dispatchId)].filter( - (row): row is DispatchContextRow => row !== undefined - ) - : (this.db - .prepare( - `SELECT * FROM dispatch_contexts - WHERE run_id = ? AND contract_version = ? - AND status IN ('pending', 'dispatched') - ORDER BY rowid` - ) - .all(params.runId, LEGACY_CONTRACT_VERSION) as DispatchContextRow[]) - ).filter( - (dispatch) => - dispatch.run_id === params.runId && - dispatch.contract_version === LEGACY_CONTRACT_VERSION && - (!params.taskId || dispatch.task_id === params.taskId) && - (params.paneKey - ? Boolean( - dispatch.assignee_pane_key && - isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) - ) - : dispatch.assignee_handle === params.terminalHandle) - ) - if (rows.length > 1) { - throw new OrchestrationError( - 'operation_unknown', - 'Multiple active legacy Dispatches match this process identity.' - ) - } - if (params.dispatchId && rows.length === 0) { - const target = this.getDispatchContextById(params.dispatchId) - if (target?.contract_version === LEGACY_CONTRACT_VERSION) { - throw new OrchestrationError( - 'legacy_read_only', - `Dispatch ${params.dispatchId} is retained but this process cannot prove ownership.` - ) - } - } - return rows[0] ? { dispatch: rows[0] } : undefined - } - - resolveLegacyCoordinatorCandidate(params: { - runId: string - terminalHandle?: string - paneKey?: string - }): { terminalHandle: string; paneKey: string } | undefined { - if (!params.terminalHandle || !params.paneKey) { - return undefined - } - const run = this.getRunRaw(params.runId) - const principal = this.getLegacyCoordinatorPrincipal(params.runId) - if (principal) { - if ( - principal.status !== 'committed' || - principal.terminal_handle !== params.terminalHandle || - !isEquivalentPaneKey(principal.pane_key, params.paneKey) || - (run?.coordinator_pane_key !== null && - (run?.coordinator_handle !== principal.terminal_handle || - !isEquivalentPaneKey(run.coordinator_pane_key, principal.pane_key))) - ) { - return undefined - } - return { terminalHandle: params.terminalHandle, paneKey: params.paneKey } - } - // Why: the first current binding durably fences uncommitted legacy processes. - if ( - !run || - run.coordinator_pane_key !== null || - this.getUniqueLegacyCoordinatorHandle(params.runId) !== params.terminalHandle - ) { - return undefined - } - return { terminalHandle: params.terminalHandle, paneKey: params.paneKey } - } - - // Why: caller-side fence jurisdiction. Widening this fences more callers, so keep it narrow. - isLegacyCoordinatorHandle(runId: string, terminalHandle: string): boolean { - const principal = this.getLegacyCoordinatorPrincipal(runId) - if (principal) { - return principal.terminal_handle === terminalHandle - } - return this.getUniqueLegacyCoordinatorHandle(runId) === terminalHandle - } - - // Why: recipient-side permit for legacy lifecycle mail. After a takeover revokes the old principal - // the replacement coordinator is only reachable through the Run binding. - isLegacyCoordinatorDeliveryTarget(runId: string, terminalHandle: string): boolean { - if (this.isLegacyCoordinatorHandle(runId, terminalHandle)) { - return true - } - if (this.getRunRaw(runId)?.coordinator_handle !== terminalHandle) { - return false - } - // Why: must match resolveLegacyWorkerCoordinatorDelivery's takeover test. A still-committed - // principal routes legacy_direct to this handle, and no reader can see that mailbox. - return this.getLegacyCoordinatorPrincipal(runId)?.status !== 'committed' - } - - findLegacyWorkerCompletion(params: { - principalId: string - taskId: string - recipientHandle: string - subject: string - body: string - payload: string | null - }): MessageRow | undefined { - const principal = this.getLegacyCompatibilityPrincipal(params.principalId) - if (!principal || principal.role !== 'worker' || !principal.dispatch_id) { - throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.') - } - const runAddress = `run:${principal.run_id}` - const rows = this.db - .prepare( - `SELECT * FROM messages - WHERE run_id = ? - AND ( - (delivery_contract = 'legacy_direct' AND to_handle = ?) OR - (delivery_contract = 'current_delivery' AND to_handle = ?) - ) - AND from_handle = ? AND type = 'worker_done' - AND subject = ? AND body = ? AND payload IS ? - ORDER BY sequence` - ) - .all( - principal.run_id, - params.recipientHandle, - runAddress, - principal.terminal_handle, - params.subject, - params.body, - params.payload - ) as MessageRow[] - const matches = rows.filter((message) => { - try { - const payload = JSON.parse(message.payload ?? '{}') as { - taskId?: unknown - dispatchId?: unknown - } - return payload.taskId === params.taskId && payload.dispatchId === principal.dispatch_id - } catch { - return false - } - }) - if (matches.length > 1) { - throw new OrchestrationError( - 'operation_unknown', - 'Multiple matching legacy worker completions exist.' - ) - } - return matches[0] ? exposeMessageTimestamps(matches[0]) : undefined - } - - hasPendingCurrentDelivery(runId: string): boolean { - return Boolean( - this.db - .prepare( - `SELECT 1 FROM messages - WHERE run_id = ? AND to_handle = ? - AND delivery_contract = 'current_delivery' AND read = 0 - LIMIT 1` - ) - .get(runId, `run:${runId}`) - ) - } - - setLegacyCompatibilityPrincipalStatus( - id: string, - status: 'settled' | 'revoked' - ): LegacyCompatibilityPrincipalRow | undefined { - this.db - .prepare( - `UPDATE legacy_compatibility_principals - SET status = ? - WHERE id = ? AND status = 'committed'` - ) - .run(status, id) - return this.getLegacyCompatibilityPrincipal(id) - } - - getLegacyOperationReceipt( - principalId: string, - operationKey: string - ): LegacyOperationReceiptRow | undefined { - return this.db - .prepare( - `SELECT * FROM legacy_operation_receipts - WHERE principal_id = ? AND operation_key = ?` - ) - .get(principalId, operationKey) as LegacyOperationReceiptRow | undefined - } - - private requireCommittedLegacyPrincipal( - principalId: string, - role?: LegacyPrincipalRole - ): LegacyCompatibilityPrincipalRow { - const principal = this.getLegacyCompatibilityPrincipal(principalId) - if (!principal || principal.status !== 'committed' || (role && principal.role !== role)) { - throw new OrchestrationError( - 'request_mismatch', - `Legacy compatibility principal ${principalId} is not committed for this operation.` - ) - } - return principal - } - - private requireLegacyMailPrincipal( - principalId: string, - role?: LegacyPrincipalRole - ): LegacyCompatibilityPrincipalRow { - const principal = this.getLegacyCompatibilityPrincipal(principalId) - if ( - !principal || - !['committed', 'settled'].includes(principal.status) || - (role && principal.role !== role) - ) { - throw new OrchestrationError( - 'request_mismatch', - `Legacy compatibility principal ${principalId} cannot access retained mail.` - ) - } - return principal - } - - private initializeLegacyRecoveryCohort(principal: LegacyCompatibilityPrincipalRow): void { - if (principal.role === 'worker') { - this.db - .prepare( - `INSERT OR IGNORE INTO legacy_mail_receipts ( - principal_id, message_id, acknowledged_at - ) - SELECT ?, m.id, NULL - FROM messages m - INNER JOIN dispatch_contexts d ON d.id = ? - WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' AND m.read = 1 - AND d.status IN ('pending', 'dispatched') - AND m.created_at >= d.created_at - AND (m.to_handle = ? OR m.to_handle = ?)` - ) - .run( - principal.id, - principal.dispatch_id, - principal.run_id, - principal.terminal_handle, - `dispatch:${principal.dispatch_id}` - ) - return - } - this.db - .prepare( - `INSERT OR IGNORE INTO legacy_mail_receipts ( - principal_id, message_id, acknowledged_at - ) - SELECT ?, m.id, NULL - FROM messages m - WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' AND m.read = 1 - AND m.to_handle = ? - AND EXISTS( - SELECT 1 FROM dispatch_contexts d - WHERE d.run_id = m.run_id - AND d.contract_version = ? - AND d.status IN ('pending', 'dispatched') - AND m.created_at >= d.created_at - AND (m.from_handle = d.assignee_handle OR m.from_handle = 'dispatch:' || d.id) - )` - ) - .run(principal.id, principal.run_id, principal.terminal_handle, LEGACY_CONTRACT_VERSION) - } - - getLegacyMailPage(params: { principalId: string; limit?: number; types?: MessageType[] }): { - messages: MessageRow[] - recovery: boolean - } { - const principal = this.requireLegacyMailPrincipal(params.principalId) - const limit = Math.min( - Math.max(params.limit ?? ORCHESTRATION_DELIVERY_BATCH_LIMIT, 1), - ORCHESTRATION_DELIVERY_BATCH_LIMIT - ) - const addressSql = - principal.role === 'worker' ? '(m.to_handle = ? OR m.to_handle = ?)' : 'm.to_handle = ?' - const addressParams = - principal.role === 'worker' - ? [principal.terminal_handle, `dispatch:${principal.dispatch_id}`] - : [principal.terminal_handle] - const typeSql = - params.types && params.types.length > 0 - ? `AND m.type IN (${params.types.map(() => '?').join(',')})` - : '' - const typeParams = params.types ?? [] - const recovery = this.db - .prepare( - `SELECT m.* - FROM legacy_mail_receipts r - INNER JOIN messages m ON m.id = r.message_id - WHERE r.principal_id = ? AND r.acknowledged_at IS NULL - AND m.run_id = ? AND m.delivery_contract = 'legacy_direct' - AND ${addressSql} - ${typeSql} - ORDER BY m.sequence ASC LIMIT ?` - ) - .all( - params.principalId, - principal.run_id, - ...addressParams, - ...typeParams, - limit - ) as MessageRow[] - if (recovery.length > 0) { - return { messages: exposeMessageListTimestamps(recovery), recovery: true } - } - - const unread = this.db - .prepare( - `SELECT m.* - FROM messages m - LEFT JOIN legacy_mail_receipts r - ON r.principal_id = ? AND r.message_id = m.id - WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' - AND m.read = 0 AND r.message_id IS NULL AND ${addressSql} - ${typeSql} - ORDER BY m.sequence ASC LIMIT ?` - ) - .all( - params.principalId, - principal.run_id, - ...addressParams, - ...typeParams, - limit - ) as MessageRow[] - return { messages: exposeMessageListTimestamps(unread), recovery: false } - } - - getLegacyMailHistory(params: { principalId: string; limit?: number; types?: MessageType[] }): { - messages: MessageRow[] - recovery: false - } { - const principal = this.requireLegacyMailPrincipal(params.principalId) - const limit = Math.min(Math.max(params.limit ?? 100, 1), 100) - const addressSql = - principal.role === 'worker' ? '(to_handle = ? OR to_handle = ?)' : 'to_handle = ?' - const addressParams = - principal.role === 'worker' - ? [principal.terminal_handle, `dispatch:${principal.dispatch_id}`] - : [principal.terminal_handle] - const typeSql = - params.types && params.types.length > 0 - ? `AND type IN (${params.types.map(() => '?').join(',')})` - : '' - const messages = this.db - .prepare( - `SELECT * FROM messages - WHERE run_id = ? AND delivery_contract = 'legacy_direct' - AND ${addressSql} ${typeSql} - ORDER BY sequence ASC LIMIT ?` - ) - .all(principal.run_id, ...addressParams, ...(params.types ?? []), limit) as MessageRow[] - return { messages: exposeMessageListTimestamps(messages), recovery: false } - } - - acknowledgeLegacyMail(params: { - principalId: string - messageIds: string[] - types?: MessageType[] - }): { - receipts: LegacyMailReceiptRow[] - duplicate: boolean - } { - if (params.messageIds.length === 0) { - return { receipts: [], duplicate: true } - } - this.db.exec('BEGIN IMMEDIATE') - try { - const principal = this.requireLegacyMailPrincipal(params.principalId) - const uniqueIds = [...new Set(params.messageIds)] - const placeholders = uniqueIds.map(() => '?').join(',') - const prior = this.db - .prepare( - `SELECT COUNT(*) AS count FROM legacy_mail_receipts - WHERE principal_id = ? AND message_id IN (${placeholders}) - AND acknowledged_at IS NOT NULL` - ) - .get(params.principalId, ...uniqueIds) as { count: number } - if (prior.count !== uniqueIds.length) { - const actionable = this.getLegacyMailPage({ - principalId: params.principalId, - limit: uniqueIds.length, - types: params.types - }).messages - if ( - actionable.length !== uniqueIds.length || - actionable.some((message, index) => message.id !== uniqueIds[index]) - ) { - throw new OrchestrationError( - 'request_mismatch', - 'Legacy mail acknowledgment does not match the current replay page.' - ) - } - } - const rows = this.db - .prepare( - `SELECT * FROM messages - WHERE id IN (${placeholders}) AND run_id = ? - AND delivery_contract = 'legacy_direct'` - ) - .all(...uniqueIds, principal.run_id) as MessageRow[] - const validIds = new Set( - rows - .filter( - (message) => - message.to_handle === principal.terminal_handle || - (principal.role === 'worker' && - message.to_handle === `dispatch:${principal.dispatch_id}`) - ) - .map((message) => message.id) - ) - if (validIds.size !== uniqueIds.length || uniqueIds.some((id) => !validIds.has(id))) { - throw new OrchestrationError( - 'request_mismatch', - 'Legacy mail acknowledgment contains a message outside this principal inbox.' - ) - } - - this.db - .prepare( - `UPDATE messages - SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) - WHERE id IN (${placeholders})` - ) - .run(...uniqueIds) - const insert = this.db.prepare( - `INSERT INTO legacy_mail_receipts ( - principal_id, message_id, acknowledged_at - ) VALUES (?, ?, datetime('now')) - ON CONFLICT(principal_id, message_id) - DO UPDATE SET acknowledged_at = COALESCE( - legacy_mail_receipts.acknowledged_at, excluded.acknowledged_at - )` - ) - for (const messageId of uniqueIds) { - insert.run(params.principalId, messageId) - } - const receipts = this.db - .prepare( - `SELECT * FROM legacy_mail_receipts - WHERE principal_id = ? AND message_id IN (${placeholders}) - ORDER BY message_id` - ) - .all(params.principalId, ...uniqueIds) as LegacyMailReceiptRow[] - this.db.exec('COMMIT') - return { receipts, duplicate: prior.count === uniqueIds.length } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - acknowledgeLegacyQuestionAnswer(params: { - principalId: string - questionId: string - answerMessageId: string - }): { receipt: LegacyMailReceiptRow; duplicate: boolean } { - this.db.exec('BEGIN IMMEDIATE') - try { - const principal = this.requireLegacyMailPrincipal(params.principalId, 'worker') - const question = this.getQuestionRaw(params.questionId) - const source = this.getMessageById(params.questionId) - const answer = this.getMessageById(params.answerMessageId) - const dispatch = principal.dispatch_id - ? this.getDispatchContextById(principal.dispatch_id) - : undefined - const exactLegacyAnswer = - answer?.delivery_contract === 'legacy_direct' && - (answer.to_handle === principal.terminal_handle || - answer.to_handle === `dispatch:${principal.dispatch_id}`) - const adoption = this.getLegacyAdoption() - const exactTakenOverAnswer = - adoption?.adopted_run_id === principal.run_id && - dispatch?.run_id === principal.run_id && - dispatch.contract_version === LEGACY_CONTRACT_VERSION && - source?.run_id === principal.run_id && - source.from_handle === principal.terminal_handle && - source.to_handle === `run:${principal.run_id}` && - source.delivery_contract === 'current_delivery' && - answer?.run_id === principal.run_id && - answer?.delivery_contract === 'current_delivery' && - answer.from_handle === `run:${principal.run_id}` && - answer.to_handle === `dispatch:${principal.dispatch_id}` && - answer.thread_id === question?.message_id - if ( - !question || - !answer || - question.run_id !== principal.run_id || - question.dispatch_id !== principal.dispatch_id || - question.answer_message_id !== params.answerMessageId || - (!exactLegacyAnswer && !exactTakenOverAnswer) - ) { - throw new OrchestrationError( - 'request_mismatch', - 'Legacy answer acknowledgment does not match this principal question.' - ) - } - const existing = this.db - .prepare( - `SELECT * FROM legacy_mail_receipts - WHERE principal_id = ? AND message_id = ?` - ) - .get(params.principalId, params.answerMessageId) as LegacyMailReceiptRow | undefined - this.db - .prepare( - `UPDATE messages - SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) - WHERE id = ?` - ) - .run(params.answerMessageId) - this.db - .prepare( - `INSERT INTO legacy_mail_receipts ( - principal_id, message_id, acknowledged_at - ) VALUES (?, ?, datetime('now')) - ON CONFLICT(principal_id, message_id) - DO UPDATE SET acknowledged_at = COALESCE( - legacy_mail_receipts.acknowledged_at, excluded.acknowledged_at - )` - ) - .run(params.principalId, params.answerMessageId) - const receipt = this.db - .prepare( - `SELECT * FROM legacy_mail_receipts - WHERE principal_id = ? AND message_id = ?` - ) - .get(params.principalId, params.answerMessageId) as LegacyMailReceiptRow - this.db.exec('COMMIT') - return { receipt, duplicate: Boolean(existing?.acknowledged_at) } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - // ── Runs ── - - createRun(params: { - objective: string - coordinatorHandle: string - coordinatorPaneKey: string - }): RunRow { - const id = generateId('run') - this.db.exec('BEGIN IMMEDIATE') - try { - this.unbindOtherRunsForPane(params.coordinatorPaneKey) - this.db - .prepare( - `INSERT INTO runs ( - id, objective, coordinator_handle, coordinator_pane_key, - consumer_generation, legacy - ) VALUES (?, ?, ?, ?, 1, 0)` - ) - .run(id, params.objective, params.coordinatorHandle, params.coordinatorPaneKey) - this.rememberRunCoordinatorHandle(id, params.coordinatorHandle) - this.db.exec('COMMIT') - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - return this.getRun(id) as RunRow - } - - bindRun(params: { - runId: string - coordinatorHandle: string - coordinatorPaneKey: string - takeoverLegacy?: boolean - legacyCoordinatorAuthority?: { - runId: string - principalId: string | null - terminalHandle: string - paneKey: string - consumerGeneration: number - } - }): RunRow | undefined { - this.db.exec('BEGIN IMMEDIATE') - try { - const run = this.getRunRaw(params.runId) - if (!run || run.legacy === 1) { - this.db.exec('ROLLBACK') - return undefined - } - const sameBinding = - run.coordinator_pane_key !== null && - isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey) - const adoption = this.getLegacyAdoption() - const adoptedRun = adoption?.adopted_run_id === params.runId - const legacyAuthority = params.legacyCoordinatorAuthority - const legacyPrincipalId = legacyAuthority?.principalId - const legacyPrincipal = legacyPrincipalId - ? this.getLegacyCompatibilityPrincipal(legacyPrincipalId) - : undefined - const provenLegacyBinding = Boolean( - adoptedRun && - legacyAuthority && - legacyAuthority.principalId !== null && - legacyAuthority.runId === params.runId && - legacyAuthority.consumerGeneration === run.consumer_generation && - legacyPrincipal?.run_id === params.runId && - legacyPrincipal.role === 'coordinator' && - legacyPrincipal.status === 'committed' && - legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle && - isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) && - params.coordinatorHandle === legacyAuthority.terminalHandle && - isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey) - ) - if (legacyAuthority && !provenLegacyBinding) { - throw new OrchestrationError( - 'legacy_read_only', - 'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.', - { effectsApplied: false } - ) - } - const activeLegacyAssignment = - adoptedRun && - Boolean( - this.db - .prepare( - `SELECT 1 FROM dispatch_contexts - WHERE run_id = ? AND contract_version = ? - AND status IN ('pending', 'dispatched') - LIMIT 1` - ) - .get(params.runId, LEGACY_CONTRACT_VERSION) - ) - const coordinatorPrincipal = adoptedRun - ? this.getLegacyCoordinatorPrincipal(params.runId) - : undefined - const retainedCoordinatorHandle = - coordinatorPrincipal?.terminal_handle ?? - run.coordinator_handle ?? - this.getUniqueLegacyCoordinatorHandle(params.runId) - const takeoverAlreadyApplied = Boolean( - params.takeoverLegacy && - sameBinding && - run.coordinator_handle === params.coordinatorHandle && - coordinatorPrincipal?.status !== 'committed' - ) - const replacesLegacyCoordinator = Boolean( - adoptedRun && - !provenLegacyBinding && - retainedCoordinatorHandle && - (params.takeoverLegacy || - retainedCoordinatorHandle !== params.coordinatorHandle || - !sameBinding) - ) - if (params.takeoverLegacy && !adoptedRun) { - throw new OrchestrationError( - 'invalid_argument', - 'Legacy takeover is only available for the automatically adopted Run.' - ) - } - // Why: only LIVE legacy work needs the flag — settled work has no competing authority left, and - // fencing it would strand the recovered graph behind an attestation the caller may not have. - if ( - activeLegacyAssignment && - !sameBinding && - !provenLegacyBinding && - !params.takeoverLegacy - ) { - throw new OrchestrationError( - 'consumer_fenced', - 'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.', - { - effectsApplied: false, - recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy` - } - ) - } - this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId) - for (const handle of new Set( - [run.coordinator_handle, params.coordinatorHandle].filter((value): value is string => - Boolean(value) - ) - )) { - this.rememberRunCoordinatorHandle(params.runId, handle) - this.routeAllUnreadDirectMessagesToRunMailbox(params.runId, handle) - } - if ( - (params.takeoverLegacy && !takeoverAlreadyApplied) || - !sameBinding || - run.coordinator_handle !== params.coordinatorHandle - ) { - if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) { - if ( - coordinatorPrincipal?.status === 'committed' && - (params.takeoverLegacy || - coordinatorPrincipal.terminal_handle !== params.coordinatorHandle || - !isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey)) - ) { - this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked') - } - } - this.db - .prepare( - `UPDATE runs - SET coordinator_handle = ?, coordinator_pane_key = ?, - consumer_generation = consumer_generation + 1, - updated_at = datetime('now') - WHERE id = ?` - ) - .run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId) - this.fenceOutstandingDelivery(params.runId) - if (params.takeoverLegacy || replacesLegacyCoordinator) { - this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle) - } - } - this.db.exec('COMMIT') - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - return this.getRun(params.runId) - } - - getRun(id: string): RunRow | undefined { - const run = this.getRunRaw(id) - return run ? exposeRunTimestamps(run) : undefined - } - - listRuns(params: { limit?: number; cursor?: string } = {}): RunListPage { - if (params.limit === undefined && params.cursor === undefined) { - const rows = this.db - .prepare('SELECT * FROM runs ORDER BY created_at DESC, id DESC') - .all() as RunRow[] - return { runs: rows.map(exposeRunTimestamps), nextCursor: null } - } - const limit = Math.min( - Math.max(1, params.limit ?? ORCHESTRATION_RUN_PAGE_LIMIT), - ORCHESTRATION_RUN_PAGE_LIMIT - ) - const cursor = params.cursor ? decodeRunListCursor(params.cursor) : undefined - const rows = ( - cursor - ? this.db - .prepare( - `SELECT * FROM runs - WHERE created_at < ? OR (created_at = ? AND id < ?) - ORDER BY created_at DESC, id DESC - LIMIT ?` - ) - .all(cursor.createdAt, cursor.createdAt, cursor.id, limit + 1) - : this.db - .prepare('SELECT * FROM runs ORDER BY created_at DESC, id DESC LIMIT ?') - .all(limit + 1) - ) as RunRow[] - const hasMore = rows.length > limit - const pageRows = hasMore ? rows.slice(0, limit) : rows - return { - runs: pageRows.map(exposeRunTimestamps), - nextCursor: hasMore ? encodeRunListCursor(pageRows.at(-1) as RunRow) : null - } - } - - getCurrentRunForPane(paneKey: string): RunRow | undefined { - const run = this.runsBoundToPane(paneKey)[0] - return run ? exposeRunTimestamps(run) : undefined - } - - // Why: the indexed suffix only narrows candidates; isEquivalentPaneKey still decides, so - // reminted tab halves keep matching and unparseable keys keep requiring an exact match. - private runsBoundToPane(paneKey: string): RunRow[] { - return ( - this.db - .prepare( - `SELECT * FROM runs - WHERE coordinator_pane_key IS NOT NULL AND legacy = 0 - AND ${RUN_PANE_KEY_MATCH_SUFFIX_SQL} = ? - ORDER BY rowid` - ) - .all(paneKeyMatchSuffix(paneKey)) as RunRow[] - ).filter( - (run) => - run.coordinator_pane_key !== null && isEquivalentPaneKey(run.coordinator_pane_key, paneKey) - ) - } - - private getRunRaw(id: string): RunRow | undefined { - return this.db.prepare('SELECT * FROM runs WHERE id = ?').get(id) as RunRow | undefined - } - - private unbindOtherRunsForPane(paneKey: string, exceptRunId?: string): void { - for (const run of this.runsBoundToPane(paneKey)) { - if (run.id !== exceptRunId) { - if (run.coordinator_handle) { - this.routeAllUnreadDirectMessagesToRunMailbox(run.id, run.coordinator_handle) - } - this.db - .prepare( - `UPDATE runs - SET coordinator_handle = NULL, coordinator_pane_key = NULL, - consumer_generation = consumer_generation + 1, - updated_at = datetime('now') - WHERE id = ?` - ) - .run(run.id) - this.fenceOutstandingDelivery(run.id) - } - } - } - - private requireRun(runId: string): void { - if (!this.getRunRaw(runId)) { - throw new Error(`Run not found: ${runId}`) - } - } - - private rememberRunCoordinatorHandle(runId: string, terminalHandle: string): void { - this.db - .prepare( - `INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) VALUES (?, ?)` - ) - .run(runId, terminalHandle) - } - - private rememberCurrentRunCoordinatorHandles(): void { - this.db.exec(` - INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) - SELECT id, coordinator_handle FROM runs - WHERE legacy = 0 AND coordinator_handle IS NOT NULL - `) - } - - private createCoordinatorMailRoutingTrigger(): void { - this.db.exec('BEGIN IMMEDIATE') - try { - this.db.exec(` - DROP TRIGGER IF EXISTS trg_messages_route_coordinator_mail; - CREATE TRIGGER trg_messages_route_coordinator_mail - AFTER INSERT ON messages - WHEN NEW.read = 0 AND NEW.delivery_contract = 'current_delivery' - AND EXISTS ( - SELECT 1 FROM runs - WHERE runs.id = NEW.run_id AND runs.legacy = 0 - ) - AND EXISTS ( - SELECT 1 FROM run_coordinator_handles - WHERE run_id = NEW.run_id AND terminal_handle = NEW.to_handle - ) - AND NOT EXISTS ( - SELECT 1 FROM dispatch_contexts - WHERE run_id = NEW.run_id AND assignee_handle = NEW.to_handle - AND status IN ('pending', 'dispatched') - ) - BEGIN - UPDATE messages SET to_handle = 'run:' || NEW.run_id WHERE sequence = NEW.sequence; - END; - `) - this.db.exec('COMMIT') - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - private fenceOutstandingDelivery(runId: string): void { - this.db - .prepare( - "UPDATE deliveries SET status = 'fenced' WHERE run_id = ? AND status = 'outstanding'" - ) - .run(runId) - } - - private promoteLegacyCoordinatorMailForTakeover( - runId: string, - retainedCoordinatorHandle: string | null - ): void { - if (!retainedCoordinatorHandle) { - return - } - this.db - .prepare( - `UPDATE messages - SET to_handle = ?, delivery_contract = 'current_delivery', - read = 0, delivered_at = NULL - WHERE run_id = ? AND delivery_contract = 'legacy_direct' - AND to_handle = ? - AND EXISTS( - SELECT 1 FROM dispatch_contexts d - WHERE d.run_id = messages.run_id - AND d.contract_version = ? - AND ( - messages.from_handle = d.assignee_handle OR - messages.from_handle = 'dispatch:' || d.id - ) - ) - AND ( - read = 0 OR EXISTS( - SELECT 1 FROM question_threads q - WHERE q.message_id = messages.id AND q.status = 'pending' - ) OR EXISTS( - SELECT 1 - FROM legacy_mail_receipts r - INNER JOIN legacy_compatibility_principals p - ON p.id = r.principal_id - WHERE r.message_id = messages.id - AND r.acknowledged_at IS NULL - AND p.run_id = messages.run_id - AND p.role = 'coordinator' - AND p.terminal_handle = ? - ) OR ( - read = 1 - AND NOT EXISTS( - SELECT 1 FROM legacy_compatibility_principals p - WHERE p.run_id = messages.run_id AND p.role = 'coordinator' - ) - AND EXISTS( - SELECT 1 FROM dispatch_contexts d - WHERE d.run_id = messages.run_id - AND d.contract_version = ? - AND d.status IN ('pending', 'dispatched') - AND messages.created_at >= d.created_at - AND ( - messages.from_handle = d.assignee_handle OR - messages.from_handle = 'dispatch:' || d.id - ) - ) - ) - )` - ) - .run( - `run:${runId}`, - runId, - retainedCoordinatorHandle, - LEGACY_CONTRACT_VERSION, - retainedCoordinatorHandle, - LEGACY_CONTRACT_VERSION - ) - } - - private getUniqueLegacyCoordinatorHandle(runId: string): string | null { - const adoption = this.getLegacyAdoption() - if (!adoption || adoption.adopted_run_id !== runId) { - return null - } - const workerHandles = new Set( - ( - this.db - .prepare( - `SELECT DISTINCT assignee_handle AS handle - FROM dispatch_contexts - WHERE run_id = ? AND contract_version = ? - AND assignee_handle IS NOT NULL - UNION - SELECT DISTINCT terminal_handle AS handle - FROM legacy_compatibility_principals - WHERE run_id = ? AND role = 'worker' - AND status IN ('committed', 'settled')` - ) - .all(runId, LEGACY_CONTRACT_VERSION, runId) as { handle: string }[] - ).map((row) => row.handle) - ) - const durableRows = this.db - .prepare( - `SELECT coordinator_handle AS handle - FROM coordinator_runs - WHERE scheduler_lost_at = ? - UNION - SELECT created_by_terminal_handle AS handle - FROM tasks t - WHERE t.run_id = ? AND t.created_by_terminal_handle IS NOT NULL - AND t.created_at <= ? - AND EXISTS( - SELECT 1 FROM dispatch_contexts d - WHERE d.task_id = t.id AND d.run_id = t.run_id - AND d.contract_version = ? - )` - ) - .all(adoption.adopted_at, runId, adoption.adopted_at, LEGACY_CONTRACT_VERSION) as { - handle: string - }[] - if (durableRows.some((row) => workerHandles.has(row.handle))) { - return null - } - const candidates = new Set(durableRows.map((row) => row.handle)) - const mailRows = this.db - .prepare( - `SELECT m.to_handle AS handle - FROM messages m - INNER JOIN dispatch_contexts d - ON d.run_id = m.run_id AND d.contract_version = ? - AND (m.from_handle = d.assignee_handle OR m.from_handle = 'dispatch:' || d.id) - WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' - AND m.created_at <= ? - UNION - SELECT m.from_handle AS handle - FROM messages m - INNER JOIN dispatch_contexts d - ON d.run_id = m.run_id AND d.contract_version = ? - AND (m.to_handle = d.assignee_handle OR m.to_handle = 'dispatch:' || d.id) - WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' - AND m.created_at <= ?` - ) - .all( - LEGACY_CONTRACT_VERSION, - runId, - adoption.adopted_at, - LEGACY_CONTRACT_VERSION, - runId, - adoption.adopted_at - ) as { - handle: string - }[] - for (const row of mailRows) { - if ( - !workerHandles.has(row.handle) && - !row.handle.startsWith('dispatch:') && - !row.handle.startsWith('run:') - ) { - candidates.add(row.handle) - } - } - return candidates.size === 1 ? ([...candidates][0] ?? null) : null - } - - private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow { - const run = this.getRunRaw(runId) - if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) { - throw new OrchestrationError( - 'consumer_fenced', - 'This mailbox consumer has been replaced. Rebind with orchestration run-use.' - ) - } - return run - } - - private getDeliveryRaw(id: string): DeliveryRow | undefined { - return this.db.prepare('SELECT * FROM deliveries WHERE id = ?').get(id) as - | DeliveryRow - | undefined - } - - private getDeliveryMessages(delivery: DeliveryRow): MessageRow[] { - const ids = JSON.parse(delivery.message_ids) as string[] - if (ids.length === 0) { - return [] - } - const rows = this.db - .prepare(`SELECT * FROM messages WHERE id IN (${ids.map(() => '?').join(',')})`) - .all(...ids) as MessageRow[] - const byId = new Map(rows.map((row) => [row.id, row])) - return exposeMessageListTimestamps( - ids.map((id) => byId.get(id)).filter((row): row is MessageRow => row !== undefined) - ) - } - - getOrCreateRunDelivery(params: { - runId: string - consumerGeneration: number - limit?: number - wakeTypes?: MessageType[] - }): { delivery: DeliveryRow; messages: MessageRow[]; replayed: boolean } | undefined { - const limit = Math.min( - Math.max(params.limit ?? ORCHESTRATION_DELIVERY_BATCH_LIMIT, 1), - ORCHESTRATION_DELIVERY_BATCH_LIMIT - ) - this.db.exec('BEGIN IMMEDIATE') - try { - this.requireCurrentConsumer(params.runId, params.consumerGeneration) - const existing = this.db - .prepare("SELECT * FROM deliveries WHERE run_id = ? AND status = 'outstanding'") - .get(params.runId) as DeliveryRow | undefined - if (existing) { - if (existing.consumer_generation !== params.consumerGeneration) { - throw new OrchestrationError( - 'consumer_fenced', - 'This mailbox Delivery belongs to a fenced consumer generation.' - ) - } - const messages = this.getDeliveryMessages(existing) - this.db.exec('COMMIT') - return { delivery: exposeDeliveryTimestamps(existing), messages, replayed: true } - } - - const address = `run:${params.runId}` - if (params.wakeTypes && params.wakeTypes.length > 0) { - const placeholders = params.wakeTypes.map(() => '?').join(',') - const matching = this.db - .prepare( - `SELECT 1 FROM messages - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery' - AND type IN (${placeholders}) LIMIT 1` - ) - .get(params.runId, address, ...params.wakeTypes) - if (!matching) { - this.db.exec('COMMIT') - return undefined - } - } - - const messages = exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery' - ORDER BY sequence ASC LIMIT ?` - ) - .all(params.runId, address, limit) as MessageRow[] - ) - if (messages.length === 0) { - this.db.exec('COMMIT') - return undefined - } - - const deliveryId = generateId('delivery') - this.db - .prepare( - `INSERT INTO deliveries (id, run_id, consumer_generation, message_ids) - VALUES (?, ?, ?, ?)` - ) - .run( - deliveryId, - params.runId, - params.consumerGeneration, - JSON.stringify(messages.map((message) => message.id)) - ) - const delivery = this.getDeliveryRaw(deliveryId) as DeliveryRow - this.db.exec('COMMIT') - return { delivery: exposeDeliveryTimestamps(delivery), messages, replayed: false } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - acknowledgeRunDelivery(params: { - runId: string - consumerGeneration: number - deliveryId: string - }): { delivery: DeliveryRow; duplicate: boolean } { - this.db.exec('BEGIN IMMEDIATE') - try { - this.requireCurrentConsumer(params.runId, params.consumerGeneration) - const delivery = this.getDeliveryRaw(params.deliveryId) - if (!delivery || delivery.run_id !== params.runId) { - throw new OrchestrationError( - 'stale_delivery', - `Delivery ${params.deliveryId} does not belong to this Run.` - ) - } - if ( - delivery.consumer_generation !== params.consumerGeneration || - delivery.status === 'fenced' - ) { - throw new OrchestrationError( - 'consumer_fenced', - 'This mailbox Delivery belongs to a fenced consumer generation.' - ) - } - if (delivery.status === 'acknowledged') { - this.db.exec('COMMIT') - return { delivery: exposeDeliveryTimestamps(delivery), duplicate: true } - } - - const messageIds = JSON.parse(delivery.message_ids) as string[] - if (messageIds.length > 0) { - const placeholders = messageIds.map(() => '?').join(',') - this.db - .prepare(`UPDATE messages SET read = 1 WHERE id IN (${placeholders})`) - .run(...messageIds) - } - this.db - .prepare( - "UPDATE deliveries SET status = 'acknowledged', acknowledged_at = datetime('now') WHERE id = ?" - ) - .run(delivery.id) - const acknowledged = this.getDeliveryRaw(delivery.id) as DeliveryRow - this.db.exec('COMMIT') - return { delivery: exposeDeliveryTimestamps(acknowledged), duplicate: false } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getRunMailboxHistory(runId: string, limit = 100, types?: MessageType[]): MessageRow[] { - const address = `run:${runId}` - if (types && types.length > 0) { - const placeholders = types.map(() => '?').join(',') - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages WHERE run_id = ? AND to_handle = ? - AND type IN (${placeholders}) ORDER BY sequence DESC LIMIT ?` - ) - .all(runId, address, ...types, limit) as MessageRow[] - ) - } - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages WHERE run_id = ? AND to_handle = ? - ORDER BY sequence DESC LIMIT ?` - ) - .all(runId, address, limit) as MessageRow[] - ) - } - - getUnreadRunMailbox(runId: string, limit = 100, types?: MessageType[]): MessageRow[] { - const address = `run:${runId}` - const conditions = [ - 'run_id = ?', - 'to_handle = ?', - 'read = 0', - "delivery_contract = 'current_delivery'" - ] - const params: (string | number)[] = [runId, address] - if (types?.length) { - conditions.push(`type IN (${types.map(() => '?').join(',')})`) - params.push(...types) - } - const indexClause = types?.length ? ' INDEXED BY idx_messages_unread_current_run_type' : '' - params.push(Math.max(1, Math.floor(limit))) - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages${indexClause} WHERE ${conditions.join(' AND ')} - ORDER BY sequence ASC LIMIT ?` - ) - .all(...params) as MessageRow[] - ) - } - - // ── Messages ── - - insertMessage(msg: { - id?: string - from: string - to: string - subject: string - body?: string - type?: MessageType - priority?: MessagePriority - threadId?: string - payload?: string - senderPaneKey?: string - runId?: string - deliveryContract?: MessageDeliveryContract - }): MessageRow { - const runId = msg.runId ?? LEGACY_RUN_ID - const deliveryContract = msg.deliveryContract ?? 'current_delivery' - this.requireRun(runId) - const id = msg.id ?? generateId('msg') - const stmt = this.db.prepare(` - INSERT INTO messages ( - id, run_id, delivery_contract, from_handle, to_handle, subject, body, - type, priority, thread_id, payload, sender_pane_key - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - stmt.run( - id, - runId, - deliveryContract, - msg.from, - msg.to, - msg.subject, - msg.body ?? '', - msg.type ?? 'status', - msg.priority ?? 'normal', - msg.threadId ?? null, - msg.payload ?? null, - msg.senderPaneKey ?? null - ) - return exposeMessageTimestamps( - this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow - ) - } - - commitLegacyLifecycleOperation(params: { - principalId: string - operationKey: string - method: string - payloadHash: string - message: { - existingId?: string - to: string - subject: string - body?: string - type: MessageType - priority?: MessagePriority - payload?: string - } - lifecycle: - | { kind: 'message_only' } - | { kind: 'heartbeat'; at: string } - | { - kind: 'worker_report' - taskId: string - outcome: WorkerReportOutcome - result: string - } - }): { - receipt: LegacyOperationReceiptRow - message: MessageRow - settlement?: WorkerReportSettlement - duplicate: boolean - } { - this.db.exec('BEGIN IMMEDIATE') - try { - const principal = this.getLegacyCompatibilityPrincipal(params.principalId) - if ( - !principal || - principal.role !== 'worker' || - !['committed', 'settled'].includes(principal.status) - ) { - throw new OrchestrationError( - 'request_mismatch', - `Legacy compatibility principal ${params.principalId} cannot send lifecycle work.` - ) - } - const dispatchId = principal.dispatch_id as string - const existingReceipt = this.requireMatchingLegacyOperationReceipt(params) - if (existingReceipt) { - const response = JSON.parse(existingReceipt.response_json) as { - messageId: string - settlement?: WorkerReportSettlement - } - const message = this.getMessageById(response.messageId) - if (!message) { - throw new OrchestrationError( - 'operation_unknown', - `Legacy operation ${params.operationKey} lost its recorded message.` - ) - } - this.db.exec('COMMIT') - return { - receipt: existingReceipt, - message, - settlement: response.settlement, - duplicate: true - } - } - - const dispatch = this.getDispatchContextById(dispatchId) - if ( - !dispatch || - dispatch.run_id !== principal.run_id || - dispatch.contract_version !== LEGACY_CONTRACT_VERSION - ) { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${dispatchId} is not this principal's legacy attempt.` - ) - } - if ( - (principal.status === 'settled' || !['pending', 'dispatched'].includes(dispatch.status)) && - (!params.message.existingId || params.lifecycle.kind !== 'worker_report') - ) { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${dispatchId} is settled and only matching completion reconstruction is allowed.` - ) - } - let message = params.message.existingId - ? this.getMessageById(params.message.existingId) - : undefined - const delivery = this.resolveLegacyWorkerCoordinatorDelivery( - principal.run_id, - params.message.to - ) - if (params.message.existingId) { - const matchesOriginalLegacyRoute = - message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to - const matchesCurrentRoute = - message?.delivery_contract === delivery.contract && message.to_handle === delivery.to - if ( - !message || - message.run_id !== principal.run_id || - message.from_handle !== principal.terminal_handle || - (!matchesOriginalLegacyRoute && !matchesCurrentRoute) - ) { - throw new OrchestrationError( - 'request_mismatch', - `Existing legacy message ${params.message.existingId} does not match this principal.` - ) - } - } else { - message = this.insertMessage({ - from: principal.terminal_handle, - to: delivery.to, - subject: params.message.subject, - body: params.message.body, - type: params.message.type, - priority: params.message.priority, - payload: params.message.payload, - senderPaneKey: principal.pane_key, - runId: principal.run_id, - deliveryContract: delivery.contract - }) - } - - let settlement: WorkerReportSettlement | undefined - if (params.lifecycle.kind === 'heartbeat') { - this.recordHeartbeat(dispatchId, params.lifecycle.at) - } else if (params.lifecycle.kind === 'worker_report') { - const persistedOutcome = - params.message.existingId && - dispatch.task_id === params.lifecycle.taskId && - dispatch.status === 'completed' - ? 'succeeded' - : params.message.existingId && - dispatch.task_id === params.lifecycle.taskId && - dispatch.status === 'failed' - ? 'failed' - : undefined - settlement = persistedOutcome - ? { action: 'settled', outcome: persistedOutcome, duplicate: true } - : this.settleWorkerReportInTransaction({ - taskId: params.lifecycle.taskId, - dispatchId, - outcome: params.lifecycle.outcome, - result: params.lifecycle.result - }) - if (settlement.action === 'rejected') { - throw new OrchestrationError(settlement.code, settlement.reason) - } - this.db - .prepare( - `UPDATE legacy_compatibility_principals - SET status = 'settled' WHERE id = ? AND status = 'committed'` - ) - .run(principal.id) - } - const responseJson = JSON.stringify({ messageId: message.id, settlement }) - const receipt = this.insertLegacyOperationReceipt({ - principalId: principal.id, - operationKey: params.operationKey, - method: params.method, - payloadHash: params.payloadHash, - effectId: message.id, - responseJson - }) - this.db.exec('COMMIT') - return { receipt, message, settlement, duplicate: false } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - commitLegacyAskOperation(params: { - principalId: string - operationKey: string - method: string - payloadHash: string - question: string - options?: string[] - recipientHandle: string - existingQuestionId?: string - }): { - receipt: LegacyOperationReceiptRow - question: QuestionRow - message: MessageRow - duplicate: boolean - } { - this.db.exec('BEGIN IMMEDIATE') - try { - const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker') - const receipt = this.requireMatchingLegacyOperationReceipt(params) - if (receipt) { - const response = JSON.parse(receipt.response_json) as { questionId: string } - const question = this.getQuestion(response.questionId) - const message = this.getMessageById(response.questionId) - if (!question || !message) { - throw new OrchestrationError( - 'operation_unknown', - `Legacy ask ${params.operationKey} lost its durable question.` - ) - } - this.db.exec('COMMIT') - return { receipt, question, message, duplicate: true } - } - - const dispatchId = principal.dispatch_id as string - const dispatch = this.getDispatchContextById(dispatchId) - if ( - !dispatch || - dispatch.run_id !== principal.run_id || - dispatch.contract_version !== LEGACY_CONTRACT_VERSION || - !['pending', 'dispatched'].includes(dispatch.status) - ) { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${dispatchId} is not an active legacy attempt.` - ) - } - - const existingQuestionId = - params.existingQuestionId && - !this.db - .prepare( - `SELECT 1 FROM legacy_operation_receipts - WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ? - LIMIT 1` - ) - .get(principal.id, params.existingQuestionId) - ? params.existingQuestionId - : undefined - let question: QuestionRow - let message: MessageRow - const delivery = this.resolveLegacyWorkerCoordinatorDelivery( - principal.run_id, - params.recipientHandle - ) - if (existingQuestionId) { - const existingQuestion = this.getQuestion(existingQuestionId) - const existingMessage = this.getMessageById(existingQuestionId) - if ( - !existingQuestion || - !existingMessage || - existingQuestion.run_id !== principal.run_id || - existingQuestion.dispatch_id !== dispatchId || - existingQuestion.status !== 'pending' || - existingMessage.delivery_contract !== delivery.contract || - !legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [ - delivery.to - ]) - ) { - throw new OrchestrationError( - 'request_mismatch', - `Question ${params.existingQuestionId} is not a pending ask for this principal.` - ) - } - question = existingQuestion - message = existingMessage - } else { - message = this.insertMessage({ - from: principal.terminal_handle, - to: delivery.to, - subject: 'Question', - body: params.question, - type: delivery.contract === 'legacy_direct' ? 'decision_gate' : 'question', - payload: JSON.stringify({ - taskId: dispatch.task_id, - dispatchId, - question: params.question, - options: params.options ?? [] - }), - senderPaneKey: principal.pane_key, - runId: principal.run_id, - deliveryContract: delivery.contract - }) - this.db - .prepare('UPDATE messages SET thread_id = ? WHERE id = ?') - .run(message.id, message.id) - this.db - .prepare( - `INSERT INTO question_threads ( - message_id, run_id, dispatch_id, asker_handle - ) VALUES (?, ?, ?, ?)` - ) - .run(message.id, principal.run_id, dispatchId, principal.terminal_handle) - question = this.getQuestion(message.id) as QuestionRow - message = this.getMessageById(message.id) as MessageRow - } - - const committedReceipt = this.insertLegacyOperationReceipt({ - principalId: principal.id, - operationKey: params.operationKey, - method: params.method, - payloadHash: params.payloadHash, - effectId: question.message_id, - responseJson: JSON.stringify({ questionId: question.message_id }) - }) - this.db.exec('COMMIT') - return { receipt: committedReceipt, question, message, duplicate: false } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - findPendingLegacyQuestions(params: { - principalId: string - question: string - options?: string[] - recipientHandle: string - }): { question: QuestionRow; message: MessageRow }[] { - return this.findLegacyQuestionsBySemanticIdentity(params) - .filter((row) => row.question.status === 'pending') - .map(({ question, message }) => ({ question, message })) - } - - findLegacyQuestionsBySemanticIdentity(params: { - principalId: string - question: string - options?: string[] - recipientHandle: string - }): { - question: QuestionRow - message: MessageRow - answerAcknowledged: boolean - claimedByOperation: boolean - }[] { - const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker') - const runAddress = `run:${principal.run_id}` - const rows = this.db - .prepare( - `SELECT q.*, m.id AS source_message_id, - EXISTS( - SELECT 1 FROM legacy_operation_receipts lor - WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask' - AND lor.effect_id = q.message_id - ) AS claimed_by_operation - FROM question_threads q - INNER JOIN messages m ON m.id = q.message_id - WHERE q.run_id = ? AND q.dispatch_id = ? - AND ( - (m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR - (m.delivery_contract = 'current_delivery' AND m.to_handle = ?) - ) - ORDER BY m.sequence - LIMIT 501` - ) - .all( - principal.id, - principal.run_id, - principal.dispatch_id, - params.recipientHandle, - runAddress - ) as (QuestionRow & { - source_message_id: string - claimed_by_operation: number - })[] - if (rows.length > 500) { - throw new OrchestrationError( - 'operation_unknown', - 'Legacy ask identity is too ambiguous to reconstruct safely.' - ) - } - return rows - .filter((row) => { - const message = this.getMessageById(row.source_message_id) - return Boolean( - message && - legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [ - params.recipientHandle, - runAddress - ]) - ) - }) - .map((row) => ({ - question: exposeQuestionTimestamps(row), - message: this.getMessageById(row.message_id) as MessageRow, - claimedByOperation: row.claimed_by_operation === 1, - answerAcknowledged: row.answer_message_id - ? Boolean( - this.db - .prepare( - `SELECT 1 FROM legacy_mail_receipts - WHERE principal_id = ? AND message_id = ? - AND acknowledged_at IS NOT NULL` - ) - .get(principal.id, row.answer_message_id) - ) - : false - })) - } - - private resolveLegacyWorkerCoordinatorDelivery( - runId: string, - retainedCoordinatorHandle: string - ): { to: string; contract: MessageDeliveryContract } { - const run = this.getRunRaw(runId) - const principal = this.getLegacyCoordinatorPrincipal(runId) - const takenOver = run?.coordinator_handle !== null && principal?.status !== 'committed' - return takenOver - ? { to: `run:${runId}`, contract: 'current_delivery' } - : { to: retainedCoordinatorHandle, contract: 'legacy_direct' } - } - - commitLegacyReplyOperation(params: { - principalId: string - operationKey: string - method: string - payloadHash: string - questionId: string - body: string - }): { - receipt: LegacyOperationReceiptRow - question: QuestionRow - message: MessageRow - duplicate: boolean - } { - this.db.exec('BEGIN IMMEDIATE') - try { - const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'coordinator') - const receipt = this.requireMatchingLegacyOperationReceipt(params) - if (receipt) { - const response = JSON.parse(receipt.response_json) as { - questionId: string - messageId: string - } - const question = this.getQuestion(response.questionId) - const message = this.getMessageById(response.messageId) - if (!question || !message) { - throw new OrchestrationError( - 'operation_unknown', - `Legacy reply ${params.operationKey} lost its durable effect.` - ) - } - this.db.exec('COMMIT') - return { receipt, question, message, duplicate: true } - } - - const question = this.getQuestionRaw(params.questionId) - const sourceMessage = this.getMessageById(params.questionId) - const dispatch = question ? this.getDispatchContextById(question.dispatch_id) : undefined - if ( - !question || - !sourceMessage || - !dispatch || - question.run_id !== principal.run_id || - sourceMessage.delivery_contract !== 'legacy_direct' || - dispatch.run_id !== principal.run_id || - dispatch.contract_version !== LEGACY_CONTRACT_VERSION || - question.status === 'closed' - ) { - throw new OrchestrationError( - 'question_not_found', - `Question ${params.questionId} is not actionable in the adopted Run.` - ) - } - let message: MessageRow - if (question.status === 'answered') { - if (question.answer_body !== params.body || !question.answer_message_id) { - throw new OrchestrationError( - 'answer_conflict', - `Question ${params.questionId} already has a different answer.` - ) - } - message = this.getMessageById(question.answer_message_id) as MessageRow - if ( - !message || - message.run_id !== principal.run_id || - message.delivery_contract !== 'legacy_direct' - ) { - throw new OrchestrationError( - 'operation_unknown', - `Question ${params.questionId} lost its recorded answer message.` - ) - } - } else { - message = this.insertMessage({ - from: principal.terminal_handle, - to: question.asker_handle, - subject: 'Re: Question', - body: params.body, - threadId: question.message_id, - runId: principal.run_id, - deliveryContract: 'legacy_direct' - }) - this.markAsRead([question.message_id]) - this.db - .prepare( - `UPDATE question_threads - SET status = 'answered', answer_message_id = ?, answer_body = ?, - answered_at = datetime('now') - WHERE message_id = ? AND status = 'pending'` - ) - .run(message.id, params.body, question.message_id) - } - - const answered = this.getQuestion(params.questionId) as QuestionRow - const committedReceipt = this.insertLegacyOperationReceipt({ - principalId: principal.id, - operationKey: params.operationKey, - method: params.method, - payloadHash: params.payloadHash, - effectId: message.id, - responseJson: JSON.stringify({ - questionId: answered.message_id, - messageId: message.id - }) - }) - this.db.exec('COMMIT') - return { - receipt: committedReceipt, - question: answered, - message, - duplicate: question.status === 'answered' - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - private requireMatchingLegacyOperationReceipt(params: { - principalId: string - operationKey: string - method: string - payloadHash: string - }): LegacyOperationReceiptRow | undefined { - const receipt = this.getLegacyOperationReceipt(params.principalId, params.operationKey) - if ( - receipt && - (receipt.method !== params.method || receipt.payload_hash !== params.payloadHash) - ) { - throw new OrchestrationError( - 'request_mismatch', - `Legacy operation ${params.operationKey} was already used with different input.` - ) - } - return receipt - } - - private insertLegacyOperationReceipt(params: { - principalId: string - operationKey: string - method: string - payloadHash: string - effectId: string - responseJson: string - }): LegacyOperationReceiptRow { - this.db - .prepare( - `INSERT INTO legacy_operation_receipts ( - principal_id, operation_key, method, payload_hash, effect_id, response_json - ) VALUES (?, ?, ?, ?, ?, ?)` - ) - .run( - params.principalId, - params.operationKey, - params.method, - params.payloadHash, - params.effectId, - params.responseJson - ) - return this.getLegacyOperationReceipt( - params.principalId, - params.operationKey - ) as LegacyOperationReceiptRow - } - - getUnreadMessages(toHandle: string, types?: MessageType[]): MessageRow[] { - if (types && types.length > 0) { - const placeholders = types.map(() => '?').join(',') - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages - WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' - AND type IN (${placeholders}) ORDER BY sequence` - ) - .all(toHandle, ...types) as MessageRow[] - ) - } - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages - WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' - ORDER BY sequence` - ) - .all(toHandle) as MessageRow[] - ) - } - - convertLifecycleMessageToRejection( - messageId: string, - code: string, - reason: string - ): MessageRow | undefined { - const message = this.getMessageById(messageId) - if (!message || (message.type !== 'worker_done' && message.type !== 'heartbeat')) { - return message - } - - const originalBody = message.body ? `\n\nOriginal body:\n${message.body}` : '' - const body = `Orca rejected this ${message.type}: ${reason}${originalBody}` - const payload = addLifecycleRejectionMarker(message.payload, code, reason) - // Why: rejected lifecycle signals stay auditable but must not reach read paths as actionable completion/liveness events. - this.db - .prepare( - `UPDATE messages - SET priority = 'high', subject = ?, body = ?, payload = ? - WHERE id = ?` - ) - .run(`Rejected ${message.type}: ${message.subject}`, body, payload, messageId) - return this.getMessageById(messageId) - } - - // Why: delivered_at IS NULL filter — push-on-idle delivers each row at most once; read (set only by check) wouldn't prevent replay. - getUndeliveredUnreadMessages( - toHandle: string, - types?: MessageType[], - options?: { excludeTypes?: readonly string[]; limit?: number } - ): MessageRow[] { - const conditions = [ - 'to_handle = ?', - 'read = 0', - 'delivered_at IS NULL', - "delivery_contract = 'current_delivery'" - ] - const params: (string | number)[] = [toHandle] - if (types?.length) { - conditions.push(`type IN (${types.map(() => '?').join(',')})`) - params.push(...types) - } - if (options?.excludeTypes?.length) { - conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`) - params.push(...options.excludeTypes) - } - const limitSql = options?.limit === undefined ? '' : ' LIMIT ?' - if (options?.limit !== undefined) { - params.push(Math.max(1, Math.floor(options.limit))) - } - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages - WHERE ${conditions.join(' AND ')} - ORDER BY sequence${limitSql}` - ) - .all(...params) as MessageRow[] - ) - } - - getUndeliveredUnreadMailboxHandles(): string[] { - return ( - this.db - .prepare( - `SELECT DISTINCT to_handle FROM messages - WHERE read = 0 AND delivered_at IS NULL - AND delivery_contract = 'current_delivery'` - ) - .all() as { to_handle: string }[] - ).map((row) => row.to_handle) - } - - hasUndeliveredDirectMessageForRun(runId: string, directHandle: string): boolean { - return Boolean( - this.db - .prepare( - `SELECT 1 FROM messages INDEXED BY idx_messages_undelivered_direct_run - WHERE run_id = ? AND to_handle = ? AND read = 0 AND delivered_at IS NULL - AND delivery_contract = 'current_delivery' - LIMIT 1` - ) - .get(runId, directHandle) - ) - } - - hasOutstandingRunDelivery(runId: string): boolean { - return Boolean( - this.db - .prepare("SELECT 1 FROM deliveries WHERE run_id = ? AND status = 'outstanding' LIMIT 1") - .get(runId) - ) - } - - getLatestUnreadDirectMessageSequenceForRun( - runId: string, - directHandle: string - ): number | undefined { - const row = this.db - .prepare( - `SELECT sequence FROM messages - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery' - ORDER BY sequence DESC LIMIT 1` - ) - .get(runId, directHandle) as { sequence: number } | undefined - return row?.sequence - } - - // Why: change mailbox ownership without changing unread or acknowledgment state. - private routeDirectMessagePage( - mailboxHandle: string, - runId: string, - directHandle: string, - throughSequence?: number, - preserveActiveDispatchOwnership = false - ): MailboxRoutingPage { - this.db.exec('BEGIN IMMEDIATE') - try { - const throughClause = throughSequence === undefined ? '' : ' AND sequence <= ?' - const dispatchOwnershipClause = preserveActiveDispatchOwnership - ? ` AND NOT EXISTS ( - SELECT 1 FROM dispatch_contexts - WHERE dispatch_contexts.run_id = messages.run_id - AND dispatch_contexts.assignee_handle = messages.to_handle - AND dispatch_contexts.status IN ('pending', 'dispatched') - )` - : '' - const params: (string | number)[] = [runId, directHandle] - if (throughSequence !== undefined) { - params.push(throughSequence) - } - params.push(ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1) - const rows = this.db - .prepare( - `SELECT id, type FROM messages - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery'${throughClause}${dispatchOwnershipClause} - ORDER BY sequence LIMIT ?` - ) - .all(...params) as { - id: string - type: MessageType - }[] - const page = rows.slice(0, ORCHESTRATION_DELIVERY_BATCH_LIMIT) - if (page.length === 0) { - this.db.exec('COMMIT') - return { routedCount: 0, hasMore: false, types: [] } - } - const placeholders = page.map(() => '?').join(',') - const result = this.db - .prepare( - `UPDATE messages INDEXED BY idx_messages_id SET to_handle = ? - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery' AND id IN (${placeholders})` - ) - .run(mailboxHandle, runId, directHandle, ...page.map((row) => row.id)) - this.db.exec('COMMIT') - return { - routedCount: Number(result.changes), - hasMore: rows.length > ORCHESTRATION_DELIVERY_BATCH_LIMIT, - types: [...new Set(page.map((row) => row.type))] - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - private routeAllUnreadDirectMessagesToRunMailbox(runId: string, directHandle: string): void { - this.db - .prepare( - `UPDATE messages SET to_handle = ? - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery' - AND NOT EXISTS ( - SELECT 1 FROM dispatch_contexts - WHERE dispatch_contexts.run_id = messages.run_id - AND dispatch_contexts.assignee_handle = messages.to_handle - AND dispatch_contexts.status IN ('pending', 'dispatched') - )` - ) - .run(`run:${runId}`, runId, directHandle) - } - - routeUnreadDirectMessagesToRunMailbox( - runId: string, - directHandle: string, - throughSequence?: number - ): MailboxRoutingPage { - return this.routeDirectMessagePage(`run:${runId}`, runId, directHandle, throughSequence, true) - } - - routeUnreadDirectMessagesToDispatchMailbox( - dispatchId: string, - runId: string, - directHandle: string, - throughSequence?: number - ): MailboxRoutingPage { - return this.routeDirectMessagePage( - `dispatch:${dispatchId}`, - runId, - directHandle, - throughSequence - ) - } - - routeUnreadDispatchMailboxToRunMailbox( - dispatchId: string, - runId: string, - throughSequence?: number - ): MailboxRoutingPage { - const dispatchMailbox = `dispatch:${dispatchId}` - this.db.exec('BEGIN IMMEDIATE') - try { - const throughClause = throughSequence === undefined ? '' : ' AND sequence <= ?' - const params: (string | number)[] = [runId, dispatchMailbox] - if (throughSequence !== undefined) { - params.push(throughSequence) - } - params.push(ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1) - const rows = this.db - .prepare( - `SELECT id, type FROM messages INDEXED BY idx_messages_unread_current_inbox - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery'${throughClause} - ORDER BY sequence LIMIT ?` - ) - .all(...params) as { - id: string - type: MessageType - }[] - const page = rows.slice(0, ORCHESTRATION_DELIVERY_BATCH_LIMIT) - if (page.length === 0) { - this.db.exec('COMMIT') - return { routedCount: 0, hasMore: false, types: [] } - } - const placeholders = page.map(() => '?').join(',') - const result = this.db - .prepare( - `UPDATE messages INDEXED BY idx_messages_id SET to_handle = ? - WHERE run_id = ? AND to_handle = ? AND read = 0 - AND delivery_contract = 'current_delivery' - AND id IN (${placeholders})` - ) - .run(`run:${runId}`, runId, dispatchMailbox, ...page.map((row) => row.id)) - this.db.exec('COMMIT') - return { - routedCount: Number(result.changes), - hasMore: rows.length > ORCHESTRATION_DELIVERY_BATCH_LIMIT, - types: [...new Set(page.map((row) => row.type))] - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getLatestUnreadMessageSequence(mailboxHandle: string): number | undefined { - const row = this.db - .prepare( - `SELECT sequence FROM messages INDEXED BY idx_messages_unread_current_inbox - WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' - ORDER BY sequence DESC LIMIT 1` - ) - .get(mailboxHandle) as { sequence: number } | undefined - return row?.sequence - } - - private findActiveDispatchForDirectMessageOwner( - runId: string, - directHandle: string, - paneKey?: string - ): DispatchContextRow | undefined { - const exact = this.db - .prepare( - `SELECT * FROM dispatch_contexts - WHERE run_id = ? AND assignee_handle = ? AND status IN ('pending', 'dispatched') - ORDER BY rowid DESC LIMIT 1` - ) - .get(runId, directHandle) as DispatchContextRow | undefined - if (exact || !paneKey || !parsePaneKey(paneKey)) { - return exact - } - return this.db - .prepare( - `SELECT * FROM dispatch_contexts - WHERE run_id = ? AND assignee_pane_key IS NOT NULL - AND status IN ('pending', 'dispatched') AND instr(assignee_pane_key, ':') > 1 - AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ? - ORDER BY rowid DESC LIMIT 1` - ) - .get(runId, paneKeyMatchSuffix(paneKey)) as DispatchContextRow | undefined - } - - routeForeignDirectMessagesToOwnedMailboxes( - directHandle: string, - currentRunId?: string, - paneKey?: string - ): ForeignDirectMailboxRoutingPage { - this.db.exec('BEGIN IMMEDIATE') - try { - const runExclusion = currentRunId === undefined ? '' : ' AND candidate.run_id <> ?' - const paneSuffix = paneKey && parsePaneKey(paneKey) ? paneKeyMatchSuffix(paneKey) : undefined - const exclusionParams = currentRunId === undefined ? [] : [currentRunId] - const branches = [ - `SELECT candidate.id, candidate.run_id, candidate.type, candidate.sequence - FROM run_coordinator_handles AS coordinator - JOIN runs AS owner_run ON owner_run.id = coordinator.run_id AND owner_run.legacy = 0 - JOIN messages AS candidate INDEXED BY idx_messages_undelivered_direct_run - ON candidate.run_id = coordinator.run_id - AND candidate.to_handle = coordinator.terminal_handle - WHERE coordinator.terminal_handle = ?${runExclusion} - AND candidate.read = 0 AND candidate.delivered_at IS NULL - AND candidate.delivery_contract = 'current_delivery'`, - `SELECT candidate.id, candidate.run_id, candidate.type, candidate.sequence - FROM ( - SELECT direct_dispatch.run_id - FROM dispatch_contexts AS direct_dispatch - INDEXED BY idx_dispatch_active_assignee_handle - JOIN runs AS owner_run - ON owner_run.id = direct_dispatch.run_id AND owner_run.legacy = 0 - WHERE direct_dispatch.assignee_handle = ? - AND direct_dispatch.status IN ('pending', 'dispatched') - GROUP BY direct_dispatch.run_id - ) AS dispatch_owner - JOIN messages AS candidate INDEXED BY idx_messages_undelivered_direct_run - ON candidate.run_id = dispatch_owner.run_id AND candidate.to_handle = ? - WHERE 1 = 1${runExclusion} - AND candidate.read = 0 AND candidate.delivered_at IS NULL - AND candidate.delivery_contract = 'current_delivery'` - ] - const branchParams: (string | number)[][] = [ - [directHandle, ...exclusionParams, ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1], - [directHandle, directHandle, ...exclusionParams, ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1] - ] - if (paneSuffix !== undefined) { - branches.push( - `SELECT candidate.id, candidate.run_id, candidate.type, candidate.sequence - FROM ( - SELECT pane_dispatch.run_id - FROM dispatch_contexts AS pane_dispatch INDEXED BY idx_dispatch_assignee_pane_leaf - JOIN runs AS owner_run - ON owner_run.id = pane_dispatch.run_id AND owner_run.legacy = 0 - WHERE pane_dispatch.assignee_pane_key IS NOT NULL - AND pane_dispatch.status IN ('pending', 'dispatched') - AND instr(pane_dispatch.assignee_pane_key, ':') > 1 - AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ? - GROUP BY pane_dispatch.run_id - ) AS pane_owner - JOIN messages AS candidate INDEXED BY idx_messages_undelivered_direct_run - ON candidate.run_id = pane_owner.run_id AND candidate.to_handle = ? - WHERE 1 = 1${runExclusion} - AND candidate.read = 0 AND candidate.delivered_at IS NULL - AND candidate.delivery_contract = 'current_delivery'` - ) - branchParams.push([ - paneSuffix, - directHandle, - ...exclusionParams, - ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1 - ]) - } - const limitedBranches = branches.map( - (branch) => `SELECT * FROM (${branch} ORDER BY candidate.sequence LIMIT ?)` - ) - const rows = this.db - .prepare( - `SELECT id, run_id, type FROM (${limitedBranches.join(' UNION ')}) - ORDER BY sequence LIMIT ?` - ) - .all(...branchParams.flat(), ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1) as { - id: string - run_id: string - type: MessageType - }[] - const page = rows.slice(0, ORCHESTRATION_DELIVERY_BATCH_LIMIT) - if (page.length === 0) { - this.db.exec('COMMIT') - return { routedCount: 0, hasMore: false, types: [], mailboxes: [] } - } - const runIds = [...new Set(page.map((row) => row.run_id))] - const dispatchByRun = new Map() - for (const runId of runIds) { - const dispatch = this.findActiveDispatchForDirectMessageOwner(runId, directHandle, paneKey) - if (dispatch) { - dispatchByRun.set(runId, dispatch) - } - } - const idsByMailbox = new Map() - const byMailbox = new Map>() - for (const row of page) { - const dispatch = dispatchByRun.get(row.run_id) - const mailboxHandle = dispatch ? `dispatch:${dispatch.id}` : `run:${row.run_id}` - const ids = idsByMailbox.get(mailboxHandle) ?? [] - ids.push(row.id) - idsByMailbox.set(mailboxHandle, ids) - const types = byMailbox.get(mailboxHandle) ?? new Set() - types.add(row.type) - byMailbox.set(mailboxHandle, types) - } - for (const [mailboxHandle, ids] of idsByMailbox) { - const placeholders = ids.map(() => '?').join(',') - this.db - .prepare( - `UPDATE messages INDEXED BY idx_messages_id SET to_handle = ? - WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL - AND delivery_contract = 'current_delivery' AND id IN (${placeholders})` - ) - .run(mailboxHandle, directHandle, ...ids) - } - this.db.exec('COMMIT') - return { - routedCount: page.length, - hasMore: rows.length > ORCHESTRATION_DELIVERY_BATCH_LIMIT, - types: [...new Set(page.map((row) => row.type))], - mailboxes: [...byMailbox].map(([mailboxHandle, types]) => ({ - mailboxHandle, - types: [...types] - })) - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getUnreadDirectMessageTypes(directHandle: string): MessageType[] { - return ( - this.db - .prepare( - `SELECT DISTINCT type FROM messages INDEXED BY idx_messages_unread_current_inbox_type - WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' - ORDER BY type` - ) - .all(directHandle) as { type: MessageType }[] - ).map((row) => row.type) - } - - areUnreadMessages(toHandle: string, ids: string[]): boolean { - let matched = 0 - for (let offset = 0; offset < ids.length; offset += MESSAGE_ID_UPDATE_BATCH_SIZE) { - const batch = ids.slice(offset, offset + MESSAGE_ID_UPDATE_BATCH_SIZE) - const placeholders = batch.map(() => '?').join(',') - const row = this.db - .prepare( - `SELECT COUNT(*) AS count FROM messages INDEXED BY idx_messages_id - WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' - AND id IN (${placeholders})` - ) - .get(toHandle, ...batch) as { count: number } - matched += row.count - } - return matched === ids.length - } - - getAllMessages(toHandle: string, limit = 20): MessageRow[] { - return exposeMessageListTimestamps( - this.db - .prepare('SELECT * FROM messages WHERE to_handle = ? ORDER BY sequence DESC LIMIT ?') - .all(toHandle, limit) as MessageRow[] - ) - } - - getMessageById(id: string): MessageRow | undefined { - const message = this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as - | MessageRow - | undefined - return message ? exposeMessageTimestamps(message) : undefined - } - - markAsRead(ids: string[]): void { - if (ids.length === 0) { - return - } - const placeholders = ids.map(() => '?').join(',') - this.db.prepare(`UPDATE messages SET read = 1 WHERE id IN (${placeholders})`).run(...ids) - } - - // Why: use datetime('now') so delivered_at matches the space-format UTC shape of the table's other timestamps for correct ordering (§3.2). - markAsDelivered(ids: string[]): void { - for (let offset = 0; offset < ids.length; offset += MESSAGE_ID_UPDATE_BATCH_SIZE) { - const batch = ids.slice(offset, offset + MESSAGE_ID_UPDATE_BATCH_SIZE) - const placeholders = batch.map(() => '?').join(',') - this.db - .prepare(`UPDATE messages SET delivered_at = datetime('now') WHERE id IN (${placeholders})`) - .run(...batch) - } - } - - markAsUndelivered(ids: string[]): void { - for (let offset = 0; offset < ids.length; offset += MESSAGE_ID_UPDATE_BATCH_SIZE) { - const batch = ids.slice(offset, offset + MESSAGE_ID_UPDATE_BATCH_SIZE) - const placeholders = batch.map(() => '?').join(',') - this.db - .prepare( - `UPDATE messages SET delivered_at = NULL - WHERE read = 0 AND id IN (${placeholders})` - ) - .run(...batch) - } - } - - markAsReadAndDelivered(ids: string[]): void { - if (ids.length === 0) { - return - } - const placeholders = ids.map(() => '?').join(',') - // Why: superseded lifecycle messages stay in history but must not be consumed or injected after their dispatch finished. - this.db - .prepare( - `UPDATE messages SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) WHERE id IN (${placeholders})` - ) - .run(...ids) - } - - getInbox(limit = 20): MessageRow[] { - return exposeMessageListTimestamps( - this.db - .prepare('SELECT * FROM messages ORDER BY sequence DESC LIMIT ?') - .all(limit) as MessageRow[] - ) - } - - // Why: read-only history for a handle — returns every message regardless of read/delivered state, never flips the read bit (§3.3). - getAllMessagesForHandle(toHandle: string, limit = 100, types?: MessageType[]): MessageRow[] { - if (types && types.length > 0) { - const placeholders = types.map(() => '?').join(',') - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages WHERE to_handle = ? AND type IN (${placeholders}) ORDER BY sequence DESC LIMIT ?` - ) - .all(toHandle, ...types, limit) as MessageRow[] - ) - } - return exposeMessageListTimestamps( - this.db - .prepare('SELECT * FROM messages WHERE to_handle = ? ORDER BY sequence DESC LIMIT ?') - .all(toHandle, limit) as MessageRow[] - ) - } - - // Why: ask wait-loop read — to_handle filter shows only replies to the worker; afterSequence resumes past its own outbound ask. - getThreadMessagesFor(threadId: string, toHandle: string, afterSequence?: number): MessageRow[] { - if (afterSequence !== undefined) { - return exposeMessageListTimestamps( - this.db - .prepare( - 'SELECT * FROM messages WHERE thread_id = ? AND to_handle = ? AND sequence > ? ORDER BY sequence ASC' - ) - .all(threadId, toHandle, afterSequence) as MessageRow[] - ) - } - return exposeMessageListTimestamps( - this.db - .prepare( - 'SELECT * FROM messages WHERE thread_id = ? AND to_handle = ? ORDER BY sequence ASC' - ) - .all(threadId, toHandle) as MessageRow[] - ) - } - - createQuestion(params: { - runId: string - dispatchId: string - askerHandle: string - question: string - options?: string[] - }): { question: QuestionRow; message: MessageRow } { - this.db.exec('BEGIN IMMEDIATE') - try { - this.requireRun(params.runId) - const dispatch = this.getDispatchContextById(params.dispatchId) - if ( - !dispatch || - dispatch.run_id !== params.runId || - (dispatch.status !== 'pending' && dispatch.status !== 'dispatched') - ) { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${params.dispatchId} is not active in Run ${params.runId}.` - ) - } - const message = this.insertMessage({ - from: `dispatch:${params.dispatchId}`, - to: `run:${params.runId}`, - subject: 'Question', - body: params.question, - type: 'question', - payload: JSON.stringify({ - taskId: dispatch.task_id, - dispatchId: dispatch.id, - question: params.question, - options: params.options ?? [] - }), - runId: params.runId - }) - this.db.prepare('UPDATE messages SET thread_id = ? WHERE id = ?').run(message.id, message.id) - this.db - .prepare( - `INSERT INTO question_threads ( - message_id, run_id, dispatch_id, asker_handle - ) VALUES (?, ?, ?, ?)` - ) - .run(message.id, params.runId, params.dispatchId, params.askerHandle) - const question = this.getQuestionRaw(message.id) as QuestionRow - const storedMessage = this.getMessageById(message.id) as MessageRow - this.db.exec('COMMIT') - return { question: exposeQuestionTimestamps(question), message: storedMessage } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getQuestion(messageId: string): QuestionRow | undefined { - const question = this.getQuestionRaw(messageId) - return question ? exposeQuestionTimestamps(question) : undefined - } - - private getQuestionRaw(messageId: string): QuestionRow | undefined { - return this.db.prepare('SELECT * FROM question_threads WHERE message_id = ?').get(messageId) as - | QuestionRow - | undefined - } - - answerQuestion(params: { - messageId: string - runId: string - consumerGeneration: number - body: string - }): { question: QuestionRow; message: MessageRow; duplicate: boolean } { - this.db.exec('BEGIN IMMEDIATE') - try { - this.requireCurrentConsumer(params.runId, params.consumerGeneration) - const question = this.getQuestionRaw(params.messageId) - if (!question || question.run_id !== params.runId) { - throw new OrchestrationError( - 'question_not_found', - `Question ${params.messageId} was not found in Run ${params.runId}.` - ) - } - if (question.status === 'closed') { - throw new OrchestrationError( - 'dispatch_inactive', - `Question ${params.messageId} is closed because its Dispatch is inactive.` - ) - } - if (question.status === 'answered') { - if (question.answer_body !== params.body || !question.answer_message_id) { - throw new OrchestrationError( - 'answer_conflict', - `Question ${params.messageId} already has a different answer.` - ) - } - const message = this.getMessageById(question.answer_message_id) - if (!message) { - throw new Error(`Recorded answer message ${question.answer_message_id} was not found.`) - } - this.db.exec('COMMIT') - return { question: exposeQuestionTimestamps(question), message, duplicate: true } - } - - const message = this.insertMessage({ - from: `run:${params.runId}`, - to: `dispatch:${question.dispatch_id}`, - subject: 'Re: Question', - body: params.body, - threadId: question.message_id, - runId: params.runId - }) - // Why: ask returns thread state directly; leaving its answer unread would deliver it again via check. - this.markAsRead([message.id]) - this.db - .prepare( - `UPDATE question_threads - SET status = 'answered', answer_message_id = ?, answer_body = ?, - answered_by_generation = ?, answered_at = datetime('now') - WHERE message_id = ? AND status = 'pending'` - ) - .run(message.id, params.body, params.consumerGeneration, question.message_id) - const answered = this.getQuestionRaw(question.message_id) as QuestionRow - const storedMessage = this.getMessageById(message.id) as MessageRow - this.db.exec('COMMIT') - return { - question: exposeQuestionTimestamps(answered), - message: storedMessage, - duplicate: false - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - closeQuestionsForDispatch(dispatchId: string): string[] { - const rows = this.db - .prepare( - "SELECT message_id FROM question_threads WHERE dispatch_id = ? AND status = 'pending'" - ) - .all(dispatchId) as { message_id: string }[] - if (rows.length === 0) { - return [] - } - this.db - .prepare( - "UPDATE question_threads SET status = 'closed', closed_at = datetime('now') WHERE dispatch_id = ? AND status = 'pending'" - ) - .run(dispatchId) - return rows.map((row) => row.message_id) - } - - // ── Tasks ── - - createTask(task: { - spec: string - taskTitle?: string - displayName?: string - deps?: string[] - parentId?: string - createdByTerminalHandle?: string - createdByPaneKey?: string - createdByProcessIncarnation?: string - createdByRunGeneration?: number - runId?: string - }): TaskRow { - const runId = task.runId ?? LEGACY_RUN_ID - this.requireRun(runId) - if (task.parentId) { - const parent = this.getTask(task.parentId) - if (!parent || parent.run_id !== runId) { - throw new Error(`Parent task ${task.parentId} must belong to run ${runId}`) - } - } - for (const depId of task.deps ?? []) { - const dependency = this.getTask(depId) - if (!dependency || dependency.run_id !== runId) { - throw new Error(`Dependency task ${depId} must belong to run ${runId}`) - } - } - const id = generateId('task') - const depsJson = JSON.stringify(task.deps ?? []) - const display = buildOrchestrationTaskDisplayMetadata({ - spec: task.spec, - taskTitle: task.taskTitle, - displayName: task.displayName - }) - this.db - .prepare( - `INSERT INTO tasks ( - id, run_id, parent_id, created_by_terminal_handle, created_by_pane_key, - created_by_process_incarnation, created_by_run_generation, - task_title, display_name, spec, status, deps - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - CASE WHEN EXISTS ( - SELECT 1 - FROM json_each(?) requested - LEFT JOIN tasks dependency ON dependency.id = requested.value - WHERE dependency.id IS NULL - OR dependency.run_id <> ? - OR dependency.status <> 'completed' - ) THEN 'pending' ELSE 'ready' END, - ? - )` - ) - .run( - id, - runId, - task.parentId ?? null, - task.createdByTerminalHandle ?? null, - task.createdByPaneKey ?? null, - task.createdByProcessIncarnation ?? null, - task.createdByRunGeneration ?? null, - display.taskTitle || null, - display.displayName || null, - task.spec, - depsJson, - runId, - depsJson - ) - return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow - } - - // Why: return the active creator Dispatch proof with the Task read; runtime still owns pane/process currency. - getTask(id: string): TaskRow | undefined - getTask(id: string, dispatchRunId: string): TaskRuntimeLineageRow | undefined - getTask(id: string, dispatchRunId?: string): TaskRow | TaskRuntimeLineageRow | undefined { - if (dispatchRunId === undefined) { - return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow | undefined - } - return this.db - .prepare( - `SELECT t.*, - creator.id AS creator_dispatch_id, - creator.run_id AS creator_dispatch_run_id, - creator.assignee_pane_key AS creator_dispatch_pane_key, - creator.process_incarnation AS creator_dispatch_process_incarnation - FROM tasks t - LEFT JOIN dispatch_contexts creator ON creator.rowid = ( - SELECT candidate.rowid - FROM dispatch_contexts candidate - WHERE candidate.assignee_handle = t.created_by_terminal_handle - AND candidate.run_id = ? - AND candidate.status IN ('pending', 'dispatched') - ORDER BY candidate.rowid DESC - LIMIT 1 - ) - WHERE t.id = ?` - ) - .get(dispatchRunId, id) as TaskRuntimeLineageRow | undefined - } - - listTasks(filter?: { status?: TaskStatus; ready?: boolean; runId?: string }): TaskRow[] { - const runWhere = filter?.runId ? 'run_id = ? AND ' : '' - const runParams: Database.BindValue[] = filter?.runId ? [filter.runId] : [] - if (filter?.ready) { - return this.db - .prepare(`SELECT * FROM tasks WHERE ${runWhere}status = 'ready' ORDER BY created_at`) - .all(...runParams) as TaskRow[] - } - if (filter?.status) { - return this.db - .prepare(`SELECT * FROM tasks WHERE ${runWhere}status = ? ORDER BY created_at`) - .all(...runParams, filter.status) as TaskRow[] - } - if (filter?.runId) { - return this.db - .prepare('SELECT * FROM tasks WHERE run_id = ? ORDER BY created_at') - .all(filter.runId) as TaskRow[] - } - return this.db.prepare('SELECT * FROM tasks ORDER BY created_at').all() as TaskRow[] - } - - // Why: the correlated indexed lookup avoids materializing every retained Dispatch before filtering Tasks. - listTasksWithDispatch(filter?: { - status?: TaskStatus - ready?: boolean - runId?: string - }): (TaskRow & { - assignee_handle: string | null - dispatch_id: string | null - })[] { - const whereClauses: string[] = [] - const params: Database.BindValue[] = [] - if (filter?.runId) { - whereClauses.push('t.run_id = ?') - params.push(filter.runId) - } - if (filter?.ready) { - whereClauses.push("t.status = 'ready'") - } else if (filter?.status) { - whereClauses.push('t.status = ?') - params.push(filter.status) - } - const where = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '' - const sql = ` - SELECT - t.*, - d.assignee_handle AS assignee_handle, - d.id AS dispatch_id - FROM tasks t - LEFT JOIN dispatch_contexts d ON d.rowid = ( - SELECT candidate.rowid - FROM dispatch_contexts candidate - WHERE candidate.task_id = t.id - AND candidate.status IN ('pending', 'dispatched') - ORDER BY candidate.rowid DESC - LIMIT 1 - ) - ${where} - ORDER BY t.created_at - ` - return this.db.prepare(sql).all(...params) as (TaskRow & { - assignee_handle: string | null - dispatch_id: string | null - })[] - } - - updateTaskStatus(id: string, status: TaskStatus, result?: string): TaskRow | undefined { - const completedAt = - status === 'completed' || status === 'failed' ? new Date().toISOString() : null - this.db - .prepare( - 'UPDATE tasks SET status = ?, result = COALESCE(?, result), completed_at = COALESCE(?, completed_at) WHERE id = ?' - ) - .run(status, result ?? null, completedAt, id) - - if (status === 'completed') { - this.promoteReadyTasks(id) - this.completeActiveDispatchForTask(id) - } - - return this.getTask(id) - } - - // Why: runs in the status-update transaction, so a completed task never leaves its ready children unpromoted. - private promoteReadyTasks(completedTaskId: string): void { - const candidates = this.db - .prepare("SELECT * FROM tasks WHERE status = 'pending'") - .all() as TaskRow[] - - for (const task of candidates) { - const deps: string[] = JSON.parse(task.deps) - if (!deps.includes(completedTaskId)) { - continue - } - - const allDepsCompleted = deps.every((depId) => { - const dep = this.getTask(depId) - return dep?.status === 'completed' - }) - if (allDepsCompleted) { - this.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id) - } - } - } - - // ── Dispatch Contexts ── - - createStartingWorkerDispatch(params: { - taskId: string - startOptions: unknown - launchTokenHash?: string - retryOf?: string - runtimeEpoch?: string - federation?: { - environmentId: string - environmentName: string - peerFingerprint: string - protocolVersion: number - } - mutationReceipt?: { - callerFingerprint: string - requestId: string - method: string - payloadHash: string - } - }): { dispatch: DispatchContextRow; worker: WorkerDispatchRow } { - this.db.exec('BEGIN IMMEDIATE') - try { - if (params.mutationReceipt) { - const receipt = params.mutationReceipt - const existing = this.getMutationReceipt(receipt.callerFingerprint, receipt.requestId) - if (existing) { - if (existing.method !== receipt.method || existing.payload_hash !== receipt.payloadHash) { - throw new OrchestrationError( - 'request_mismatch', - `Mutation request ${receipt.requestId} was already used with different input.` - ) - } - throw new OrchestrationError( - 'operation_unknown', - `Mutation ${receipt.requestId} already has a durable acceptance record.` - ) - } - ensureMutationReceiptCapacity(this.db) - this.db - .prepare( - `INSERT INTO mutation_receipts ( - caller_fingerprint, request_id, method, payload_hash, state - ) VALUES (?, ?, ?, ?, 'pending')` - ) - .run(receipt.callerFingerprint, receipt.requestId, receipt.method, receipt.payloadHash) - } - const task = this.getTask(params.taskId) - if (!task) { - throw new OrchestrationError('task_not_found', `Task ${params.taskId} was not found.`) - } - if (params.retryOf) { - const prior = this.getDispatchContextById(params.retryOf) - const priorWorker = this.getWorkerDispatch(params.retryOf) - const latest = this.getDispatchContext(task.id) - if ( - !prior || - prior.task_id !== task.id || - latest?.id !== prior.id || - !priorWorker || - !['failed', 'stopped', 'abandoned'].includes(priorWorker.state) || - !['failed', 'blocked'].includes(task.status) - ) { - throw new OrchestrationError( - 'task_not_startable', - `Task ${task.id} cannot retry from Dispatch ${params.retryOf}.` - ) - } - } else if (task.status !== 'ready') { - throw new OrchestrationError( - 'task_not_startable', - `Task ${task.id} is ${task.status}; only a ready Task can start.` - ) - } - - const id = generateId('ctx') - if (params.mutationReceipt) { - this.db - .prepare( - `UPDATE mutation_receipts - SET receipt = ?, updated_at = datetime('now') - WHERE caller_fingerprint = ? AND request_id = ? AND state = 'pending'` - ) - .run( - JSON.stringify({ accepted: { dispatchId: id } }), - params.mutationReceipt.callerFingerprint, - params.mutationReceipt.requestId - ) - } - this.db - .prepare( - `INSERT INTO dispatch_contexts ( - id, run_id, task_id, contract_version, launch_token_hash, status, dispatched_at - ) VALUES (?, ?, ?, ?, ?, 'pending', datetime('now'))` - ) - .run(id, task.run_id, task.id, CURRENT_CONTRACT_VERSION, params.launchTokenHash ?? null) - this.db - .prepare( - `INSERT INTO worker_dispatches ( - dispatch_id, runtime_epoch, state, stage, start_options - ) VALUES (?, ?, 'starting', 'accepted', ?)` - ) - .run(id, params.runtimeEpoch ?? null, JSON.stringify(params.startOptions)) - if (params.federation) { - this.db - .prepare( - `INSERT INTO federated_dispatches ( - dispatch_id, environment_id, environment_name, peer_fingerprint, protocol_version - ) VALUES (?, ?, ?, ?, ?)` - ) - .run( - id, - params.federation.environmentId, - params.federation.environmentName, - params.federation.peerFingerprint, - params.federation.protocolVersion - ) - } - this.db - .prepare( - "UPDATE tasks SET status = 'dispatched', result = NULL, completed_at = NULL WHERE id = ?" - ) - .run(task.id) - this.db.exec('COMMIT') - return { - dispatch: this.getDispatchContextById(id) as DispatchContextRow, - worker: this.getWorkerDispatch(id) as WorkerDispatchRow - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - recordWorkerStage(params: { - dispatchId: string - stage: string - worktreeId?: string - terminalHandle?: string - setupState?: string - effects?: unknown[] - residualResources?: unknown[] - lastError?: string - state?: WorkerDispatchState - }): WorkerDispatchRow { - const current = this.getWorkerDispatch(params.dispatchId) - if (!current) { - throw new OrchestrationError( - 'dispatch_not_found', - `Dispatch ${params.dispatchId} was not found.` - ) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET stage = ?, state = ?, worktree_id = ?, agent_terminal_handle = ?, - setup_state = ?, effects = ?, residual_resources = ?, last_error = ?, - updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run( - params.stage, - params.state ?? current.state, - params.worktreeId ?? current.worktree_id, - params.terminalHandle ?? current.agent_terminal_handle, - params.setupState ?? current.setup_state, - params.effects ? JSON.stringify(params.effects) : current.effects, - params.residualResources - ? JSON.stringify(params.residualResources) - : current.residual_resources, - params.lastError ?? current.last_error, - params.dispatchId - ) - return this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow - } - - updateWorkerSetupEvidence(params: { - dispatchId: string - setupState: string - effects: unknown[] - }): { worker: WorkerDispatchRow; changed: boolean } { - const current = this.getWorkerDispatch(params.dispatchId) - if (!current) { - throw new OrchestrationError( - 'dispatch_not_found', - `Dispatch ${params.dispatchId} was not found.` - ) - } - const effects = JSON.stringify(params.effects) - if (current.setup_state === params.setupState && current.effects === effects) { - return { worker: current, changed: false } - } - this.db - .prepare( - `UPDATE worker_dispatches - SET setup_state = ?, effects = ?, updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(params.setupState, effects, params.dispatchId) - return { - worker: this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow, - changed: true - } - } - - prepareStartingWorkerAuthority(params: { - dispatchId: string - handle: string - paneKey: string - processIncarnation: string - launchTokenHash?: string - worktreeId: string - effects: unknown[] - setupState: string - hostScope?: string | null - // 'created': this worker-start operation created the agent terminal (including agent-first - // worktree creation, whose effects receipt says 'reused_agent_terminal'). 'external': an - // explicit --terminal reuse; ownership transfers only from an exact owned settled resource. - terminalOwnership?: 'created' | 'external' - }): string { - const dispatch = this.getDispatchContextById(params.dispatchId) - const worker = this.getWorkerDispatch(params.dispatchId) - if (!dispatch || dispatch.status !== 'pending' || worker?.state !== 'starting') { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${params.dispatchId} is not starting.` - ) - } - if ( - dispatch.launch_token_hash && - params.launchTokenHash && - dispatch.launch_token_hash !== params.launchTokenHash - ) { - throw new OrchestrationError( - 'request_mismatch', - `Dispatch ${params.dispatchId} already has a different launch-token commitment.` - ) - } - const capability = `dcap_${randomBytes(32).toString('base64url')}` - this.db.exec('BEGIN IMMEDIATE') - try { - this.db - .prepare( - `UPDATE dispatch_contexts - SET assignee_handle = ?, assignee_pane_key = ?, process_incarnation = ?, - capability_hash = ?, launch_token_hash = COALESCE(launch_token_hash, ?), - capability_revoked_at = NULL - WHERE id = ? AND status = 'pending'` - ) - .run( - params.handle, - params.paneKey, - params.processIncarnation, - hashDispatchCapability(capability), - params.launchTokenHash ?? null, - params.dispatchId - ) - this.db - .prepare( - `UPDATE worker_dispatches - SET stage = 'authority_attached', worktree_id = ?, agent_terminal_handle = ?, - setup_state = ?, effects = ?, residual_resources = ?, updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'starting'` - ) - .run( - params.worktreeId, - params.handle, - params.setupState, - JSON.stringify(params.effects), - JSON.stringify( - params.effects.filter((effect) => - Boolean( - effect && - typeof effect === 'object' && - ((effect as { action?: string }).action?.startsWith('created') || - (effect as { action?: string }).action === 'reused_agent_terminal') - ) - ) - ), - params.dispatchId - ) - if (params.terminalOwnership && !this.getWorkerTerminalResourceByOwner(params.dispatchId)) { - if (params.terminalOwnership === 'created') { - this.createWorkerTerminalResourceStatement({ - dispatchId: params.dispatchId, - worktreeId: params.worktreeId, - terminalHandle: params.handle, - paneKey: params.paneKey, - processIncarnation: params.processIncarnation, - hostScope: params.hostScope, - ownership: 'owned' - }) - } else { - const transferable = this.findTransferableWorkerTerminalResource({ - terminalHandle: params.handle, - paneKey: params.paneKey, - processIncarnation: params.processIncarnation, - hostScope: params.hostScope ?? null - }) - if (transferable) { - this.transferWorkerTerminalResourceStatement({ - resourceId: transferable.id, - toDispatchId: params.dispatchId, - terminalHandle: params.handle, - paneKey: params.paneKey, - processIncarnation: params.processIncarnation, - hostScope: params.hostScope ?? null - }) - } else { - this.createWorkerTerminalResourceStatement({ - dispatchId: params.dispatchId, - worktreeId: params.worktreeId, - terminalHandle: params.handle, - paneKey: params.paneKey, - processIncarnation: params.processIncarnation, - hostScope: params.hostScope, - ownership: 'external' - }) - } - } - } - this.db.exec('COMMIT') - return capability - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - markWorkerDispatchReady(dispatchId: string, effects?: unknown[]): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const dispatch = this.getDispatchContextById(dispatchId) - const worker = this.getWorkerDispatch(dispatchId) - if (!dispatch || dispatch.status !== 'pending' || worker?.state !== 'starting') { - throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) - } - this.db - .prepare("UPDATE dispatch_contexts SET status = 'dispatched' WHERE id = ?") - .run(dispatchId) - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'ready', stage = 'input_accepted', - effects = COALESCE(?, effects), updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(effects ? JSON.stringify(effects) : null, dispatchId) - this.db.exec('COMMIT') - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - failWorkerStart(dispatchId: string, stage: string, reason: string): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const dispatch = this.getDispatchContextById(dispatchId) - const worker = this.getWorkerDispatch(dispatchId) - if (!dispatch || !worker || worker.state !== 'starting') { - throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) - } - this.db - .prepare( - `UPDATE dispatch_contexts - SET status = 'failed', last_failure = ?, completed_at = datetime('now'), - capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ?` - ) - .run(reason, dispatchId) - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'failed', stage = ?, last_error = ?, updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(stage, reason, dispatchId) - this.db - .prepare("UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE id = ?") - .run(dispatch.task_id) - this.closeQuestionsForDispatch(dispatchId) - this.db.exec('COMMIT') - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - markWorkerStartUnknown(dispatchId: string, stage: string, reason: string): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const dispatch = this.getDispatchContextById(dispatchId) - const worker = this.getWorkerDispatch(dispatchId) - if (!dispatch || !worker || worker.state !== 'starting') { - throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'start_unknown', stage = ?, last_error = ?, updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(stage, reason, dispatchId) - this.db - .prepare( - `UPDATE dispatch_contexts - SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ?` - ) - .run(dispatchId) - this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) - this.closeQuestionsForDispatch(dispatchId) - this.db.exec('COMMIT') - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - reconcileFederatedWorkerStart(params: { - dispatchId: string - state: 'ready' | 'failed' | 'stopped' | 'start_unknown' - stage: string - lastError?: string | null - worktreeId?: string | null - terminalHandle?: string | null - setupState?: string - effects?: unknown[] - residualResources?: unknown[] - }): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const dispatch = this.getDispatchContextById(params.dispatchId) - const worker = this.getWorkerDispatch(params.dispatchId) - if (!dispatch || !worker) { - throw new OrchestrationError( - 'dispatch_not_found', - `Federated Dispatch ${params.dispatchId} was not found.` - ) - } - if (!['starting', 'start_unknown'].includes(worker.state)) { - this.db.exec('COMMIT') - return worker - } - - if (params.state === 'ready') { - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'ready', stage = ?, worktree_id = COALESCE(?, worktree_id), - agent_terminal_handle = COALESCE(?, agent_terminal_handle), setup_state = ?, - effects = ?, residual_resources = ?, last_error = NULL, - updated_at = datetime('now') - WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` - ) - .run( - params.stage, - params.worktreeId ?? null, - params.terminalHandle ?? null, - params.setupState ?? worker.setup_state, - JSON.stringify(params.effects ?? JSON.parse(worker.effects)), - JSON.stringify(params.residualResources ?? JSON.parse(worker.residual_resources)), - params.dispatchId - ) - this.db - .prepare( - "UPDATE dispatch_contexts SET status = 'dispatched' WHERE id = ? AND status = 'pending'" - ) - .run(params.dispatchId) - this.db - .prepare( - "UPDATE tasks SET status = 'dispatched', completed_at = NULL WHERE id = ? AND status = 'blocked'" - ) - .run(dispatch.task_id) - } else if (params.state === 'start_unknown') { - this.db - .prepare( - `UPDATE worker_dispatches - SET stage = ?, last_error = ?, updated_at = datetime('now') - WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` - ) - .run(params.stage, params.lastError ?? worker.last_error, params.dispatchId) - } else { - const reason = params.lastError ?? `The worker server reported ${params.state}.` - this.db - .prepare( - `UPDATE worker_dispatches - SET state = ?, stage = ?, last_error = ?, updated_at = datetime('now') - WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` - ) - .run(params.state, params.stage, reason, params.dispatchId) - this.db - .prepare( - `UPDATE dispatch_contexts - SET status = 'failed', last_failure = ?, completed_at = datetime('now'), - capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ? AND status IN ('pending', 'dispatched')` - ) - .run(reason, params.dispatchId) - this.db - .prepare( - "UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE id = ? AND status IN ('blocked', 'dispatched')" - ) - .run(dispatch.task_id) - this.closeQuestionsForDispatch(params.dispatchId) - } - this.db.exec('COMMIT') - return this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getWorkerDispatch(dispatchId: string): WorkerDispatchRow | undefined { - return this.db - .prepare('SELECT * FROM worker_dispatches WHERE dispatch_id = ?') - .get(dispatchId) as WorkerDispatchRow | undefined - } - - listLegacyWorkerTerminalRecoveryRows(): LegacyWorkerTerminalRecoveryRow[] { - return this.db - .prepare( - `SELECT dc.id AS dispatch_id, dc.task_id, dc.status AS dispatch_status, - dc.contract_version, dc.assignee_handle, dc.assignee_pane_key, - dc.process_incarnation, wd.state AS worker_state, wd.worktree_id, - wd.agent_terminal_handle - FROM dispatch_contexts dc - INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id - WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown') - ORDER BY dc.rowid` - ) - .all() as LegacyWorkerTerminalRecoveryRow[] - } - - reconcileMissingWorkerTerminal(dispatchId: string, reason: string): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const dispatch = this.getDispatchContextById(dispatchId) - const worker = this.getWorkerDispatch(dispatchId) - if (!dispatch || !worker) { - throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) - } - if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(worker.state)) { - this.db.exec('COMMIT') - return worker - } - - const activeDispatch = dispatch.status === 'pending' || dispatch.status === 'dispatched' - const stopWasPending = worker.state === 'stopping' || worker.state === 'stop_unknown' - if (activeDispatch) { - const failureCount = dispatch.failure_count + 1 - const dispatchStatus: DispatchStatus = failureCount >= 3 ? 'circuit_broken' : 'failed' - this.db - .prepare( - `UPDATE dispatch_contexts - SET status = ?, failure_count = ?, last_failure = ?, - completed_at = datetime('now'), - capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ? AND status IN ('pending', 'dispatched')` - ) - .run(dispatchStatus, failureCount, reason, dispatchId) - if (!stopWasPending) { - const taskStatus: TaskStatus = dispatchStatus === 'circuit_broken' ? 'failed' : 'ready' - this.db - .prepare( - `UPDATE tasks - SET status = ?, completed_at = CASE WHEN ? = 'failed' THEN datetime('now') ELSE NULL END - WHERE id = ? AND status IN ('dispatched', 'blocked')` - ) - .run(taskStatus, taskStatus, dispatch.task_id) - } - this.closeQuestionsForDispatch(dispatchId) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = ?, stage = 'terminal_missing', last_error = ?, updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(stopWasPending ? 'stopped' : 'abandoned', reason, dispatchId) - this.db.exec('COMMIT') - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined { - return this.db - .prepare('SELECT * FROM federated_dispatches WHERE dispatch_id = ?') - .get(dispatchId) as FederatedDispatchRow | undefined - } - - listActiveFederatedDispatches(runId?: string): FederatedDispatchRow[] { - return this.db - .prepare( - `SELECT fd.* - FROM federated_dispatches fd - INNER JOIN dispatch_contexts dc ON dc.id = fd.dispatch_id - INNER JOIN worker_dispatches wd ON wd.dispatch_id = fd.dispatch_id - WHERE wd.state IN ('starting', 'ready', 'stopping', 'start_unknown', 'stop_unknown') - AND (? IS NULL OR dc.run_id = ?) - ORDER BY fd.rowid` - ) - .all(runId ?? null, runId ?? null) as FederatedDispatchRow[] - } - - findNextTerminalFederatedDispatchPendingAcknowledgment( - afterRowId: number - ): { dispatchId: string; rowId: number } | undefined { - return this.db - .prepare( - `SELECT fd.dispatch_id AS dispatchId, fd.rowid AS rowId - FROM federated_dispatches fd - INNER JOIN worker_dispatches wd ON wd.dispatch_id = fd.dispatch_id - WHERE wd.state NOT IN ('starting', 'ready', 'stopping', 'start_unknown', 'stop_unknown') - AND fd.to_home_acknowledged_sequence < fd.to_home_imported_sequence - AND fd.rowid > ? - ORDER BY fd.rowid - LIMIT 1` - ) - .get(afterRowId) as { dispatchId: string; rowId: number } | undefined - } - - isFederatedDispatchRelayEligible(dispatchId: string): boolean { - return Boolean( - this.db - .prepare( - `SELECT 1 - FROM federated_dispatches fd - INNER JOIN worker_dispatches wd ON wd.dispatch_id = fd.dispatch_id - WHERE fd.dispatch_id = ? - AND ( - wd.state IN ('starting', 'ready', 'stopping', 'start_unknown', 'stop_unknown') - OR ( - fd.to_home_acknowledged_sequence < fd.to_home_imported_sequence - ) - )` - ) - .get(dispatchId) - ) - } - - updateFederatedDispatchResources(params: { - dispatchId: string - remoteRuntimeEpoch: string - worktreeId: string - terminalHandle: string - }): FederatedDispatchRow { - this.db - .prepare( - `UPDATE federated_dispatches - SET remote_runtime_epoch = ?, remote_worktree_id = ?, remote_terminal_handle = ?, - updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(params.remoteRuntimeEpoch, params.worktreeId, params.terminalHandle, params.dispatchId) - const row = this.getFederatedDispatch(params.dispatchId) - if (!row) { - throw new OrchestrationError( - 'dispatch_not_found', - `Federated Dispatch ${params.dispatchId} was not found.` - ) - } - return row - } - - createRemoteDispatchAttachment(params: { - dispatchId: string - taskId: string - homePeerFingerprint: string - protocolVersion: number - runtimeEpoch: string - mutationReceipt: { - callerFingerprint: string - requestId: string - method: string - payloadHash: string - } - }): RemoteDispatchAttachmentRow { - this.db.exec('BEGIN IMMEDIATE') - try { - if (params.homePeerFingerprint !== params.mutationReceipt.callerFingerprint) { - throw new OrchestrationError( - 'resource_server_mismatch', - 'The authenticated Run-home peer does not match the attachment request.' - ) - } - const existingReceipt = this.getMutationReceipt( - params.mutationReceipt.callerFingerprint, - params.mutationReceipt.requestId - ) - if (existingReceipt) { - throw new OrchestrationError( - existingReceipt.method === params.mutationReceipt.method && - existingReceipt.payload_hash === params.mutationReceipt.payloadHash - ? 'operation_unknown' - : 'request_mismatch', - `Remote attachment request ${params.mutationReceipt.requestId} already exists.` - ) - } - ensureMutationReceiptCapacity(this.db) - this.db - .prepare( - `INSERT INTO mutation_receipts ( - caller_fingerprint, request_id, method, payload_hash, state, receipt - ) VALUES (?, ?, ?, ?, 'pending', ?)` - ) - .run( - params.mutationReceipt.callerFingerprint, - params.mutationReceipt.requestId, - params.mutationReceipt.method, - params.mutationReceipt.payloadHash, - JSON.stringify({ accepted: { dispatchId: params.dispatchId } }) - ) - this.db - .prepare( - `INSERT INTO remote_dispatch_attachments ( - dispatch_id, task_id, home_peer_fingerprint, protocol_version, runtime_epoch - ) VALUES (?, ?, ?, ?, ?)` - ) - .run( - params.dispatchId, - params.taskId, - params.homePeerFingerprint, - params.protocolVersion, - params.runtimeEpoch - ) - this.db.exec('COMMIT') - return this.getRemoteDispatchAttachment(params.dispatchId) as RemoteDispatchAttachmentRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getRemoteDispatchAttachment(dispatchId: string): RemoteDispatchAttachmentRow | undefined { - return this.db - .prepare('SELECT * FROM remote_dispatch_attachments WHERE dispatch_id = ?') - .get(dispatchId) as RemoteDispatchAttachmentRow | undefined - } - - recordRemoteAttachmentStage(params: { - dispatchId: string - stage: string - state?: WorkerDispatchState - worktreeId?: string - terminalHandle?: string - setupState?: string - effects?: unknown[] - residualResources?: unknown[] - lastError?: string - }): RemoteDispatchAttachmentRow { - const current = this.getRemoteDispatchAttachment(params.dispatchId) - if (!current) { - throw new OrchestrationError( - 'dispatch_not_found', - `Remote Dispatch ${params.dispatchId} was not found.` - ) - } - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET stage = ?, state = ?, worktree_id = ?, terminal_handle = ?, setup_state = ?, - effects = ?, residual_resources = ?, last_error = ?, updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run( - params.stage, - params.state ?? current.state, - params.worktreeId ?? current.worktree_id, - params.terminalHandle ?? current.terminal_handle, - params.setupState ?? current.setup_state, - params.effects ? JSON.stringify(params.effects) : current.effects, - params.residualResources - ? JSON.stringify(params.residualResources) - : current.residual_resources, - params.lastError ?? current.last_error, - params.dispatchId - ) - return this.getRemoteDispatchAttachment(params.dispatchId) as RemoteDispatchAttachmentRow - } - - updateRemoteAttachmentSetupEvidence(params: { - dispatchId: string - setupState: string - effects: unknown[] - }): { attachment: RemoteDispatchAttachmentRow; changed: boolean } { - const current = this.getRemoteDispatchAttachment(params.dispatchId) - if (!current) { - throw new OrchestrationError( - 'dispatch_not_found', - `Remote Dispatch ${params.dispatchId} was not found.` - ) - } - const effects = JSON.stringify(params.effects) - if (current.setup_state === params.setupState && current.effects === effects) { - return { attachment: current, changed: false } - } - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET setup_state = ?, effects = ?, updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(params.setupState, effects, params.dispatchId) - return { - attachment: this.getRemoteDispatchAttachment( - params.dispatchId - ) as RemoteDispatchAttachmentRow, - changed: true - } - } - - prepareRemoteAttachmentAuthority(params: { - dispatchId: string - paneKey: string - processIncarnation: string - worktreeId: string - terminalHandle: string - setupState: string - effects: unknown[] - }): string { - const attachment = this.getRemoteDispatchAttachment(params.dispatchId) - if (!attachment || attachment.state !== 'starting') { - throw new OrchestrationError( - 'dispatch_inactive', - `Remote Dispatch ${params.dispatchId} is not starting.` - ) - } - const capability = `dcap_${randomBytes(32).toString('base64url')}` - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET stage = 'authority_attached', capability_hash = ?, pane_key = ?, - process_incarnation = ?, worktree_id = ?, terminal_handle = ?, setup_state = ?, - effects = ?, residual_resources = ?, updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'starting'` - ) - .run( - hashDispatchCapability(capability), - params.paneKey, - params.processIncarnation, - params.worktreeId, - params.terminalHandle, - params.setupState, - JSON.stringify(params.effects), - JSON.stringify( - params.effects.filter((effect) => - Boolean( - effect && - typeof effect === 'object' && - ((effect as { action?: string }).action?.startsWith('created') || - (effect as { action?: string }).action === 'reused_agent_terminal') - ) - ) - ), - params.dispatchId - ) - return capability - } - - markRemoteAttachmentReady(dispatchId: string, effects?: unknown[]): RemoteDispatchAttachmentRow { - const result = this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET state = 'ready', stage = 'input_accepted', - effects = COALESCE(?, effects), updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'starting'` - ) - .run(effects ? JSON.stringify(effects) : null, dispatchId) - if (result.changes !== 1) { - throw new OrchestrationError( - 'dispatch_inactive', - `Remote Dispatch ${dispatchId} is not starting.` - ) - } - return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow - } - - failRemoteAttachment( - dispatchId: string, - stage: string, - reason: string, - unknown: boolean - ): RemoteDispatchAttachmentRow { - const state = unknown ? 'start_unknown' : 'failed' - const result = this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET state = ?, stage = ?, last_error = ?, capability_hash = NULL, - updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'starting'` - ) - .run(state, stage, reason, dispatchId) - if (result.changes !== 1) { - throw new OrchestrationError( - 'dispatch_inactive', - `Remote Dispatch ${dispatchId} is not starting.` - ) - } - return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow - } - - verifyRemoteAttachmentAuthority(params: { - dispatchId: string - capability: string | undefined - paneKey: string | null - processIncarnation: string | null - }): boolean { - const attachment = this.getRemoteDispatchAttachment(params.dispatchId) - if ( - !attachment?.capability_hash || - !params.capability || - !attachment.pane_key || - !params.paneKey || - !isEquivalentPaneKey(attachment.pane_key, params.paneKey) || - !attachment.process_incarnation || - attachment.process_incarnation !== params.processIncarnation - ) { - return false - } - const expected = Buffer.from(attachment.capability_hash, 'hex') - const observed = Buffer.from(hashDispatchCapability(params.capability), 'hex') - return expected.length === observed.length && timingSafeEqual(expected, observed) - } - - isRemoteAttachmentProcessCurrent(params: { - dispatchId: string - paneKey: string | null - processIncarnation: string | null - }): boolean { - const attachment = this.getRemoteDispatchAttachment(params.dispatchId) - return Boolean( - attachment?.pane_key && - params.paneKey && - isEquivalentPaneKey(attachment.pane_key, params.paneKey) && - attachment.process_incarnation && - attachment.process_incarnation === params.processIncarnation - ) - } - - beginRemoteAttachmentStop(dispatchId: string): RemoteDispatchAttachmentRow { - const attachment = this.getRemoteDispatchAttachment(dispatchId) - if (!attachment) { - throw new OrchestrationError( - 'dispatch_not_found', - `Remote Dispatch ${dispatchId} was not found.` - ) - } - if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(attachment.state)) { - return attachment - } - if (!['ready', 'start_unknown'].includes(attachment.state)) { - throw new OrchestrationError( - 'dispatch_inactive', - `Remote Dispatch ${dispatchId} cannot stop from ${attachment.state}.` - ) - } - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET state = 'stopping', stage = 'stop_requested', capability_hash = NULL, - updated_at = datetime('now') - WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')` - ) - .run(dispatchId) - return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow - } - - settleRemoteAttachmentStop(dispatchId: string): RemoteDispatchAttachmentRow { - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET state = 'stopped', stage = 'process_stopped', updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'stopping'` - ) - .run(dispatchId) - return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow - } - - markRemoteAttachmentStopUnknown(dispatchId: string, reason: string): RemoteDispatchAttachmentRow { - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET state = 'stop_unknown', stage = 'stop_outcome_unknown', last_error = ?, - updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'stopping'` - ) - .run(reason, dispatchId) - return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow - } - - findActiveRemoteAttachmentForPane(paneKey: string): RemoteDispatchAttachmentRow | undefined { - if (!parsePaneKey(paneKey)) { - return this.db - .prepare( - `SELECT * FROM remote_dispatch_attachments - WHERE state IN ('starting', 'ready') AND pane_key = ? - ORDER BY rowid DESC LIMIT 1` - ) - .get(paneKey) as RemoteDispatchAttachmentRow | undefined - } - return this.db - .prepare( - `SELECT * FROM remote_dispatch_attachments - WHERE state IN ('starting', 'ready') AND pane_key IS NOT NULL - AND instr(pane_key, ':') > 1 - AND ${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL} = ? - ORDER BY rowid DESC LIMIT 1` - ) - .get(paneKeyMatchSuffix(paneKey)) as RemoteDispatchAttachmentRow | undefined - } - - enqueueFederationRelay(params: { - dispatchId: string - direction: FederationRelayDirection - kind: string - payload: string - messageId?: string - settleRemoteOutcome?: WorkerReportOutcome - remoteQuestion?: true - }): FederationRelayItemRow { - const byteCount = Buffer.byteLength(params.payload, 'utf8') - const messageId = params.messageId ?? generateId('relay') - if (byteCount > 64 * 1024) { - throw new OrchestrationError( - 'relay_quota_exceeded', - 'A federated orchestration message cannot exceed 64 KiB.' - ) - } - this.db.exec('BEGIN IMMEDIATE') - try { - if (params.settleRemoteOutcome) { - const attachment = this.getRemoteDispatchAttachment(params.dispatchId) - if (!attachment || attachment.state !== 'ready') { - throw new OrchestrationError( - 'dispatch_inactive', - `Remote Dispatch ${params.dispatchId} is not active.` - ) - } - } - if (params.kind === 'heartbeat') { - const heartbeat = this.db - .prepare( - `SELECT * FROM federation_relay_items - WHERE dispatch_id = ? AND direction = ? AND kind = 'heartbeat' - AND acked_at IS NULL - ORDER BY sequence DESC LIMIT 1` - ) - .get(params.dispatchId, params.direction) as FederationRelayItemRow | undefined - if (heartbeat) { - this.db - .prepare( - `UPDATE federation_relay_items - SET payload = ?, byte_count = ?, created_at = datetime('now') - WHERE dispatch_id = ? AND direction = ? AND sequence = ?` - ) - .run(params.payload, byteCount, params.dispatchId, params.direction, heartbeat.sequence) - this.db.exec('COMMIT') - return this.getFederationRelayItem( - params.dispatchId, - params.direction, - heartbeat.sequence - ) as FederationRelayItemRow - } - } - if (params.kind === 'worker_done') { - const identicalReport = this.db - .prepare( - `SELECT * FROM federation_relay_items - WHERE dispatch_id = ? AND direction = ? AND kind = 'worker_done' - AND payload = ? AND acked_at IS NULL - ORDER BY sequence DESC LIMIT 1` - ) - .get(params.dispatchId, params.direction, params.payload) as - | FederationRelayItemRow - | undefined - if (identicalReport) { - this.settleRemoteAttachmentInRelayTransaction( - params.dispatchId, - params.settleRemoteOutcome - ) - this.db.exec('COMMIT') - return identicalReport - } - } - const quota = this.db - .prepare( - `SELECT COUNT(*) AS count, COALESCE(SUM(byte_count), 0) AS bytes - FROM federation_relay_items - WHERE dispatch_id = ? AND direction = ? AND acked_at IS NULL` - ) - .get(params.dispatchId, params.direction) as { count: number; bytes: number } - if (quota.count >= 256 || quota.bytes + byteCount > 1024 * 1024) { - if (params.kind === 'worker_done') { - const heartbeat = this.db - .prepare( - `SELECT * FROM federation_relay_items - WHERE dispatch_id = ? AND direction = ? AND kind = 'heartbeat' - AND acked_at IS NULL - ORDER BY sequence LIMIT 1` - ) - .get(params.dispatchId, params.direction) as FederationRelayItemRow | undefined - if (heartbeat) { - this.db - .prepare( - `UPDATE federation_relay_items - SET message_id = ?, kind = ?, payload = ?, byte_count = ?, - created_at = datetime('now') - WHERE dispatch_id = ? AND direction = ? AND sequence = ?` - ) - .run( - messageId, - params.kind, - params.payload, - byteCount, - params.dispatchId, - params.direction, - heartbeat.sequence - ) - this.settleRemoteAttachmentInRelayTransaction( - params.dispatchId, - params.settleRemoteOutcome - ) - this.db.exec('COMMIT') - return this.getFederationRelayItem( - params.dispatchId, - params.direction, - heartbeat.sequence - ) as FederationRelayItemRow - } - } - throw new OrchestrationError( - 'relay_quota_exceeded', - `Federated Dispatch ${params.dispatchId} has no relay capacity.` - ) - } - const latest = this.db - .prepare( - `SELECT COALESCE(MAX(sequence), 0) AS sequence - FROM federation_relay_items WHERE dispatch_id = ? AND direction = ?` - ) - .get(params.dispatchId, params.direction) as { sequence: number } - const sequence = latest.sequence + 1 - this.db - .prepare( - `INSERT INTO federation_relay_items ( - dispatch_id, direction, sequence, message_id, kind, payload, byte_count - ) VALUES (?, ?, ?, ?, ?, ?, ?)` - ) - .run( - params.dispatchId, - params.direction, - sequence, - messageId, - params.kind, - params.payload, - byteCount - ) - if (params.remoteQuestion) { - this.db - .prepare( - `INSERT INTO remote_questions (message_id, dispatch_id) - VALUES (?, ?)` - ) - .run(messageId, params.dispatchId) - } - this.settleRemoteAttachmentInRelayTransaction(params.dispatchId, params.settleRemoteOutcome) - this.db.exec('COMMIT') - return this.getFederationRelayItem( - params.dispatchId, - params.direction, - sequence - ) as FederationRelayItemRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - listFederationRelay(params: { - dispatchId: string - direction: FederationRelayDirection - afterSequence: number - limit?: number - }): FederationRelayItemRow[] { - return this.db - .prepare( - `SELECT * FROM federation_relay_items - WHERE dispatch_id = ? AND direction = ? AND sequence > ? - ORDER BY sequence LIMIT ?` - ) - .all( - params.dispatchId, - params.direction, - params.afterSequence, - Math.min(Math.max(params.limit ?? 50, 1), 50) - ) as FederationRelayItemRow[] - } - - listPendingFederationRelay( - dispatchId: string, - direction: FederationRelayDirection, - limit = 50 - ): FederationRelayItemRow[] { - return this.db - .prepare( - `SELECT * FROM federation_relay_items - WHERE dispatch_id = ? AND direction = ? AND acked_at IS NULL - ORDER BY sequence LIMIT ?` - ) - .all(dispatchId, direction, Math.min(Math.max(limit, 1), 50)) as FederationRelayItemRow[] - } - - acknowledgeFederationRelay(params: { - dispatchId: string - direction: FederationRelayDirection - throughSequence: number - settleRemoteReports?: { sequence: number; outcome?: WorkerReportOutcome }[] - }): void { - this.db.exec('BEGIN IMMEDIATE') - try { - const settledReports = params.settleRemoteReports ?? [] - for (const settledReport of settledReports) { - const report = this.getFederationRelayItem( - params.dispatchId, - params.direction, - settledReport.sequence - ) - if ( - params.direction !== 'to_home' || - settledReport.sequence > params.throughSequence || - report?.kind !== 'worker_done' || - (settledReport.outcome !== undefined && - parseFederatedWorkerReportOutcome(report.payload) !== settledReport.outcome) - ) { - throw new OrchestrationError( - 'request_mismatch', - `Federation acknowledgment for ${params.dispatchId} does not match its queued worker_done.` - ) - } - } - const attachment = this.getRemoteDispatchAttachment(params.dispatchId) - if ( - params.direction === 'to_home' && - attachment !== undefined && - attachment.protocol_version >= - ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION - ) { - const acknowledgedReports = this.db - .prepare( - `SELECT sequence FROM federation_relay_items - WHERE dispatch_id = ? AND direction = 'to_home' AND kind = 'worker_done' - AND acked_at IS NULL AND sequence <= ?` - ) - .all(params.dispatchId, params.throughSequence) as { sequence: number }[] - const settledSequences = new Set(settledReports.map((report) => report.sequence)) - if (acknowledgedReports.some((report) => !settledSequences.has(report.sequence))) { - throw new OrchestrationError( - 'request_mismatch', - `Federation acknowledgment for ${params.dispatchId} omits a worker_done settlement.` - ) - } - } - const terminalOutcomes = new Set( - settledReports.flatMap((report) => (report.outcome ? [report.outcome] : [])) - ) - if (terminalOutcomes.size > 1) { - throw new OrchestrationError( - 'request_mismatch', - `Federation acknowledgment for ${params.dispatchId} contains conflicting settlements.` - ) - } - const terminalOutcome = settledReports.find((report) => report.outcome)?.outcome - if (terminalOutcome) { - this.settleRemoteAttachmentInRelayTransaction( - params.dispatchId, - terminalOutcome, - 'worker_report_settled' - ) - } - this.db - .prepare( - `UPDATE federation_relay_items SET acked_at = COALESCE(acked_at, datetime('now')) - WHERE dispatch_id = ? AND direction = ? AND sequence <= ?` - ) - .run(params.dispatchId, params.direction, params.throughSequence) - this.db.exec('COMMIT') - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - setFederatedHomeImportSequence(dispatchId: string, sequence: number): void { - this.db - .prepare( - `UPDATE federated_dispatches - SET to_home_imported_sequence = ?, updated_at = datetime('now') - WHERE dispatch_id = ? AND to_home_imported_sequence < ?` - ) - .run(sequence, dispatchId, sequence) - } - - recordFederatedHomeAcknowledgment(params: { - dispatchId: string - remoteRuntimeEpoch: string - sequence: number - }): void { - const federated = this.getFederatedDispatch(params.dispatchId) - if ( - !federated || - !Number.isInteger(params.sequence) || - params.sequence < 0 || - params.sequence > federated.to_home_imported_sequence - ) { - throw new OrchestrationError( - 'request_mismatch', - `Federation acknowledgment for ${params.dispatchId} exceeds imported relay state.` - ) - } - this.db - .prepare( - `UPDATE federated_dispatches - SET remote_runtime_epoch = ?, - to_home_acknowledged_sequence = CASE - WHEN remote_runtime_epoch = ? - THEN MAX(to_home_acknowledged_sequence, ?) - ELSE ? - END, - updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run( - params.remoteRuntimeEpoch, - params.remoteRuntimeEpoch, - params.sequence, - params.sequence, - params.dispatchId - ) - } - - importFederatedRelayItem(params: { - dispatchId: string - sequence: number - message: { - id: string - runId: string - from: string - to: string - subject: string - body: string - type: MessageType - priority: MessagePriority - threadId?: string - payload?: string - } - lifecycle: - | { kind: 'none' } - | { kind: 'heartbeat'; at: string } - | { - kind: 'worker_report' - taskId: string - outcome: WorkerReportOutcome - result: string - } - | { kind: 'rejected'; code: string; reason: string } - }): { - message: MessageRow - duplicate: boolean - lifecycle?: WorkerReportSettlement | { action: 'rejected'; code: string; reason: string } - } { - this.db.exec('BEGIN IMMEDIATE') - try { - const federated = this.getFederatedDispatch(params.dispatchId) - if (!federated) { - throw new OrchestrationError( - 'dispatch_not_found', - `Federated Dispatch ${params.dispatchId} was not found.` - ) - } - const duplicate = params.sequence <= federated.to_home_imported_sequence - if (duplicate && !this.getMessageById(params.message.id)) { - throw new OrchestrationError( - 'operation_unknown', - `Federated relay sequence ${params.sequence} was committed without its message.` - ) - } - if (!duplicate && params.sequence !== federated.to_home_imported_sequence + 1) { - throw new OrchestrationError( - 'operation_unknown', - `Federated relay for ${params.dispatchId} is not contiguous after sequence ${federated.to_home_imported_sequence}.` - ) - } - - let message = this.getMessageById(params.message.id) - if (!message) { - message = this.insertMessage(params.message) - } else if ( - message.run_id !== params.message.runId || - message.to_handle !== params.message.to || - message.type !== params.message.type - ) { - throw new OrchestrationError( - 'request_mismatch', - `Federated relay message ${params.message.id} conflicts with an existing message.` - ) - } - if (message.type === 'question') { - this.registerFederatedQuestion({ - messageId: message.id, - runId: params.message.runId, - dispatchId: params.dispatchId - }) - } - let lifecycle: - | WorkerReportSettlement - | { action: 'rejected'; code: string; reason: string } - | undefined - if (params.lifecycle.kind === 'heartbeat' && !duplicate) { - this.recordHeartbeat(params.dispatchId, params.lifecycle.at) - } else if (params.lifecycle.kind === 'worker_report') { - lifecycle = this.settleWorkerReportInTransaction({ - taskId: params.lifecycle.taskId, - dispatchId: params.dispatchId, - outcome: params.lifecycle.outcome, - result: params.lifecycle.result - }) - if (lifecycle.action === 'rejected' && !duplicate) { - message = this.convertLifecycleMessageToRejection( - message.id, - lifecycle.code, - lifecycle.reason - ) as MessageRow - } - } else if (params.lifecycle.kind === 'rejected') { - lifecycle = { - action: 'rejected', - code: params.lifecycle.code, - reason: params.lifecycle.reason - } - if (!duplicate) { - message = this.convertLifecycleMessageToRejection( - message.id, - params.lifecycle.code, - params.lifecycle.reason - ) as MessageRow - } - } - if (!duplicate) { - this.setFederatedHomeImportSequence(params.dispatchId, params.sequence) - } - this.db.exec('COMMIT') - return { message, duplicate, ...(lifecycle ? { lifecycle } : {}) } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getRemoteQuestion(messageId: string): - | { - message_id: string - dispatch_id: string - status: 'pending' | 'answered' | 'closed' - answer_message_id: string | null - answer_body: string | null - } - | undefined { - return this.db.prepare('SELECT * FROM remote_questions WHERE message_id = ?').get(messageId) as - | { - message_id: string - dispatch_id: string - status: 'pending' | 'answered' | 'closed' - answer_message_id: string | null - answer_body: string | null - } - | undefined - } - - answerRemoteQuestion(params: { - messageId: string - dispatchId: string - answerMessageId: string - body: string - }): void { - const question = this.getRemoteQuestion(params.messageId) - if (!question || question.dispatch_id !== params.dispatchId) { - throw new OrchestrationError( - 'question_not_found', - `Remote Question ${params.messageId} was not found.` - ) - } - if (question.status === 'answered') { - if ( - question.answer_message_id !== params.answerMessageId || - question.answer_body !== params.body - ) { - throw new OrchestrationError( - 'answer_conflict', - `Remote Question ${params.messageId} already has a different answer.` - ) - } - return - } - this.db - .prepare( - `UPDATE remote_questions - SET status = 'answered', answer_message_id = ?, answer_body = ?, - answered_at = datetime('now') - WHERE message_id = ? AND status = 'pending'` - ) - .run(params.answerMessageId, params.body, params.messageId) - } - - setRemoteWorkerImportSequence(dispatchId: string, sequence: number): void { - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET to_worker_imported_sequence = ?, updated_at = datetime('now') - WHERE dispatch_id = ? AND to_worker_imported_sequence < ?` - ) - .run(sequence, dispatchId, sequence) - } - - registerFederatedQuestion(params: { - messageId: string - runId: string - dispatchId: string - }): void { - this.db - .prepare( - `INSERT OR IGNORE INTO question_threads ( - message_id, run_id, dispatch_id, asker_handle - ) VALUES (?, ?, ?, ?)` - ) - .run(params.messageId, params.runId, params.dispatchId, `dispatch:${params.dispatchId}`) - } - - private getFederationRelayItem( - dispatchId: string, - direction: FederationRelayDirection, - sequence: number - ): FederationRelayItemRow | undefined { - return this.db - .prepare( - `SELECT * FROM federation_relay_items - WHERE dispatch_id = ? AND direction = ? AND sequence = ?` - ) - .get(dispatchId, direction, sequence) as FederationRelayItemRow | undefined - } - - private settleRemoteAttachmentInRelayTransaction( - dispatchId: string, - outcome: WorkerReportOutcome | undefined, - stage = 'worker_report_queued' - ): void { - if (!outcome) { - return - } - const attachment = this.getRemoteDispatchAttachment(dispatchId) - const state = outcome === 'succeeded' ? 'succeeded' : 'failed' - if (!attachment) { - throw new OrchestrationError( - 'dispatch_not_found', - `Remote Dispatch ${dispatchId} was not found.` - ) - } - if (attachment.state === state) { - return - } - if (attachment.state !== 'ready') { - throw new OrchestrationError( - 'request_mismatch', - `Remote Dispatch ${dispatchId} cannot settle as ${state} from ${attachment.state}.` - ) - } - this.db - .prepare( - `UPDATE remote_dispatch_attachments - SET state = ?, stage = ?, capability_hash = NULL, - updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'ready'` - ) - .run(state, stage, dispatchId) - } - - isDispatchProcessCurrent(params: { - dispatchId: string - paneKey: string | null - processIncarnation: string | null - }): boolean { - const dispatch = this.getDispatchContextById(params.dispatchId) - return Boolean( - dispatch?.assignee_pane_key && - params.paneKey && - isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) && - dispatch.process_incarnation && - params.processIncarnation === dispatch.process_incarnation - ) - } - - beginWorkerStop( - dispatchId: string - ): - | { disposition: 'stopping'; worker: WorkerDispatchRow; dispatch: DispatchContextRow } - | { disposition: 'already_settled'; worker: WorkerDispatchRow; dispatch: DispatchContextRow } - | ({ disposition: 'context_only' } & ContextOnlyDispatchReleaseResult) { - this.db.exec('BEGIN IMMEDIATE') - try { - const dispatch = this.getDispatchContextById(dispatchId) - const worker = this.getWorkerDispatch(dispatchId) - if (!dispatch) { - throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) - } - if (!worker) { - const released = releaseContextOnlyDispatch( - this.db, - dispatch, - this.getDispatchContext(dispatch.task_id)?.id, - 'stopped' - ) - if (!released.alreadySettled) { - this.closeQuestionsForDispatch(dispatchId) - } - this.db.exec('COMMIT') - return { disposition: 'context_only', ...released } - } - if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(worker.state)) { - this.db.exec('COMMIT') - return { disposition: 'already_settled', worker, dispatch } - } - if (!['ready', 'start_unknown'].includes(worker.state)) { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${dispatchId} cannot stop from ${worker.state}.` - ) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'stopping', stage = 'stop_requested', updated_at = datetime('now') - WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')` - ) - .run(dispatchId) - this.db - .prepare( - `UPDATE dispatch_contexts - SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ?` - ) - .run(dispatchId) - this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) - this.closeQuestionsForDispatch(dispatchId) - this.db.exec('COMMIT') - return { - disposition: 'stopping', - worker: this.getWorkerDispatch(dispatchId) as WorkerDispatchRow, - dispatch: this.getDispatchContextById(dispatchId) as DispatchContextRow - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - settleWorkerStop(dispatchId: string): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const worker = this.getWorkerDispatch(dispatchId) - const dispatch = this.getDispatchContextById(dispatchId) - if (!worker || !dispatch || worker.state !== 'stopping') { - throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not stopping.`) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'stopped', stage = 'process_stopped', updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'stopping'` - ) - .run(dispatchId) - this.db - .prepare( - `UPDATE dispatch_contexts - SET status = 'failed', completed_at = datetime('now'), last_failure = 'stopped' - WHERE id = ? AND status IN ('pending', 'dispatched')` - ) - .run(dispatchId) - this.db.exec('COMMIT') - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - reconcileFederatedWorkerStop(dispatchId: string): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const worker = this.getWorkerDispatch(dispatchId) - const dispatch = this.getDispatchContextById(dispatchId) - if (!worker || !dispatch || !this.getFederatedDispatch(dispatchId)) { - throw new OrchestrationError( - 'dispatch_not_found', - `Federated Dispatch ${dispatchId} was not found.` - ) - } - if (worker.state === 'stopped') { - this.db.exec('COMMIT') - return worker - } - if (!['stopping', 'stop_unknown'].includes(worker.state)) { - throw new OrchestrationError( - 'dispatch_inactive', - `Federated Dispatch ${dispatchId} cannot reconcile stop from ${worker.state}.` - ) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'stopped', stage = 'process_stopped', last_error = NULL, - updated_at = datetime('now') - WHERE dispatch_id = ? AND state IN ('stopping', 'stop_unknown')` - ) - .run(dispatchId) - this.db - .prepare( - `UPDATE dispatch_contexts - SET status = 'failed', completed_at = COALESCE(completed_at, datetime('now')), - last_failure = 'stopped' - WHERE id = ? AND status IN ('pending', 'dispatched')` - ) - .run(dispatchId) - this.db.exec('COMMIT') - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - resumeFederatedWorkerForTerminalRelay(dispatchId: string): WorkerDispatchRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const worker = this.getWorkerDispatch(dispatchId) - const dispatch = this.getDispatchContextById(dispatchId) - if (!worker || !dispatch || worker.state !== 'stopping') { - throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not stopping.`) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'ready', stage = 'remote_report_pending', updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'stopping'` - ) - .run(dispatchId) - this.db - .prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ? AND status = 'blocked'") - .run(dispatch.task_id) - this.db.exec('COMMIT') - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - markWorkerStopUnknown(dispatchId: string, reason: string): WorkerDispatchRow { - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'stop_unknown', stage = 'stop_outcome_unknown', last_error = ?, - updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'stopping'` - ) - .run(reason, dispatchId) - return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } - - abandonWorkerDispatch(dispatchId: string): - | { - disposition: 'abandoned' | 'already_abandoned' | 'stale' - worker: WorkerDispatchRow - } - | ({ disposition: 'context_only' } & ContextOnlyDispatchReleaseResult) { - this.db.exec('BEGIN IMMEDIATE') - try { - const worker = this.getWorkerDispatch(dispatchId) - const dispatch = this.getDispatchContextById(dispatchId) - if (!dispatch) { - throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) - } - if (!worker) { - const released = releaseContextOnlyDispatch( - this.db, - dispatch, - this.getDispatchContext(dispatch.task_id)?.id, - 'abandoned' - ) - if (!released.alreadySettled) { - this.closeQuestionsForDispatch(dispatchId) - } - this.db.exec('COMMIT') - return { disposition: 'context_only', ...released } - } - if (worker.state === 'abandoned') { - this.db.exec('COMMIT') - return { disposition: 'already_abandoned', worker } - } - if (this.getDispatchContext(dispatch.task_id)?.id !== dispatchId) { - this.db.exec('COMMIT') - return { disposition: 'stale', worker } - } - if (worker.state === 'succeeded') { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${dispatchId} already succeeded and cannot be abandoned.` - ) - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = 'abandoned', stage = 'abandoned', updated_at = datetime('now') - WHERE dispatch_id = ?` - ) - .run(dispatchId) - this.db - .prepare( - `UPDATE dispatch_contexts - SET status = CASE WHEN status IN ('pending', 'dispatched') THEN 'failed' ELSE status END, - capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')), - completed_at = COALESCE(completed_at, datetime('now')) - WHERE id = ?` - ) - .run(dispatchId) - this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) - this.closeQuestionsForDispatch(dispatchId) - this.db.exec('COMMIT') - return { - disposition: 'abandoned', - worker: this.getWorkerDispatch(dispatchId) as WorkerDispatchRow - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - // --- Worker terminal resources (schema v23) --------------------------------------------------- - - // Historical renderer input and reuse cannot be proven, so pre-v23 terminals stay external. - private backfillWorkerTerminalResources(): void { - const rows = this.db - .prepare( - `SELECT w.dispatch_id, w.worktree_id, w.agent_terminal_handle, - d.assignee_pane_key, d.process_incarnation - FROM worker_dispatches w - JOIN dispatch_contexts d ON d.id = w.dispatch_id - WHERE w.agent_terminal_handle IS NOT NULL - AND NOT EXISTS (SELECT 1 FROM federated_dispatches f WHERE f.dispatch_id = w.dispatch_id) - AND NOT EXISTS ( - SELECT 1 FROM worker_terminal_resources r WHERE r.owner_dispatch_id = w.dispatch_id - )` - ) - .all() as { - dispatch_id: string - worktree_id: string | null - agent_terminal_handle: string - assignee_pane_key: string | null - process_incarnation: string | null - }[] - const insert = this.db.prepare( - `INSERT INTO worker_terminal_resources ( - id, origin_dispatch_id, owner_dispatch_id, worktree_id, terminal_handle, - pane_key, process_incarnation, ownership_state, release_state, retained_reason - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const row of rows) { - insert.run( - generateId('wtr'), - row.dispatch_id, - row.dispatch_id, - row.worktree_id, - row.agent_terminal_handle, - row.assignee_pane_key, - row.process_incarnation, - 'external', - 'retained', - 'legacy_ambiguous' - ) - } - } - - // No transaction: composes inside worker-start's authority transaction. - createWorkerTerminalResourceStatement(params: { - dispatchId: string - worktreeId: string | null - terminalHandle: string - paneKey: string | null - processIncarnation: string | null - hostScope?: string | null - ownership: Extract - }): WorkerTerminalResourceRow { - const id = generateId('wtr') - this.db - .prepare( - `INSERT INTO worker_terminal_resources ( - id, origin_dispatch_id, owner_dispatch_id, worktree_id, terminal_handle, - pane_key, process_incarnation, host_scope, ownership_state, release_state, - retained_reason - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'not_requested', ?)` - ) - .run( - id, - params.dispatchId, - params.dispatchId, - params.worktreeId, - params.terminalHandle, - params.paneKey, - params.processIncarnation, - params.hostScope ?? null, - params.ownership, - params.ownership === 'external' ? 'external_terminal' : null - ) - return this.getWorkerTerminalResource(id) as WorkerTerminalResourceRow - } - - getWorkerTerminalResource(id: string): WorkerTerminalResourceRow | undefined { - return this.db.prepare('SELECT * FROM worker_terminal_resources WHERE id = ?').get(id) as - | WorkerTerminalResourceRow - | undefined - } - - getWorkerTerminalResourceByOwner(dispatchId: string): WorkerTerminalResourceRow | undefined { - return this.db - .prepare('SELECT * FROM worker_terminal_resources WHERE owner_dispatch_id = ?') - .get(dispatchId) as WorkerTerminalResourceRow | undefined - } - - getWorkerTerminalResourceFormerlyOwnedBy( - dispatchId: string - ): WorkerTerminalResourceRow | undefined { - return this.db - .prepare( - `SELECT * FROM worker_terminal_resources - WHERE prior_owner_dispatch_ids LIKE ? - ORDER BY updated_at DESC LIMIT 1` - ) - .get(`%"${dispatchId}"%`) as WorkerTerminalResourceRow | undefined - } - - // Reusable exact settled terminal: transfers cleanup ownership to the new Dispatch and fences - // release through the old owner. No transaction: composes inside the authority transaction. - transferWorkerTerminalResourceStatement(params: { - resourceId: string - toDispatchId: string - terminalHandle: string - paneKey: string - processIncarnation: string - hostScope: string | null - }): WorkerTerminalResourceRow { - const resource = this.getWorkerTerminalResource(params.resourceId) - if (!resource) { - throw new OrchestrationError( - 'dispatch_not_found', - `Worker terminal resource ${params.resourceId} was not found.` - ) - } - const priorOwners = JSON.parse(resource.prior_owner_dispatch_ids) as string[] - priorOwners.push(resource.owner_dispatch_id) - this.db - .prepare( - `UPDATE worker_terminal_resources - SET owner_dispatch_id = ?, prior_owner_dispatch_ids = ?, release_state = 'not_requested', - retained_reason = NULL, release_requested_at = NULL, release_completed_at = NULL, - release_error = NULL, terminal_handle = ?, pane_key = ?, process_incarnation = ?, - host_scope = ?, updated_at = datetime('now') - WHERE id = ? AND ownership_state = 'owned'` - ) - .run( - params.toDispatchId, - JSON.stringify(priorOwners), - params.terminalHandle, - params.paneKey, - params.processIncarnation, - params.hostScope, - params.resourceId - ) - return this.getWorkerTerminalResource(params.resourceId) as WorkerTerminalResourceRow - } - - // Finds an owned, settled, exact-match resource for an explicitly reused terminal. - findTransferableWorkerTerminalResource(params: { - terminalHandle: string - paneKey: string | null - processIncarnation: string | null - hostScope: string | null - }): WorkerTerminalResourceRow | undefined { - if (!params.paneKey || !params.processIncarnation) { - return undefined - } - const candidates = this.db - .prepare( - `SELECT r.* FROM worker_terminal_resources r - JOIN worker_dispatches w ON w.dispatch_id = r.owner_dispatch_id - WHERE r.process_incarnation = ? AND r.host_scope IS ? - AND r.ownership_state != 'released'` - ) - .all(params.processIncarnation, params.hostScope) as WorkerTerminalResourceRow[] - const exact = candidates.filter( - (candidate) => - candidate.pane_key && - params.paneKey && - isEquivalentPaneKey(candidate.pane_key, params.paneKey) && - candidate.process_incarnation === params.processIncarnation && - candidate.host_scope === params.hostScope - ) - if ( - exact.some((candidate) => - ['requested', 'releasing', 'unknown'].includes(candidate.release_state) - ) - ) { - throw new OrchestrationError( - 'terminal_release_in_progress', - `Terminal ${params.terminalHandle} has a release in progress; wait for cleanup or use another terminal.` - ) - } - return exact.find( - (candidate) => - candidate.ownership_state === 'owned' && - ['not_requested', 'retained'].includes(candidate.release_state) && - ['succeeded', 'failed', 'stopped', 'abandoned'].includes( - this.getWorkerDispatch(candidate.owner_dispatch_id)?.state ?? '' - ) - ) - } - - workerTerminalResourceHasIdentityConflict(resourceId: string): boolean { - const resource = this.getWorkerTerminalResource(resourceId) - if (!resource?.pane_key || !resource.process_incarnation) { - return true - } - const candidates = this.db - .prepare( - `SELECT * FROM worker_terminal_resources - WHERE process_incarnation = ? AND host_scope IS ? - AND id != ? AND ownership_state != 'released'` - ) - .all( - resource.process_incarnation, - resource.host_scope, - resource.id - ) as WorkerTerminalResourceRow[] - return candidates.some( - (candidate) => - candidate.pane_key && - isEquivalentPaneKey(candidate.pane_key, resource.pane_key as string) && - candidate.process_incarnation === resource.process_incarnation && - candidate.host_scope === resource.host_scope - ) - } - - requestWorkerTerminalRelease(dispatchId: string): - | { disposition: 'requested'; resource: WorkerTerminalResourceRow } - | { disposition: 'already_released'; resource: WorkerTerminalResourceRow } - | { - disposition: 'retained' - resource: WorkerTerminalResourceRow | null - reason: WorkerTerminalRetainedReason - } { - this.db.exec('BEGIN IMMEDIATE') - try { - const worker = this.getWorkerDispatch(dispatchId) - if (!worker) { - throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) - } - if (!WORKER_SETTLED_STATES.includes(worker.state)) { - // Why: release is post-completion cleanup only; recording intent for an unsettled or - // uncertain worker would let recovery close a terminal the coordinator never reviewed. - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${dispatchId} is ${worker.state}; only a settled worker can release. Use worker-stop to cancel an active worker.` - ) - } - const resource = this.getWorkerTerminalResourceByOwner(dispatchId) - if (!resource) { - const transferred = this.getWorkerTerminalResourceFormerlyOwnedBy(dispatchId) - this.db.exec('COMMIT') - return transferred - ? { disposition: 'retained', resource: transferred, reason: 'ownership_transferred' } - : { disposition: 'retained', resource: null, reason: 'no_owned_resource' } - } - if (resource.release_state === 'released' || resource.ownership_state === 'released') { - this.db.exec('COMMIT') - return { disposition: 'already_released', resource } - } - if (worker.state === 'stopped' || worker.state === 'abandoned') { - this.db.exec('COMMIT') - return { disposition: 'retained', resource, reason: 'identity_unproven' } - } - if (resource.ownership_state === 'external') { - this.db.exec('COMMIT') - return { - disposition: 'retained', - resource, - reason: (resource.retained_reason as WorkerTerminalRetainedReason) ?? 'external_terminal' - } - } - if (resource.ownership_state === 'user_owned') { - this.db.exec('COMMIT') - return { disposition: 'retained', resource, reason: 'user_takeover' } - } - if (resource.ownership_state === 'transferred') { - this.db.exec('COMMIT') - return { disposition: 'retained', resource, reason: 'ownership_transferred' } - } - if ( - resource.release_state === 'unknown' || - (resource.release_state === 'retained' && resource.retained_reason === 'user_requested') - ) { - this.db - .prepare('DELETE FROM worker_terminal_archives WHERE dispatch_id = ?') - .run(dispatchId) - } - this.db - .prepare( - `UPDATE worker_terminal_resources - SET release_state = CASE - WHEN release_state = 'releasing' THEN 'releasing' - ELSE 'requested' - END, - retained_reason = NULL, - release_requested_at = COALESCE(release_requested_at, datetime('now')), - release_error = NULL, updated_at = datetime('now') - WHERE id = ? AND release_state IN ('not_requested', 'retained', 'requested', 'releasing', 'unknown')` - ) - .run(resource.id) - this.db.exec('COMMIT') - return { - disposition: 'requested', - resource: this.getWorkerTerminalResource(resource.id) as WorkerTerminalResourceRow - } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - settleDeadWorkerTerminalRelease(params: { - requestingDispatchId: string - resourceId: string - processIncarnation: string - }): - | { disposition: 'released'; resource: WorkerTerminalResourceRow } - | { disposition: 'retained'; resource: WorkerTerminalResourceRow } { - this.db.exec('BEGIN IMMEDIATE') - try { - const resource = this.getWorkerTerminalResource(params.resourceId) - if (!resource) { - throw new OrchestrationError( - 'dispatch_not_found', - `Worker terminal resource ${params.resourceId} was not found.` - ) - } - const priorOwners = parseWorkerTerminalPriorOwnerIds(resource.prior_owner_dispatch_ids) - const requesterRelated = - resource.owner_dispatch_id === params.requestingDispatchId || - priorOwners?.includes(params.requestingDispatchId) === true - const requester = this.getWorkerDispatch(params.requestingDispatchId) - const owner = this.getWorkerDispatch(resource.owner_dispatch_id) - const requesterSettled = Boolean(requester && WORKER_SETTLED_STATES.includes(requester.state)) - const ownerSettled = Boolean(owner && WORKER_SETTLED_STATES.includes(owner.state)) - if ( - !priorOwners || - !requesterRelated || - !requesterSettled || - !ownerSettled || - resource.process_incarnation !== params.processIncarnation || - resource.ownership_state === 'released' || - !['not_requested', 'retained', 'unknown'].includes(resource.release_state) - ) { - this.db.exec('COMMIT') - return { disposition: 'retained', resource } - } - this.db - .prepare( - `UPDATE worker_terminal_resources - SET release_state = 'released', ownership_state = 'released', retained_reason = NULL, - release_requested_at = COALESCE(release_requested_at, datetime('now')), - release_completed_at = datetime('now'), release_error = NULL, - updated_at = datetime('now') - WHERE id = ? AND process_incarnation = ? AND ownership_state != 'released' - AND release_state IN ('not_requested', 'retained', 'unknown')` - ) - .run(params.resourceId, params.processIncarnation) - const released = this.getWorkerTerminalResource( - params.resourceId - ) as WorkerTerminalResourceRow - this.db.exec('COMMIT') - return released.release_state === 'released' - ? { disposition: 'released', resource: released } - : { disposition: 'retained', resource: released } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - storeWorkerTerminalArchive(params: { - dispatchId: string - resourceId: string - kind: 'transcript_pin' | 'terminal_tail' - content: string - }): void { - this.db - .prepare( - `INSERT INTO worker_terminal_archives (dispatch_id, resource_id, kind, content) - VALUES (?, ?, ?, ?) - ON CONFLICT(dispatch_id) DO UPDATE SET - resource_id = excluded.resource_id, kind = excluded.kind, content = excluded.content` - ) - .run(params.dispatchId, params.resourceId, params.kind, params.content) - } - - commitWorkerTerminalArchiveForRelease(params: { - dispatchId: string - resourceId: string - kind?: 'transcript_pin' | 'terminal_tail' - content?: string - archiveSource: 'transcript' | 'terminal' - archiveStatus: Extract - }): WorkerTerminalResourceRow { - this.db.exec('BEGIN IMMEDIATE') - try { - const resource = this.getWorkerTerminalResource(params.resourceId) - if ( - resource?.owner_dispatch_id === params.dispatchId && - resource.ownership_state === 'owned' && - resource.release_state === 'requested' - ) { - if (params.kind && params.content !== undefined) { - this.storeWorkerTerminalArchive({ - dispatchId: params.dispatchId, - resourceId: params.resourceId, - kind: params.kind, - content: params.content - }) - } - const archive = this.getWorkerTerminalArchive(params.dispatchId) - if (!archive || archive.resource_id !== params.resourceId) { - throw new OrchestrationError( - 'archive_failed', - `Output could not be preserved for Dispatch ${params.dispatchId}; the terminal was retained.` - ) - } - this.db - .prepare( - `UPDATE worker_terminal_resources - SET release_state = 'releasing', archive_source = ?, archive_status = ?, - updated_at = datetime('now') - WHERE id = ? AND owner_dispatch_id = ? AND ownership_state = 'owned' - AND release_state = 'requested'` - ) - .run(params.archiveSource, params.archiveStatus, params.resourceId, params.dispatchId) - } - const updated = this.getWorkerTerminalResource(params.resourceId) as WorkerTerminalResourceRow - this.db.exec('COMMIT') - return updated - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - getWorkerTerminalArchive(dispatchId: string): WorkerTerminalArchiveRow | undefined { - return this.db - .prepare('SELECT * FROM worker_terminal_archives WHERE dispatch_id = ?') - .get(dispatchId) as WorkerTerminalArchiveRow | undefined - } - - settleWorkerTerminalRelease(resourceId: string): WorkerTerminalResourceRow { - this.db - .prepare( - `UPDATE worker_terminal_resources - SET release_state = 'released', ownership_state = 'released', - release_completed_at = datetime('now'), release_error = NULL, - updated_at = datetime('now') - WHERE id = ? AND release_state IN ('requested', 'releasing', 'unknown')` - ) - .run(resourceId) - return this.getWorkerTerminalResource(resourceId) as WorkerTerminalResourceRow - } - - markWorkerTerminalReleaseUnknown(resourceId: string, reason: string): WorkerTerminalResourceRow { - this.db - .prepare( - `UPDATE worker_terminal_resources - SET release_state = 'unknown', release_error = ?, updated_at = datetime('now') - WHERE id = ? AND release_state IN ('requested', 'releasing')` - ) - .run(reason, resourceId) - return this.getWorkerTerminalResource(resourceId) as WorkerTerminalResourceRow - } - - revertWorkerTerminalReleaseToRetained( - resourceId: string, - reason: WorkerTerminalRetainedReason - ): WorkerTerminalResourceRow { - this.db - .prepare( - `UPDATE worker_terminal_resources - SET release_state = 'retained', retained_reason = ?, updated_at = datetime('now') - WHERE id = ? AND release_state IN ('requested', 'releasing')` - ) - .run(reason, resourceId) - return this.getWorkerTerminalResource(resourceId) as WorkerTerminalResourceRow - } - - retainWorkerTerminalResource( - dispatchId: string - ): - | { disposition: 'retained'; resource: WorkerTerminalResourceRow } - | { disposition: 'already_released'; resource: WorkerTerminalResourceRow } - | { disposition: 'release_committed'; resource: WorkerTerminalResourceRow } - | { disposition: 'no_owned_resource'; resource: null } { - this.db.exec('BEGIN IMMEDIATE') - try { - const resource = this.getWorkerTerminalResourceByOwner(dispatchId) - if (!resource) { - this.db.exec('COMMIT') - return { disposition: 'no_owned_resource', resource: null } - } - if (resource.release_state === 'released') { - this.db.exec('COMMIT') - return { disposition: 'already_released', resource } - } - this.db - .prepare( - `UPDATE worker_terminal_resources - SET release_state = 'retained', retained_reason = 'user_requested', - updated_at = datetime('now') - WHERE id = ? AND release_state IN ('not_requested', 'retained', 'requested')` - ) - .run(resource.id) - const updated = this.getWorkerTerminalResource(resource.id) as WorkerTerminalResourceRow - if (updated.release_state !== 'retained') { - this.db.exec('COMMIT') - return { disposition: 'release_committed', resource: updated } - } - this.db.prepare('DELETE FROM worker_terminal_archives WHERE dispatch_id = ?').run(dispatchId) - this.db.exec('COMMIT') - return { disposition: 'retained', resource: updated } - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - // Real user input relinquishes orchestration ownership durably; programmatic prompt delivery, - // query auto-replies, resize, and output never reach this path. - markWorkerTerminalUserOwned(paneKey: string): number { - this.db.exec('BEGIN IMMEDIATE') - try { - const exact = this.db - .prepare( - `SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources - WHERE pane_key = ? AND ownership_state = 'owned' - AND release_state IN ('not_requested', 'retained', 'requested')` - ) - .all(paneKey) as { id: string; owner_dispatch_id: string; pane_key: string }[] - const candidates = - exact.length > 0 - ? exact - : ( - this.db - .prepare( - `SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources - WHERE ownership_state = 'owned' - AND release_state IN ('not_requested', 'retained', 'requested') - AND pane_key IS NOT NULL` - ) - .all() as { id: string; owner_dispatch_id: string; pane_key: string }[] - ).filter((candidate) => isEquivalentPaneKey(candidate.pane_key, paneKey)) - const update = this.db.prepare( - `UPDATE worker_terminal_resources - SET ownership_state = 'user_owned', release_state = 'retained', - retained_reason = 'user_takeover', updated_at = datetime('now') - WHERE id = ? AND ownership_state = 'owned' - AND release_state IN ('not_requested', 'retained', 'requested')` - ) - let changed = 0 - for (const candidate of candidates) { - const result = Number(update.run(candidate.id).changes) - if (result > 0) { - this.db - .prepare('DELETE FROM worker_terminal_archives WHERE dispatch_id = ?') - .run(candidate.owner_dispatch_id) - changed += result - } - } - this.db.exec('COMMIT') - return changed - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - listWorkerTerminalReleaseBacklog(): WorkerTerminalResourceRow[] { - return this.db - .prepare( - `SELECT * FROM worker_terminal_resources - WHERE release_state IN ('requested', 'releasing') - ORDER BY release_requested_at ASC` - ) - .all() as WorkerTerminalResourceRow[] - } - - listWorkerTerminalResources(params: { runId?: string } = {}): { - dispatchId: string - taskId: string - runId: string - workerState: WorkerDispatchState - dispatchStatus: DispatchStatus - agentTerminalHandle: string | null - terminalState: WorkerTerminalListState | null - resource: WorkerTerminalResourceRow | null - }[] { - const rows = this.db - .prepare( - `SELECT w.dispatch_id, w.state AS worker_state, w.agent_terminal_handle, - d.task_id, d.run_id, d.status AS dispatch_status - FROM worker_dispatches w - JOIN dispatch_contexts d ON d.id = w.dispatch_id - ${params.runId ? 'WHERE d.run_id = ?' : ''} - ORDER BY w.created_at ASC` - ) - .all(...(params.runId ? [params.runId] : [])) as { - dispatch_id: string - worker_state: WorkerDispatchState - agent_terminal_handle: string | null - task_id: string - run_id: string - dispatch_status: DispatchStatus - }[] - const resources = this.db - .prepare( - `SELECT r.* FROM worker_terminal_resources r - JOIN dispatch_contexts d ON d.id = r.owner_dispatch_id - ${params.runId ? 'WHERE d.run_id = ?' : ''}` - ) - .all(...(params.runId ? [params.runId] : [])) as WorkerTerminalResourceRow[] - const resourceByOwner = new Map( - resources.map((resource) => [resource.owner_dispatch_id, resource]) - ) - return rows.map((row) => { - const resource = resourceByOwner.get(row.dispatch_id) ?? null - return { - dispatchId: row.dispatch_id, - taskId: row.task_id, - runId: row.run_id, - workerState: row.worker_state, - dispatchStatus: row.dispatch_status, - agentTerminalHandle: row.agent_terminal_handle, - terminalState: deriveWorkerTerminalListState({ - workerState: row.worker_state, - agentTerminalHandle: row.agent_terminal_handle, - resource - }), - resource - } - }) - } - - createDispatchContext( - taskId: string, - assigneeHandle: string, - // Why: pane key is the remint-stable identity behind the handle — lets worker_done ownership survive handle reissue. - assigneePaneKey?: string, - launchTokenHash?: string, - processIncarnation?: string - ): DispatchContextRow { - const task = this.getTask(taskId) - if (!task) { - throw new Error(`Task not found: ${taskId}`) - } - if (task.status !== 'ready') { - throw new Error(`Task ${taskId} is ${task.status}; only ready tasks can be dispatched`) - } - - // Why: lock on pane identity too, so a reminted handle can't open a second concurrent dispatch on the same pane. - const existing = this.findActiveDispatchForAssignee(assigneeHandle, assigneePaneKey) - - if (existing) { - throw new Error( - `Terminal ${assigneeHandle} already has an active dispatch (${existing.id} for task ${existing.task_id})` - ) - } - - // Carry forward failure_count so the circuit breaker accumulates across retries for the same task. - const prior = this.db - .prepare('SELECT MAX(failure_count) as max_failures FROM dispatch_contexts WHERE task_id = ?') - .get(taskId) as { max_failures: number | null } | undefined - const priorFailures = prior?.max_failures ?? 0 - - const id = generateId('ctx') - this.db - .prepare( - `INSERT INTO dispatch_contexts ( - id, run_id, task_id, contract_version, launch_token_hash, - assignee_handle, assignee_pane_key, process_incarnation, - status, failure_count, dispatched_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'dispatched', ?, datetime('now'))` - ) - .run( - id, - task.run_id, - taskId, - CURRENT_CONTRACT_VERSION, - launchTokenHash ?? null, - assigneeHandle, - assigneePaneKey ?? null, - processIncarnation ?? null, - priorFailures - ) - this.hasAnyDispatchContextsCache = true - - this.db.prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ?").run(taskId) - - return this.db - .prepare('SELECT * FROM dispatch_contexts WHERE id = ?') - .get(id) as DispatchContextRow - } - - getDispatchContext(taskId: string): DispatchContextRow | undefined { - return this.db - .prepare('SELECT * FROM dispatch_contexts WHERE task_id = ? ORDER BY rowid DESC LIMIT 1') - .get(taskId) as DispatchContextRow | undefined - } - - getDispatchContextById(dispatchId: string): DispatchContextRow | undefined { - return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(dispatchId) as - | DispatchContextRow - | undefined - } - - commitDispatchLaunchTokenHash(dispatchId: string, launchTokenHash: string): DispatchContextRow { - const dispatch = this.getDispatchContextById(dispatchId) - if (!dispatch) { - throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) - } - if (dispatch.contract_version !== CURRENT_CONTRACT_VERSION) { - throw new OrchestrationError( - 'request_mismatch', - `Dispatch ${dispatchId} does not use the current contract.` - ) - } - if (dispatch.launch_token_hash && dispatch.launch_token_hash !== launchTokenHash) { - throw new OrchestrationError( - 'request_mismatch', - `Dispatch ${dispatchId} already has a different launch-token commitment.` - ) - } - this.db - .prepare( - `UPDATE dispatch_contexts - SET launch_token_hash = COALESCE(launch_token_hash, ?) - WHERE id = ?` - ) - .run(launchTokenHash, dispatchId) - return this.getDispatchContextById(dispatchId) as DispatchContextRow - } - - mintDispatchCapability(params: { - dispatchId: string - paneKey: string - processIncarnation: string - }): string { - const dispatch = this.getDispatchContextById(params.dispatchId) - if (!dispatch || (dispatch.status !== 'pending' && dispatch.status !== 'dispatched')) { - throw new OrchestrationError( - 'dispatch_inactive', - `Dispatch ${params.dispatchId} is not active.` - ) - } - const capability = `dcap_${randomBytes(32).toString('base64url')}` - this.db - .prepare( - `UPDATE dispatch_contexts - SET capability_hash = ?, assignee_pane_key = ?, process_incarnation = ?, - capability_revoked_at = NULL - WHERE id = ?` - ) - .run( - hashDispatchCapability(capability), - params.paneKey, - params.processIncarnation, - params.dispatchId - ) - return capability - } - - verifyDispatchCapability(params: { - dispatchId: string - capability: string | undefined - paneKey: string | undefined - processIncarnation: string | undefined - }): { valid: true } | { valid: false; reason: string } { - const dispatch = this.getDispatchContextById(params.dispatchId) - if (!dispatch) { - return { valid: false, reason: `Dispatch ${params.dispatchId} was not found.` } - } - if (!dispatch.capability_hash) { - return { valid: false, reason: `Dispatch ${params.dispatchId} has no lifecycle capability.` } - } - if (dispatch.capability_revoked_at) { - return { valid: false, reason: `Dispatch ${params.dispatchId} capability is revoked.` } - } - if (!params.capability) { - return { valid: false, reason: 'The Dispatch capability is missing.' } - } - const expected = Buffer.from(dispatch.capability_hash, 'hex') - const observed = Buffer.from(hashDispatchCapability(params.capability), 'hex') - if (expected.length !== observed.length || !timingSafeEqual(expected, observed)) { - return { valid: false, reason: 'The Dispatch capability is invalid.' } - } - if ( - !dispatch.assignee_pane_key || - !params.paneKey || - !isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) - ) { - return { valid: false, reason: 'The caller is not the Dispatch pane.' } - } - if ( - !dispatch.process_incarnation || - !params.processIncarnation || - dispatch.process_incarnation !== params.processIncarnation - ) { - return { valid: false, reason: 'The Dispatch process incarnation changed.' } - } - return { valid: true } - } - - revokeDispatchCapability(dispatchId: string): void { - this.db - .prepare( - `UPDATE dispatch_contexts - SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ?` - ) - .run(dispatchId) - } - - getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined { - return this.findActiveDispatchForAssignee(handle) - } - - /** - * Cheap "are there any dispatch rows at all" probe. When false, no terminal - * can have an active or recent-completed dispatch, so orchestration-context - * builders can skip their per-terminal query fan-out entirely. Cached after - * the first probe; createDispatchContext marks it true, resets clear it. - */ - hasAnyDispatchContexts(): boolean { - if (this.hasAnyDispatchContextsCache === undefined) { - const row = this.db.prepare('SELECT 1 FROM dispatch_contexts LIMIT 1').get() - this.hasAnyDispatchContextsCache = row !== undefined - } - return this.hasAnyDispatchContextsCache - } - - getActiveDispatchForIdentity(handle: string, paneKey?: string): DispatchContextRow | undefined { - return this.findActiveDispatchForAssignee(handle, paneKey) - } - - private findActiveDispatchForAssignee( - assigneeHandle: string, - assigneePaneKey?: string - ): DispatchContextRow | undefined { - const byHandle = this.db - .prepare( - "SELECT * FROM dispatch_contexts WHERE assignee_handle = ? AND status IN ('pending', 'dispatched') LIMIT 1" - ) - .get(assigneeHandle) as DispatchContextRow | undefined - if (byHandle) { - return byHandle - } - - if (!assigneePaneKey) { - return undefined - } - - const exactPane = this.db - .prepare( - `SELECT * FROM dispatch_contexts - WHERE assignee_pane_key = ? AND status IN ('pending', 'dispatched') - ORDER BY rowid DESC LIMIT 1` - ) - .get(assigneePaneKey) as DispatchContextRow | undefined - if (exactPane) { - return exactPane - } - if (!parsePaneKey(assigneePaneKey)) { - return undefined - } - return this.db - .prepare( - `SELECT * FROM dispatch_contexts - WHERE assignee_pane_key IS NOT NULL - AND status IN ('pending', 'dispatched') AND instr(assignee_pane_key, ':') > 1 - AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ? - ORDER BY rowid DESC LIMIT 1` - ) - .get(paneKeyMatchSuffix(assigneePaneKey)) as DispatchContextRow | undefined - } - - getLatestDispatchForTerminal(handle: string): DispatchContextRow | undefined { - return this.db - .prepare( - 'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1' - ) - .get(handle) as DispatchContextRow | undefined - } - - completeDispatch(ctxId: string): void { - this.db - .prepare( - "UPDATE dispatch_contexts SET status = 'completed', completed_at = datetime('now'), capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) WHERE id = ?" - ) - .run(ctxId) - } - - completeActiveDispatchForTask(taskId: string): void { - const active = this.db - .prepare( - "SELECT * FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" - ) - .get(taskId) as DispatchContextRow | undefined - if (active) { - this.completeDispatch(active.id) - } - } - - settleWorkerReport(params: { - taskId: string - dispatchId: string - outcome: WorkerReportOutcome - result: string - }): WorkerReportSettlement { - this.db.exec('BEGIN IMMEDIATE') - try { - const settlement = this.settleWorkerReportInTransaction(params) - this.db.exec('COMMIT') - return settlement - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - private settleWorkerReportInTransaction(params: { - taskId: string - dispatchId: string - outcome: WorkerReportOutcome - result: string - }): WorkerReportSettlement { - const task = this.getTask(params.taskId) - if (!task) { - return { action: 'rejected', code: 'unknown_task', reason: `Unknown task ${params.taskId}.` } - } - const dispatch = this.getDispatchContextById(params.dispatchId) - if (!dispatch) { - return { - action: 'rejected', - code: 'unknown_dispatch', - reason: `Unknown dispatch ${params.dispatchId}.` - } - } - if (dispatch.task_id !== params.taskId) { - return { - action: 'rejected', - code: 'task_dispatch_mismatch', - reason: `Dispatch ${params.dispatchId} belongs to task ${dispatch.task_id}, not ${params.taskId}.` - } - } - - const expectedDispatchStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' - const expectedTaskStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' - if (dispatch.status === expectedDispatchStatus && task.status === expectedTaskStatus) { - return { action: 'settled', outcome: params.outcome, duplicate: true } - } - if (dispatch.status !== 'dispatched' || task.status !== 'dispatched') { - return { - action: 'rejected', - code: 'inactive_dispatch', - reason: `inactive dispatch ${params.dispatchId}: it or task ${params.taskId} is already settled.` - } - } - const latest = this.getDispatchContext(params.taskId) - if (latest?.id !== params.dispatchId) { - return { - action: 'rejected', - code: 'stale_dispatch', - reason: `Dispatch ${params.dispatchId} is not the current dispatch for task ${params.taskId}.` - } - } - - this.db.exec('SAVEPOINT settle_worker_report') - const dispatchUpdate = this.db - .prepare( - `UPDATE dispatch_contexts - SET status = ?, completed_at = datetime('now'), - last_failure = CASE WHEN ? = 'failed' THEN ? ELSE last_failure END, - capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ? AND status = 'dispatched'` - ) - .run(expectedDispatchStatus, expectedDispatchStatus, params.result, params.dispatchId) - const taskUpdate = this.db - .prepare( - `UPDATE tasks - SET status = ?, result = ?, completed_at = datetime('now') - WHERE id = ? AND status = 'dispatched'` - ) - .run(expectedTaskStatus, params.result, params.taskId) - if (dispatchUpdate.changes !== 1 || taskUpdate.changes !== 1) { - this.db.exec('ROLLBACK TO settle_worker_report') - this.db.exec('RELEASE settle_worker_report') - return { - action: 'rejected', - code: 'inactive_dispatch', - reason: `Dispatch ${params.dispatchId} changed while its worker report was settling.` - } - } - this.db - .prepare( - `UPDATE worker_dispatches - SET state = ?, stage = 'settled', updated_at = datetime('now') - WHERE dispatch_id = ? AND state = 'ready'` - ) - .run(params.outcome === 'succeeded' ? 'succeeded' : 'failed', params.dispatchId) - this.closeQuestionsForDispatch(params.dispatchId) - if (params.outcome === 'succeeded') { - this.promoteReadyTasks(params.taskId) - } - this.db.exec('RELEASE settle_worker_report') - return { action: 'settled', outcome: params.outcome, duplicate: false } - } - - failActiveDispatchForTask(taskId: string, error: string): DispatchContextRow | undefined { - const active = this.db - .prepare( - "SELECT * FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" - ) - .get(taskId) as DispatchContextRow | undefined - return active ? this.failDispatch(active.id, error) : undefined - } - - // Why: only bump status='dispatched' — a zombie heartbeat from a finished dispatch would mask a hung retry from the stale detector (§5.3.4). - recordHeartbeat(dispatchId: string, at: string): void { - this.db - .prepare( - "UPDATE dispatch_contexts SET last_heartbeat_at = ? WHERE id = ? AND status = 'dispatched'" - ) - .run(at, dispatchId) - } - - // Why: dispatched_at grace skips workers still within their first heartbeat interval; julianday() vs raw-TEXT compare avoids misflagging space-format timestamps as stale (#8452). - getStaleDispatches(thresholdIso: string): DispatchContextRow[] { - return this.db - .prepare( - `SELECT * FROM dispatch_contexts - WHERE status = 'dispatched' - AND dispatched_at IS NOT NULL - AND julianday(dispatched_at) < julianday(?) - AND (last_heartbeat_at IS NULL OR julianday(last_heartbeat_at) < julianday(?))` - ) - .all(thresholdIso, thresholdIso) as DispatchContextRow[] - } - - failDispatch(ctxId: string, error: string): DispatchContextRow | undefined { - const ctx = this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(ctxId) as - | DispatchContextRow - | undefined - if (!ctx) { - return undefined - } - - const newFailureCount = ctx.failure_count + 1 - const newStatus: DispatchStatus = newFailureCount >= 3 ? 'circuit_broken' : 'failed' - - this.db - .prepare( - `UPDATE dispatch_contexts - SET status = ?, failure_count = ?, last_failure = ?, - completed_at = COALESCE(completed_at, datetime('now')), - capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) - WHERE id = ?` - ) - .run(newStatus, newFailureCount, error, ctxId) - - // Why: back to 'ready' not 'pending' — 'pending' would strand it since promoteReadyTasks only runs when a dep completes. - const taskStatus: TaskStatus = newStatus === 'circuit_broken' ? 'failed' : 'ready' - this.db.prepare('UPDATE tasks SET status = ? WHERE id = ?').run(taskStatus, ctx.task_id) - - return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(ctxId) as - | DispatchContextRow - | undefined - } - - // ── Decision Gates ── - - createGate(gate: { taskId: string; question: string; options?: string[] }): DecisionGateRow { - const id = generateId('gate') - const optionsJson = JSON.stringify(gate.options ?? []) - this.db - .prepare( - 'INSERT INTO decision_gates (id, run_id, task_id, question, options) VALUES (?, ?, ?, ?, ?)' - ) - .run( - id, - this.getTask(gate.taskId)?.run_id ?? LEGACY_RUN_ID, - gate.taskId, - gate.question, - optionsJson - ) - - this.completeActiveDispatchForTask(gate.taskId) - this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(gate.taskId) - - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(id) as DecisionGateRow - } - - resolveGate(gateId: string, resolution: string): DecisionGateRow | undefined { - const gate = this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as - | DecisionGateRow - | undefined - if (!gate) { - return undefined - } - - this.db - .prepare( - "UPDATE decision_gates SET status = 'resolved', resolution = ?, resolved_at = datetime('now') WHERE id = ?" - ) - .run(resolution, gateId) - - // Why: set to 'ready' (not the previous status) so the coordinator re-dispatches the worker with the resolution context. - this.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(gate.task_id) - - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as - | DecisionGateRow - | undefined - } - - timeoutGate(gateId: string): DecisionGateRow | undefined { - this.db - .prepare( - "UPDATE decision_gates SET status = 'timeout', resolved_at = datetime('now') WHERE id = ?" - ) - .run(gateId) - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as - | DecisionGateRow - | undefined - } - - listGates(filter?: { taskId?: string; status?: GateStatus }): DecisionGateRow[] { - if (filter?.taskId && filter?.status) { - return this.db - .prepare( - 'SELECT * FROM decision_gates WHERE task_id = ? AND status = ? ORDER BY created_at' - ) - .all(filter.taskId, filter.status) as DecisionGateRow[] - } - if (filter?.taskId) { - return this.db - .prepare('SELECT * FROM decision_gates WHERE task_id = ? ORDER BY created_at') - .all(filter.taskId) as DecisionGateRow[] - } - if (filter?.status) { - return this.db - .prepare('SELECT * FROM decision_gates WHERE status = ? ORDER BY created_at') - .all(filter.status) as DecisionGateRow[] - } - return this.db - .prepare('SELECT * FROM decision_gates ORDER BY created_at') - .all() as DecisionGateRow[] - } - - getGate(id: string): DecisionGateRow | undefined { - return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(id) as - | DecisionGateRow - | undefined - } - - // ── Coordinator Runs ── - - createCoordinatorRun(run: { - spec: string - coordinatorHandle: string - pollIntervalMs?: number - }): CoordinatorRun { - const id = generateId('run') - this.db - .prepare( - "INSERT INTO coordinator_runs (id, spec, status, coordinator_handle, poll_interval_ms) VALUES (?, ?, 'running', ?, ?)" - ) - .run(id, run.spec, run.coordinatorHandle, run.pollIntervalMs ?? 2000) - return this.db.prepare('SELECT * FROM coordinator_runs WHERE id = ?').get(id) as CoordinatorRun - } - - getCoordinatorRun(id: string): CoordinatorRun | undefined { - return this.db.prepare('SELECT * FROM coordinator_runs WHERE id = ?').get(id) as - | CoordinatorRun - | undefined - } - - updateCoordinatorRun(id: string, status: CoordinatorStatus): CoordinatorRun | undefined { - const completedAt = - status === 'completed' || status === 'failed' ? new Date().toISOString() : null - this.db - .prepare( - 'UPDATE coordinator_runs SET status = ?, completed_at = COALESCE(?, completed_at) WHERE id = ?' - ) - .run(status, completedAt, id) - return this.getCoordinatorRun(id) - } - - getActiveCoordinatorRun(): CoordinatorRun | undefined { - return this.db - .prepare( - "SELECT * FROM coordinator_runs WHERE status = 'running' ORDER BY created_at DESC LIMIT 1" - ) - .get() as CoordinatorRun | undefined - } - - // ── Queries for Coordinator ── - - getIdleTerminals(excludeHandles: string[] = []): string[] { - const active = this.db - .prepare( - "SELECT DISTINCT assignee_handle FROM dispatch_contexts WHERE status IN ('pending', 'dispatched')" - ) - .all() as { assignee_handle: string }[] - const busyHandles = new Set(active.map((r) => r.assignee_handle)) - for (const h of excludeHandles) { - busyHandles.add(h) - } - // Return handles from message history that aren't busy - const allHandles = this.db - .prepare( - 'SELECT DISTINCT to_handle FROM messages UNION SELECT DISTINCT from_handle FROM messages' - ) - .all() as { to_handle: string }[] - return [...new Set(allHandles.map((r) => r.to_handle))].filter((h) => !busyHandles.has(h)) - } - - // ── Lifecycle ── - - private runResetTransaction(statements: string): void { - this.db.exec('BEGIN IMMEDIATE') - try { - this.db.exec(statements) - this.db.exec('COMMIT') - } catch (error) { - this.db.exec('ROLLBACK') - throw error - } - } - - resetAll(): void { - // Why: retain mutation receipts so a lost reset response cannot replay as a new mutation. - this.runResetTransaction(` - DELETE FROM coordinator_runs; - DELETE FROM decision_gates; - DELETE FROM remote_questions; - DELETE FROM question_threads; - DELETE FROM deliveries; - DELETE FROM legacy_mail_receipts; - DELETE FROM legacy_operation_receipts; - DELETE FROM legacy_compatibility_principals; - DELETE FROM legacy_adoptions; - DELETE FROM federation_relay_items; - DELETE FROM remote_dispatch_attachments; - DELETE FROM federated_dispatches; - DELETE FROM worker_terminal_archives; - DELETE FROM worker_terminal_resources; - DELETE FROM worker_dispatches; - DELETE FROM dispatch_contexts; - DELETE FROM tasks; - DELETE FROM messages; - DELETE FROM run_coordinator_handles; - DELETE FROM runs; - INSERT INTO runs (id, objective, home_database, consumer_generation, legacy) - VALUES ('${LEGACY_RUN_ID}', 'Legacy orchestration state (inspect only)', 'this_database', 0, 1); - `) - this.hasAnyDispatchContextsCache = undefined - } - - resetTasks(): void { - this.runResetTransaction(` - DELETE FROM coordinator_runs; - DELETE FROM decision_gates; - DELETE FROM remote_questions; - DELETE FROM question_threads; - DELETE FROM legacy_mail_receipts; - DELETE FROM legacy_operation_receipts; - DELETE FROM legacy_compatibility_principals; - DELETE FROM legacy_adoptions; - DELETE FROM federation_relay_items; - DELETE FROM remote_dispatch_attachments; - DELETE FROM federated_dispatches; - DELETE FROM worker_terminal_archives; - DELETE FROM worker_terminal_resources; - DELETE FROM worker_dispatches; - DELETE FROM dispatch_contexts; - DELETE FROM tasks; - `) - this.hasAnyDispatchContextsCache = undefined - } - - resetMessages(): void { - // Why: relay rows carry contiguous cross-server cursors, not just inbox history. - this.runResetTransaction(` - DELETE FROM legacy_mail_receipts; - DELETE FROM question_threads; - DELETE FROM deliveries; - DELETE FROM messages; - `) - } - - close(): void { - this.db.close() - } -} - -function parseFederatedWorkerReportOutcome(payload: string): WorkerReportOutcome | undefined { - try { - const message = JSON.parse(payload) as { payload?: unknown } - if (typeof message.payload !== 'string') { - return undefined - } - const report = JSON.parse(message.payload) as { outcome?: unknown } - return report.outcome === 'succeeded' || report.outcome === 'failed' - ? report.outcome - : undefined - } catch { - return undefined - } -} +} from './types' diff --git a/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts b/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts new file mode 100644 index 00000000000..22bf627a673 --- /dev/null +++ b/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts @@ -0,0 +1,121 @@ +import { attachCoordinatorRunStore } from './coordinator-runs/coordinator-run-store' +import { attachDecisionGateStore } from './decision-gates/decision-gate-store' +import { attachDispatchCapability } from './dispatch-context/dispatch-capability' +import { attachDispatchCompletion } from './dispatch-context/dispatch-completion' +import { attachDispatchContextStore } from './dispatch-context/dispatch-context-store' +import { attachDispatchLookup } from './dispatch-context/dispatch-lookup' +import { attachWorkerReportSettlement } from './dispatch-context/worker-report-settlement' +import { attachFederatedDispatchStore } from './federation/federated-dispatch-store' +import { attachFederationRelayAck } from './federation/federation-relay-ack' +import { attachFederationRelayEnqueue } from './federation/federation-relay-enqueue' +import { attachFederationRelayImport } from './federation/federation-relay-import' +import { attachFederationRelayItem } from './federation/federation-relay-item' +import { attachRemoteDispatchAttachmentAuthority } from './federation/remote-dispatch-attachment-authority' +import { attachRemoteDispatchAttachmentCreate } from './federation/remote-dispatch-attachment-create' +import { attachRemoteDispatchAttachmentStop } from './federation/remote-dispatch-attachment-stop' +import { attachRemoteQuestionStore } from './federation/remote-question-store' +import { attachLegacyAskOperation } from './legacy/legacy-ask-operation' +import { attachLegacyCompatibilityCandidates } from './legacy/legacy-compatibility-candidates' +import { attachLegacyCompatibilityPrincipals } from './legacy/legacy-compatibility-principals' +import { attachLegacyLifecycleOperation } from './legacy/legacy-lifecycle-operation' +import { attachLegacyMailAcknowledge } from './legacy/legacy-mail-acknowledge' +import { attachLegacyQuestionAcknowledge } from './legacy/legacy-question-acknowledge' +import { attachLegacyQuestionLookup } from './legacy/legacy-question-lookup' +import { attachLegacyRecoveryCohort } from './legacy/legacy-recovery-cohort' +import { attachLegacyReplyOperation } from './legacy/legacy-reply-operation' +import { attachLegacyWorkerCompletion } from './legacy/legacy-worker-completion' +import { attachDirectMailboxRouting } from './messages/direct-mailbox-routing' +import { attachForeignDirectMailboxRouting } from './messages/foreign-direct-mailbox-routing' +import { attachMessageInbox } from './messages/message-inbox' +import { attachMessageInsert } from './messages/message-insert' +import { attachMutationReceiptStore } from './mutation-receipts/mutation-receipt-store' +import { attachQuestionThreads } from './questions/question-threads' +import { attachOrchestrationReset } from './reset/orchestration-reset' +import { attachRunBinding } from './runs/run-binding' +import { attachRunCoordinatorMailRouting } from './runs/run-coordinator-mail-routing' +import { attachRunCreate } from './runs/run-create' +import { attachRunDelivery } from './runs/run-delivery' +import { attachRunLookup } from './runs/run-lookup' +import { attachLegacyCoordinatorMailTakeover } from './runs/legacy-coordinator-mail-takeover' +import { attachAdoptLegacyRun } from './schema/adopt-legacy-run' +import { attachBackfillLegacyQuestionThreads } from './schema/backfill-legacy-question-threads' +import { attachCreateTables } from './schema/create-tables' +import { attachMigrateLegacyContractStorage } from './schema/migrate-legacy-contract-storage' +import { attachSchemaMigrate } from './schema/migrate' +import { attachSchemaColumnProbes } from './schema/schema-column-probes' +import { attachTaskStore } from './tasks/task-store' +import { attachFederatedWorkerStartReconcile } from './worker-dispatch/federated-worker-start-reconcile' +import { attachWorkerDispatchAbandon } from './worker-dispatch/worker-dispatch-abandon' +import { attachWorkerDispatchAuthority } from './worker-dispatch/worker-dispatch-authority' +import { attachWorkerDispatchOutcome } from './worker-dispatch/worker-dispatch-outcome' +import { attachWorkerDispatchStage } from './worker-dispatch/worker-dispatch-stage' +import { attachWorkerDispatchStart } from './worker-dispatch/worker-dispatch-start' +import { attachWorkerDispatchStop } from './worker-dispatch/worker-dispatch-stop' +import { attachWorkerTerminalRecovery } from './worker-dispatch/worker-terminal-recovery' +import { attachWorkerTerminalArchive } from './worker-terminal/worker-terminal-archive' +import { attachWorkerTerminalListing } from './worker-terminal/worker-terminal-listing' +import { attachWorkerTerminalRelease } from './worker-terminal/worker-terminal-release' +import { attachWorkerTerminalResourceStore } from './worker-terminal/worker-terminal-resource-store' +import { attachWorkerTerminalTransfer } from './worker-terminal/worker-terminal-transfer' + +export function attachOrchestrationDbMethods(ctor: { prototype: object }): void { + attachCreateTables(ctor) + attachSchemaMigrate(ctor) + attachSchemaColumnProbes(ctor) + attachMigrateLegacyContractStorage(ctor) + attachBackfillLegacyQuestionThreads(ctor) + attachAdoptLegacyRun(ctor) + attachMutationReceiptStore(ctor) + attachLegacyCompatibilityPrincipals(ctor) + attachLegacyCompatibilityCandidates(ctor) + attachLegacyWorkerCompletion(ctor) + attachLegacyRecoveryCohort(ctor) + attachLegacyMailAcknowledge(ctor) + attachLegacyQuestionAcknowledge(ctor) + attachLegacyLifecycleOperation(ctor) + attachLegacyAskOperation(ctor) + attachLegacyQuestionLookup(ctor) + attachLegacyReplyOperation(ctor) + attachRunCreate(ctor) + attachRunBinding(ctor) + attachRunLookup(ctor) + attachRunCoordinatorMailRouting(ctor) + attachLegacyCoordinatorMailTakeover(ctor) + attachRunDelivery(ctor) + attachMessageInsert(ctor) + attachMessageInbox(ctor) + attachDirectMailboxRouting(ctor) + attachForeignDirectMailboxRouting(ctor) + attachQuestionThreads(ctor) + attachTaskStore(ctor) + attachWorkerDispatchStart(ctor) + attachWorkerDispatchStage(ctor) + attachWorkerDispatchAuthority(ctor) + attachWorkerDispatchOutcome(ctor) + attachFederatedWorkerStartReconcile(ctor) + attachWorkerTerminalRecovery(ctor) + attachWorkerDispatchStop(ctor) + attachWorkerDispatchAbandon(ctor) + attachFederatedDispatchStore(ctor) + attachRemoteDispatchAttachmentCreate(ctor) + attachRemoteDispatchAttachmentAuthority(ctor) + attachRemoteDispatchAttachmentStop(ctor) + attachFederationRelayEnqueue(ctor) + attachFederationRelayAck(ctor) + attachFederationRelayImport(ctor) + attachRemoteQuestionStore(ctor) + attachFederationRelayItem(ctor) + attachWorkerTerminalResourceStore(ctor) + attachWorkerTerminalTransfer(ctor) + attachWorkerTerminalRelease(ctor) + attachWorkerTerminalArchive(ctor) + attachWorkerTerminalListing(ctor) + attachDispatchContextStore(ctor) + attachDispatchCapability(ctor) + attachDispatchLookup(ctor) + attachDispatchCompletion(ctor) + attachWorkerReportSettlement(ctor) + attachDecisionGateStore(ctor) + attachCoordinatorRunStore(ctor) + attachOrchestrationReset(ctor) +} diff --git a/src/main/runtime/orchestration/db/contract-constants.ts b/src/main/runtime/orchestration/db/contract-constants.ts new file mode 100644 index 00000000000..e64082cde4f --- /dev/null +++ b/src/main/runtime/orchestration/db/contract-constants.ts @@ -0,0 +1,10 @@ +import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../../shared/orchestration-rpc-contract' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' + +export const LEGACY_RUN_ID = ORCHESTRATION_LEGACY_RUN_ID + +export const LEGACY_CONTRACT_VERSION = 0 +export const CURRENT_CONTRACT_VERSION = ORCHESTRATION_CONTRACT_VERSION + +// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity. +export const SCHEMA_VERSION = 28 diff --git a/src/main/runtime/orchestration/db/coordinator-runs/coordinator-run-store.ts b/src/main/runtime/orchestration/db/coordinator-runs/coordinator-run-store.ts new file mode 100644 index 00000000000..d4c2d984389 --- /dev/null +++ b/src/main/runtime/orchestration/db/coordinator-runs/coordinator-run-store.ts @@ -0,0 +1,91 @@ +import type { CoordinatorStatus, CoordinatorRun } from '../../types' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Coordinator Runs ── + +export function createCoordinatorRun( + this: OrchestrationDb, + run: { + spec: string + coordinatorHandle: string + pollIntervalMs?: number + } +): CoordinatorRun { + const id = generateId('run') + this.db + .prepare( + "INSERT INTO coordinator_runs (id, spec, status, coordinator_handle, poll_interval_ms) VALUES (?, ?, 'running', ?, ?)" + ) + .run(id, run.spec, run.coordinatorHandle, run.pollIntervalMs ?? 2000) + return this.db.prepare('SELECT * FROM coordinator_runs WHERE id = ?').get(id) as CoordinatorRun +} + +export function getCoordinatorRun(this: OrchestrationDb, id: string): CoordinatorRun | undefined { + return this.db.prepare('SELECT * FROM coordinator_runs WHERE id = ?').get(id) as + | CoordinatorRun + | undefined +} + +export function updateCoordinatorRun( + this: OrchestrationDb, + id: string, + status: CoordinatorStatus +): CoordinatorRun | undefined { + const completedAt = + status === 'completed' || status === 'failed' ? new Date().toISOString() : null + this.db + .prepare( + 'UPDATE coordinator_runs SET status = ?, completed_at = COALESCE(?, completed_at) WHERE id = ?' + ) + .run(status, completedAt, id) + return this.getCoordinatorRun(id) +} + +export function getActiveCoordinatorRun(this: OrchestrationDb): CoordinatorRun | undefined { + return this.db + .prepare( + "SELECT * FROM coordinator_runs WHERE status = 'running' ORDER BY created_at DESC LIMIT 1" + ) + .get() as CoordinatorRun | undefined +} + +// ── Queries for Coordinator ── + +export function getIdleTerminals(this: OrchestrationDb, excludeHandles: string[] = []): string[] { + const active = this.db + .prepare( + "SELECT DISTINCT assignee_handle FROM dispatch_contexts WHERE status IN ('pending', 'dispatched')" + ) + .all() as { assignee_handle: string }[] + const busyHandles = new Set(active.map((r) => r.assignee_handle)) + for (const h of excludeHandles) { + busyHandles.add(h) + } + // Return handles from message history that aren't busy + const allHandles = this.db + .prepare( + // Why: alias the UNION column — otherwise the row shape depends on the first branch's column name. + 'SELECT DISTINCT to_handle AS handle FROM messages UNION SELECT DISTINCT from_handle FROM messages' + ) + .all() as { handle: string }[] + return [...new Set(allHandles.map((r) => r.handle))].filter((h) => !busyHandles.has(h)) +} + +export type CoordinatorRunStoreMethods = { + createCoordinatorRun: typeof createCoordinatorRun + getCoordinatorRun: typeof getCoordinatorRun + updateCoordinatorRun: typeof updateCoordinatorRun + getActiveCoordinatorRun: typeof getActiveCoordinatorRun + getIdleTerminals: typeof getIdleTerminals +} + +export function attachCoordinatorRunStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createCoordinatorRun, + getCoordinatorRun, + updateCoordinatorRun, + getActiveCoordinatorRun, + getIdleTerminals + }) +} diff --git a/src/main/runtime/orchestration/db/database-file-permissions.ts b/src/main/runtime/orchestration/db/database-file-permissions.ts new file mode 100644 index 00000000000..1bbd640ba77 --- /dev/null +++ b/src/main/runtime/orchestration/db/database-file-permissions.ts @@ -0,0 +1,17 @@ +import { chmodSync, existsSync } from 'node:fs' + +export function hardenOrchestrationDatabaseFiles(dbPath: (string & {}) | ':memory:'): void { + if (dbPath === ':memory:' || process.platform === 'win32') { + // Why: Windows protects these files through Orca's current-user-only userData DACL; POSIX mode bits are inert there. + return + } + for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) { + try { + if (existsSync(path)) { + chmodSync(path, 0o600) + } + } catch { + // Why: best-effort — a mount that rejects chmod (SSHFS, some network shares) must not fail DB startup. + } + } +} diff --git a/src/main/runtime/orchestration/db/decision-gates/decision-gate-store.ts b/src/main/runtime/orchestration/db/decision-gates/decision-gate-store.ts new file mode 100644 index 00000000000..7a7851bbfe5 --- /dev/null +++ b/src/main/runtime/orchestration/db/decision-gates/decision-gate-store.ts @@ -0,0 +1,116 @@ +import type { GateStatus, DecisionGateRow } from '../../types' +import { LEGACY_RUN_ID } from '../contract-constants' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Decision Gates ── + +export function createGate( + this: OrchestrationDb, + gate: { taskId: string; question: string; options?: string[] } +): DecisionGateRow { + const id = generateId('gate') + const optionsJson = JSON.stringify(gate.options ?? []) + this.db + .prepare( + 'INSERT INTO decision_gates (id, run_id, task_id, question, options) VALUES (?, ?, ?, ?, ?)' + ) + .run( + id, + this.getTask(gate.taskId)?.run_id ?? LEGACY_RUN_ID, + gate.taskId, + gate.question, + optionsJson + ) + + this.completeActiveDispatchForTask(gate.taskId) + this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(gate.taskId) + + return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(id) as DecisionGateRow +} + +export function resolveGate( + this: OrchestrationDb, + gateId: string, + resolution: string +): DecisionGateRow | undefined { + const gate = this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as + | DecisionGateRow + | undefined + if (!gate) { + return undefined + } + + this.db + .prepare( + "UPDATE decision_gates SET status = 'resolved', resolution = ?, resolved_at = datetime('now') WHERE id = ?" + ) + .run(resolution, gateId) + + // Why: set to 'ready' (not the previous status) so the coordinator re-dispatches the worker with the resolution context. + this.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(gate.task_id) + + return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as + | DecisionGateRow + | undefined +} + +export function timeoutGate(this: OrchestrationDb, gateId: string): DecisionGateRow | undefined { + this.db + .prepare( + // Why: without the status guard a late timeout overwrites a gate the user already resolved. + "UPDATE decision_gates SET status = 'timeout', resolved_at = datetime('now') WHERE id = ? AND status = 'pending'" + ) + .run(gateId) + return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(gateId) as + | DecisionGateRow + | undefined +} + +export function listGates( + this: OrchestrationDb, + filter?: { taskId?: string; status?: GateStatus } +): DecisionGateRow[] { + if (filter?.taskId && filter?.status) { + return this.db + .prepare('SELECT * FROM decision_gates WHERE task_id = ? AND status = ? ORDER BY created_at') + .all(filter.taskId, filter.status) as DecisionGateRow[] + } + if (filter?.taskId) { + return this.db + .prepare('SELECT * FROM decision_gates WHERE task_id = ? ORDER BY created_at') + .all(filter.taskId) as DecisionGateRow[] + } + if (filter?.status) { + return this.db + .prepare('SELECT * FROM decision_gates WHERE status = ? ORDER BY created_at') + .all(filter.status) as DecisionGateRow[] + } + return this.db + .prepare('SELECT * FROM decision_gates ORDER BY created_at') + .all() as DecisionGateRow[] +} + +export function getGate(this: OrchestrationDb, id: string): DecisionGateRow | undefined { + return this.db.prepare('SELECT * FROM decision_gates WHERE id = ?').get(id) as + | DecisionGateRow + | undefined +} + +export type DecisionGateStoreMethods = { + createGate: typeof createGate + resolveGate: typeof resolveGate + timeoutGate: typeof timeoutGate + listGates: typeof listGates + getGate: typeof getGate +} + +export function attachDecisionGateStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createGate, + resolveGate, + timeoutGate, + listGates, + getGate + }) +} diff --git a/src/main/runtime/orchestration/db/dispatch-capability-hash.ts b/src/main/runtime/orchestration/db/dispatch-capability-hash.ts new file mode 100644 index 00000000000..c4e6631920c --- /dev/null +++ b/src/main/runtime/orchestration/db/dispatch-capability-hash.ts @@ -0,0 +1,5 @@ +import { createHash } from 'node:crypto' + +export function hashDispatchCapability(capability: string): string { + return createHash('sha256').update(capability).digest('hex') +} diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-capability.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-capability.ts new file mode 100644 index 00000000000..e75371b369c --- /dev/null +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-capability.ts @@ -0,0 +1,105 @@ +import { randomBytes, timingSafeEqual } from 'node:crypto' +import { OrchestrationError } from '../../orchestration-error' +import { hashDispatchCapability } from '../dispatch-capability-hash' +import { isEquivalentPaneKey } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function mintDispatchCapability( + this: OrchestrationDb, + params: { + dispatchId: string + paneKey: string + processIncarnation: string + } +): string { + const dispatch = this.getDispatchContextById(params.dispatchId) + if (!dispatch || (dispatch.status !== 'pending' && dispatch.status !== 'dispatched')) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not active.` + ) + } + const capability = `dcap_${randomBytes(32).toString('base64url')}` + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_hash = ?, assignee_pane_key = ?, process_incarnation = ?, + capability_revoked_at = NULL + WHERE id = ?` + ) + .run( + hashDispatchCapability(capability), + params.paneKey, + params.processIncarnation, + params.dispatchId + ) + return capability +} + +export function verifyDispatchCapability( + this: OrchestrationDb, + params: { + dispatchId: string + capability: string | undefined + paneKey: string | undefined + processIncarnation: string | undefined + } +): { valid: true } | { valid: false; reason: string } { + const dispatch = this.getDispatchContextById(params.dispatchId) + if (!dispatch) { + return { valid: false, reason: `Dispatch ${params.dispatchId} was not found.` } + } + if (!dispatch.capability_hash) { + return { valid: false, reason: `Dispatch ${params.dispatchId} has no lifecycle capability.` } + } + if (dispatch.capability_revoked_at) { + return { valid: false, reason: `Dispatch ${params.dispatchId} capability is revoked.` } + } + if (!params.capability) { + return { valid: false, reason: 'The Dispatch capability is missing.' } + } + const expected = Buffer.from(dispatch.capability_hash, 'hex') + const observed = Buffer.from(hashDispatchCapability(params.capability), 'hex') + if (expected.length !== observed.length || !timingSafeEqual(expected, observed)) { + return { valid: false, reason: 'The Dispatch capability is invalid.' } + } + if ( + !dispatch.assignee_pane_key || + !params.paneKey || + !isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) + ) { + return { valid: false, reason: 'The caller is not the Dispatch pane.' } + } + if ( + !dispatch.process_incarnation || + !params.processIncarnation || + dispatch.process_incarnation !== params.processIncarnation + ) { + return { valid: false, reason: 'The Dispatch process incarnation changed.' } + } + return { valid: true } +} + +export function revokeDispatchCapability(this: OrchestrationDb, dispatchId: string): void { + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) +} + +export type DispatchCapabilityMethods = { + mintDispatchCapability: typeof mintDispatchCapability + verifyDispatchCapability: typeof verifyDispatchCapability + revokeDispatchCapability: typeof revokeDispatchCapability +} + +export function attachDispatchCapability(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + mintDispatchCapability, + verifyDispatchCapability, + revokeDispatchCapability + }) +} diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-circuit-breaker.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-circuit-breaker.ts new file mode 100644 index 00000000000..1b4e3be6891 --- /dev/null +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-circuit-breaker.ts @@ -0,0 +1,2 @@ +// Why: shared so every dispatch failure path breaks the circuit at the same accumulated failure_count. +export const DISPATCH_CIRCUIT_BREAK_FAILURES = 3 diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-completion.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-completion.ts new file mode 100644 index 00000000000..fa3006cd5b6 --- /dev/null +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-completion.ts @@ -0,0 +1,134 @@ +import type { TaskStatus, DispatchStatus, DispatchContextRow } from '../../types' +import { DISPATCH_CIRCUIT_BREAK_FAILURES } from './dispatch-circuit-breaker' +import type { OrchestrationDb } from '../orchestration-db' + +const FAIL_DISPATCH_SAVEPOINT = 'fail_dispatch' + +export function completeDispatch(this: OrchestrationDb, ctxId: string): void { + this.db + .prepare( + // Why: the status guard keeps a late completion from reviving a dispatch already failed or circuit-broken. + "UPDATE dispatch_contexts SET status = 'completed', completed_at = datetime('now'), capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) WHERE id = ? AND status IN ('pending', 'dispatched')" + ) + .run(ctxId) +} + +export function completeActiveDispatchForTask(this: OrchestrationDb, taskId: string): void { + const active = this.db + .prepare( + "SELECT * FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" + ) + .get(taskId) as DispatchContextRow | undefined + if (active) { + this.completeDispatch(active.id) + } +} + +export function failActiveDispatchForTask( + this: OrchestrationDb, + taskId: string, + error: string +): DispatchContextRow | undefined { + const active = this.db + .prepare( + "SELECT * FROM dispatch_contexts WHERE task_id = ? AND status IN ('pending', 'dispatched') ORDER BY rowid DESC LIMIT 1" + ) + .get(taskId) as DispatchContextRow | undefined + return active ? this.failDispatch(active.id, error) : undefined +} + +// Why: only bump status='dispatched' — a zombie heartbeat from a finished dispatch would mask a hung retry from the stale detector (§5.3.4). +export function recordHeartbeat(this: OrchestrationDb, dispatchId: string, at: string): void { + this.db + .prepare( + "UPDATE dispatch_contexts SET last_heartbeat_at = ? WHERE id = ? AND status = 'dispatched'" + ) + .run(at, dispatchId) +} + +// Why: dispatched_at grace skips workers still within their first heartbeat interval; julianday() vs raw-TEXT compare avoids misflagging space-format timestamps as stale (#8452). +export function getStaleDispatches( + this: OrchestrationDb, + thresholdIso: string +): DispatchContextRow[] { + return this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE status = 'dispatched' + AND dispatched_at IS NOT NULL + AND julianday(dispatched_at) < julianday(?) + AND (last_heartbeat_at IS NULL OR julianday(last_heartbeat_at) < julianday(?))` + ) + .all(thresholdIso, thresholdIso) as DispatchContextRow[] +} + +export function failDispatch( + this: OrchestrationDb, + ctxId: string, + error: string +): DispatchContextRow | undefined { + this.db.exec(`SAVEPOINT ${FAIL_DISPATCH_SAVEPOINT}`) + try { + const ctx = this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(ctxId) as + | DispatchContextRow + | undefined + if (!ctx || (ctx.status !== 'pending' && ctx.status !== 'dispatched')) { + this.db.exec(`RELEASE ${FAIL_DISPATCH_SAVEPOINT}`) + return ctx + } + + const newFailureCount = ctx.failure_count + 1 + const newStatus: DispatchStatus = + newFailureCount >= DISPATCH_CIRCUIT_BREAK_FAILURES ? 'circuit_broken' : 'failed' + const result = this.db + .prepare( + `UPDATE dispatch_contexts + SET status = ?, failure_count = ?, last_failure = ?, + completed_at = COALESCE(completed_at, datetime('now')), + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(newStatus, newFailureCount, error, ctxId) + if (result.changes !== 1) { + this.db.exec(`RELEASE ${FAIL_DISPATCH_SAVEPOINT}`) + return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(ctxId) as + | DispatchContextRow + | undefined + } + + // Why: back to 'ready' not 'pending' — 'pending' would strand it since promoteReadyTasks only runs when a dep completes. + const taskStatus: TaskStatus = newStatus === 'circuit_broken' ? 'failed' : 'ready' + // Why: the status guard keeps a late failure from reopening a task that already completed or was retried elsewhere. + this.db + .prepare("UPDATE tasks SET status = ? WHERE id = ? AND status IN ('dispatched', 'blocked')") + .run(taskStatus, ctx.task_id) + this.db.exec(`RELEASE ${FAIL_DISPATCH_SAVEPOINT}`) + return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(ctxId) as + | DispatchContextRow + | undefined + } catch (cause) { + this.db.exec(`ROLLBACK TO ${FAIL_DISPATCH_SAVEPOINT}`) + this.db.exec(`RELEASE ${FAIL_DISPATCH_SAVEPOINT}`) + throw cause + } +} + +export type DispatchCompletionMethods = { + completeDispatch: typeof completeDispatch + completeActiveDispatchForTask: typeof completeActiveDispatchForTask + failActiveDispatchForTask: typeof failActiveDispatchForTask + recordHeartbeat: typeof recordHeartbeat + getStaleDispatches: typeof getStaleDispatches + failDispatch: typeof failDispatch +} + +export function attachDispatchCompletion(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + completeDispatch, + completeActiveDispatchForTask, + failActiveDispatchForTask, + recordHeartbeat, + getStaleDispatches, + failDispatch + }) +} diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts new file mode 100644 index 00000000000..f80c421c24a --- /dev/null +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts @@ -0,0 +1,131 @@ +import type { DispatchContextRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { CURRENT_CONTRACT_VERSION } from '../contract-constants' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +export function createDispatchContext( + this: OrchestrationDb, + taskId: string, + assigneeHandle: string, + // Why: pane key is the remint-stable identity behind the handle — lets worker_done ownership survive handle reissue. + assigneePaneKey?: string, + launchTokenHash?: string, + processIncarnation?: string +): DispatchContextRow { + const task = this.getTask(taskId) + if (!task) { + throw new Error(`Task not found: ${taskId}`) + } + if (task.status !== 'ready') { + throw new Error(`Task ${taskId} is ${task.status}; only ready tasks can be dispatched`) + } + + // Why: lock on pane identity too, so a reminted handle can't open a second concurrent dispatch on the same pane. + const existing = this.findActiveDispatchForAssignee(assigneeHandle, assigneePaneKey) + + if (existing) { + throw new Error( + `Terminal ${assigneeHandle} already has an active dispatch (${existing.id} for task ${existing.task_id})` + ) + } + + // Carry forward failure_count so the circuit breaker accumulates across retries for the same task. + const prior = this.db + .prepare('SELECT MAX(failure_count) as max_failures FROM dispatch_contexts WHERE task_id = ?') + .get(taskId) as { max_failures: number | null } | undefined + const priorFailures = prior?.max_failures ?? 0 + + const id = generateId('ctx') + this.db + .prepare( + `INSERT INTO dispatch_contexts ( + id, run_id, task_id, contract_version, launch_token_hash, + assignee_handle, assignee_pane_key, process_incarnation, + status, failure_count, dispatched_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'dispatched', ?, datetime('now'))` + ) + .run( + id, + task.run_id, + taskId, + CURRENT_CONTRACT_VERSION, + launchTokenHash ?? null, + assigneeHandle, + assigneePaneKey ?? null, + processIncarnation ?? null, + priorFailures + ) + this.hasAnyDispatchContextsCache = true + + this.db.prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ?").run(taskId) + + return this.db + .prepare('SELECT * FROM dispatch_contexts WHERE id = ?') + .get(id) as DispatchContextRow +} + +export function getDispatchContext( + this: OrchestrationDb, + taskId: string +): DispatchContextRow | undefined { + return this.db + .prepare('SELECT * FROM dispatch_contexts WHERE task_id = ? ORDER BY rowid DESC LIMIT 1') + .get(taskId) as DispatchContextRow | undefined +} + +export function getDispatchContextById( + this: OrchestrationDb, + dispatchId: string +): DispatchContextRow | undefined { + return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(dispatchId) as + | DispatchContextRow + | undefined +} + +export function commitDispatchLaunchTokenHash( + this: OrchestrationDb, + dispatchId: string, + launchTokenHash: string +): DispatchContextRow { + const dispatch = this.getDispatchContextById(dispatchId) + if (!dispatch) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (dispatch.contract_version !== CURRENT_CONTRACT_VERSION) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${dispatchId} does not use the current contract.` + ) + } + if (dispatch.launch_token_hash && dispatch.launch_token_hash !== launchTokenHash) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${dispatchId} already has a different launch-token commitment.` + ) + } + this.db + .prepare( + `UPDATE dispatch_contexts + SET launch_token_hash = COALESCE(launch_token_hash, ?) + WHERE id = ?` + ) + .run(launchTokenHash, dispatchId) + return this.getDispatchContextById(dispatchId) as DispatchContextRow +} + +export type DispatchContextStoreMethods = { + createDispatchContext: typeof createDispatchContext + getDispatchContext: typeof getDispatchContext + getDispatchContextById: typeof getDispatchContextById + commitDispatchLaunchTokenHash: typeof commitDispatchLaunchTokenHash +} + +export function attachDispatchContextStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createDispatchContext, + getDispatchContext, + getDispatchContextById, + commitDispatchLaunchTokenHash + }) +} diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts new file mode 100644 index 00000000000..178d17dd88a --- /dev/null +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts @@ -0,0 +1,108 @@ +import { parsePaneKey } from '../../../../../shared/stable-pane-id' +import type { DispatchContextRow } from '../../types' +import { DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL, paneKeyMatchSuffix } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function getActiveDispatchForTerminal( + this: OrchestrationDb, + handle: string +): DispatchContextRow | undefined { + return this.findActiveDispatchForAssignee(handle) +} + +/** + * Cheap "are there any dispatch rows at all" probe. When false, no terminal + * can have an active or recent-completed dispatch, so orchestration-context + * builders can skip their per-terminal query fan-out entirely. Cached after + * the first probe; createDispatchContext marks it true, resets clear it. + */ + +export function hasAnyDispatchContexts(this: OrchestrationDb): boolean { + if (this.hasAnyDispatchContextsCache === undefined) { + const row = this.db.prepare('SELECT 1 FROM dispatch_contexts LIMIT 1').get() + this.hasAnyDispatchContextsCache = row !== undefined + } + return this.hasAnyDispatchContextsCache +} + +export function getActiveDispatchForIdentity( + this: OrchestrationDb, + handle: string, + paneKey?: string +): DispatchContextRow | undefined { + return this.findActiveDispatchForAssignee(handle, paneKey) +} + +export function findActiveDispatchForAssignee( + this: OrchestrationDb, + assigneeHandle: string, + assigneePaneKey?: string +): DispatchContextRow | undefined { + const byHandle = this.db + .prepare( + // Why: newest-first like the pane lookups below — an unordered LIMIT 1 could pin a stale row if a handle ever has two active dispatches. + `SELECT * FROM dispatch_contexts + WHERE assignee_handle = ? AND status IN ('pending', 'dispatched') + ORDER BY rowid DESC LIMIT 1` + ) + .get(assigneeHandle) as DispatchContextRow | undefined + if (byHandle) { + return byHandle + } + + if (!assigneePaneKey) { + return undefined + } + + const exactPane = this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE assignee_pane_key = ? AND status IN ('pending', 'dispatched') + ORDER BY rowid DESC LIMIT 1` + ) + .get(assigneePaneKey) as DispatchContextRow | undefined + if (exactPane) { + return exactPane + } + if (!parsePaneKey(assigneePaneKey)) { + return undefined + } + return this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE assignee_pane_key IS NOT NULL + AND status IN ('pending', 'dispatched') AND instr(assignee_pane_key, ':') > 1 + AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ? + ORDER BY rowid DESC LIMIT 1` + ) + .get(paneKeyMatchSuffix(assigneePaneKey)) as DispatchContextRow | undefined +} + +export function getLatestDispatchForTerminal( + this: OrchestrationDb, + handle: string +): DispatchContextRow | undefined { + return this.db + .prepare( + 'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1' + ) + .get(handle) as DispatchContextRow | undefined +} + +export type DispatchLookupMethods = { + getActiveDispatchForTerminal: typeof getActiveDispatchForTerminal + hasAnyDispatchContexts: typeof hasAnyDispatchContexts + getActiveDispatchForIdentity: typeof getActiveDispatchForIdentity + findActiveDispatchForAssignee: typeof findActiveDispatchForAssignee + getLatestDispatchForTerminal: typeof getLatestDispatchForTerminal +} + +export function attachDispatchLookup(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getActiveDispatchForTerminal, + hasAnyDispatchContexts, + getActiveDispatchForIdentity, + findActiveDispatchForAssignee, + getLatestDispatchForTerminal + }) +} diff --git a/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts b/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts new file mode 100644 index 00000000000..7c1fc5f9429 --- /dev/null +++ b/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts @@ -0,0 +1,125 @@ +import type { WorkerReportOutcome, WorkerReportSettlement } from '../../types' +import type { OrchestrationDb } from '../orchestration-db' + +export function settleWorkerReport( + this: OrchestrationDb, + params: { + taskId: string + dispatchId: string + outcome: WorkerReportOutcome + result: string + } +): WorkerReportSettlement { + this.db.exec('BEGIN IMMEDIATE') + try { + const settlement = this.settleWorkerReportInTransaction(params) + this.db.exec('COMMIT') + return settlement + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function settleWorkerReportInTransaction( + this: OrchestrationDb, + params: { + taskId: string + dispatchId: string + outcome: WorkerReportOutcome + result: string + } +): WorkerReportSettlement { + const task = this.getTask(params.taskId) + if (!task) { + return { action: 'rejected', code: 'unknown_task', reason: `Unknown task ${params.taskId}.` } + } + const dispatch = this.getDispatchContextById(params.dispatchId) + if (!dispatch) { + return { + action: 'rejected', + code: 'unknown_dispatch', + reason: `Unknown dispatch ${params.dispatchId}.` + } + } + if (dispatch.task_id !== params.taskId) { + return { + action: 'rejected', + code: 'task_dispatch_mismatch', + reason: `Dispatch ${params.dispatchId} belongs to task ${dispatch.task_id}, not ${params.taskId}.` + } + } + + const expectedDispatchStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' + const expectedTaskStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' + if (dispatch.status === expectedDispatchStatus && task.status === expectedTaskStatus) { + return { action: 'settled', outcome: params.outcome, duplicate: true } + } + if (dispatch.status !== 'dispatched' || task.status !== 'dispatched') { + return { + action: 'rejected', + code: 'inactive_dispatch', + reason: `inactive dispatch ${params.dispatchId}: it or task ${params.taskId} is already settled.` + } + } + const latest = this.getDispatchContext(params.taskId) + if (latest?.id !== params.dispatchId) { + return { + action: 'rejected', + code: 'stale_dispatch', + reason: `Dispatch ${params.dispatchId} is not the current dispatch for task ${params.taskId}.` + } + } + + this.db.exec('SAVEPOINT settle_worker_report') + const dispatchUpdate = this.db + .prepare( + `UPDATE dispatch_contexts + SET status = ?, completed_at = datetime('now'), + last_failure = CASE WHEN ? = 'failed' THEN ? ELSE last_failure END, + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ? AND status = 'dispatched'` + ) + .run(expectedDispatchStatus, expectedDispatchStatus, params.result, params.dispatchId) + const taskUpdate = this.db + .prepare( + `UPDATE tasks + SET status = ?, result = ?, completed_at = datetime('now') + WHERE id = ? AND status = 'dispatched'` + ) + .run(expectedTaskStatus, params.result, params.taskId) + if (dispatchUpdate.changes !== 1 || taskUpdate.changes !== 1) { + this.db.exec('ROLLBACK TO settle_worker_report') + this.db.exec('RELEASE settle_worker_report') + return { + action: 'rejected', + code: 'inactive_dispatch', + reason: `Dispatch ${params.dispatchId} changed while its worker report was settling.` + } + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = ?, stage = 'settled', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'ready'` + ) + .run(params.outcome === 'succeeded' ? 'succeeded' : 'failed', params.dispatchId) + this.closeQuestionsForDispatch(params.dispatchId) + if (params.outcome === 'succeeded') { + this.promoteReadyTasks(params.taskId) + } + this.db.exec('RELEASE settle_worker_report') + return { action: 'settled', outcome: params.outcome, duplicate: false } +} + +export type WorkerReportSettlementMethods = { + settleWorkerReport: typeof settleWorkerReport + settleWorkerReportInTransaction: typeof settleWorkerReportInTransaction +} + +export function attachWorkerReportSettlement(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + settleWorkerReport, + settleWorkerReportInTransaction + }) +} diff --git a/src/main/runtime/orchestration/db/federated-worker-report-outcome.test.ts b/src/main/runtime/orchestration/db/federated-worker-report-outcome.test.ts new file mode 100644 index 00000000000..314c4d269e5 --- /dev/null +++ b/src/main/runtime/orchestration/db/federated-worker-report-outcome.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest' +import { parseFederatedWorkerReportOutcome } from './federated-worker-report-outcome' + +describe('federated-worker-report-outcome', () => { + it('reads succeeded or failed from a nested report payload', () => { + const succeeded = JSON.stringify({ payload: JSON.stringify({ outcome: 'succeeded' }) }) + const failed = JSON.stringify({ payload: JSON.stringify({ outcome: 'failed' }) }) + expect(parseFederatedWorkerReportOutcome(succeeded)).toBe('succeeded') + expect(parseFederatedWorkerReportOutcome(failed)).toBe('failed') + }) + + it('returns undefined for malformed or non-terminal outcomes', () => { + expect(parseFederatedWorkerReportOutcome('not-json')).toBeUndefined() + expect(parseFederatedWorkerReportOutcome(JSON.stringify({ payload: {} }))).toBeUndefined() + const other = JSON.stringify({ payload: JSON.stringify({ outcome: 'running' }) }) + expect(parseFederatedWorkerReportOutcome(other)).toBeUndefined() + }) +}) diff --git a/src/main/runtime/orchestration/db/federated-worker-report-outcome.ts b/src/main/runtime/orchestration/db/federated-worker-report-outcome.ts new file mode 100644 index 00000000000..cfcd39fc2ed --- /dev/null +++ b/src/main/runtime/orchestration/db/federated-worker-report-outcome.ts @@ -0,0 +1,18 @@ +import type { WorkerReportOutcome } from '../types' + +export function parseFederatedWorkerReportOutcome( + payload: string +): WorkerReportOutcome | undefined { + try { + const message = JSON.parse(payload) as { payload?: unknown } + if (typeof message.payload !== 'string') { + return undefined + } + const report = JSON.parse(message.payload) as { outcome?: unknown } + return report.outcome === 'succeeded' || report.outcome === 'failed' + ? report.outcome + : undefined + } catch { + return undefined + } +} diff --git a/src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts b/src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts new file mode 100644 index 00000000000..ca5bb66ba5e --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts @@ -0,0 +1,114 @@ +import type { FederatedDispatchRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function getFederatedDispatch( + this: OrchestrationDb, + dispatchId: string +): FederatedDispatchRow | undefined { + return this.db + .prepare('SELECT * FROM federated_dispatches WHERE dispatch_id = ?') + .get(dispatchId) as FederatedDispatchRow | undefined +} + +export function listActiveFederatedDispatches( + this: OrchestrationDb, + runId?: string +): FederatedDispatchRow[] { + return this.db + .prepare( + `SELECT fd.* + FROM federated_dispatches fd + INNER JOIN dispatch_contexts dc ON dc.id = fd.dispatch_id + INNER JOIN worker_dispatches wd ON wd.dispatch_id = fd.dispatch_id + WHERE wd.state IN ('starting', 'ready', 'stopping', 'start_unknown', 'stop_unknown') + AND (? IS NULL OR dc.run_id = ?) + ORDER BY fd.rowid` + ) + .all(runId ?? null, runId ?? null) as FederatedDispatchRow[] +} + +export function findNextTerminalFederatedDispatchPendingAcknowledgment( + this: OrchestrationDb, + afterRowId: number +): { dispatchId: string; rowId: number } | undefined { + return this.db + .prepare( + `SELECT fd.dispatch_id AS dispatchId, fd.rowid AS rowId + FROM federated_dispatches fd + INNER JOIN worker_dispatches wd ON wd.dispatch_id = fd.dispatch_id + WHERE wd.state NOT IN ('starting', 'ready', 'stopping', 'start_unknown', 'stop_unknown') + AND fd.to_home_acknowledged_sequence < fd.to_home_imported_sequence + AND fd.rowid > ? + ORDER BY fd.rowid + LIMIT 1` + ) + .get(afterRowId) as { dispatchId: string; rowId: number } | undefined +} + +export function isFederatedDispatchRelayEligible( + this: OrchestrationDb, + dispatchId: string +): boolean { + return Boolean( + this.db + .prepare( + `SELECT 1 + FROM federated_dispatches fd + INNER JOIN worker_dispatches wd ON wd.dispatch_id = fd.dispatch_id + WHERE fd.dispatch_id = ? + AND ( + wd.state IN ('starting', 'ready', 'stopping', 'start_unknown', 'stop_unknown') + OR ( + fd.to_home_acknowledged_sequence < fd.to_home_imported_sequence + ) + )` + ) + .get(dispatchId) + ) +} + +export function updateFederatedDispatchResources( + this: OrchestrationDb, + params: { + dispatchId: string + remoteRuntimeEpoch: string + worktreeId: string + terminalHandle: string + } +): FederatedDispatchRow { + this.db + .prepare( + `UPDATE federated_dispatches + SET remote_runtime_epoch = ?, remote_worktree_id = ?, remote_terminal_handle = ?, + updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(params.remoteRuntimeEpoch, params.worktreeId, params.terminalHandle, params.dispatchId) + const row = this.getFederatedDispatch(params.dispatchId) + if (!row) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${params.dispatchId} was not found.` + ) + } + return row +} + +export type FederatedDispatchStoreMethods = { + getFederatedDispatch: typeof getFederatedDispatch + listActiveFederatedDispatches: typeof listActiveFederatedDispatches + findNextTerminalFederatedDispatchPendingAcknowledgment: typeof findNextTerminalFederatedDispatchPendingAcknowledgment + isFederatedDispatchRelayEligible: typeof isFederatedDispatchRelayEligible + updateFederatedDispatchResources: typeof updateFederatedDispatchResources +} + +export function attachFederatedDispatchStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getFederatedDispatch, + listActiveFederatedDispatches, + findNextTerminalFederatedDispatchPendingAcknowledgment, + isFederatedDispatchRelayEligible, + updateFederatedDispatchResources + }) +} diff --git a/src/main/runtime/orchestration/db/federation/federation-relay-ack.ts b/src/main/runtime/orchestration/db/federation/federation-relay-ack.ts new file mode 100644 index 00000000000..37ebd8b2b6f --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/federation-relay-ack.ts @@ -0,0 +1,202 @@ +import type { + WorkerReportOutcome, + FederationRelayDirection, + FederationRelayItemRow +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION } from '../../../../../shared/protocol-version' +import { parseFederatedWorkerReportOutcome } from '../federated-worker-report-outcome' +import type { OrchestrationDb } from '../orchestration-db' + +export function listFederationRelay( + this: OrchestrationDb, + params: { + dispatchId: string + direction: FederationRelayDirection + afterSequence: number + limit?: number + } +): FederationRelayItemRow[] { + return this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND sequence > ? + ORDER BY sequence LIMIT ?` + ) + .all( + params.dispatchId, + params.direction, + params.afterSequence, + Math.min(Math.max(params.limit ?? 50, 1), 50) + ) as FederationRelayItemRow[] +} + +export function listPendingFederationRelay( + this: OrchestrationDb, + dispatchId: string, + direction: FederationRelayDirection, + limit = 50 +): FederationRelayItemRow[] { + return this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND acked_at IS NULL + ORDER BY sequence LIMIT ?` + ) + .all(dispatchId, direction, Math.min(Math.max(limit, 1), 50)) as FederationRelayItemRow[] +} + +export function acknowledgeFederationRelay( + this: OrchestrationDb, + params: { + dispatchId: string + direction: FederationRelayDirection + throughSequence: number + settleRemoteReports?: { sequence: number; outcome?: WorkerReportOutcome }[] + } +): void { + this.db.exec('BEGIN IMMEDIATE') + try { + const settledReports = params.settleRemoteReports ?? [] + for (const settledReport of settledReports) { + const report = this.getFederationRelayItem( + params.dispatchId, + params.direction, + settledReport.sequence + ) + if ( + params.direction !== 'to_home' || + settledReport.sequence > params.throughSequence || + report?.kind !== 'worker_done' || + (settledReport.outcome !== undefined && + parseFederatedWorkerReportOutcome(report.payload) !== settledReport.outcome) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Federation acknowledgment for ${params.dispatchId} does not match its queued worker_done.` + ) + } + } + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + if ( + params.direction === 'to_home' && + attachment !== undefined && + attachment.protocol_version >= ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION + ) { + const acknowledgedReports = this.db + .prepare( + `SELECT sequence FROM federation_relay_items + WHERE dispatch_id = ? AND direction = 'to_home' AND kind = 'worker_done' + AND acked_at IS NULL AND sequence <= ?` + ) + .all(params.dispatchId, params.throughSequence) as { sequence: number }[] + const settledSequences = new Set(settledReports.map((report) => report.sequence)) + if (acknowledgedReports.some((report) => !settledSequences.has(report.sequence))) { + throw new OrchestrationError( + 'request_mismatch', + `Federation acknowledgment for ${params.dispatchId} omits a worker_done settlement.` + ) + } + } + const terminalOutcomes = new Set( + settledReports.flatMap((report) => (report.outcome ? [report.outcome] : [])) + ) + if (terminalOutcomes.size > 1) { + throw new OrchestrationError( + 'request_mismatch', + `Federation acknowledgment for ${params.dispatchId} contains conflicting settlements.` + ) + } + const terminalOutcome = settledReports.find((report) => report.outcome)?.outcome + if (terminalOutcome) { + this.settleRemoteAttachmentInRelayTransaction( + params.dispatchId, + terminalOutcome, + 'worker_report_settled' + ) + } + this.db + .prepare( + `UPDATE federation_relay_items SET acked_at = COALESCE(acked_at, datetime('now')) + WHERE dispatch_id = ? AND direction = ? AND sequence <= ?` + ) + .run(params.dispatchId, params.direction, params.throughSequence) + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function setFederatedHomeImportSequence( + this: OrchestrationDb, + dispatchId: string, + sequence: number +): void { + this.db + .prepare( + `UPDATE federated_dispatches + SET to_home_imported_sequence = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND to_home_imported_sequence < ?` + ) + .run(sequence, dispatchId, sequence) +} + +export function recordFederatedHomeAcknowledgment( + this: OrchestrationDb, + params: { + dispatchId: string + remoteRuntimeEpoch: string + sequence: number + } +): void { + const federated = this.getFederatedDispatch(params.dispatchId) + if ( + !federated || + !Number.isInteger(params.sequence) || + params.sequence < 0 || + params.sequence > federated.to_home_imported_sequence + ) { + throw new OrchestrationError( + 'request_mismatch', + `Federation acknowledgment for ${params.dispatchId} exceeds imported relay state.` + ) + } + this.db + .prepare( + `UPDATE federated_dispatches + SET remote_runtime_epoch = ?, + to_home_acknowledged_sequence = CASE + WHEN remote_runtime_epoch = ? + THEN MAX(to_home_acknowledged_sequence, ?) + ELSE ? + END, + updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run( + params.remoteRuntimeEpoch, + params.remoteRuntimeEpoch, + params.sequence, + params.sequence, + params.dispatchId + ) +} + +export type FederationRelayAckMethods = { + listFederationRelay: typeof listFederationRelay + listPendingFederationRelay: typeof listPendingFederationRelay + acknowledgeFederationRelay: typeof acknowledgeFederationRelay + setFederatedHomeImportSequence: typeof setFederatedHomeImportSequence + recordFederatedHomeAcknowledgment: typeof recordFederatedHomeAcknowledgment +} + +export function attachFederationRelayAck(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + listFederationRelay, + listPendingFederationRelay, + acknowledgeFederationRelay, + setFederatedHomeImportSequence, + recordFederatedHomeAcknowledgment + }) +} diff --git a/src/main/runtime/orchestration/db/federation/federation-relay-enqueue.ts b/src/main/runtime/orchestration/db/federation/federation-relay-enqueue.ts new file mode 100644 index 00000000000..28841f44e6e --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/federation-relay-enqueue.ts @@ -0,0 +1,185 @@ +import type { + WorkerReportOutcome, + FederationRelayDirection, + FederationRelayItemRow +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +export function enqueueFederationRelay( + this: OrchestrationDb, + params: { + dispatchId: string + direction: FederationRelayDirection + kind: string + payload: string + messageId?: string + settleRemoteOutcome?: WorkerReportOutcome + remoteQuestion?: true + } +): FederationRelayItemRow { + const byteCount = Buffer.byteLength(params.payload, 'utf8') + const messageId = params.messageId ?? generateId('relay') + if (byteCount > 64 * 1024) { + throw new OrchestrationError( + 'relay_quota_exceeded', + 'A federated orchestration message cannot exceed 64 KiB.' + ) + } + this.db.exec('BEGIN IMMEDIATE') + try { + if (params.settleRemoteOutcome) { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + if (!attachment || attachment.state !== 'ready') { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${params.dispatchId} is not active.` + ) + } + } + if (params.kind === 'heartbeat') { + const heartbeat = this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND kind = 'heartbeat' + AND acked_at IS NULL + ORDER BY sequence DESC LIMIT 1` + ) + .get(params.dispatchId, params.direction) as FederationRelayItemRow | undefined + if (heartbeat) { + this.db + .prepare( + `UPDATE federation_relay_items + SET payload = ?, byte_count = ?, created_at = datetime('now') + WHERE dispatch_id = ? AND direction = ? AND sequence = ?` + ) + .run(params.payload, byteCount, params.dispatchId, params.direction, heartbeat.sequence) + this.db.exec('COMMIT') + return this.getFederationRelayItem( + params.dispatchId, + params.direction, + heartbeat.sequence + ) as FederationRelayItemRow + } + } + if (params.kind === 'worker_done') { + const identicalReport = this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND kind = 'worker_done' + AND payload = ? AND acked_at IS NULL + ORDER BY sequence DESC LIMIT 1` + ) + .get(params.dispatchId, params.direction, params.payload) as + | FederationRelayItemRow + | undefined + if (identicalReport) { + this.settleRemoteAttachmentInRelayTransaction(params.dispatchId, params.settleRemoteOutcome) + this.db.exec('COMMIT') + return identicalReport + } + } + const quota = this.db + .prepare( + `SELECT COUNT(*) AS count, COALESCE(SUM(byte_count), 0) AS bytes + FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND acked_at IS NULL` + ) + .get(params.dispatchId, params.direction) as { count: number; bytes: number } + if (quota.count >= 256 || quota.bytes + byteCount > 1024 * 1024) { + if (params.kind === 'worker_done') { + const heartbeat = this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND kind = 'heartbeat' + AND acked_at IS NULL + ORDER BY sequence LIMIT 1` + ) + .get(params.dispatchId, params.direction) as FederationRelayItemRow | undefined + if (heartbeat) { + this.db + .prepare( + `UPDATE federation_relay_items + SET message_id = ?, kind = ?, payload = ?, byte_count = ?, + created_at = datetime('now') + WHERE dispatch_id = ? AND direction = ? AND sequence = ?` + ) + .run( + messageId, + params.kind, + params.payload, + byteCount, + params.dispatchId, + params.direction, + heartbeat.sequence + ) + this.settleRemoteAttachmentInRelayTransaction( + params.dispatchId, + params.settleRemoteOutcome + ) + this.db.exec('COMMIT') + return this.getFederationRelayItem( + params.dispatchId, + params.direction, + heartbeat.sequence + ) as FederationRelayItemRow + } + } + throw new OrchestrationError( + 'relay_quota_exceeded', + `Federated Dispatch ${params.dispatchId} has no relay capacity.` + ) + } + const latest = this.db + .prepare( + `SELECT COALESCE(MAX(sequence), 0) AS sequence + FROM federation_relay_items WHERE dispatch_id = ? AND direction = ?` + ) + .get(params.dispatchId, params.direction) as { sequence: number } + const sequence = latest.sequence + 1 + this.db + .prepare( + `INSERT INTO federation_relay_items ( + dispatch_id, direction, sequence, message_id, kind, payload, byte_count + ) VALUES (?, ?, ?, ?, ?, ?, ?)` + ) + .run( + params.dispatchId, + params.direction, + sequence, + messageId, + params.kind, + params.payload, + byteCount + ) + if (params.remoteQuestion) { + this.db + .prepare( + `INSERT INTO remote_questions (message_id, dispatch_id) + VALUES (?, ?)` + ) + .run(messageId, params.dispatchId) + } + this.settleRemoteAttachmentInRelayTransaction(params.dispatchId, params.settleRemoteOutcome) + this.db.exec('COMMIT') + return this.getFederationRelayItem( + params.dispatchId, + params.direction, + sequence + ) as FederationRelayItemRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type FederationRelayEnqueueMethods = { + enqueueFederationRelay: typeof enqueueFederationRelay +} + +export function attachFederationRelayEnqueue(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + enqueueFederationRelay + }) +} diff --git a/src/main/runtime/orchestration/db/federation/federation-relay-import.ts b/src/main/runtime/orchestration/db/federation/federation-relay-import.ts new file mode 100644 index 00000000000..c6b70e8fe54 --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/federation-relay-import.ts @@ -0,0 +1,140 @@ +import type { + MessageType, + MessagePriority, + MessageRow, + WorkerReportOutcome, + WorkerReportSettlement +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function importFederatedRelayItem( + this: OrchestrationDb, + params: { + dispatchId: string + sequence: number + message: { + id: string + runId: string + from: string + to: string + subject: string + body: string + type: MessageType + priority: MessagePriority + threadId?: string + payload?: string + } + lifecycle: + | { kind: 'none' } + | { kind: 'heartbeat'; at: string } + | { + kind: 'worker_report' + taskId: string + outcome: WorkerReportOutcome + result: string + } + | { kind: 'rejected'; code: string; reason: string } + } +): { + message: MessageRow + duplicate: boolean + lifecycle?: WorkerReportSettlement | { action: 'rejected'; code: string; reason: string } +} { + this.db.exec('BEGIN IMMEDIATE') + try { + const federated = this.getFederatedDispatch(params.dispatchId) + if (!federated) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${params.dispatchId} was not found.` + ) + } + const duplicate = params.sequence <= federated.to_home_imported_sequence + if (duplicate && !this.getMessageById(params.message.id)) { + throw new OrchestrationError( + 'operation_unknown', + `Federated relay sequence ${params.sequence} was committed without its message.` + ) + } + if (!duplicate && params.sequence !== federated.to_home_imported_sequence + 1) { + throw new OrchestrationError( + 'operation_unknown', + `Federated relay for ${params.dispatchId} is not contiguous after sequence ${federated.to_home_imported_sequence}.` + ) + } + + let message = this.getMessageById(params.message.id) + if (!message) { + message = this.insertMessage(params.message) + } else if ( + message.run_id !== params.message.runId || + message.to_handle !== params.message.to || + message.type !== params.message.type + ) { + throw new OrchestrationError( + 'request_mismatch', + `Federated relay message ${params.message.id} conflicts with an existing message.` + ) + } + if (message.type === 'question') { + this.registerFederatedQuestion({ + messageId: message.id, + runId: params.message.runId, + dispatchId: params.dispatchId + }) + } + let lifecycle: + | WorkerReportSettlement + | { action: 'rejected'; code: string; reason: string } + | undefined + if (params.lifecycle.kind === 'heartbeat' && !duplicate) { + this.recordHeartbeat(params.dispatchId, params.lifecycle.at) + } else if (params.lifecycle.kind === 'worker_report') { + lifecycle = this.settleWorkerReportInTransaction({ + taskId: params.lifecycle.taskId, + dispatchId: params.dispatchId, + outcome: params.lifecycle.outcome, + result: params.lifecycle.result + }) + if (lifecycle.action === 'rejected' && !duplicate) { + message = this.convertLifecycleMessageToRejection( + message.id, + lifecycle.code, + lifecycle.reason + ) as MessageRow + } + } else if (params.lifecycle.kind === 'rejected') { + lifecycle = { + action: 'rejected', + code: params.lifecycle.code, + reason: params.lifecycle.reason + } + if (!duplicate) { + message = this.convertLifecycleMessageToRejection( + message.id, + params.lifecycle.code, + params.lifecycle.reason + ) as MessageRow + } + } + if (!duplicate) { + this.setFederatedHomeImportSequence(params.dispatchId, params.sequence) + } + this.db.exec('COMMIT') + return { message, duplicate, ...(lifecycle ? { lifecycle } : {}) } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type FederationRelayImportMethods = { + importFederatedRelayItem: typeof importFederatedRelayItem +} + +export function attachFederationRelayImport(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + importFederatedRelayItem + }) +} diff --git a/src/main/runtime/orchestration/db/federation/federation-relay-item.ts b/src/main/runtime/orchestration/db/federation/federation-relay-item.ts new file mode 100644 index 00000000000..1ae13b6b6ae --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/federation-relay-item.ts @@ -0,0 +1,69 @@ +import type { + WorkerReportOutcome, + FederationRelayDirection, + FederationRelayItemRow +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function getFederationRelayItem( + this: OrchestrationDb, + dispatchId: string, + direction: FederationRelayDirection, + sequence: number +): FederationRelayItemRow | undefined { + return this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND sequence = ?` + ) + .get(dispatchId, direction, sequence) as FederationRelayItemRow | undefined +} + +export function settleRemoteAttachmentInRelayTransaction( + this: OrchestrationDb, + dispatchId: string, + outcome: WorkerReportOutcome | undefined, + stage = 'worker_report_queued' +): void { + if (!outcome) { + return + } + const attachment = this.getRemoteDispatchAttachment(dispatchId) + const state = outcome === 'succeeded' ? 'succeeded' : 'failed' + if (!attachment) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${dispatchId} was not found.` + ) + } + if (attachment.state === state) { + return + } + if (attachment.state !== 'ready') { + throw new OrchestrationError( + 'request_mismatch', + `Remote Dispatch ${dispatchId} cannot settle as ${state} from ${attachment.state}.` + ) + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = ?, stage = ?, capability_hash = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'ready'` + ) + .run(state, stage, dispatchId) +} + +export type FederationRelayItemMethods = { + getFederationRelayItem: typeof getFederationRelayItem + settleRemoteAttachmentInRelayTransaction: typeof settleRemoteAttachmentInRelayTransaction +} + +export function attachFederationRelayItem(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getFederationRelayItem, + settleRemoteAttachmentInRelayTransaction + }) +} diff --git a/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-authority.ts b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-authority.ts new file mode 100644 index 00000000000..84166e28631 --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-authority.ts @@ -0,0 +1,173 @@ +import { randomBytes, timingSafeEqual } from 'node:crypto' +import type { RemoteDispatchAttachmentRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { hashDispatchCapability } from '../dispatch-capability-hash' +import { isEquivalentPaneKey } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function prepareRemoteAttachmentAuthority( + this: OrchestrationDb, + params: { + dispatchId: string + paneKey: string + processIncarnation: string + worktreeId: string + terminalHandle: string + setupState: string + effects: unknown[] + } +): string { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + if (!attachment || attachment.state !== 'starting') { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${params.dispatchId} is not starting.` + ) + } + const capability = `dcap_${randomBytes(32).toString('base64url')}` + const result = this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET stage = 'authority_attached', capability_hash = ?, pane_key = ?, + process_incarnation = ?, worktree_id = ?, terminal_handle = ?, setup_state = ?, + effects = ?, residual_resources = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run( + hashDispatchCapability(capability), + params.paneKey, + params.processIncarnation, + params.worktreeId, + params.terminalHandle, + params.setupState, + JSON.stringify(params.effects), + JSON.stringify( + params.effects.filter((effect) => + Boolean( + effect && + typeof effect === 'object' && + ((effect as { action?: string }).action?.startsWith('created') || + (effect as { action?: string }).action === 'reused_agent_terminal') + ) + ) + ), + params.dispatchId + ) + // Why: without this the caller keeps a capability whose hash was never stored, surfacing later as an authority mismatch. + if (result.changes !== 1) { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${params.dispatchId} is not starting.` + ) + } + return capability +} + +export function markRemoteAttachmentReady( + this: OrchestrationDb, + dispatchId: string, + effects?: unknown[] +): RemoteDispatchAttachmentRow { + const result = this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'ready', stage = 'input_accepted', + effects = COALESCE(?, effects), updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run(effects ? JSON.stringify(effects) : null, dispatchId) + if (result.changes !== 1) { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${dispatchId} is not starting.` + ) + } + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow +} + +export function failRemoteAttachment( + this: OrchestrationDb, + dispatchId: string, + stage: string, + reason: string, + unknown: boolean +): RemoteDispatchAttachmentRow { + const state = unknown ? 'start_unknown' : 'failed' + const result = this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = ?, stage = ?, last_error = ?, capability_hash = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run(state, stage, reason, dispatchId) + if (result.changes !== 1) { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${dispatchId} is not starting.` + ) + } + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow +} + +export function verifyRemoteAttachmentAuthority( + this: OrchestrationDb, + params: { + dispatchId: string + capability: string | undefined + paneKey: string | null + processIncarnation: string | null + } +): boolean { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + if ( + !attachment?.capability_hash || + !params.capability || + !attachment.pane_key || + !params.paneKey || + !isEquivalentPaneKey(attachment.pane_key, params.paneKey) || + !attachment.process_incarnation || + attachment.process_incarnation !== params.processIncarnation + ) { + return false + } + const expected = Buffer.from(attachment.capability_hash, 'hex') + const observed = Buffer.from(hashDispatchCapability(params.capability), 'hex') + return expected.length === observed.length && timingSafeEqual(expected, observed) +} + +export function isRemoteAttachmentProcessCurrent( + this: OrchestrationDb, + params: { + dispatchId: string + paneKey: string | null + processIncarnation: string | null + } +): boolean { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + return Boolean( + attachment?.pane_key && + params.paneKey && + isEquivalentPaneKey(attachment.pane_key, params.paneKey) && + attachment.process_incarnation && + attachment.process_incarnation === params.processIncarnation + ) +} + +export type RemoteDispatchAttachmentAuthorityMethods = { + prepareRemoteAttachmentAuthority: typeof prepareRemoteAttachmentAuthority + markRemoteAttachmentReady: typeof markRemoteAttachmentReady + failRemoteAttachment: typeof failRemoteAttachment + verifyRemoteAttachmentAuthority: typeof verifyRemoteAttachmentAuthority + isRemoteAttachmentProcessCurrent: typeof isRemoteAttachmentProcessCurrent +} + +export function attachRemoteDispatchAttachmentAuthority(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + prepareRemoteAttachmentAuthority, + markRemoteAttachmentReady, + failRemoteAttachment, + verifyRemoteAttachmentAuthority, + isRemoteAttachmentProcessCurrent + }) +} diff --git a/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-create.ts b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-create.ts new file mode 100644 index 00000000000..bb724f74cb8 --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-create.ts @@ -0,0 +1,177 @@ +import type { WorkerDispatchState, RemoteDispatchAttachmentRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { ensureMutationReceiptCapacity } from '../../mutation-receipt-capacity' +import type { OrchestrationDb } from '../orchestration-db' + +export function createRemoteDispatchAttachment( + this: OrchestrationDb, + params: { + dispatchId: string + taskId: string + homePeerFingerprint: string + protocolVersion: number + runtimeEpoch: string + mutationReceipt: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } + } +): RemoteDispatchAttachmentRow { + this.db.exec('BEGIN IMMEDIATE') + try { + if (params.homePeerFingerprint !== params.mutationReceipt.callerFingerprint) { + throw new OrchestrationError( + 'resource_server_mismatch', + 'The authenticated Run-home peer does not match the attachment request.' + ) + } + const existingReceipt = this.getMutationReceipt( + params.mutationReceipt.callerFingerprint, + params.mutationReceipt.requestId + ) + if (existingReceipt) { + throw new OrchestrationError( + existingReceipt.method === params.mutationReceipt.method && + existingReceipt.payload_hash === params.mutationReceipt.payloadHash + ? 'operation_unknown' + : 'request_mismatch', + `Remote attachment request ${params.mutationReceipt.requestId} already exists.` + ) + } + ensureMutationReceiptCapacity(this.db) + this.db + .prepare( + `INSERT INTO mutation_receipts ( + caller_fingerprint, request_id, method, payload_hash, state, receipt + ) VALUES (?, ?, ?, ?, 'pending', ?)` + ) + .run( + params.mutationReceipt.callerFingerprint, + params.mutationReceipt.requestId, + params.mutationReceipt.method, + params.mutationReceipt.payloadHash, + JSON.stringify({ accepted: { dispatchId: params.dispatchId } }) + ) + this.db + .prepare( + `INSERT INTO remote_dispatch_attachments ( + dispatch_id, task_id, home_peer_fingerprint, protocol_version, runtime_epoch + ) VALUES (?, ?, ?, ?, ?)` + ) + .run( + params.dispatchId, + params.taskId, + params.homePeerFingerprint, + params.protocolVersion, + params.runtimeEpoch + ) + this.db.exec('COMMIT') + return this.getRemoteDispatchAttachment(params.dispatchId) as RemoteDispatchAttachmentRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getRemoteDispatchAttachment( + this: OrchestrationDb, + dispatchId: string +): RemoteDispatchAttachmentRow | undefined { + return this.db + .prepare('SELECT * FROM remote_dispatch_attachments WHERE dispatch_id = ?') + .get(dispatchId) as RemoteDispatchAttachmentRow | undefined +} + +export function recordRemoteAttachmentStage( + this: OrchestrationDb, + params: { + dispatchId: string + stage: string + state?: WorkerDispatchState + worktreeId?: string + terminalHandle?: string + setupState?: string + effects?: unknown[] + residualResources?: unknown[] + lastError?: string + } +): RemoteDispatchAttachmentRow { + const current = this.getRemoteDispatchAttachment(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${params.dispatchId} was not found.` + ) + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET stage = ?, state = ?, worktree_id = ?, terminal_handle = ?, setup_state = ?, + effects = ?, residual_resources = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run( + params.stage, + params.state ?? current.state, + params.worktreeId ?? current.worktree_id, + params.terminalHandle ?? current.terminal_handle, + params.setupState ?? current.setup_state, + params.effects ? JSON.stringify(params.effects) : current.effects, + params.residualResources + ? JSON.stringify(params.residualResources) + : current.residual_resources, + params.lastError ?? current.last_error, + params.dispatchId + ) + return this.getRemoteDispatchAttachment(params.dispatchId) as RemoteDispatchAttachmentRow +} + +export function updateRemoteAttachmentSetupEvidence( + this: OrchestrationDb, + params: { + dispatchId: string + setupState: string + effects: unknown[] + } +): { attachment: RemoteDispatchAttachmentRow; changed: boolean } { + const current = this.getRemoteDispatchAttachment(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${params.dispatchId} was not found.` + ) + } + const effects = JSON.stringify(params.effects) + if (current.setup_state === params.setupState && current.effects === effects) { + return { attachment: current, changed: false } + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET setup_state = ?, effects = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(params.setupState, effects, params.dispatchId) + return { + attachment: this.getRemoteDispatchAttachment(params.dispatchId) as RemoteDispatchAttachmentRow, + changed: true + } +} + +export type RemoteDispatchAttachmentCreateMethods = { + createRemoteDispatchAttachment: typeof createRemoteDispatchAttachment + getRemoteDispatchAttachment: typeof getRemoteDispatchAttachment + recordRemoteAttachmentStage: typeof recordRemoteAttachmentStage + updateRemoteAttachmentSetupEvidence: typeof updateRemoteAttachmentSetupEvidence +} + +export function attachRemoteDispatchAttachmentCreate(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createRemoteDispatchAttachment, + getRemoteDispatchAttachment, + recordRemoteAttachmentStage, + updateRemoteAttachmentSetupEvidence + }) +} diff --git a/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts new file mode 100644 index 00000000000..a0774b12d1f --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts @@ -0,0 +1,106 @@ +import { parsePaneKey } from '../../../../../shared/stable-pane-id' +import type { RemoteDispatchAttachmentRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { paneKeyMatchSuffix, REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function beginRemoteAttachmentStop( + this: OrchestrationDb, + dispatchId: string +): RemoteDispatchAttachmentRow { + const attachment = this.getRemoteDispatchAttachment(dispatchId) + if (!attachment) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${dispatchId} was not found.` + ) + } + if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(attachment.state)) { + return attachment + } + if (!['ready', 'start_unknown'].includes(attachment.state)) { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${dispatchId} cannot stop from ${attachment.state}.` + ) + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'stopping', stage = 'stop_requested', capability_hash = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')` + ) + .run(dispatchId) + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow +} + +export function settleRemoteAttachmentStop( + this: OrchestrationDb, + dispatchId: string +): RemoteDispatchAttachmentRow { + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'stopped', stage = 'process_stopped', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(dispatchId) + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow +} + +export function markRemoteAttachmentStopUnknown( + this: OrchestrationDb, + dispatchId: string, + reason: string +): RemoteDispatchAttachmentRow { + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'stop_unknown', stage = 'stop_outcome_unknown', last_error = ?, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(reason, dispatchId) + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow +} + +export function findActiveRemoteAttachmentForPane( + this: OrchestrationDb, + paneKey: string +): RemoteDispatchAttachmentRow | undefined { + if (!parsePaneKey(paneKey)) { + return this.db + .prepare( + `SELECT * FROM remote_dispatch_attachments + WHERE state IN ('starting', 'ready') AND pane_key = ? + ORDER BY rowid DESC LIMIT 1` + ) + .get(paneKey) as RemoteDispatchAttachmentRow | undefined + } + return this.db + .prepare( + `SELECT * FROM remote_dispatch_attachments + WHERE state IN ('starting', 'ready') AND pane_key IS NOT NULL + AND instr(pane_key, ':') > 1 + AND ${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL} = ? + ORDER BY rowid DESC LIMIT 1` + ) + .get(paneKeyMatchSuffix(paneKey)) as RemoteDispatchAttachmentRow | undefined +} + +export type RemoteDispatchAttachmentStopMethods = { + beginRemoteAttachmentStop: typeof beginRemoteAttachmentStop + settleRemoteAttachmentStop: typeof settleRemoteAttachmentStop + markRemoteAttachmentStopUnknown: typeof markRemoteAttachmentStopUnknown + findActiveRemoteAttachmentForPane: typeof findActiveRemoteAttachmentForPane +} + +export function attachRemoteDispatchAttachmentStop(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + beginRemoteAttachmentStop, + settleRemoteAttachmentStop, + markRemoteAttachmentStopUnknown, + findActiveRemoteAttachmentForPane + }) +} diff --git a/src/main/runtime/orchestration/db/federation/remote-question-store.ts b/src/main/runtime/orchestration/db/federation/remote-question-store.ts new file mode 100644 index 00000000000..aa3082bf8af --- /dev/null +++ b/src/main/runtime/orchestration/db/federation/remote-question-store.ts @@ -0,0 +1,139 @@ +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function getRemoteQuestion( + this: OrchestrationDb, + messageId: string +): + | { + message_id: string + dispatch_id: string + status: 'pending' | 'answered' | 'closed' + answer_message_id: string | null + answer_body: string | null + } + | undefined { + return this.db.prepare('SELECT * FROM remote_questions WHERE message_id = ?').get(messageId) as + | { + message_id: string + dispatch_id: string + status: 'pending' | 'answered' | 'closed' + answer_message_id: string | null + answer_body: string | null + } + | undefined +} + +type RemoteAnswerParams = { + messageId: string + dispatchId: string + answerMessageId: string + body: string +} + +// Returns 'settled' when the stored row already carries this exact answer, 'writable' when the guarded UPDATE may apply. +function classifyRemoteQuestion( + this: OrchestrationDb, + params: RemoteAnswerParams +): 'settled' | 'writable' { + const question = this.getRemoteQuestion(params.messageId) + if (!question || question.dispatch_id !== params.dispatchId) { + throw new OrchestrationError( + 'question_not_found', + `Remote Question ${params.messageId} was not found.` + ) + } + if (question.status === 'answered') { + if ( + question.answer_message_id !== params.answerMessageId || + question.answer_body !== params.body + ) { + throw new OrchestrationError( + 'answer_conflict', + `Remote Question ${params.messageId} already has a different answer.` + ) + } + return 'settled' + } + // Why: the guarded UPDATE is a silent no-op for a closed question, which the caller would read as stored. + if (question.status === 'closed') { + throw new OrchestrationError( + 'question_not_found', + `Remote Question ${params.messageId} is closed.` + ) + } + return 'writable' +} + +export function answerRemoteQuestion(this: OrchestrationDb, params: RemoteAnswerParams): void { + if (classifyRemoteQuestion.call(this, params) === 'settled') { + return + } + const changes = this.db + .prepare( + `UPDATE remote_questions + SET status = 'answered', answer_message_id = ?, answer_body = ?, + answered_at = datetime('now') + WHERE message_id = ? AND status = 'pending'` + ) + .run(params.answerMessageId, params.body, params.messageId).changes + if (changes > 0) { + return + } + // Why: a concurrent answer or close won the guarded UPDATE after our read; re-classify so the caller + // sees the stored outcome instead of a success that never happened. + if (classifyRemoteQuestion.call(this, params) === 'settled') { + return + } + throw new OrchestrationError( + 'answer_conflict', + `Remote Question ${params.messageId} could not be answered.` + ) +} + +export function setRemoteWorkerImportSequence( + this: OrchestrationDb, + dispatchId: string, + sequence: number +): void { + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET to_worker_imported_sequence = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND to_worker_imported_sequence < ?` + ) + .run(sequence, dispatchId, sequence) +} + +export function registerFederatedQuestion( + this: OrchestrationDb, + params: { + messageId: string + runId: string + dispatchId: string + } +): void { + this.db + .prepare( + `INSERT OR IGNORE INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle + ) VALUES (?, ?, ?, ?)` + ) + .run(params.messageId, params.runId, params.dispatchId, `dispatch:${params.dispatchId}`) +} + +export type RemoteQuestionStoreMethods = { + getRemoteQuestion: typeof getRemoteQuestion + answerRemoteQuestion: typeof answerRemoteQuestion + setRemoteWorkerImportSequence: typeof setRemoteWorkerImportSequence + registerFederatedQuestion: typeof registerFederatedQuestion +} + +export function attachRemoteQuestionStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getRemoteQuestion, + answerRemoteQuestion, + setRemoteWorkerImportSequence, + registerFederatedQuestion + }) +} diff --git a/src/main/runtime/orchestration/db/generated-id.ts b/src/main/runtime/orchestration/db/generated-id.ts new file mode 100644 index 00000000000..069ca09172d --- /dev/null +++ b/src/main/runtime/orchestration/db/generated-id.ts @@ -0,0 +1,5 @@ +import { randomBytes } from 'node:crypto' + +export function generateId(prefix: string): string { + return `${prefix}_${randomBytes(6).toString('hex')}` +} diff --git a/src/main/runtime/orchestration/db/legacy-question-identity.test.ts b/src/main/runtime/orchestration/db/legacy-question-identity.test.ts new file mode 100644 index 00000000000..5d60e1a67b3 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy-question-identity.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import type { MessageRow } from '../types' +import { + legacyMessageMatchesQuestion, + normalizeLegacyQuestionOptions, + normalizeLegacyQuestionText +} from './legacy-question-identity' + +function message(overrides: Partial = {}): MessageRow { + return { + id: 'msg_1', + run_id: 'run_1', + delivery_contract: 'legacy_direct', + from_handle: 'from', + to_handle: 'to', + subject: 'q', + body: 'Ask this?\r\n', + type: 'question', + priority: 'normal', + thread_id: null, + payload: '{"options":[" yes ","no"]}', + read: 0, + sequence: 1, + created_at: '2026-01-01 00:00:00', + delivered_at: null, + sender_pane_key: null, + ...overrides + } +} + +describe('legacy-question-identity', () => { + it('normalizes question text and option arrays', () => { + expect(normalizeLegacyQuestionText(' a\r\nb ')).toBe('a\nb') + expect(normalizeLegacyQuestionOptions([' yes ', 'no'])).toBe('["yes","no"]') + expect(normalizeLegacyQuestionOptions(['yes', 1])).toBe('[]') + }) + + it('matches a question only when recipient, body, and options agree', () => { + expect(legacyMessageMatchesQuestion(message(), 'Ask this?', ['yes', 'no'], ['to'])).toBe(true) + expect(legacyMessageMatchesQuestion(message(), 'Ask this?', ['yes', 'no'], ['other'])).toBe( + false + ) + expect(legacyMessageMatchesQuestion(message(), 'Different', ['yes', 'no'], ['to'])).toBe(false) + expect( + legacyMessageMatchesQuestion(message({ payload: '{' }), 'Ask this?', ['yes'], ['to']) + ).toBe(false) + }) +}) diff --git a/src/main/runtime/orchestration/db/legacy-question-identity.ts b/src/main/runtime/orchestration/db/legacy-question-identity.ts new file mode 100644 index 00000000000..055c40e0970 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy-question-identity.ts @@ -0,0 +1,34 @@ +import type { MessageRow } from '../types' + +export function normalizeLegacyQuestionText(value: string): string { + return value.replace(/\r\n/g, '\n').trim() +} + +export function normalizeLegacyQuestionOptions(options: unknown): string { + if (!Array.isArray(options) || !options.every((option) => typeof option === 'string')) { + return '[]' + } + return JSON.stringify(options.map((option) => option.trim())) +} + +export function legacyMessageMatchesQuestion( + message: MessageRow, + question: string, + options: string[], + recipientHandles: readonly string[] +): boolean { + if ( + !recipientHandles.includes(message.to_handle) || + normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question) + ) { + return false + } + try { + const payload = JSON.parse(message.payload ?? '{}') as { options?: unknown } + return ( + normalizeLegacyQuestionOptions(payload.options) === normalizeLegacyQuestionOptions(options) + ) + } catch { + return false + } +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-ask-operation.ts b/src/main/runtime/orchestration/db/legacy/legacy-ask-operation.ts new file mode 100644 index 00000000000..c4b7dbf3556 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-ask-operation.ts @@ -0,0 +1,148 @@ +import type { MessageRow, LegacyOperationReceiptRow, QuestionRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import { legacyMessageMatchesQuestion } from '../legacy-question-identity' +import type { OrchestrationDb } from '../orchestration-db' + +export function commitLegacyAskOperation( + this: OrchestrationDb, + params: { + principalId: string + operationKey: string + method: string + payloadHash: string + question: string + options?: string[] + recipientHandle: string + existingQuestionId?: string + } +): { + receipt: LegacyOperationReceiptRow + question: QuestionRow + message: MessageRow + duplicate: boolean +} { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker') + const receipt = this.requireMatchingLegacyOperationReceipt(params) + if (receipt) { + const response = JSON.parse(receipt.response_json) as { questionId: string } + const question = this.getQuestion(response.questionId) + const message = this.getMessageById(response.questionId) + if (!question || !message) { + throw new OrchestrationError( + 'operation_unknown', + `Legacy ask ${params.operationKey} lost its durable question.` + ) + } + this.db.exec('COMMIT') + return { receipt, question, message, duplicate: true } + } + + const dispatchId = principal.dispatch_id as string + const dispatch = this.getDispatchContextById(dispatchId) + if ( + !dispatch || + dispatch.run_id !== principal.run_id || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION || + !['pending', 'dispatched'].includes(dispatch.status) + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} is not an active legacy attempt.` + ) + } + + const existingQuestionId = + params.existingQuestionId && + !this.db + .prepare( + `SELECT 1 FROM legacy_operation_receipts + WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ? + LIMIT 1` + ) + .get(principal.id, params.existingQuestionId) + ? params.existingQuestionId + : undefined + let question: QuestionRow + let message: MessageRow + const delivery = this.resolveLegacyWorkerCoordinatorDelivery( + principal.run_id, + params.recipientHandle + ) + if (existingQuestionId) { + const existingQuestion = this.getQuestion(existingQuestionId) + const existingMessage = this.getMessageById(existingQuestionId) + if ( + !existingQuestion || + !existingMessage || + existingQuestion.run_id !== principal.run_id || + existingQuestion.dispatch_id !== dispatchId || + existingQuestion.status !== 'pending' || + existingMessage.delivery_contract !== delivery.contract || + !legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [ + delivery.to + ]) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Question ${params.existingQuestionId} is not a pending ask for this principal.` + ) + } + question = existingQuestion + message = existingMessage + } else { + message = this.insertMessage({ + from: principal.terminal_handle, + to: delivery.to, + subject: 'Question', + body: params.question, + type: delivery.contract === 'legacy_direct' ? 'decision_gate' : 'question', + payload: JSON.stringify({ + taskId: dispatch.task_id, + dispatchId, + question: params.question, + options: params.options ?? [] + }), + senderPaneKey: principal.pane_key, + runId: principal.run_id, + deliveryContract: delivery.contract + }) + this.db.prepare('UPDATE messages SET thread_id = ? WHERE id = ?').run(message.id, message.id) + this.db + .prepare( + `INSERT INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle + ) VALUES (?, ?, ?, ?)` + ) + .run(message.id, principal.run_id, dispatchId, principal.terminal_handle) + question = this.getQuestion(message.id) as QuestionRow + message = this.getMessageById(message.id) as MessageRow + } + + const committedReceipt = this.insertLegacyOperationReceipt({ + principalId: principal.id, + operationKey: params.operationKey, + method: params.method, + payloadHash: params.payloadHash, + effectId: question.message_id, + responseJson: JSON.stringify({ questionId: question.message_id }) + }) + this.db.exec('COMMIT') + return { receipt: committedReceipt, question, message, duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type LegacyAskOperationMethods = { + commitLegacyAskOperation: typeof commitLegacyAskOperation +} + +export function attachLegacyAskOperation(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + commitLegacyAskOperation + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-compatibility-candidates.ts b/src/main/runtime/orchestration/db/legacy/legacy-compatibility-candidates.ts new file mode 100644 index 00000000000..d94c89ac4b2 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-compatibility-candidates.ts @@ -0,0 +1,185 @@ +import type { + DispatchContextRow, + LegacyCompatibilityPrincipalRow, + LegacyPrincipalRole +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import { isEquivalentPaneKey } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function resolveLegacyCompatibilityPrincipalByIdentity( + this: OrchestrationDb, + params: { + runId: string + role: LegacyPrincipalRole + terminalHandle?: string + paneKey?: string + } +): LegacyCompatibilityPrincipalRow | undefined { + if (!params.terminalHandle && !params.paneKey) { + return undefined + } + const rows = ( + this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE run_id = ? AND role = ? AND status IN ('committed', 'settled') + ORDER BY rowid` + ) + .all(params.runId, params.role) as LegacyCompatibilityPrincipalRow[] + ).filter((principal) => + params.paneKey + ? isEquivalentPaneKey(principal.pane_key, params.paneKey) + : principal.terminal_handle === params.terminalHandle + ) + if (rows.length > 1) { + throw new OrchestrationError( + 'operation_unknown', + 'Multiple legacy principals match this process identity.' + ) + } + return rows[0] +} + +export function resolveLegacyWorkerCandidate( + this: OrchestrationDb, + params: { + runId?: string + terminalHandle?: string + paneKey?: string + dispatchId?: string + taskId?: string + } +): { dispatch: DispatchContextRow } | undefined { + if (!params.runId || (!params.terminalHandle && !params.paneKey)) { + return undefined + } + const rows = ( + params.dispatchId + ? [this.getDispatchContextById(params.dispatchId)].filter( + (row): row is DispatchContextRow => row !== undefined + ) + : (this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE run_id = ? AND contract_version = ? + AND status IN ('pending', 'dispatched') + ORDER BY rowid` + ) + .all(params.runId, LEGACY_CONTRACT_VERSION) as DispatchContextRow[]) + ).filter( + (dispatch) => + dispatch.run_id === params.runId && + dispatch.contract_version === LEGACY_CONTRACT_VERSION && + (!params.taskId || dispatch.task_id === params.taskId) && + (params.paneKey + ? Boolean( + dispatch.assignee_pane_key && + isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) + ) + : dispatch.assignee_handle === params.terminalHandle) + ) + if (rows.length > 1) { + throw new OrchestrationError( + 'operation_unknown', + 'Multiple active legacy Dispatches match this process identity.' + ) + } + if (params.dispatchId && rows.length === 0) { + const target = this.getDispatchContextById(params.dispatchId) + if (target?.contract_version === LEGACY_CONTRACT_VERSION) { + throw new OrchestrationError( + 'legacy_read_only', + `Dispatch ${params.dispatchId} is retained but this process cannot prove ownership.` + ) + } + } + return rows[0] ? { dispatch: rows[0] } : undefined +} + +export function resolveLegacyCoordinatorCandidate( + this: OrchestrationDb, + params: { + runId: string + terminalHandle?: string + paneKey?: string + } +): { terminalHandle: string; paneKey: string } | undefined { + if (!params.terminalHandle || !params.paneKey) { + return undefined + } + const run = this.getRunRaw(params.runId) + const principal = this.getLegacyCoordinatorPrincipal(params.runId) + if (principal) { + if ( + principal.status !== 'committed' || + principal.terminal_handle !== params.terminalHandle || + !isEquivalentPaneKey(principal.pane_key, params.paneKey) || + (run?.coordinator_pane_key !== null && + (run?.coordinator_handle !== principal.terminal_handle || + !isEquivalentPaneKey(run.coordinator_pane_key, principal.pane_key))) + ) { + return undefined + } + return { terminalHandle: params.terminalHandle, paneKey: params.paneKey } + } + // Why: the first current binding durably fences uncommitted legacy processes. + if ( + !run || + run.coordinator_pane_key !== null || + this.getUniqueLegacyCoordinatorHandle(params.runId) !== params.terminalHandle + ) { + return undefined + } + return { terminalHandle: params.terminalHandle, paneKey: params.paneKey } +} + +// Why: caller-side fence jurisdiction. Widening this fences more callers, so keep it narrow. +export function isLegacyCoordinatorHandle( + this: OrchestrationDb, + runId: string, + terminalHandle: string +): boolean { + const principal = this.getLegacyCoordinatorPrincipal(runId) + if (principal) { + return principal.terminal_handle === terminalHandle + } + return this.getUniqueLegacyCoordinatorHandle(runId) === terminalHandle +} + +// Why: recipient-side permit for legacy lifecycle mail. After a takeover revokes the old principal +// the replacement coordinator is only reachable through the Run binding. +export function isLegacyCoordinatorDeliveryTarget( + this: OrchestrationDb, + runId: string, + terminalHandle: string +): boolean { + if (this.isLegacyCoordinatorHandle(runId, terminalHandle)) { + return true + } + if (this.getRunRaw(runId)?.coordinator_handle !== terminalHandle) { + return false + } + // Why: must match resolveLegacyWorkerCoordinatorDelivery's takeover test. A still-committed + // principal routes legacy_direct to this handle, and no reader can see that mailbox. + return this.getLegacyCoordinatorPrincipal(runId)?.status !== 'committed' +} + +export type LegacyCompatibilityCandidatesMethods = { + resolveLegacyCompatibilityPrincipalByIdentity: typeof resolveLegacyCompatibilityPrincipalByIdentity + resolveLegacyWorkerCandidate: typeof resolveLegacyWorkerCandidate + resolveLegacyCoordinatorCandidate: typeof resolveLegacyCoordinatorCandidate + isLegacyCoordinatorHandle: typeof isLegacyCoordinatorHandle + isLegacyCoordinatorDeliveryTarget: typeof isLegacyCoordinatorDeliveryTarget +} + +export function attachLegacyCompatibilityCandidates(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + resolveLegacyCompatibilityPrincipalByIdentity, + resolveLegacyWorkerCandidate, + resolveLegacyCoordinatorCandidate, + isLegacyCoordinatorHandle, + isLegacyCoordinatorDeliveryTarget + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts b/src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts new file mode 100644 index 00000000000..6d2379cfbe4 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts @@ -0,0 +1,188 @@ +import type { + LegacyAdoptionRow, + LegacyCompatibilityPrincipalRow, + LegacyPrincipalRole +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { LEGACY_RUN_ID, LEGACY_CONTRACT_VERSION } from '../contract-constants' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Legacy adoption and compatibility principals ── + +export function getLegacyAdoption(this: OrchestrationDb): LegacyAdoptionRow | undefined { + return this.db + .prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?') + .get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined +} + +export function commitLegacyCompatibilityPrincipal( + this: OrchestrationDb, + params: { + runId: string + dispatchId?: string + role: LegacyPrincipalRole + hostScope: string + terminalHandle: string + paneKey: string + launchTokenHash: string + processIncarnation?: string + } +): { principal: LegacyCompatibilityPrincipalRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + const adoption = this.getLegacyAdoption() + if (!adoption || adoption.adopted_run_id !== params.runId) { + throw new OrchestrationError( + 'request_mismatch', + `Run ${params.runId} is not the adopted legacy Run.` + ) + } + const dispatchId = params.role === 'worker' ? (params.dispatchId ?? null) : null + let initialStatus: 'committed' | 'settled' = 'committed' + if (params.role === 'worker') { + const dispatch = dispatchId ? this.getDispatchContextById(dispatchId) : undefined + if ( + !dispatch || + dispatch.run_id !== params.runId || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION + ) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${dispatchId ?? '(missing)'} is not a legacy attempt in this Run.` + ) + } + initialStatus = ['pending', 'dispatched'].includes(dispatch.status) ? 'committed' : 'settled' + } else if (params.dispatchId) { + throw new OrchestrationError( + 'request_mismatch', + 'A coordinator compatibility principal cannot name a Dispatch.' + ) + } + + const existing = this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE role = ? AND run_id = ? AND dispatch_id IS ?` + ) + .get(params.role, params.runId, dispatchId) as LegacyCompatibilityPrincipalRow | undefined + if (existing) { + const same = + existing.host_scope === params.hostScope && + existing.terminal_handle === params.terminalHandle && + existing.pane_key === params.paneKey && + existing.launch_token_hash === params.launchTokenHash && + existing.process_incarnation === (params.processIncarnation ?? null) + if (!same) { + throw new OrchestrationError( + 'request_mismatch', + `The ${params.role} compatibility principal is already committed to different proof.` + ) + } + if (existing.status === 'revoked') { + throw new OrchestrationError( + 'legacy_read_only', + `The ${params.role} compatibility principal has been revoked. No effects were applied.`, + { effectsApplied: false } + ) + } + this.db.exec('COMMIT') + return { principal: existing, duplicate: true } + } + if ( + params.role === 'coordinator' && + !this.resolveLegacyCoordinatorCandidate({ + runId: params.runId, + terminalHandle: params.terminalHandle, + paneKey: params.paneKey + }) + ) { + throw new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.', + { effectsApplied: false } + ) + } + + const id = generateId('legacy_principal') + this.db + .prepare( + `INSERT INTO legacy_compatibility_principals ( + id, run_id, dispatch_id, role, host_scope, terminal_handle, + pane_key, launch_token_hash, process_incarnation, status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + id, + params.runId, + dispatchId, + params.role, + params.hostScope, + params.terminalHandle, + params.paneKey, + params.launchTokenHash, + params.processIncarnation ?? null, + initialStatus + ) + const principal = this.getLegacyCompatibilityPrincipal(id) as LegacyCompatibilityPrincipalRow + if (principal.status === 'committed') { + this.initializeLegacyRecoveryCohort(principal) + } + this.db.exec('COMMIT') + return { principal, duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getLegacyCompatibilityPrincipal( + this: OrchestrationDb, + id: string +): LegacyCompatibilityPrincipalRow | undefined { + return this.db.prepare('SELECT * FROM legacy_compatibility_principals WHERE id = ?').get(id) as + | LegacyCompatibilityPrincipalRow + | undefined +} + +export function listLegacyCompatibilityPrincipals( + this: OrchestrationDb, + runId: string +): LegacyCompatibilityPrincipalRow[] { + return this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE run_id = ? ORDER BY rowid` + ) + .all(runId) as LegacyCompatibilityPrincipalRow[] +} + +export function getLegacyCoordinatorPrincipal( + this: OrchestrationDb, + runId: string +): LegacyCompatibilityPrincipalRow | undefined { + return this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE run_id = ? AND role = 'coordinator'` + ) + .get(runId) as LegacyCompatibilityPrincipalRow | undefined +} + +export type LegacyCompatibilityPrincipalsMethods = { + getLegacyAdoption: typeof getLegacyAdoption + commitLegacyCompatibilityPrincipal: typeof commitLegacyCompatibilityPrincipal + getLegacyCompatibilityPrincipal: typeof getLegacyCompatibilityPrincipal + listLegacyCompatibilityPrincipals: typeof listLegacyCompatibilityPrincipals + getLegacyCoordinatorPrincipal: typeof getLegacyCoordinatorPrincipal +} + +export function attachLegacyCompatibilityPrincipals(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getLegacyAdoption, + commitLegacyCompatibilityPrincipal, + getLegacyCompatibilityPrincipal, + listLegacyCompatibilityPrincipals, + getLegacyCoordinatorPrincipal + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-lifecycle-operation.ts b/src/main/runtime/orchestration/db/legacy/legacy-lifecycle-operation.ts new file mode 100644 index 00000000000..793efcbcb52 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-lifecycle-operation.ts @@ -0,0 +1,196 @@ +import type { + MessageType, + MessagePriority, + MessageRow, + WorkerReportOutcome, + WorkerReportSettlement, + LegacyOperationReceiptRow +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' + +export function commitLegacyLifecycleOperation( + this: OrchestrationDb, + params: { + principalId: string + operationKey: string + method: string + payloadHash: string + message: { + existingId?: string + to: string + subject: string + body?: string + type: MessageType + priority?: MessagePriority + payload?: string + } + lifecycle: + | { kind: 'message_only' } + | { kind: 'heartbeat'; at: string } + | { + kind: 'worker_report' + taskId: string + outcome: WorkerReportOutcome + result: string + } + } +): { + receipt: LegacyOperationReceiptRow + message: MessageRow + settlement?: WorkerReportSettlement + duplicate: boolean +} { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.getLegacyCompatibilityPrincipal(params.principalId) + if ( + !principal || + principal.role !== 'worker' || + !['committed', 'settled'].includes(principal.status) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy compatibility principal ${params.principalId} cannot send lifecycle work.` + ) + } + const dispatchId = principal.dispatch_id as string + const existingReceipt = this.requireMatchingLegacyOperationReceipt(params) + if (existingReceipt) { + const response = JSON.parse(existingReceipt.response_json) as { + messageId: string + settlement?: WorkerReportSettlement + } + const message = this.getMessageById(response.messageId) + if (!message) { + throw new OrchestrationError( + 'operation_unknown', + `Legacy operation ${params.operationKey} lost its recorded message.` + ) + } + this.db.exec('COMMIT') + return { + receipt: existingReceipt, + message, + settlement: response.settlement, + duplicate: true + } + } + + const dispatch = this.getDispatchContextById(dispatchId) + if ( + !dispatch || + dispatch.run_id !== principal.run_id || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} is not this principal's legacy attempt.` + ) + } + if ( + (principal.status === 'settled' || !['pending', 'dispatched'].includes(dispatch.status)) && + (!params.message.existingId || params.lifecycle.kind !== 'worker_report') + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} is settled and only matching completion reconstruction is allowed.` + ) + } + let message = params.message.existingId + ? this.getMessageById(params.message.existingId) + : undefined + const delivery = this.resolveLegacyWorkerCoordinatorDelivery( + principal.run_id, + params.message.to + ) + if (params.message.existingId) { + const matchesOriginalLegacyRoute = + message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to + const matchesCurrentRoute = + message?.delivery_contract === delivery.contract && message.to_handle === delivery.to + if ( + !message || + message.run_id !== principal.run_id || + message.from_handle !== principal.terminal_handle || + (!matchesOriginalLegacyRoute && !matchesCurrentRoute) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Existing legacy message ${params.message.existingId} does not match this principal.` + ) + } + } else { + message = this.insertMessage({ + from: principal.terminal_handle, + to: delivery.to, + subject: params.message.subject, + body: params.message.body, + type: params.message.type, + priority: params.message.priority, + payload: params.message.payload, + senderPaneKey: principal.pane_key, + runId: principal.run_id, + deliveryContract: delivery.contract + }) + } + + let settlement: WorkerReportSettlement | undefined + if (params.lifecycle.kind === 'heartbeat') { + this.recordHeartbeat(dispatchId, params.lifecycle.at) + } else if (params.lifecycle.kind === 'worker_report') { + const persistedOutcome = + params.message.existingId && + dispatch.task_id === params.lifecycle.taskId && + dispatch.status === 'completed' + ? 'succeeded' + : params.message.existingId && + dispatch.task_id === params.lifecycle.taskId && + dispatch.status === 'failed' + ? 'failed' + : undefined + settlement = persistedOutcome + ? { action: 'settled', outcome: persistedOutcome, duplicate: true } + : this.settleWorkerReportInTransaction({ + taskId: params.lifecycle.taskId, + dispatchId, + outcome: params.lifecycle.outcome, + result: params.lifecycle.result + }) + if (settlement.action === 'rejected') { + throw new OrchestrationError(settlement.code, settlement.reason) + } + this.db + .prepare( + `UPDATE legacy_compatibility_principals + SET status = 'settled' WHERE id = ? AND status = 'committed'` + ) + .run(principal.id) + } + const responseJson = JSON.stringify({ messageId: message.id, settlement }) + const receipt = this.insertLegacyOperationReceipt({ + principalId: principal.id, + operationKey: params.operationKey, + method: params.method, + payloadHash: params.payloadHash, + effectId: message.id, + responseJson + }) + this.db.exec('COMMIT') + return { receipt, message, settlement, duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type LegacyLifecycleOperationMethods = { + commitLegacyLifecycleOperation: typeof commitLegacyLifecycleOperation +} + +export function attachLegacyLifecycleOperation(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + commitLegacyLifecycleOperation + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-mail-acknowledge.ts b/src/main/runtime/orchestration/db/legacy/legacy-mail-acknowledge.ts new file mode 100644 index 00000000000..7f77c23474c --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-mail-acknowledge.ts @@ -0,0 +1,113 @@ +import type { MessageType, MessageRow, LegacyMailReceiptRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function acknowledgeLegacyMail( + this: OrchestrationDb, + params: { + principalId: string + messageIds: string[] + types?: MessageType[] + } +): { + receipts: LegacyMailReceiptRow[] + duplicate: boolean +} { + if (params.messageIds.length === 0) { + return { receipts: [], duplicate: true } + } + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireLegacyMailPrincipal(params.principalId) + const uniqueIds = [...new Set(params.messageIds)] + const placeholders = uniqueIds.map(() => '?').join(',') + const prior = this.db + .prepare( + `SELECT COUNT(*) AS count FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id IN (${placeholders}) + AND acknowledged_at IS NOT NULL` + ) + .get(params.principalId, ...uniqueIds) as { count: number } + if (prior.count !== uniqueIds.length) { + const actionable = this.getLegacyMailPage({ + principalId: params.principalId, + limit: uniqueIds.length, + types: params.types + }).messages + if ( + actionable.length !== uniqueIds.length || + actionable.some((message, index) => message.id !== uniqueIds[index]) + ) { + throw new OrchestrationError( + 'request_mismatch', + 'Legacy mail acknowledgment does not match the current replay page.' + ) + } + } + const rows = this.db + .prepare( + `SELECT * FROM messages + WHERE id IN (${placeholders}) AND run_id = ? + AND delivery_contract = 'legacy_direct'` + ) + .all(...uniqueIds, principal.run_id) as MessageRow[] + const validIds = new Set( + rows + .filter( + (message) => + message.to_handle === principal.terminal_handle || + (principal.role === 'worker' && + message.to_handle === `dispatch:${principal.dispatch_id}`) + ) + .map((message) => message.id) + ) + if (validIds.size !== uniqueIds.length || uniqueIds.some((id) => !validIds.has(id))) { + throw new OrchestrationError( + 'request_mismatch', + 'Legacy mail acknowledgment contains a message outside this principal inbox.' + ) + } + + this.db + .prepare( + `UPDATE messages + SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) + WHERE id IN (${placeholders})` + ) + .run(...uniqueIds) + const insert = this.db.prepare( + `INSERT INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) VALUES (?, ?, datetime('now')) + ON CONFLICT(principal_id, message_id) + DO UPDATE SET acknowledged_at = COALESCE( + legacy_mail_receipts.acknowledged_at, excluded.acknowledged_at + )` + ) + for (const messageId of uniqueIds) { + insert.run(params.principalId, messageId) + } + const receipts = this.db + .prepare( + `SELECT * FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id IN (${placeholders}) + ORDER BY message_id` + ) + .all(params.principalId, ...uniqueIds) as LegacyMailReceiptRow[] + this.db.exec('COMMIT') + return { receipts, duplicate: prior.count === uniqueIds.length } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type LegacyMailAcknowledgeMethods = { + acknowledgeLegacyMail: typeof acknowledgeLegacyMail +} + +export function attachLegacyMailAcknowledge(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + acknowledgeLegacyMail + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-question-acknowledge.ts b/src/main/runtime/orchestration/db/legacy/legacy-question-acknowledge.ts new file mode 100644 index 00000000000..ec1b3484791 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-question-acknowledge.ts @@ -0,0 +1,100 @@ +import type { LegacyMailReceiptRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' + +export function acknowledgeLegacyQuestionAnswer( + this: OrchestrationDb, + params: { + principalId: string + questionId: string + answerMessageId: string + } +): { receipt: LegacyMailReceiptRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireLegacyMailPrincipal(params.principalId, 'worker') + const question = this.getQuestionRaw(params.questionId) + const source = this.getMessageById(params.questionId) + const answer = this.getMessageById(params.answerMessageId) + const dispatch = principal.dispatch_id + ? this.getDispatchContextById(principal.dispatch_id) + : undefined + const exactLegacyAnswer = + answer?.delivery_contract === 'legacy_direct' && + (answer.to_handle === principal.terminal_handle || + answer.to_handle === `dispatch:${principal.dispatch_id}`) + const adoption = this.getLegacyAdoption() + const exactTakenOverAnswer = + adoption?.adopted_run_id === principal.run_id && + dispatch?.run_id === principal.run_id && + dispatch.contract_version === LEGACY_CONTRACT_VERSION && + source?.run_id === principal.run_id && + source.from_handle === principal.terminal_handle && + source.to_handle === `run:${principal.run_id}` && + source.delivery_contract === 'current_delivery' && + answer?.run_id === principal.run_id && + answer?.delivery_contract === 'current_delivery' && + answer.from_handle === `run:${principal.run_id}` && + answer.to_handle === `dispatch:${principal.dispatch_id}` && + answer.thread_id === question?.message_id + if ( + !question || + !answer || + question.run_id !== principal.run_id || + question.dispatch_id !== principal.dispatch_id || + question.answer_message_id !== params.answerMessageId || + (!exactLegacyAnswer && !exactTakenOverAnswer) + ) { + throw new OrchestrationError( + 'request_mismatch', + 'Legacy answer acknowledgment does not match this principal question.' + ) + } + const existing = this.db + .prepare( + `SELECT * FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id = ?` + ) + .get(params.principalId, params.answerMessageId) as LegacyMailReceiptRow | undefined + this.db + .prepare( + `UPDATE messages + SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) + WHERE id = ?` + ) + .run(params.answerMessageId) + this.db + .prepare( + `INSERT INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) VALUES (?, ?, datetime('now')) + ON CONFLICT(principal_id, message_id) + DO UPDATE SET acknowledged_at = COALESCE( + legacy_mail_receipts.acknowledged_at, excluded.acknowledged_at + )` + ) + .run(params.principalId, params.answerMessageId) + const receipt = this.db + .prepare( + `SELECT * FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id = ?` + ) + .get(params.principalId, params.answerMessageId) as LegacyMailReceiptRow + this.db.exec('COMMIT') + return { receipt, duplicate: Boolean(existing?.acknowledged_at) } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type LegacyQuestionAcknowledgeMethods = { + acknowledgeLegacyQuestionAnswer: typeof acknowledgeLegacyQuestionAnswer +} + +export function attachLegacyQuestionAcknowledge(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + acknowledgeLegacyQuestionAnswer + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-question-lookup.ts b/src/main/runtime/orchestration/db/legacy/legacy-question-lookup.ts new file mode 100644 index 00000000000..5a669716696 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-question-lookup.ts @@ -0,0 +1,126 @@ +import type { MessageDeliveryContract, MessageRow, QuestionRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { exposeQuestionTimestamps } from '../utc-timestamp' +import { legacyMessageMatchesQuestion } from '../legacy-question-identity' +import type { OrchestrationDb } from '../orchestration-db' + +export function findPendingLegacyQuestions( + this: OrchestrationDb, + params: { + principalId: string + question: string + options?: string[] + recipientHandle: string + } +): { question: QuestionRow; message: MessageRow }[] { + return this.findLegacyQuestionsBySemanticIdentity(params) + .filter((row) => row.question.status === 'pending') + .map(({ question, message }) => ({ question, message })) +} + +export function findLegacyQuestionsBySemanticIdentity( + this: OrchestrationDb, + params: { + principalId: string + question: string + options?: string[] + recipientHandle: string + } +): { + question: QuestionRow + message: MessageRow + answerAcknowledged: boolean + claimedByOperation: boolean +}[] { + const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker') + const runAddress = `run:${principal.run_id}` + const rows = this.db + .prepare( + `SELECT q.*, m.id AS source_message_id, + EXISTS( + SELECT 1 FROM legacy_operation_receipts lor + WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask' + AND lor.effect_id = q.message_id + ) AS claimed_by_operation + FROM question_threads q + INNER JOIN messages m ON m.id = q.message_id + WHERE q.run_id = ? AND q.dispatch_id = ? + AND ( + (m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR + (m.delivery_contract = 'current_delivery' AND m.to_handle = ?) + ) + ORDER BY m.sequence + LIMIT 501` + ) + .all( + principal.id, + principal.run_id, + principal.dispatch_id, + params.recipientHandle, + runAddress + ) as (QuestionRow & { + source_message_id: string + claimed_by_operation: number + })[] + if (rows.length > 500) { + throw new OrchestrationError( + 'operation_unknown', + 'Legacy ask identity is too ambiguous to reconstruct safely.' + ) + } + return rows + .filter((row) => { + const message = this.getMessageById(row.source_message_id) + return Boolean( + message && + legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [ + params.recipientHandle, + runAddress + ]) + ) + }) + .map((row) => ({ + question: exposeQuestionTimestamps(row), + message: this.getMessageById(row.message_id) as MessageRow, + claimedByOperation: row.claimed_by_operation === 1, + answerAcknowledged: row.answer_message_id + ? Boolean( + this.db + .prepare( + `SELECT 1 FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id = ? + AND acknowledged_at IS NOT NULL` + ) + .get(principal.id, row.answer_message_id) + ) + : false + })) +} + +export function resolveLegacyWorkerCoordinatorDelivery( + this: OrchestrationDb, + runId: string, + retainedCoordinatorHandle: string +): { to: string; contract: MessageDeliveryContract } { + const run = this.getRunRaw(runId) + const principal = this.getLegacyCoordinatorPrincipal(runId) + // Why: `!== null` alone reads an unknown run (undefined) as taken over and misroutes it to run: delivery. + const takenOver = run?.coordinator_handle != null && principal?.status !== 'committed' + return takenOver + ? { to: `run:${runId}`, contract: 'current_delivery' } + : { to: retainedCoordinatorHandle, contract: 'legacy_direct' } +} + +export type LegacyQuestionLookupMethods = { + findPendingLegacyQuestions: typeof findPendingLegacyQuestions + findLegacyQuestionsBySemanticIdentity: typeof findLegacyQuestionsBySemanticIdentity + resolveLegacyWorkerCoordinatorDelivery: typeof resolveLegacyWorkerCoordinatorDelivery +} + +export function attachLegacyQuestionLookup(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + findPendingLegacyQuestions, + findLegacyQuestionsBySemanticIdentity, + resolveLegacyWorkerCoordinatorDelivery + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-recovery-cohort.ts b/src/main/runtime/orchestration/db/legacy/legacy-recovery-cohort.ts new file mode 100644 index 00000000000..c63cbe939c6 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-recovery-cohort.ts @@ -0,0 +1,163 @@ +import type { MessageType, MessageRow, LegacyCompatibilityPrincipalRow } from '../../types' +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import { ORCHESTRATION_DELIVERY_BATCH_LIMIT } from '../messages/mailbox-routing-page' +import { exposeMessageListTimestamps } from '../utc-timestamp' +import type { OrchestrationDb } from '../orchestration-db' + +export function initializeLegacyRecoveryCohort( + this: OrchestrationDb, + principal: LegacyCompatibilityPrincipalRow +): void { + if (principal.role === 'worker') { + this.db + .prepare( + `INSERT OR IGNORE INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) + SELECT ?, m.id, NULL + FROM messages m + INNER JOIN dispatch_contexts d ON d.id = ? + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' AND m.read = 1 + AND d.status IN ('pending', 'dispatched') + AND m.created_at >= d.created_at + AND (m.to_handle = ? OR m.to_handle = ?)` + ) + .run( + principal.id, + principal.dispatch_id, + principal.run_id, + principal.terminal_handle, + `dispatch:${principal.dispatch_id}` + ) + return + } + this.db + .prepare( + `INSERT OR IGNORE INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) + SELECT ?, m.id, NULL + FROM messages m + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' AND m.read = 1 + AND m.to_handle = ? + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.run_id = m.run_id + AND d.contract_version = ? + AND d.status IN ('pending', 'dispatched') + AND m.created_at >= d.created_at + AND (m.from_handle = d.assignee_handle OR m.from_handle = 'dispatch:' || d.id) + )` + ) + .run(principal.id, principal.run_id, principal.terminal_handle, LEGACY_CONTRACT_VERSION) +} + +export function getLegacyMailPage( + this: OrchestrationDb, + params: { principalId: string; limit?: number; types?: MessageType[] } +): { + messages: MessageRow[] + recovery: boolean +} { + const principal = this.requireLegacyMailPrincipal(params.principalId) + const limit = Math.min( + Math.max(params.limit ?? ORCHESTRATION_DELIVERY_BATCH_LIMIT, 1), + ORCHESTRATION_DELIVERY_BATCH_LIMIT + ) + const addressSql = + principal.role === 'worker' ? '(m.to_handle = ? OR m.to_handle = ?)' : 'm.to_handle = ?' + const addressParams = + principal.role === 'worker' + ? [principal.terminal_handle, `dispatch:${principal.dispatch_id}`] + : [principal.terminal_handle] + const typeSql = + params.types && params.types.length > 0 + ? `AND m.type IN (${params.types.map(() => '?').join(',')})` + : '' + const typeParams = params.types ?? [] + const recovery = this.db + .prepare( + `SELECT m.* + FROM legacy_mail_receipts r + INNER JOIN messages m ON m.id = r.message_id + WHERE r.principal_id = ? AND r.acknowledged_at IS NULL + AND m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND ${addressSql} + ${typeSql} + ORDER BY m.sequence ASC LIMIT ?` + ) + .all( + params.principalId, + principal.run_id, + ...addressParams, + ...typeParams, + limit + ) as MessageRow[] + if (recovery.length > 0) { + return { messages: exposeMessageListTimestamps(recovery), recovery: true } + } + + const unread = this.db + .prepare( + `SELECT m.* + FROM messages m + LEFT JOIN legacy_mail_receipts r + ON r.principal_id = ? AND r.message_id = m.id + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND m.read = 0 AND r.message_id IS NULL AND ${addressSql} + ${typeSql} + ORDER BY m.sequence ASC LIMIT ?` + ) + .all( + params.principalId, + principal.run_id, + ...addressParams, + ...typeParams, + limit + ) as MessageRow[] + return { messages: exposeMessageListTimestamps(unread), recovery: false } +} + +export function getLegacyMailHistory( + this: OrchestrationDb, + params: { principalId: string; limit?: number; types?: MessageType[] } +): { + messages: MessageRow[] + recovery: false +} { + const principal = this.requireLegacyMailPrincipal(params.principalId) + const limit = Math.min(Math.max(params.limit ?? 100, 1), 100) + const addressSql = + principal.role === 'worker' ? '(to_handle = ? OR to_handle = ?)' : 'to_handle = ?' + const addressParams = + principal.role === 'worker' + ? [principal.terminal_handle, `dispatch:${principal.dispatch_id}`] + : [principal.terminal_handle] + const typeSql = + params.types && params.types.length > 0 + ? `AND type IN (${params.types.map(() => '?').join(',')})` + : '' + const messages = this.db + .prepare( + `SELECT * FROM messages + WHERE run_id = ? AND delivery_contract = 'legacy_direct' + AND ${addressSql} ${typeSql} + ORDER BY sequence ASC LIMIT ?` + ) + .all(principal.run_id, ...addressParams, ...(params.types ?? []), limit) as MessageRow[] + return { messages: exposeMessageListTimestamps(messages), recovery: false } +} + +export type LegacyRecoveryCohortMethods = { + initializeLegacyRecoveryCohort: typeof initializeLegacyRecoveryCohort + getLegacyMailPage: typeof getLegacyMailPage + getLegacyMailHistory: typeof getLegacyMailHistory +} + +export function attachLegacyRecoveryCohort(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + initializeLegacyRecoveryCohort, + getLegacyMailPage, + getLegacyMailHistory + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-reply-operation.ts b/src/main/runtime/orchestration/db/legacy/legacy-reply-operation.ts new file mode 100644 index 00000000000..9c5dff27afd --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-reply-operation.ts @@ -0,0 +1,191 @@ +import type { MessageRow, LegacyOperationReceiptRow, QuestionRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' + +export function commitLegacyReplyOperation( + this: OrchestrationDb, + params: { + principalId: string + operationKey: string + method: string + payloadHash: string + questionId: string + body: string + } +): { + receipt: LegacyOperationReceiptRow + question: QuestionRow + message: MessageRow + duplicate: boolean +} { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'coordinator') + const receipt = this.requireMatchingLegacyOperationReceipt(params) + if (receipt) { + const response = JSON.parse(receipt.response_json) as { + questionId: string + messageId: string + } + const question = this.getQuestion(response.questionId) + const message = this.getMessageById(response.messageId) + if (!question || !message) { + throw new OrchestrationError( + 'operation_unknown', + `Legacy reply ${params.operationKey} lost its durable effect.` + ) + } + this.db.exec('COMMIT') + return { receipt, question, message, duplicate: true } + } + + const question = this.getQuestionRaw(params.questionId) + const sourceMessage = this.getMessageById(params.questionId) + const dispatch = question ? this.getDispatchContextById(question.dispatch_id) : undefined + if ( + !question || + !sourceMessage || + !dispatch || + question.run_id !== principal.run_id || + sourceMessage.delivery_contract !== 'legacy_direct' || + dispatch.run_id !== principal.run_id || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION || + question.status === 'closed' + ) { + throw new OrchestrationError( + 'question_not_found', + `Question ${params.questionId} is not actionable in the adopted Run.` + ) + } + let message: MessageRow + if (question.status === 'answered') { + if (question.answer_body !== params.body || !question.answer_message_id) { + throw new OrchestrationError( + 'answer_conflict', + `Question ${params.questionId} already has a different answer.` + ) + } + message = this.getMessageById(question.answer_message_id) as MessageRow + if ( + !message || + message.run_id !== principal.run_id || + message.delivery_contract !== 'legacy_direct' + ) { + throw new OrchestrationError( + 'operation_unknown', + `Question ${params.questionId} lost its recorded answer message.` + ) + } + } else { + message = this.insertMessage({ + from: principal.terminal_handle, + to: question.asker_handle, + subject: 'Re: Question', + body: params.body, + threadId: question.message_id, + runId: principal.run_id, + deliveryContract: 'legacy_direct' + }) + this.markAsRead([question.message_id]) + this.db + .prepare( + `UPDATE question_threads + SET status = 'answered', answer_message_id = ?, answer_body = ?, + answered_at = datetime('now') + WHERE message_id = ? AND status = 'pending'` + ) + .run(message.id, params.body, question.message_id) + } + + const answered = this.getQuestion(params.questionId) as QuestionRow + const committedReceipt = this.insertLegacyOperationReceipt({ + principalId: principal.id, + operationKey: params.operationKey, + method: params.method, + payloadHash: params.payloadHash, + effectId: message.id, + responseJson: JSON.stringify({ + questionId: answered.message_id, + messageId: message.id + }) + }) + this.db.exec('COMMIT') + return { + receipt: committedReceipt, + question: answered, + message, + duplicate: question.status === 'answered' + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function requireMatchingLegacyOperationReceipt( + this: OrchestrationDb, + params: { + principalId: string + operationKey: string + method: string + payloadHash: string + } +): LegacyOperationReceiptRow | undefined { + const receipt = this.getLegacyOperationReceipt(params.principalId, params.operationKey) + if ( + receipt && + (receipt.method !== params.method || receipt.payload_hash !== params.payloadHash) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy operation ${params.operationKey} was already used with different input.` + ) + } + return receipt +} + +export function insertLegacyOperationReceipt( + this: OrchestrationDb, + params: { + principalId: string + operationKey: string + method: string + payloadHash: string + effectId: string + responseJson: string + } +): LegacyOperationReceiptRow { + this.db + .prepare( + `INSERT INTO legacy_operation_receipts ( + principal_id, operation_key, method, payload_hash, effect_id, response_json + ) VALUES (?, ?, ?, ?, ?, ?)` + ) + .run( + params.principalId, + params.operationKey, + params.method, + params.payloadHash, + params.effectId, + params.responseJson + ) + return this.getLegacyOperationReceipt( + params.principalId, + params.operationKey + ) as LegacyOperationReceiptRow +} + +export type LegacyReplyOperationMethods = { + commitLegacyReplyOperation: typeof commitLegacyReplyOperation + requireMatchingLegacyOperationReceipt: typeof requireMatchingLegacyOperationReceipt + insertLegacyOperationReceipt: typeof insertLegacyOperationReceipt +} + +export function attachLegacyReplyOperation(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + commitLegacyReplyOperation, + requireMatchingLegacyOperationReceipt, + insertLegacyOperationReceipt + }) +} diff --git a/src/main/runtime/orchestration/db/legacy/legacy-worker-completion.ts b/src/main/runtime/orchestration/db/legacy/legacy-worker-completion.ts new file mode 100644 index 00000000000..c86b3153057 --- /dev/null +++ b/src/main/runtime/orchestration/db/legacy/legacy-worker-completion.ts @@ -0,0 +1,161 @@ +import type { + MessageRow, + LegacyCompatibilityPrincipalRow, + LegacyPrincipalRole, + LegacyOperationReceiptRow +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { exposeMessageTimestamps } from '../utc-timestamp' +import type { OrchestrationDb } from '../orchestration-db' + +export function findLegacyWorkerCompletion( + this: OrchestrationDb, + params: { + principalId: string + taskId: string + recipientHandle: string + subject: string + body: string + payload: string | null + } +): MessageRow | undefined { + const principal = this.getLegacyCompatibilityPrincipal(params.principalId) + if (!principal || principal.role !== 'worker' || !principal.dispatch_id) { + throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.') + } + const runAddress = `run:${principal.run_id}` + const rows = this.db + .prepare( + `SELECT * FROM messages + WHERE run_id = ? + AND ( + (delivery_contract = 'legacy_direct' AND to_handle = ?) OR + (delivery_contract = 'current_delivery' AND to_handle = ?) + ) + AND from_handle = ? AND type = 'worker_done' + AND subject = ? AND body = ? AND payload IS ? + ORDER BY sequence` + ) + .all( + principal.run_id, + params.recipientHandle, + runAddress, + principal.terminal_handle, + params.subject, + params.body, + params.payload + ) as MessageRow[] + const matches = rows.filter((message) => { + try { + const payload = JSON.parse(message.payload ?? '{}') as { + taskId?: unknown + dispatchId?: unknown + } + return payload.taskId === params.taskId && payload.dispatchId === principal.dispatch_id + } catch { + return false + } + }) + if (matches.length > 1) { + throw new OrchestrationError( + 'operation_unknown', + 'Multiple matching legacy worker completions exist.' + ) + } + return matches[0] ? exposeMessageTimestamps(matches[0]) : undefined +} + +export function hasPendingCurrentDelivery(this: OrchestrationDb, runId: string): boolean { + return Boolean( + this.db + .prepare( + `SELECT 1 FROM messages + WHERE run_id = ? AND to_handle = ? + AND delivery_contract = 'current_delivery' AND read = 0 + LIMIT 1` + ) + .get(runId, `run:${runId}`) + ) +} + +export function setLegacyCompatibilityPrincipalStatus( + this: OrchestrationDb, + id: string, + status: 'settled' | 'revoked' +): LegacyCompatibilityPrincipalRow | undefined { + this.db + .prepare( + `UPDATE legacy_compatibility_principals + SET status = ? + WHERE id = ? AND status = 'committed'` + ) + .run(status, id) + return this.getLegacyCompatibilityPrincipal(id) +} + +export function getLegacyOperationReceipt( + this: OrchestrationDb, + principalId: string, + operationKey: string +): LegacyOperationReceiptRow | undefined { + return this.db + .prepare( + `SELECT * FROM legacy_operation_receipts + WHERE principal_id = ? AND operation_key = ?` + ) + .get(principalId, operationKey) as LegacyOperationReceiptRow | undefined +} + +export function requireCommittedLegacyPrincipal( + this: OrchestrationDb, + principalId: string, + role?: LegacyPrincipalRole +): LegacyCompatibilityPrincipalRow { + const principal = this.getLegacyCompatibilityPrincipal(principalId) + if (!principal || principal.status !== 'committed' || (role && principal.role !== role)) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy compatibility principal ${principalId} is not committed for this operation.` + ) + } + return principal +} + +export function requireLegacyMailPrincipal( + this: OrchestrationDb, + principalId: string, + role?: LegacyPrincipalRole +): LegacyCompatibilityPrincipalRow { + const principal = this.getLegacyCompatibilityPrincipal(principalId) + if ( + !principal || + !['committed', 'settled'].includes(principal.status) || + (role && principal.role !== role) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy compatibility principal ${principalId} cannot access retained mail.` + ) + } + return principal +} + +export type LegacyWorkerCompletionMethods = { + findLegacyWorkerCompletion: typeof findLegacyWorkerCompletion + hasPendingCurrentDelivery: typeof hasPendingCurrentDelivery + setLegacyCompatibilityPrincipalStatus: typeof setLegacyCompatibilityPrincipalStatus + getLegacyOperationReceipt: typeof getLegacyOperationReceipt + requireCommittedLegacyPrincipal: typeof requireCommittedLegacyPrincipal + requireLegacyMailPrincipal: typeof requireLegacyMailPrincipal +} + +export function attachLegacyWorkerCompletion(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + findLegacyWorkerCompletion, + hasPendingCurrentDelivery, + setLegacyCompatibilityPrincipalStatus, + getLegacyOperationReceipt, + requireCommittedLegacyPrincipal, + requireLegacyMailPrincipal + }) +} diff --git a/src/main/runtime/orchestration/db/lifecycle-rejection-marker.test.ts b/src/main/runtime/orchestration/db/lifecycle-rejection-marker.test.ts new file mode 100644 index 00000000000..72152e781e4 --- /dev/null +++ b/src/main/runtime/orchestration/db/lifecycle-rejection-marker.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { + addLifecycleRejectionMarker, + hasLifecycleRejectionMarker +} from './lifecycle-rejection-marker' + +describe('lifecycle-rejection-marker', () => { + it('adds a marker onto an object payload and recognizes it', () => { + const marked = addLifecycleRejectionMarker('{"keep":true}', 'rejected', 'not current') + expect(JSON.parse(marked)).toMatchObject({ + keep: true, + _orcaLifecycleRejection: { code: 'rejected', reason: 'not current' } + }) + expect(hasLifecycleRejectionMarker(marked)).toBe(true) + }) + + it('treats missing, non-object, and incomplete markers as absent', () => { + expect(hasLifecycleRejectionMarker(null)).toBe(false) + expect(hasLifecycleRejectionMarker('[]')).toBe(false) + expect(hasLifecycleRejectionMarker('{"_orcaLifecycleRejection":{"code":1}}')).toBe(false) + expect(hasLifecycleRejectionMarker('not-json')).toBe(false) + }) +}) diff --git a/src/main/runtime/orchestration/db/lifecycle-rejection-marker.ts b/src/main/runtime/orchestration/db/lifecycle-rejection-marker.ts new file mode 100644 index 00000000000..3346fd3b49c --- /dev/null +++ b/src/main/runtime/orchestration/db/lifecycle-rejection-marker.ts @@ -0,0 +1,38 @@ +export function addLifecycleRejectionMarker( + payload: string | null, + code: string, + reason: string +): string { + let parsed: Record = {} + try { + const value: unknown = payload ? JSON.parse(payload) : {} + if (value && typeof value === 'object' && !Array.isArray(value)) { + parsed = value as Record + } + } catch { + // Authority reconciliation only reaches this path with object payloads. + } + return JSON.stringify({ + ...parsed, + _orcaLifecycleRejection: { code, reason } + }) +} + +export function hasLifecycleRejectionMarker(payload: string | null): boolean { + try { + const value: unknown = JSON.parse(payload ?? 'null') + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return false + } + const marker = (value as Record)._orcaLifecycleRejection + return Boolean( + marker && + typeof marker === 'object' && + !Array.isArray(marker) && + typeof (marker as Record).code === 'string' && + typeof (marker as Record).reason === 'string' + ) + } catch { + return false + } +} diff --git a/src/main/runtime/orchestration/db/messages/direct-mailbox-routing.ts b/src/main/runtime/orchestration/db/messages/direct-mailbox-routing.ts new file mode 100644 index 00000000000..aa7d93fbea2 --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/direct-mailbox-routing.ts @@ -0,0 +1,198 @@ +import type { MessageType } from '../../types' +import type { OrchestrationDb } from '../orchestration-db' +import { ORCHESTRATION_DELIVERY_BATCH_LIMIT, type MailboxRoutingPage } from './mailbox-routing-page' + +export function hasUndeliveredDirectMessageForRun( + this: OrchestrationDb, + runId: string, + directHandle: string +): boolean { + return Boolean( + this.db + .prepare( + `SELECT 1 FROM messages INDEXED BY idx_messages_undelivered_direct_run + WHERE run_id = ? AND to_handle = ? AND read = 0 AND delivered_at IS NULL + AND delivery_contract = 'current_delivery' + LIMIT 1` + ) + .get(runId, directHandle) + ) +} + +export function getLatestUnreadDirectMessageSequenceForRun( + this: OrchestrationDb, + runId: string, + directHandle: string +): number | undefined { + const row = this.db + .prepare( + `SELECT sequence FROM messages + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' + ORDER BY sequence DESC LIMIT 1` + ) + .get(runId, directHandle) as { sequence: number } | undefined + return row?.sequence +} + +// Why: change mailbox ownership without changing unread or acknowledgment state. +export function routeDirectMessagePage( + this: OrchestrationDb, + mailboxHandle: string, + runId: string, + directHandle: string, + throughSequence?: number, + preserveActiveDispatchOwnership = false +): MailboxRoutingPage { + this.db.exec('BEGIN IMMEDIATE') + try { + const throughClause = throughSequence === undefined ? '' : ' AND sequence <= ?' + const dispatchOwnershipClause = preserveActiveDispatchOwnership + ? ` AND NOT EXISTS ( + SELECT 1 FROM dispatch_contexts + WHERE dispatch_contexts.run_id = messages.run_id + AND dispatch_contexts.assignee_handle = messages.to_handle + AND dispatch_contexts.status IN ('pending', 'dispatched') + )` + : '' + const params: (string | number)[] = [runId, directHandle] + if (throughSequence !== undefined) { + params.push(throughSequence) + } + params.push(ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1) + const rows = this.db + .prepare( + `SELECT id, type FROM messages + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery'${throughClause}${dispatchOwnershipClause} + ORDER BY sequence LIMIT ?` + ) + .all(...params) as { id: string; type: MessageType }[] + const page = rows.slice(0, ORCHESTRATION_DELIVERY_BATCH_LIMIT) + if (page.length === 0) { + this.db.exec('COMMIT') + return { routedCount: 0, hasMore: false, types: [] } + } + const placeholders = page.map(() => '?').join(',') + const result = this.db + .prepare( + `UPDATE messages INDEXED BY idx_messages_id SET to_handle = ? + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' AND id IN (${placeholders})` + ) + .run(mailboxHandle, runId, directHandle, ...page.map((row) => row.id)) + this.db.exec('COMMIT') + return { + routedCount: Number(result.changes), + hasMore: rows.length > ORCHESTRATION_DELIVERY_BATCH_LIMIT, + types: [...new Set(page.map((row) => row.type))] + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function routeUnreadDirectMessagesToRunMailbox( + this: OrchestrationDb, + runId: string, + directHandle: string, + throughSequence?: number +): MailboxRoutingPage { + return this.routeDirectMessagePage(`run:${runId}`, runId, directHandle, throughSequence, true) +} + +export function routeUnreadDirectMessagesToDispatchMailbox( + this: OrchestrationDb, + dispatchId: string, + runId: string, + directHandle: string, + throughSequence?: number +): MailboxRoutingPage { + return this.routeDirectMessagePage(`dispatch:${dispatchId}`, runId, directHandle, throughSequence) +} + +export function routeUnreadDispatchMailboxToRunMailbox( + this: OrchestrationDb, + dispatchId: string, + runId: string, + throughSequence?: number +): MailboxRoutingPage { + const dispatchMailbox = `dispatch:${dispatchId}` + this.db.exec('BEGIN IMMEDIATE') + try { + const throughClause = throughSequence === undefined ? '' : ' AND sequence <= ?' + const params: (string | number)[] = [runId, dispatchMailbox] + if (throughSequence !== undefined) { + params.push(throughSequence) + } + params.push(ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1) + const rows = this.db + .prepare( + `SELECT id, type FROM messages INDEXED BY idx_messages_unread_current_inbox + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery'${throughClause} + ORDER BY sequence LIMIT ?` + ) + .all(...params) as { id: string; type: MessageType }[] + const page = rows.slice(0, ORCHESTRATION_DELIVERY_BATCH_LIMIT) + if (page.length === 0) { + this.db.exec('COMMIT') + return { routedCount: 0, hasMore: false, types: [] } + } + const placeholders = page.map(() => '?').join(',') + const result = this.db + .prepare( + `UPDATE messages INDEXED BY idx_messages_id SET to_handle = ? + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' + AND id IN (${placeholders})` + ) + .run(`run:${runId}`, runId, dispatchMailbox, ...page.map((row) => row.id)) + this.db.exec('COMMIT') + return { + routedCount: Number(result.changes), + hasMore: rows.length > ORCHESTRATION_DELIVERY_BATCH_LIMIT, + types: [...new Set(page.map((row) => row.type))] + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getLatestUnreadMessageSequence( + this: OrchestrationDb, + mailboxHandle: string +): number | undefined { + const row = this.db + .prepare( + `SELECT sequence FROM messages INDEXED BY idx_messages_unread_current_inbox + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + ORDER BY sequence DESC LIMIT 1` + ) + .get(mailboxHandle) as { sequence: number } | undefined + return row?.sequence +} + +export type DirectMailboxRoutingMethods = { + hasUndeliveredDirectMessageForRun: typeof hasUndeliveredDirectMessageForRun + getLatestUnreadDirectMessageSequenceForRun: typeof getLatestUnreadDirectMessageSequenceForRun + routeDirectMessagePage: typeof routeDirectMessagePage + routeUnreadDirectMessagesToRunMailbox: typeof routeUnreadDirectMessagesToRunMailbox + routeUnreadDirectMessagesToDispatchMailbox: typeof routeUnreadDirectMessagesToDispatchMailbox + routeUnreadDispatchMailboxToRunMailbox: typeof routeUnreadDispatchMailboxToRunMailbox + getLatestUnreadMessageSequence: typeof getLatestUnreadMessageSequence +} + +export function attachDirectMailboxRouting(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + hasUndeliveredDirectMessageForRun, + getLatestUnreadDirectMessageSequenceForRun, + routeDirectMessagePage, + routeUnreadDirectMessagesToRunMailbox, + routeUnreadDirectMessagesToDispatchMailbox, + routeUnreadDispatchMailboxToRunMailbox, + getLatestUnreadMessageSequence + }) +} diff --git a/src/main/runtime/orchestration/db/messages/foreign-direct-mailbox-routing.ts b/src/main/runtime/orchestration/db/messages/foreign-direct-mailbox-routing.ts new file mode 100644 index 00000000000..1b5afd1fbcb --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/foreign-direct-mailbox-routing.ts @@ -0,0 +1,197 @@ +import { parsePaneKey } from '../../../../../shared/stable-pane-id' +import type { DispatchContextRow, MessageType } from '../../types' +import { DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL, paneKeyMatchSuffix } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' +import { + ORCHESTRATION_DELIVERY_BATCH_LIMIT, + type ForeignDirectMailboxRoutingPage +} from './mailbox-routing-page' + +export function findActiveDispatchForDirectMessageOwner( + this: OrchestrationDb, + runId: string, + directHandle: string, + paneKey?: string +): DispatchContextRow | undefined { + const exact = this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE run_id = ? AND assignee_handle = ? AND status IN ('pending', 'dispatched') + ORDER BY rowid DESC LIMIT 1` + ) + .get(runId, directHandle) as DispatchContextRow | undefined + if (exact || !paneKey || !parsePaneKey(paneKey)) { + return exact + } + return this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE run_id = ? AND assignee_pane_key IS NOT NULL + AND status IN ('pending', 'dispatched') AND instr(assignee_pane_key, ':') > 1 + AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ? + ORDER BY rowid DESC LIMIT 1` + ) + .get(runId, paneKeyMatchSuffix(paneKey)) as DispatchContextRow | undefined +} + +export function routeForeignDirectMessagesToOwnedMailboxes( + this: OrchestrationDb, + directHandle: string, + currentRunId?: string, + paneKey?: string +): ForeignDirectMailboxRoutingPage { + this.db.exec('BEGIN IMMEDIATE') + try { + const runExclusion = currentRunId === undefined ? '' : ' AND candidate.run_id <> ?' + const paneSuffix = paneKey && parsePaneKey(paneKey) ? paneKeyMatchSuffix(paneKey) : undefined + const exclusionParams = currentRunId === undefined ? [] : [currentRunId] + const branches = [ + `SELECT candidate.id, candidate.run_id, candidate.type, candidate.sequence + FROM run_coordinator_handles AS coordinator + JOIN runs AS owner_run ON owner_run.id = coordinator.run_id AND owner_run.legacy = 0 + JOIN messages AS candidate INDEXED BY idx_messages_undelivered_direct_run + ON candidate.run_id = coordinator.run_id + AND candidate.to_handle = coordinator.terminal_handle + WHERE coordinator.terminal_handle = ?${runExclusion} + AND candidate.read = 0 AND candidate.delivered_at IS NULL + AND candidate.delivery_contract = 'current_delivery'`, + `SELECT candidate.id, candidate.run_id, candidate.type, candidate.sequence + FROM ( + SELECT direct_dispatch.run_id + FROM dispatch_contexts AS direct_dispatch + INDEXED BY idx_dispatch_active_assignee_handle + JOIN runs AS owner_run + ON owner_run.id = direct_dispatch.run_id AND owner_run.legacy = 0 + WHERE direct_dispatch.assignee_handle = ? + AND direct_dispatch.status IN ('pending', 'dispatched') + GROUP BY direct_dispatch.run_id + ) AS dispatch_owner + JOIN messages AS candidate INDEXED BY idx_messages_undelivered_direct_run + ON candidate.run_id = dispatch_owner.run_id AND candidate.to_handle = ? + WHERE 1 = 1${runExclusion} + AND candidate.read = 0 AND candidate.delivered_at IS NULL + AND candidate.delivery_contract = 'current_delivery'` + ] + const branchParams: (string | number)[][] = [ + [directHandle, ...exclusionParams, ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1], + [directHandle, directHandle, ...exclusionParams, ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1] + ] + if (paneSuffix !== undefined) { + branches.push( + `SELECT candidate.id, candidate.run_id, candidate.type, candidate.sequence + FROM ( + SELECT pane_dispatch.run_id + FROM dispatch_contexts AS pane_dispatch INDEXED BY idx_dispatch_assignee_pane_leaf + JOIN runs AS owner_run + ON owner_run.id = pane_dispatch.run_id AND owner_run.legacy = 0 + WHERE pane_dispatch.assignee_pane_key IS NOT NULL + AND pane_dispatch.status IN ('pending', 'dispatched') + AND instr(pane_dispatch.assignee_pane_key, ':') > 1 + AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ? + GROUP BY pane_dispatch.run_id + ) AS pane_owner + JOIN messages AS candidate INDEXED BY idx_messages_undelivered_direct_run + ON candidate.run_id = pane_owner.run_id AND candidate.to_handle = ? + WHERE 1 = 1${runExclusion} + AND candidate.read = 0 AND candidate.delivered_at IS NULL + AND candidate.delivery_contract = 'current_delivery'` + ) + branchParams.push([ + paneSuffix, + directHandle, + ...exclusionParams, + ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1 + ]) + } + const limitedBranches = branches.map( + (branch) => `SELECT * FROM (${branch} ORDER BY candidate.sequence LIMIT ?)` + ) + const rows = this.db + .prepare( + `SELECT id, run_id, type FROM (${limitedBranches.join(' UNION ')}) + ORDER BY sequence LIMIT ?` + ) + .all(...branchParams.flat(), ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1) as { + id: string + run_id: string + type: MessageType + }[] + const page = rows.slice(0, ORCHESTRATION_DELIVERY_BATCH_LIMIT) + if (page.length === 0) { + this.db.exec('COMMIT') + return { routedCount: 0, hasMore: false, types: [], mailboxes: [] } + } + const runIds = [...new Set(page.map((row) => row.run_id))] + const dispatchByRun = new Map() + for (const runId of runIds) { + const dispatch = this.findActiveDispatchForDirectMessageOwner(runId, directHandle, paneKey) + if (dispatch) { + dispatchByRun.set(runId, dispatch) + } + } + const idsByMailbox = new Map() + const byMailbox = new Map>() + for (const row of page) { + const dispatch = dispatchByRun.get(row.run_id) + const mailboxHandle = dispatch ? `dispatch:${dispatch.id}` : `run:${row.run_id}` + const ids = idsByMailbox.get(mailboxHandle) ?? [] + ids.push(row.id) + idsByMailbox.set(mailboxHandle, ids) + const types = byMailbox.get(mailboxHandle) ?? new Set() + types.add(row.type) + byMailbox.set(mailboxHandle, types) + } + for (const [mailboxHandle, ids] of idsByMailbox) { + const placeholders = ids.map(() => '?').join(',') + this.db + .prepare( + `UPDATE messages INDEXED BY idx_messages_id SET to_handle = ? + WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL + AND delivery_contract = 'current_delivery' AND id IN (${placeholders})` + ) + .run(mailboxHandle, directHandle, ...ids) + } + this.db.exec('COMMIT') + return { + routedCount: page.length, + hasMore: rows.length > ORCHESTRATION_DELIVERY_BATCH_LIMIT, + types: [...new Set(page.map((row) => row.type))], + mailboxes: [...byMailbox].map(([mailboxHandle, types]) => ({ + mailboxHandle, + types: [...types] + })) + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getUnreadDirectMessageTypes( + this: OrchestrationDb, + directHandle: string +): MessageType[] { + return ( + this.db + .prepare( + `SELECT DISTINCT type FROM messages INDEXED BY idx_messages_unread_current_inbox_type + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + ORDER BY type` + ) + .all(directHandle) as { type: MessageType }[] + ).map((row) => row.type) +} + +export type ForeignDirectMailboxRoutingMethods = { + findActiveDispatchForDirectMessageOwner: typeof findActiveDispatchForDirectMessageOwner + routeForeignDirectMessagesToOwnedMailboxes: typeof routeForeignDirectMessagesToOwnedMailboxes + getUnreadDirectMessageTypes: typeof getUnreadDirectMessageTypes +} + +export function attachForeignDirectMailboxRouting(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + findActiveDispatchForDirectMessageOwner, + routeForeignDirectMessagesToOwnedMailboxes, + getUnreadDirectMessageTypes + }) +} diff --git a/src/main/runtime/orchestration/db/messages/mailbox-routing-page.ts b/src/main/runtime/orchestration/db/messages/mailbox-routing-page.ts new file mode 100644 index 00000000000..c82affd7593 --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/mailbox-routing-page.ts @@ -0,0 +1,13 @@ +import type { MessageType } from '../../types' + +export const ORCHESTRATION_DELIVERY_BATCH_LIMIT = 50 + +export type MailboxRoutingPage = { + routedCount: number + hasMore: boolean + types: MessageType[] +} + +export type ForeignDirectMailboxRoutingPage = MailboxRoutingPage & { + mailboxes: { mailboxHandle: string; types: MessageType[] }[] +} diff --git a/src/main/runtime/orchestration/db/messages/message-inbox.ts b/src/main/runtime/orchestration/db/messages/message-inbox.ts new file mode 100644 index 00000000000..e5d03eea771 --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/message-inbox.ts @@ -0,0 +1,293 @@ +import type { MessageType, MessageRow } from '../../types' +import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp' +import { addLifecycleRejectionMarker } from '../lifecycle-rejection-marker' +import type { OrchestrationDb } from '../orchestration-db' + +const MESSAGE_ID_UPDATE_BATCH_SIZE = 500 +const MESSAGE_MUTATION_SAVEPOINT = 'message_id_mutation' + +function runBatchedMessageMutation( + db: OrchestrationDb, + ids: string[], + sqlForPlaceholders: (placeholders: string) => string +): void { + if (ids.length === 0) { + return + } + db.db.exec(`SAVEPOINT ${MESSAGE_MUTATION_SAVEPOINT}`) + try { + for (let offset = 0; offset < ids.length; offset += MESSAGE_ID_UPDATE_BATCH_SIZE) { + const batch = ids.slice(offset, offset + MESSAGE_ID_UPDATE_BATCH_SIZE) + const placeholders = batch.map(() => '?').join(',') + db.db.prepare(sqlForPlaceholders(placeholders)).run(...batch) + } + db.db.exec(`RELEASE ${MESSAGE_MUTATION_SAVEPOINT}`) + } catch (error) { + db.db.exec(`ROLLBACK TO ${MESSAGE_MUTATION_SAVEPOINT}`) + db.db.exec(`RELEASE ${MESSAGE_MUTATION_SAVEPOINT}`) + throw error + } +} + +export function getUnreadMessages( + this: OrchestrationDb, + toHandle: string, + types?: MessageType[] +): MessageRow[] { + if (types && types.length > 0) { + const placeholders = types.map(() => '?').join(',') + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + AND type IN (${placeholders}) ORDER BY sequence` + ) + .all(toHandle, ...types) as MessageRow[] + ) + } + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + ORDER BY sequence` + ) + .all(toHandle) as MessageRow[] + ) +} + +export function convertLifecycleMessageToRejection( + this: OrchestrationDb, + messageId: string, + code: string, + reason: string +): MessageRow | undefined { + const message = this.getMessageById(messageId) + if (!message || (message.type !== 'worker_done' && message.type !== 'heartbeat')) { + return message + } + + const originalBody = message.body ? `\n\nOriginal body:\n${message.body}` : '' + const body = `Orca rejected this ${message.type}: ${reason}${originalBody}` + const payload = addLifecycleRejectionMarker(message.payload, code, reason) + // Why: rejected lifecycle signals stay auditable but must not reach read paths as actionable completion/liveness events. + this.db + .prepare( + `UPDATE messages + SET priority = 'high', subject = ?, body = ?, payload = ? + WHERE id = ?` + ) + .run(`Rejected ${message.type}: ${message.subject}`, body, payload, messageId) + return this.getMessageById(messageId) +} + +// Why: delivered_at IS NULL filter — push-on-idle delivers each row at most once; read (set only by check) wouldn't prevent replay. +export function getUndeliveredUnreadMessages( + this: OrchestrationDb, + toHandle: string, + types?: MessageType[], + options?: { excludeTypes?: readonly string[]; limit?: number } +): MessageRow[] { + const conditions = [ + 'to_handle = ?', + 'read = 0', + 'delivered_at IS NULL', + "delivery_contract = 'current_delivery'" + ] + const params: (string | number)[] = [toHandle] + if (types?.length) { + conditions.push(`type IN (${types.map(() => '?').join(',')})`) + params.push(...types) + } + if (options?.excludeTypes?.length) { + conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`) + params.push(...options.excludeTypes) + } + const limitSql = options?.limit === undefined ? '' : ' LIMIT ?' + if (options?.limit !== undefined) { + params.push(Math.max(1, Math.floor(options.limit))) + } + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages + WHERE ${conditions.join(' AND ')} + ORDER BY sequence${limitSql}` + ) + .all(...params) as MessageRow[] + ) +} + +export function getUndeliveredUnreadMailboxHandles(this: OrchestrationDb): string[] { + return ( + this.db + .prepare( + `SELECT DISTINCT to_handle FROM messages + WHERE read = 0 AND delivered_at IS NULL + AND delivery_contract = 'current_delivery'` + ) + .all() as { to_handle: string }[] + ).map((row) => row.to_handle) +} + +export function getAllMessages(this: OrchestrationDb, toHandle: string, limit = 20): MessageRow[] { + return exposeMessageListTimestamps( + this.db + .prepare('SELECT * FROM messages WHERE to_handle = ? ORDER BY sequence DESC LIMIT ?') + .all(toHandle, limit) as MessageRow[] + ) +} + +export function getMessageById(this: OrchestrationDb, id: string): MessageRow | undefined { + const message = this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as + | MessageRow + | undefined + return message ? exposeMessageTimestamps(message) : undefined +} + +export function markAsRead(this: OrchestrationDb, ids: string[]): void { + runBatchedMessageMutation( + this, + ids, + (placeholders) => `UPDATE messages SET read = 1 WHERE id IN (${placeholders})` + ) +} + +// Why: use datetime('now') so delivered_at matches the space-format UTC shape of the table's other timestamps for correct ordering (§3.2). +export function markAsDelivered(this: OrchestrationDb, ids: string[]): void { + runBatchedMessageMutation( + this, + ids, + (placeholders) => + `UPDATE messages SET delivered_at = datetime('now') WHERE id IN (${placeholders})` + ) +} + +export function markAsUndelivered(this: OrchestrationDb, ids: string[]): void { + runBatchedMessageMutation( + this, + ids, + (placeholders) => + `UPDATE messages SET delivered_at = NULL + WHERE read = 0 AND id IN (${placeholders})` + ) +} + +export function areUnreadMessages(this: OrchestrationDb, toHandle: string, ids: string[]): boolean { + let matched = 0 + for (let offset = 0; offset < ids.length; offset += MESSAGE_ID_UPDATE_BATCH_SIZE) { + const batch = ids.slice(offset, offset + MESSAGE_ID_UPDATE_BATCH_SIZE) + const placeholders = batch.map(() => '?').join(',') + const row = this.db + .prepare( + `SELECT COUNT(*) AS count FROM messages INDEXED BY idx_messages_id + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + AND id IN (${placeholders})` + ) + .get(toHandle, ...batch) as { count: number } + matched += row.count + } + return matched === ids.length +} + +// Why: superseded lifecycle messages stay in history but must not be consumed or injected after their dispatch finished. +export function markAsReadAndDelivered(this: OrchestrationDb, ids: string[]): void { + runBatchedMessageMutation( + this, + ids, + (placeholders) => + `UPDATE messages SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) WHERE id IN (${placeholders})` + ) +} + +export function getInbox(this: OrchestrationDb, limit = 20): MessageRow[] { + return exposeMessageListTimestamps( + this.db + .prepare('SELECT * FROM messages ORDER BY sequence DESC LIMIT ?') + .all(limit) as MessageRow[] + ) +} + +// Why: read-only history for a handle — returns every message regardless of read/delivered state, never flips the read bit (§3.3). +export function getAllMessagesForHandle( + this: OrchestrationDb, + toHandle: string, + limit = 100, + types?: MessageType[] +): MessageRow[] { + if (types && types.length > 0) { + const placeholders = types.map(() => '?').join(',') + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages WHERE to_handle = ? AND type IN (${placeholders}) ORDER BY sequence DESC LIMIT ?` + ) + .all(toHandle, ...types, limit) as MessageRow[] + ) + } + return exposeMessageListTimestamps( + this.db + .prepare('SELECT * FROM messages WHERE to_handle = ? ORDER BY sequence DESC LIMIT ?') + .all(toHandle, limit) as MessageRow[] + ) +} + +// Why: ask wait-loop read — to_handle filter shows only replies to the worker; afterSequence resumes past its own outbound ask. +export function getThreadMessagesFor( + this: OrchestrationDb, + threadId: string, + toHandle: string, + afterSequence?: number +): MessageRow[] { + if (afterSequence !== undefined) { + return exposeMessageListTimestamps( + this.db + .prepare( + 'SELECT * FROM messages WHERE thread_id = ? AND to_handle = ? AND sequence > ? ORDER BY sequence ASC' + ) + .all(threadId, toHandle, afterSequence) as MessageRow[] + ) + } + return exposeMessageListTimestamps( + this.db + .prepare('SELECT * FROM messages WHERE thread_id = ? AND to_handle = ? ORDER BY sequence ASC') + .all(threadId, toHandle) as MessageRow[] + ) +} + +export type MessageInboxMethods = { + getUnreadMessages: typeof getUnreadMessages + convertLifecycleMessageToRejection: typeof convertLifecycleMessageToRejection + getUndeliveredUnreadMessages: typeof getUndeliveredUnreadMessages + getUndeliveredUnreadMailboxHandles: typeof getUndeliveredUnreadMailboxHandles + getAllMessages: typeof getAllMessages + getMessageById: typeof getMessageById + markAsRead: typeof markAsRead + markAsDelivered: typeof markAsDelivered + markAsUndelivered: typeof markAsUndelivered + areUnreadMessages: typeof areUnreadMessages + markAsReadAndDelivered: typeof markAsReadAndDelivered + getInbox: typeof getInbox + getAllMessagesForHandle: typeof getAllMessagesForHandle + getThreadMessagesFor: typeof getThreadMessagesFor +} + +export function attachMessageInbox(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getUnreadMessages, + convertLifecycleMessageToRejection, + getUndeliveredUnreadMessages, + getUndeliveredUnreadMailboxHandles, + getAllMessages, + getMessageById, + markAsRead, + markAsDelivered, + markAsUndelivered, + areUnreadMessages, + markAsReadAndDelivered, + getInbox, + getAllMessagesForHandle, + getThreadMessagesFor + }) +} diff --git a/src/main/runtime/orchestration/db/messages/message-insert.ts b/src/main/runtime/orchestration/db/messages/message-insert.ts new file mode 100644 index 00000000000..d1a1b2f5611 --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/message-insert.ts @@ -0,0 +1,64 @@ +import type { MessageType, MessagePriority, MessageDeliveryContract, MessageRow } from '../../types' +import { LEGACY_RUN_ID } from '../contract-constants' +import { generateId } from '../generated-id' +import { exposeMessageTimestamps } from '../utc-timestamp' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Messages ── + +export function insertMessage( + this: OrchestrationDb, + msg: { + id?: string + from: string + to: string + subject: string + body?: string + type?: MessageType + priority?: MessagePriority + threadId?: string + payload?: string + senderPaneKey?: string + runId?: string + deliveryContract?: MessageDeliveryContract + } +): MessageRow { + const runId = msg.runId ?? LEGACY_RUN_ID + const deliveryContract = msg.deliveryContract ?? 'current_delivery' + this.requireRun(runId) + const id = msg.id ?? generateId('msg') + const stmt = this.db.prepare(` + INSERT INTO messages ( + id, run_id, delivery_contract, from_handle, to_handle, subject, body, + type, priority, thread_id, payload, sender_pane_key + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + stmt.run( + id, + runId, + deliveryContract, + msg.from, + msg.to, + msg.subject, + msg.body ?? '', + msg.type ?? 'status', + msg.priority ?? 'normal', + msg.threadId ?? null, + msg.payload ?? null, + msg.senderPaneKey ?? null + ) + return exposeMessageTimestamps( + this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow + ) +} + +export type MessageInsertMethods = { + insertMessage: typeof insertMessage +} + +export function attachMessageInsert(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + insertMessage + }) +} diff --git a/src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts b/src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts new file mode 100644 index 00000000000..7d396505d6c --- /dev/null +++ b/src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts @@ -0,0 +1,155 @@ +import { randomBytes } from 'node:crypto' +import type { MutationReceiptRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { ensureMutationReceiptCapacity } from '../../mutation-receipt-capacity' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Durable mutation receipts ── + +export function getOrCreateLocalMutationCallerFingerprint(this: OrchestrationDb): string { + if (this.localMutationCallerFingerprint) { + return this.localMutationCallerFingerprint + } + const transport = 'local_authenticated_transport' + const existing = this.db + .prepare('SELECT caller_fingerprint FROM mutation_caller_identities WHERE transport = ?') + .get(transport) as { caller_fingerprint: string } | undefined + if (existing) { + this.localMutationCallerFingerprint = existing.caller_fingerprint + return this.localMutationCallerFingerprint + } + this.db + .prepare( + `INSERT OR IGNORE INTO mutation_caller_identities (transport, caller_fingerprint) + VALUES (?, ?)` + ) + .run(transport, randomBytes(32).toString('hex')) + const created = this.db + .prepare('SELECT caller_fingerprint FROM mutation_caller_identities WHERE transport = ?') + .get(transport) as { caller_fingerprint: string } | undefined + if (!created) { + throw new Error('Failed to create the local orchestration mutation caller identity.') + } + this.localMutationCallerFingerprint = created.caller_fingerprint + return this.localMutationCallerFingerprint +} + +export function beginMutationReceipt( + this: OrchestrationDb, + params: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } +): + | { disposition: 'started'; row: MutationReceiptRow } + | { disposition: 'pending'; row: MutationReceiptRow } + | { disposition: 'completed'; row: MutationReceiptRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + const existing = this.getMutationReceipt(params.callerFingerprint, params.requestId) + if (existing) { + if (existing.method !== params.method || existing.payload_hash !== params.payloadHash) { + throw new OrchestrationError( + 'request_mismatch', + `Mutation request ${params.requestId} was already used with different input.` + ) + } + this.db.exec('COMMIT') + return { disposition: existing.state, row: existing } + } + ensureMutationReceiptCapacity(this.db) + this.db + .prepare( + `INSERT INTO mutation_receipts ( + caller_fingerprint, request_id, method, payload_hash, state + ) VALUES (?, ?, ?, ?, 'pending')` + ) + .run(params.callerFingerprint, params.requestId, params.method, params.payloadHash) + const row = this.getMutationReceipt(params.callerFingerprint, params.requestId) + this.db.exec('COMMIT') + return { disposition: 'started', row: row as MutationReceiptRow } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function completeMutationReceipt( + this: OrchestrationDb, + params: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + receipt: string + } +): MutationReceiptRow { + const result = this.db + .prepare( + `UPDATE mutation_receipts + SET state = 'completed', receipt = ?, updated_at = datetime('now') + WHERE caller_fingerprint = ? AND request_id = ? AND method = ? + AND payload_hash = ?` + ) + .run( + params.receipt, + params.callerFingerprint, + params.requestId, + params.method, + params.payloadHash + ) + const row = this.getMutationReceipt(params.callerFingerprint, params.requestId) + if (result.changes !== 1 || !row) { + throw new OrchestrationError( + 'request_mismatch', + `Mutation request ${params.requestId} no longer matches its pending operation.` + ) + } + return row +} + +export function discardPendingMutationReceipt( + this: OrchestrationDb, + callerFingerprint: string, + requestId: string +): void { + this.db + .prepare( + `DELETE FROM mutation_receipts + WHERE caller_fingerprint = ? AND request_id = ? AND state = 'pending'` + ) + .run(callerFingerprint, requestId) +} + +export function getMutationReceipt( + this: OrchestrationDb, + callerFingerprint: string, + requestId: string +): MutationReceiptRow | undefined { + return this.db + .prepare( + `SELECT * FROM mutation_receipts + WHERE caller_fingerprint = ? AND request_id = ?` + ) + .get(callerFingerprint, requestId) as MutationReceiptRow | undefined +} + +export type MutationReceiptStoreMethods = { + getOrCreateLocalMutationCallerFingerprint: typeof getOrCreateLocalMutationCallerFingerprint + beginMutationReceipt: typeof beginMutationReceipt + completeMutationReceipt: typeof completeMutationReceipt + discardPendingMutationReceipt: typeof discardPendingMutationReceipt + getMutationReceipt: typeof getMutationReceipt +} + +export function attachMutationReceiptStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getOrCreateLocalMutationCallerFingerprint, + beginMutationReceipt, + completeMutationReceipt, + discardPendingMutationReceipt, + getMutationReceipt + }) +} diff --git a/src/main/runtime/orchestration/db/orchestration-db-methods.ts b/src/main/runtime/orchestration/db/orchestration-db-methods.ts new file mode 100644 index 00000000000..129bd1ddb54 --- /dev/null +++ b/src/main/runtime/orchestration/db/orchestration-db-methods.ts @@ -0,0 +1,119 @@ +import type { CoordinatorRunStoreMethods } from './coordinator-runs/coordinator-run-store' +import type { DecisionGateStoreMethods } from './decision-gates/decision-gate-store' +import type { DispatchCapabilityMethods } from './dispatch-context/dispatch-capability' +import type { DispatchCompletionMethods } from './dispatch-context/dispatch-completion' +import type { DispatchContextStoreMethods } from './dispatch-context/dispatch-context-store' +import type { DispatchLookupMethods } from './dispatch-context/dispatch-lookup' +import type { WorkerReportSettlementMethods } from './dispatch-context/worker-report-settlement' +import type { FederatedDispatchStoreMethods } from './federation/federated-dispatch-store' +import type { FederationRelayAckMethods } from './federation/federation-relay-ack' +import type { FederationRelayEnqueueMethods } from './federation/federation-relay-enqueue' +import type { FederationRelayImportMethods } from './federation/federation-relay-import' +import type { FederationRelayItemMethods } from './federation/federation-relay-item' +import type { RemoteDispatchAttachmentAuthorityMethods } from './federation/remote-dispatch-attachment-authority' +import type { RemoteDispatchAttachmentCreateMethods } from './federation/remote-dispatch-attachment-create' +import type { RemoteDispatchAttachmentStopMethods } from './federation/remote-dispatch-attachment-stop' +import type { RemoteQuestionStoreMethods } from './federation/remote-question-store' +import type { LegacyAskOperationMethods } from './legacy/legacy-ask-operation' +import type { LegacyCompatibilityCandidatesMethods } from './legacy/legacy-compatibility-candidates' +import type { LegacyCompatibilityPrincipalsMethods } from './legacy/legacy-compatibility-principals' +import type { LegacyLifecycleOperationMethods } from './legacy/legacy-lifecycle-operation' +import type { LegacyMailAcknowledgeMethods } from './legacy/legacy-mail-acknowledge' +import type { LegacyQuestionAcknowledgeMethods } from './legacy/legacy-question-acknowledge' +import type { LegacyQuestionLookupMethods } from './legacy/legacy-question-lookup' +import type { LegacyRecoveryCohortMethods } from './legacy/legacy-recovery-cohort' +import type { LegacyReplyOperationMethods } from './legacy/legacy-reply-operation' +import type { LegacyWorkerCompletionMethods } from './legacy/legacy-worker-completion' +import type { DirectMailboxRoutingMethods } from './messages/direct-mailbox-routing' +import type { ForeignDirectMailboxRoutingMethods } from './messages/foreign-direct-mailbox-routing' +import type { MessageInboxMethods } from './messages/message-inbox' +import type { MessageInsertMethods } from './messages/message-insert' +import type { MutationReceiptStoreMethods } from './mutation-receipts/mutation-receipt-store' +import type { QuestionThreadsMethods } from './questions/question-threads' +import type { OrchestrationResetMethods } from './reset/orchestration-reset' +import type { RunBindingMethods } from './runs/run-binding' +import type { RunCoordinatorMailRoutingMethods } from './runs/run-coordinator-mail-routing' +import type { RunCreateMethods } from './runs/run-create' +import type { RunDeliveryMethods } from './runs/run-delivery' +import type { RunLookupMethods } from './runs/run-lookup' +import type { LegacyCoordinatorMailTakeoverMethods } from './runs/legacy-coordinator-mail-takeover' +import type { AdoptLegacyRunMethods } from './schema/adopt-legacy-run' +import type { BackfillLegacyQuestionThreadsMethods } from './schema/backfill-legacy-question-threads' +import type { CreateTablesMethods } from './schema/create-tables' +import type { MigrateLegacyContractStorageMethods } from './schema/migrate-legacy-contract-storage' +import type { SchemaMigrateMethods } from './schema/migrate' +import type { SchemaColumnProbesMethods } from './schema/schema-column-probes' +import type { TaskStoreMethods } from './tasks/task-store' +import type { FederatedWorkerStartReconcileMethods } from './worker-dispatch/federated-worker-start-reconcile' +import type { WorkerDispatchAbandonMethods } from './worker-dispatch/worker-dispatch-abandon' +import type { WorkerDispatchAuthorityMethods } from './worker-dispatch/worker-dispatch-authority' +import type { WorkerDispatchOutcomeMethods } from './worker-dispatch/worker-dispatch-outcome' +import type { WorkerDispatchStageMethods } from './worker-dispatch/worker-dispatch-stage' +import type { WorkerDispatchStartMethods } from './worker-dispatch/worker-dispatch-start' +import type { WorkerDispatchStopMethods } from './worker-dispatch/worker-dispatch-stop' +import type { WorkerTerminalRecoveryMethods } from './worker-dispatch/worker-terminal-recovery' +import type { WorkerTerminalArchiveMethods } from './worker-terminal/worker-terminal-archive' +import type { WorkerTerminalListingMethods } from './worker-terminal/worker-terminal-listing' +import type { WorkerTerminalReleaseMethods } from './worker-terminal/worker-terminal-release' +import type { WorkerTerminalResourceStoreMethods } from './worker-terminal/worker-terminal-resource-store' +import type { WorkerTerminalTransferMethods } from './worker-terminal/worker-terminal-transfer' + +export type OrchestrationDbMethods = CreateTablesMethods & + SchemaMigrateMethods & + SchemaColumnProbesMethods & + MigrateLegacyContractStorageMethods & + BackfillLegacyQuestionThreadsMethods & + AdoptLegacyRunMethods & + MutationReceiptStoreMethods & + LegacyCompatibilityPrincipalsMethods & + LegacyCompatibilityCandidatesMethods & + LegacyWorkerCompletionMethods & + LegacyRecoveryCohortMethods & + LegacyMailAcknowledgeMethods & + LegacyQuestionAcknowledgeMethods & + LegacyLifecycleOperationMethods & + LegacyAskOperationMethods & + LegacyQuestionLookupMethods & + LegacyReplyOperationMethods & + RunCreateMethods & + RunBindingMethods & + RunLookupMethods & + RunCoordinatorMailRoutingMethods & + LegacyCoordinatorMailTakeoverMethods & + RunDeliveryMethods & + MessageInsertMethods & + MessageInboxMethods & + DirectMailboxRoutingMethods & + ForeignDirectMailboxRoutingMethods & + QuestionThreadsMethods & + TaskStoreMethods & + WorkerDispatchStartMethods & + WorkerDispatchStageMethods & + WorkerDispatchAuthorityMethods & + WorkerDispatchOutcomeMethods & + FederatedWorkerStartReconcileMethods & + WorkerTerminalRecoveryMethods & + WorkerDispatchStopMethods & + WorkerDispatchAbandonMethods & + FederatedDispatchStoreMethods & + RemoteDispatchAttachmentCreateMethods & + RemoteDispatchAttachmentAuthorityMethods & + RemoteDispatchAttachmentStopMethods & + FederationRelayEnqueueMethods & + FederationRelayAckMethods & + FederationRelayImportMethods & + RemoteQuestionStoreMethods & + FederationRelayItemMethods & + WorkerTerminalResourceStoreMethods & + WorkerTerminalTransferMethods & + WorkerTerminalReleaseMethods & + WorkerTerminalArchiveMethods & + WorkerTerminalListingMethods & + DispatchContextStoreMethods & + DispatchCapabilityMethods & + DispatchLookupMethods & + DispatchCompletionMethods & + WorkerReportSettlementMethods & + DecisionGateStoreMethods & + CoordinatorRunStoreMethods & + OrchestrationResetMethods diff --git a/src/main/runtime/orchestration/db/orchestration-db.ts b/src/main/runtime/orchestration/db/orchestration-db.ts new file mode 100644 index 00000000000..2a970841a38 --- /dev/null +++ b/src/main/runtime/orchestration/db/orchestration-db.ts @@ -0,0 +1,48 @@ +import Database from '../../../sqlite/sync-database' +import { attachOrchestrationDbMethods } from './attach-orchestration-db-methods' +import { hardenOrchestrationDatabaseFiles } from './database-file-permissions' +import type { OrchestrationDbMethods } from './orchestration-db-methods' +import { + createCoordinatorMailRoutingTrigger, + rememberCurrentRunCoordinatorHandles +} from './runs/run-coordinator-mail-routing' +import { createTables } from './schema/create-tables' +import { migrate } from './schema/migrate' + +class OrchestrationDbCore { + db: Database.Database + + // Why: the orchestration DB is created lazily for ALL users, but only the + // small minority who dispatch work ever have dispatch_contexts rows. The + // renderer graph publish rebuilds orchestration context on every 16ms tick + // (buildAgentOrchestrationByPaneKey), issuing 2 queries per terminal. Cache + // emptiness so the non-orchestration majority short-circuits the whole + // per-terminal fan-out. Only createDispatchContext flips this false→true. + hasAnyDispatchContextsCache: boolean | undefined + localMutationCallerFingerprint: string | undefined + + constructor(dbPath: (string & {}) | ':memory:') { + this.db = new Database(dbPath) + this.db.pragma('journal_mode = WAL') + this.db.pragma('synchronous = NORMAL') + this.db.pragma('busy_timeout = 5000') + createTables.call(this as unknown as OrchestrationDb) + migrate.call(this as unknown as OrchestrationDb) + createCoordinatorMailRoutingTrigger.call(this as unknown as OrchestrationDb) + rememberCurrentRunCoordinatorHandles.call(this as unknown as OrchestrationDb) + hardenOrchestrationDatabaseFiles(dbPath) + } + + close(): void { + this.db.close() + } +} + +export type OrchestrationDb = OrchestrationDbCore & OrchestrationDbMethods + +attachOrchestrationDbMethods(OrchestrationDbCore) + +// Why: attach adds methods on the prototype; oxlint forbids class/interface merging, so the construct type is asserted. +export const OrchestrationDb = OrchestrationDbCore as new ( + dbPath: (string & {}) | ':memory:' +) => OrchestrationDb diff --git a/src/main/runtime/orchestration/db/pane-key-match.test.ts b/src/main/runtime/orchestration/db/pane-key-match.test.ts new file mode 100644 index 00000000000..875d954e2b3 --- /dev/null +++ b/src/main/runtime/orchestration/db/pane-key-match.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import { + isEquivalentPaneKey, + paneKeyMatchSuffix, + parseWorkerTerminalPriorOwnerIds +} from './pane-key-match' + +describe('pane-key-match', () => { + it('treats identical keys and same leaf UUIDs as equivalent', () => { + const leaf = '11111111-1111-4111-8111-111111111111' + expect(isEquivalentPaneKey('tab-a', 'tab-a')).toBe(true) + expect(isEquivalentPaneKey(`tab-a:${leaf}`, `tab-b:${leaf}`)).toBe(true) + expect(isEquivalentPaneKey(`tab-a:${leaf}`, `tab-a:22222222-2222-4222-8222-222222222222`)).toBe( + false + ) + }) + + it('uses the text after the first colon as the indexable suffix', () => { + expect(paneKeyMatchSuffix('no-colon')).toBe('no-colon') + expect(paneKeyMatchSuffix('tab:leaf')).toBe('leaf') + expect(paneKeyMatchSuffix('tab:leaf:extra')).toBe('leaf:extra') + }) + + it('parses a string array of prior owner ids and rejects other JSON', () => { + expect(parseWorkerTerminalPriorOwnerIds('["a","b"]')).toEqual(['a', 'b']) + expect(parseWorkerTerminalPriorOwnerIds('[1]')).toBeNull() + expect(parseWorkerTerminalPriorOwnerIds('not-json')).toBeNull() + }) +}) diff --git a/src/main/runtime/orchestration/db/pane-key-match.ts b/src/main/runtime/orchestration/db/pane-key-match.ts new file mode 100644 index 00000000000..cd938dbd3ca --- /dev/null +++ b/src/main/runtime/orchestration/db/pane-key-match.ts @@ -0,0 +1,36 @@ +import { parsePaneKey } from '../../../../shared/stable-pane-id' + +// Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys. +export function isEquivalentPaneKey(a: string, b: string): boolean { + if (a === b) { + return true + } + const aLeaf = parsePaneKey(a)?.leafId + const bLeaf = parsePaneKey(b)?.leafId + return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) +} + +export function parseWorkerTerminalPriorOwnerIds(value: string): string[] | null { + try { + const parsed: unknown = JSON.parse(value) + return Array.isArray(parsed) && parsed.every((entry) => typeof entry === 'string') + ? parsed + : null + } catch { + return null + } +} + +// Why: indexable pre-filter for isEquivalentPaneKey — equal strings and equal leaves both share the +// text after the first ':', so this narrows candidates without deciding equivalence itself. +export const RUN_PANE_KEY_MATCH_SUFFIX_SQL = + "substr(coordinator_pane_key, instr(coordinator_pane_key, ':') + 1)" +export const DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL = + "substr(assignee_pane_key, instr(assignee_pane_key, ':') + 1)" +export const REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL = + "substr(pane_key, instr(pane_key, ':') + 1)" + +export function paneKeyMatchSuffix(paneKey: string): string { + const colon = paneKey.indexOf(':') + return colon === -1 ? paneKey : paneKey.slice(colon + 1) +} diff --git a/src/main/runtime/orchestration/db/questions/question-threads.ts b/src/main/runtime/orchestration/db/questions/question-threads.ts new file mode 100644 index 00000000000..126ffa8e3b3 --- /dev/null +++ b/src/main/runtime/orchestration/db/questions/question-threads.ts @@ -0,0 +1,176 @@ +import type { MessageRow, QuestionRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { exposeQuestionTimestamps } from '../utc-timestamp' +import type { OrchestrationDb } from '../orchestration-db' + +export function createQuestion( + this: OrchestrationDb, + params: { + runId: string + dispatchId: string + askerHandle: string + question: string + options?: string[] + } +): { question: QuestionRow; message: MessageRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireRun(params.runId) + const dispatch = this.getDispatchContextById(params.dispatchId) + if ( + !dispatch || + dispatch.run_id !== params.runId || + (dispatch.status !== 'pending' && dispatch.status !== 'dispatched') + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not active in Run ${params.runId}.` + ) + } + const message = this.insertMessage({ + from: `dispatch:${params.dispatchId}`, + to: `run:${params.runId}`, + subject: 'Question', + body: params.question, + type: 'question', + payload: JSON.stringify({ + taskId: dispatch.task_id, + dispatchId: dispatch.id, + question: params.question, + options: params.options ?? [] + }), + runId: params.runId + }) + this.db.prepare('UPDATE messages SET thread_id = ? WHERE id = ?').run(message.id, message.id) + this.db + .prepare( + `INSERT INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle + ) VALUES (?, ?, ?, ?)` + ) + .run(message.id, params.runId, params.dispatchId, params.askerHandle) + const question = this.getQuestionRaw(message.id) as QuestionRow + const storedMessage = this.getMessageById(message.id) as MessageRow + this.db.exec('COMMIT') + return { question: exposeQuestionTimestamps(question), message: storedMessage } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getQuestion(this: OrchestrationDb, messageId: string): QuestionRow | undefined { + const question = this.getQuestionRaw(messageId) + return question ? exposeQuestionTimestamps(question) : undefined +} + +export function getQuestionRaw(this: OrchestrationDb, messageId: string): QuestionRow | undefined { + return this.db.prepare('SELECT * FROM question_threads WHERE message_id = ?').get(messageId) as + | QuestionRow + | undefined +} + +export function answerQuestion( + this: OrchestrationDb, + params: { + messageId: string + runId: string + consumerGeneration: number + body: string + } +): { question: QuestionRow; message: MessageRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireCurrentConsumer(params.runId, params.consumerGeneration) + const question = this.getQuestionRaw(params.messageId) + if (!question || question.run_id !== params.runId) { + throw new OrchestrationError( + 'question_not_found', + `Question ${params.messageId} was not found in Run ${params.runId}.` + ) + } + if (question.status === 'closed') { + throw new OrchestrationError( + 'dispatch_inactive', + `Question ${params.messageId} is closed because its Dispatch is inactive.` + ) + } + if (question.status === 'answered') { + if (question.answer_body !== params.body || !question.answer_message_id) { + throw new OrchestrationError( + 'answer_conflict', + `Question ${params.messageId} already has a different answer.` + ) + } + const message = this.getMessageById(question.answer_message_id) + if (!message) { + throw new Error(`Recorded answer message ${question.answer_message_id} was not found.`) + } + this.db.exec('COMMIT') + return { question: exposeQuestionTimestamps(question), message, duplicate: true } + } + + const message = this.insertMessage({ + from: `run:${params.runId}`, + to: `dispatch:${question.dispatch_id}`, + subject: 'Re: Question', + body: params.body, + threadId: question.message_id, + runId: params.runId + }) + // Why: ask returns thread state directly; leaving its answer unread would deliver it again via check. + this.markAsRead([message.id]) + this.db + .prepare( + `UPDATE question_threads + SET status = 'answered', answer_message_id = ?, answer_body = ?, + answered_by_generation = ?, answered_at = datetime('now') + WHERE message_id = ? AND status = 'pending'` + ) + .run(message.id, params.body, params.consumerGeneration, question.message_id) + const answered = this.getQuestionRaw(question.message_id) as QuestionRow + const storedMessage = this.getMessageById(message.id) as MessageRow + this.db.exec('COMMIT') + return { + question: exposeQuestionTimestamps(answered), + message: storedMessage, + duplicate: false + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function closeQuestionsForDispatch(this: OrchestrationDb, dispatchId: string): string[] { + const rows = this.db + .prepare("SELECT message_id FROM question_threads WHERE dispatch_id = ? AND status = 'pending'") + .all(dispatchId) as { message_id: string }[] + if (rows.length === 0) { + return [] + } + this.db + .prepare( + "UPDATE question_threads SET status = 'closed', closed_at = datetime('now') WHERE dispatch_id = ? AND status = 'pending'" + ) + .run(dispatchId) + return rows.map((row) => row.message_id) +} + +export type QuestionThreadsMethods = { + createQuestion: typeof createQuestion + getQuestion: typeof getQuestion + getQuestionRaw: typeof getQuestionRaw + answerQuestion: typeof answerQuestion + closeQuestionsForDispatch: typeof closeQuestionsForDispatch +} + +export function attachQuestionThreads(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createQuestion, + getQuestion, + getQuestionRaw, + answerQuestion, + closeQuestionsForDispatch + }) +} diff --git a/src/main/runtime/orchestration/db/reset/orchestration-reset.ts b/src/main/runtime/orchestration/db/reset/orchestration-reset.ts new file mode 100644 index 00000000000..e004b15d1b6 --- /dev/null +++ b/src/main/runtime/orchestration/db/reset/orchestration-reset.ts @@ -0,0 +1,100 @@ +import { LEGACY_RUN_ID } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Lifecycle ── + +export function runResetTransaction(this: OrchestrationDb, statements: string): void { + this.db.exec('BEGIN IMMEDIATE') + try { + this.db.exec(statements) + this.db.exec('COMMIT') + } catch (error) { + try { + this.db.exec('ROLLBACK') + } catch { + // Why: a failed COMMIT may already have rolled back; that ROLLBACK error must not mask the real failure. + } + throw error + } +} + +export function resetAll(this: OrchestrationDb): void { + // Why: retain mutation receipts so a lost reset response cannot replay as a new mutation. + this.runResetTransaction(` + DELETE FROM coordinator_runs; + DELETE FROM decision_gates; + DELETE FROM remote_questions; + DELETE FROM question_threads; + DELETE FROM deliveries; + DELETE FROM legacy_mail_receipts; + DELETE FROM legacy_operation_receipts; + DELETE FROM legacy_compatibility_principals; + DELETE FROM legacy_adoptions; + DELETE FROM federation_relay_items; + DELETE FROM remote_dispatch_attachments; + DELETE FROM federated_dispatches; + DELETE FROM worker_terminal_archives; + DELETE FROM worker_terminal_resources; + DELETE FROM worker_dispatches; + DELETE FROM dispatch_contexts; + DELETE FROM tasks; + DELETE FROM messages; + DELETE FROM run_coordinator_handles; + DELETE FROM runs; + INSERT INTO runs (id, objective, home_database, consumer_generation, legacy) + VALUES ('${LEGACY_RUN_ID}', 'Legacy orchestration state (inspect only)', 'this_database', 0, 1); + `) + this.hasAnyDispatchContextsCache = undefined +} + +export function resetTasks(this: OrchestrationDb): void { + // Why: messages survive this scope, so question threads are closed rather than deleted — an orphaned + // question message would otherwise answer as a generic reply while legacy acknowledgment rejects it. + this.runResetTransaction(` + DELETE FROM coordinator_runs; + DELETE FROM decision_gates; + DELETE FROM remote_questions; + UPDATE question_threads + SET status = 'closed', closed_at = COALESCE(closed_at, datetime('now')) + WHERE status = 'pending'; + DELETE FROM legacy_mail_receipts; + DELETE FROM legacy_operation_receipts; + DELETE FROM legacy_compatibility_principals; + DELETE FROM legacy_adoptions; + DELETE FROM federation_relay_items; + DELETE FROM remote_dispatch_attachments; + DELETE FROM federated_dispatches; + DELETE FROM worker_terminal_archives; + DELETE FROM worker_terminal_resources; + DELETE FROM worker_dispatches; + DELETE FROM dispatch_contexts; + DELETE FROM tasks; + `) + this.hasAnyDispatchContextsCache = undefined +} + +export function resetMessages(this: OrchestrationDb): void { + // Why: federation_relay_items is deliberately kept — relay rows carry contiguous cross-server cursors, not just inbox history. + this.runResetTransaction(` + DELETE FROM legacy_mail_receipts; + DELETE FROM question_threads; + DELETE FROM deliveries; + DELETE FROM messages; + `) +} + +export type OrchestrationResetMethods = { + runResetTransaction: typeof runResetTransaction + resetAll: typeof resetAll + resetTasks: typeof resetTasks + resetMessages: typeof resetMessages +} + +export function attachOrchestrationReset(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + runResetTransaction, + resetAll, + resetTasks, + resetMessages + }) +} diff --git a/src/main/runtime/orchestration/db/run-list-cursor.test.ts b/src/main/runtime/orchestration/db/run-list-cursor.test.ts new file mode 100644 index 00000000000..d60906b0531 --- /dev/null +++ b/src/main/runtime/orchestration/db/run-list-cursor.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { decodeRunListCursor, encodeRunListCursor } from './run-list-cursor' +import type { RunRow } from '../types' + +describe('run-list-cursor', () => { + it('round-trips createdAt and id', () => { + const run = { created_at: '2026-01-02 03:04:05', id: 'run_abc' } as RunRow + expect(decodeRunListCursor(encodeRunListCursor(run))).toEqual({ + createdAt: '2026-01-02 03:04:05', + id: 'run_abc' + }) + }) + + it('rejects an invalid cursor with cursor_invalid', () => { + expect(() => decodeRunListCursor('not-a-cursor')).toThrow(/cursor is invalid/) + }) +}) diff --git a/src/main/runtime/orchestration/db/run-list-cursor.ts b/src/main/runtime/orchestration/db/run-list-cursor.ts new file mode 100644 index 00000000000..24cb5286c66 --- /dev/null +++ b/src/main/runtime/orchestration/db/run-list-cursor.ts @@ -0,0 +1,25 @@ +import { OrchestrationError } from '../orchestration-error' +import type { RunRow } from '../types' +import type { RunListCursor } from './run-list-page' + +export function encodeRunListCursor(run: RunRow): string { + const cursor: RunListCursor = { createdAt: run.created_at, id: run.id } + return Buffer.from(JSON.stringify(cursor)).toString('base64url') +} + +export function decodeRunListCursor(value: string): RunListCursor { + try { + const parsed: unknown = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')) + if ( + !parsed || + typeof parsed !== 'object' || + typeof (parsed as RunListCursor).createdAt !== 'string' || + typeof (parsed as RunListCursor).id !== 'string' + ) { + throw new Error('invalid cursor shape') + } + return parsed as RunListCursor + } catch { + throw new OrchestrationError('cursor_invalid', 'The Run list cursor is invalid.') + } +} diff --git a/src/main/runtime/orchestration/db/run-list-page.ts b/src/main/runtime/orchestration/db/run-list-page.ts new file mode 100644 index 00000000000..c9fb918afc8 --- /dev/null +++ b/src/main/runtime/orchestration/db/run-list-page.ts @@ -0,0 +1,18 @@ +import type { RunRow, TaskRow } from '../types' + +export type RunListPage = { + runs: RunRow[] + nextCursor: string | null +} + +export type TaskRuntimeLineageRow = TaskRow & { + creator_dispatch_id: string | null + creator_dispatch_run_id: string | null + creator_dispatch_pane_key: string | null + creator_dispatch_process_incarnation: string | null +} + +export type RunListCursor = { + createdAt: string + id: string +} diff --git a/src/main/runtime/orchestration/db/runs/legacy-coordinator-mail-takeover.ts b/src/main/runtime/orchestration/db/runs/legacy-coordinator-mail-takeover.ts new file mode 100644 index 00000000000..f2892b0880b --- /dev/null +++ b/src/main/runtime/orchestration/db/runs/legacy-coordinator-mail-takeover.ts @@ -0,0 +1,170 @@ +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' + +export function promoteLegacyCoordinatorMailForTakeover( + this: OrchestrationDb, + runId: string, + retainedCoordinatorHandle: string | null +): void { + if (!retainedCoordinatorHandle) { + return + } + this.db + .prepare( + `UPDATE messages + SET to_handle = ?, delivery_contract = 'current_delivery', + read = 0, delivered_at = NULL + WHERE run_id = ? AND delivery_contract = 'legacy_direct' + AND to_handle = ? + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.run_id = messages.run_id + AND d.contract_version = ? + AND ( + messages.from_handle = d.assignee_handle OR + messages.from_handle = 'dispatch:' || d.id + ) + ) + AND ( + read = 0 OR EXISTS( + SELECT 1 FROM question_threads q + WHERE q.message_id = messages.id AND q.status = 'pending' + ) OR EXISTS( + SELECT 1 + FROM legacy_mail_receipts r + INNER JOIN legacy_compatibility_principals p + ON p.id = r.principal_id + WHERE r.message_id = messages.id + AND r.acknowledged_at IS NULL + AND p.run_id = messages.run_id + AND p.role = 'coordinator' + AND p.terminal_handle = ? + ) OR ( + read = 1 + AND NOT EXISTS( + SELECT 1 FROM legacy_compatibility_principals p + WHERE p.run_id = messages.run_id AND p.role = 'coordinator' + ) + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.run_id = messages.run_id + AND d.contract_version = ? + AND d.status IN ('pending', 'dispatched') + AND messages.created_at >= d.created_at + AND ( + messages.from_handle = d.assignee_handle OR + messages.from_handle = 'dispatch:' || d.id + ) + ) + ) + )` + ) + .run( + `run:${runId}`, + runId, + retainedCoordinatorHandle, + LEGACY_CONTRACT_VERSION, + retainedCoordinatorHandle, + LEGACY_CONTRACT_VERSION + ) +} + +export function getUniqueLegacyCoordinatorHandle( + this: OrchestrationDb, + runId: string +): string | null { + const adoption = this.getLegacyAdoption() + if (!adoption || adoption.adopted_run_id !== runId) { + return null + } + const workerHandles = new Set( + ( + this.db + .prepare( + `SELECT DISTINCT assignee_handle AS handle + FROM dispatch_contexts + WHERE run_id = ? AND contract_version = ? + AND assignee_handle IS NOT NULL + UNION + SELECT DISTINCT terminal_handle AS handle + FROM legacy_compatibility_principals + WHERE run_id = ? AND role = 'worker' + AND status IN ('committed', 'settled')` + ) + .all(runId, LEGACY_CONTRACT_VERSION, runId) as { handle: string }[] + ).map((row) => row.handle) + ) + const durableRows = this.db + .prepare( + `SELECT coordinator_handle AS handle + FROM coordinator_runs + WHERE scheduler_lost_at = ? + UNION + SELECT created_by_terminal_handle AS handle + FROM tasks t + WHERE t.run_id = ? AND t.created_by_terminal_handle IS NOT NULL + AND t.created_at <= ? + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.task_id = t.id AND d.run_id = t.run_id + AND d.contract_version = ? + )` + ) + .all(adoption.adopted_at, runId, adoption.adopted_at, LEGACY_CONTRACT_VERSION) as { + handle: string + }[] + if (durableRows.some((row) => workerHandles.has(row.handle))) { + return null + } + const candidates = new Set(durableRows.map((row) => row.handle)) + const mailRows = this.db + .prepare( + `SELECT m.to_handle AS handle + FROM messages m + INNER JOIN dispatch_contexts d + ON d.run_id = m.run_id AND d.contract_version = ? + AND (m.from_handle = d.assignee_handle OR m.from_handle = 'dispatch:' || d.id) + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND m.created_at <= ? + UNION + SELECT m.from_handle AS handle + FROM messages m + INNER JOIN dispatch_contexts d + ON d.run_id = m.run_id AND d.contract_version = ? + AND (m.to_handle = d.assignee_handle OR m.to_handle = 'dispatch:' || d.id) + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND m.created_at <= ?` + ) + .all( + LEGACY_CONTRACT_VERSION, + runId, + adoption.adopted_at, + LEGACY_CONTRACT_VERSION, + runId, + adoption.adopted_at + ) as { + handle: string + }[] + for (const row of mailRows) { + if ( + !workerHandles.has(row.handle) && + !row.handle.startsWith('dispatch:') && + !row.handle.startsWith('run:') + ) { + candidates.add(row.handle) + } + } + return candidates.size === 1 ? ([...candidates][0] ?? null) : null +} + +export type LegacyCoordinatorMailTakeoverMethods = { + promoteLegacyCoordinatorMailForTakeover: typeof promoteLegacyCoordinatorMailForTakeover + getUniqueLegacyCoordinatorHandle: typeof getUniqueLegacyCoordinatorHandle +} + +export function attachLegacyCoordinatorMailTakeover(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + promoteLegacyCoordinatorMailForTakeover, + getUniqueLegacyCoordinatorHandle + }) +} diff --git a/src/main/runtime/orchestration/db/runs/run-binding.ts b/src/main/runtime/orchestration/db/runs/run-binding.ts new file mode 100644 index 00000000000..a4110dd589f --- /dev/null +++ b/src/main/runtime/orchestration/db/runs/run-binding.ts @@ -0,0 +1,166 @@ +import type { RunRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { LEGACY_CONTRACT_VERSION } from '../contract-constants' +import { isEquivalentPaneKey } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function bindRun( + this: OrchestrationDb, + params: { + runId: string + coordinatorHandle: string + coordinatorPaneKey: string + takeoverLegacy?: boolean + legacyCoordinatorAuthority?: { + runId: string + principalId: string | null + terminalHandle: string + paneKey: string + consumerGeneration: number + } + } +): RunRow | undefined { + this.db.exec('BEGIN IMMEDIATE') + try { + const run = this.getRunRaw(params.runId) + if (!run || run.legacy === 1) { + this.db.exec('ROLLBACK') + return undefined + } + const sameBinding = + run.coordinator_pane_key !== null && + isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey) + const adoption = this.getLegacyAdoption() + const adoptedRun = adoption?.adopted_run_id === params.runId + const legacyAuthority = params.legacyCoordinatorAuthority + const legacyPrincipalId = legacyAuthority?.principalId + const legacyPrincipal = legacyPrincipalId + ? this.getLegacyCompatibilityPrincipal(legacyPrincipalId) + : undefined + const provenLegacyBinding = Boolean( + adoptedRun && + legacyAuthority && + legacyAuthority.principalId !== null && + legacyAuthority.runId === params.runId && + legacyAuthority.consumerGeneration === run.consumer_generation && + legacyPrincipal?.run_id === params.runId && + legacyPrincipal.role === 'coordinator' && + legacyPrincipal.status === 'committed' && + legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle && + isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) && + params.coordinatorHandle === legacyAuthority.terminalHandle && + isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey) + ) + if (legacyAuthority && !provenLegacyBinding) { + throw new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.', + { effectsApplied: false } + ) + } + const activeLegacyAssignment = + adoptedRun && + Boolean( + this.db + .prepare( + `SELECT 1 FROM dispatch_contexts + WHERE run_id = ? AND contract_version = ? + AND status IN ('pending', 'dispatched') + LIMIT 1` + ) + .get(params.runId, LEGACY_CONTRACT_VERSION) + ) + const coordinatorPrincipal = adoptedRun + ? this.getLegacyCoordinatorPrincipal(params.runId) + : undefined + const retainedCoordinatorHandle = + coordinatorPrincipal?.terminal_handle ?? + run.coordinator_handle ?? + this.getUniqueLegacyCoordinatorHandle(params.runId) + const takeoverAlreadyApplied = Boolean( + params.takeoverLegacy && + sameBinding && + run.coordinator_handle === params.coordinatorHandle && + coordinatorPrincipal?.status !== 'committed' + ) + const replacesLegacyCoordinator = Boolean( + adoptedRun && + !provenLegacyBinding && + retainedCoordinatorHandle && + (params.takeoverLegacy || + retainedCoordinatorHandle !== params.coordinatorHandle || + !sameBinding) + ) + if (params.takeoverLegacy && !adoptedRun) { + throw new OrchestrationError( + 'invalid_argument', + 'Legacy takeover is only available for the automatically adopted Run.' + ) + } + // Why: only LIVE legacy work needs the flag — settled work has no competing authority left, and + // fencing it would strand the recovered graph behind an attestation the caller may not have. + if (activeLegacyAssignment && !sameBinding && !provenLegacyBinding && !params.takeoverLegacy) { + throw new OrchestrationError( + 'consumer_fenced', + 'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.', + { + effectsApplied: false, + recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy` + } + ) + } + this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId) + for (const handle of new Set( + [run.coordinator_handle, params.coordinatorHandle].filter((value): value is string => + Boolean(value) + ) + )) { + this.rememberRunCoordinatorHandle(params.runId, handle) + this.routeAllUnreadDirectMessagesToRunMailbox(params.runId, handle) + } + if ( + (params.takeoverLegacy && !takeoverAlreadyApplied) || + !sameBinding || + run.coordinator_handle !== params.coordinatorHandle + ) { + if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) { + if ( + coordinatorPrincipal?.status === 'committed' && + (params.takeoverLegacy || + coordinatorPrincipal.terminal_handle !== params.coordinatorHandle || + !isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey)) + ) { + this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked') + } + } + this.db + .prepare( + `UPDATE runs + SET coordinator_handle = ?, coordinator_pane_key = ?, + consumer_generation = consumer_generation + 1, + updated_at = datetime('now') + WHERE id = ?` + ) + .run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId) + this.fenceOutstandingDelivery(params.runId) + if (params.takeoverLegacy || replacesLegacyCoordinator) { + this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle) + } + } + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + return this.getRun(params.runId) +} + +export type RunBindingMethods = { + bindRun: typeof bindRun +} + +export function attachRunBinding(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + bindRun + }) +} diff --git a/src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts b/src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts new file mode 100644 index 00000000000..121a051cba3 --- /dev/null +++ b/src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts @@ -0,0 +1,89 @@ +import type { OrchestrationDb } from '../orchestration-db' + +export function rememberRunCoordinatorHandle( + this: OrchestrationDb, + runId: string, + terminalHandle: string +): void { + this.db + .prepare( + `INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) VALUES (?, ?)` + ) + .run(runId, terminalHandle) +} + +export function rememberCurrentRunCoordinatorHandles(this: OrchestrationDb): void { + this.db.exec(` + INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) + SELECT id, coordinator_handle FROM runs + WHERE legacy = 0 AND coordinator_handle IS NOT NULL + `) +} + +export function createCoordinatorMailRoutingTrigger(this: OrchestrationDb): void { + this.db.exec('BEGIN IMMEDIATE') + try { + this.db.exec(` + DROP TRIGGER IF EXISTS trg_messages_route_coordinator_mail; + CREATE TRIGGER trg_messages_route_coordinator_mail + AFTER INSERT ON messages + WHEN NEW.read = 0 AND NEW.delivery_contract = 'current_delivery' + AND EXISTS ( + SELECT 1 FROM runs + WHERE runs.id = NEW.run_id AND runs.legacy = 0 + ) + AND EXISTS ( + SELECT 1 FROM run_coordinator_handles + WHERE run_id = NEW.run_id AND terminal_handle = NEW.to_handle + ) + AND NOT EXISTS ( + SELECT 1 FROM dispatch_contexts + WHERE run_id = NEW.run_id AND assignee_handle = NEW.to_handle + AND status IN ('pending', 'dispatched') + ) + BEGIN + UPDATE messages SET to_handle = 'run:' || NEW.run_id WHERE sequence = NEW.sequence; + END; + `) + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function routeAllUnreadDirectMessagesToRunMailbox( + this: OrchestrationDb, + runId: string, + directHandle: string +): void { + this.db + .prepare( + `UPDATE messages SET to_handle = ? + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' + AND NOT EXISTS ( + SELECT 1 FROM dispatch_contexts + WHERE dispatch_contexts.run_id = messages.run_id + AND dispatch_contexts.assignee_handle = messages.to_handle + AND dispatch_contexts.status IN ('pending', 'dispatched') + )` + ) + .run(`run:${runId}`, runId, directHandle) +} + +export type RunCoordinatorMailRoutingMethods = { + rememberRunCoordinatorHandle: typeof rememberRunCoordinatorHandle + rememberCurrentRunCoordinatorHandles: typeof rememberCurrentRunCoordinatorHandles + createCoordinatorMailRoutingTrigger: typeof createCoordinatorMailRoutingTrigger + routeAllUnreadDirectMessagesToRunMailbox: typeof routeAllUnreadDirectMessagesToRunMailbox +} + +export function attachRunCoordinatorMailRouting(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + rememberRunCoordinatorHandle, + rememberCurrentRunCoordinatorHandles, + createCoordinatorMailRoutingTrigger, + routeAllUnreadDirectMessagesToRunMailbox + }) +} diff --git a/src/main/runtime/orchestration/db/runs/run-create.ts b/src/main/runtime/orchestration/db/runs/run-create.ts new file mode 100644 index 00000000000..3f99e018936 --- /dev/null +++ b/src/main/runtime/orchestration/db/runs/run-create.ts @@ -0,0 +1,44 @@ +import type { RunRow } from '../../types' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Runs ── + +export function createRun( + this: OrchestrationDb, + params: { + objective: string + coordinatorHandle: string + coordinatorPaneKey: string + } +): RunRow { + const id = generateId('run') + this.db.exec('BEGIN IMMEDIATE') + try { + this.unbindOtherRunsForPane(params.coordinatorPaneKey) + this.db + .prepare( + `INSERT INTO runs ( + id, objective, coordinator_handle, coordinator_pane_key, + consumer_generation, legacy + ) VALUES (?, ?, ?, ?, 1, 0)` + ) + .run(id, params.objective, params.coordinatorHandle, params.coordinatorPaneKey) + this.rememberRunCoordinatorHandle(id, params.coordinatorHandle) + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + return this.getRun(id) as RunRow +} + +export type RunCreateMethods = { + createRun: typeof createRun +} + +export function attachRunCreate(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createRun + }) +} diff --git a/src/main/runtime/orchestration/db/runs/run-delivery.ts b/src/main/runtime/orchestration/db/runs/run-delivery.ts new file mode 100644 index 00000000000..48ada051339 --- /dev/null +++ b/src/main/runtime/orchestration/db/runs/run-delivery.ts @@ -0,0 +1,268 @@ +import type { MessageType, MessageRow, RunRow, DeliveryRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { generateId } from '../generated-id' +import { exposeMessageListTimestamps, exposeDeliveryTimestamps } from '../utc-timestamp' +import { ORCHESTRATION_DELIVERY_BATCH_LIMIT } from '../messages/mailbox-routing-page' +import type { OrchestrationDb } from '../orchestration-db' + +export function requireCurrentConsumer( + this: OrchestrationDb, + runId: string, + consumerGeneration: number +): RunRow { + const run = this.getRunRaw(runId) + if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) { + throw new OrchestrationError( + 'consumer_fenced', + 'This mailbox consumer has been replaced. Rebind with orchestration run-use.' + ) + } + return run +} + +export function getDeliveryRaw(this: OrchestrationDb, id: string): DeliveryRow | undefined { + return this.db.prepare('SELECT * FROM deliveries WHERE id = ?').get(id) as DeliveryRow | undefined +} + +export function getDeliveryMessages(this: OrchestrationDb, delivery: DeliveryRow): MessageRow[] { + const ids = JSON.parse(delivery.message_ids) as string[] + if (ids.length === 0) { + return [] + } + const rows = this.db + .prepare(`SELECT * FROM messages WHERE id IN (${ids.map(() => '?').join(',')})`) + .all(...ids) as MessageRow[] + const byId = new Map(rows.map((row) => [row.id, row])) + return exposeMessageListTimestamps( + ids.map((id) => byId.get(id)).filter((row): row is MessageRow => row !== undefined) + ) +} + +export function getOrCreateRunDelivery( + this: OrchestrationDb, + params: { + runId: string + consumerGeneration: number + limit?: number + wakeTypes?: MessageType[] + } +): { delivery: DeliveryRow; messages: MessageRow[]; replayed: boolean } | undefined { + const limit = Math.min( + Math.max(params.limit ?? ORCHESTRATION_DELIVERY_BATCH_LIMIT, 1), + ORCHESTRATION_DELIVERY_BATCH_LIMIT + ) + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireCurrentConsumer(params.runId, params.consumerGeneration) + const existing = this.db + .prepare("SELECT * FROM deliveries WHERE run_id = ? AND status = 'outstanding'") + .get(params.runId) as DeliveryRow | undefined + if (existing) { + if (existing.consumer_generation !== params.consumerGeneration) { + throw new OrchestrationError( + 'consumer_fenced', + 'This mailbox Delivery belongs to a fenced consumer generation.' + ) + } + const messages = this.getDeliveryMessages(existing) + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(existing), messages, replayed: true } + } + + const address = `run:${params.runId}` + if (params.wakeTypes && params.wakeTypes.length > 0) { + const placeholders = params.wakeTypes.map(() => '?').join(',') + const matching = this.db + .prepare( + `SELECT 1 FROM messages + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' + AND type IN (${placeholders}) LIMIT 1` + ) + .get(params.runId, address, ...params.wakeTypes) + if (!matching) { + this.db.exec('COMMIT') + return undefined + } + } + + const messages = exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' + ORDER BY sequence ASC LIMIT ?` + ) + .all(params.runId, address, limit) as MessageRow[] + ) + if (messages.length === 0) { + this.db.exec('COMMIT') + return undefined + } + + const deliveryId = generateId('delivery') + this.db + .prepare( + `INSERT INTO deliveries (id, run_id, consumer_generation, message_ids) + VALUES (?, ?, ?, ?)` + ) + .run( + deliveryId, + params.runId, + params.consumerGeneration, + JSON.stringify(messages.map((message) => message.id)) + ) + const delivery = this.getDeliveryRaw(deliveryId) as DeliveryRow + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(delivery), messages, replayed: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function acknowledgeRunDelivery( + this: OrchestrationDb, + params: { + runId: string + consumerGeneration: number + deliveryId: string + } +): { delivery: DeliveryRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireCurrentConsumer(params.runId, params.consumerGeneration) + const delivery = this.getDeliveryRaw(params.deliveryId) + if (!delivery || delivery.run_id !== params.runId) { + throw new OrchestrationError( + 'stale_delivery', + `Delivery ${params.deliveryId} does not belong to this Run.` + ) + } + if ( + delivery.consumer_generation !== params.consumerGeneration || + delivery.status === 'fenced' + ) { + throw new OrchestrationError( + 'consumer_fenced', + 'This mailbox Delivery belongs to a fenced consumer generation.' + ) + } + if (delivery.status === 'acknowledged') { + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(delivery), duplicate: true } + } + + const messageIds = JSON.parse(delivery.message_ids) as string[] + if (messageIds.length > 0) { + const placeholders = messageIds.map(() => '?').join(',') + this.db + .prepare(`UPDATE messages SET read = 1 WHERE id IN (${placeholders})`) + .run(...messageIds) + } + this.db + .prepare( + "UPDATE deliveries SET status = 'acknowledged', acknowledged_at = datetime('now') WHERE id = ?" + ) + .run(delivery.id) + const acknowledged = this.getDeliveryRaw(delivery.id) as DeliveryRow + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(acknowledged), duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getRunMailboxHistory( + this: OrchestrationDb, + runId: string, + limit = 100, + types?: MessageType[] +): MessageRow[] { + const address = `run:${runId}` + // Why: SQLite reads a negative LIMIT as unbounded, so an unsanitized caller value dumps the whole mailbox. + const rowLimit = Math.max(1, Math.floor(limit)) + if (types && types.length > 0) { + const placeholders = types.map(() => '?').join(',') + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages WHERE run_id = ? AND to_handle = ? + AND type IN (${placeholders}) ORDER BY sequence DESC LIMIT ?` + ) + .all(runId, address, ...types, rowLimit) as MessageRow[] + ) + } + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages WHERE run_id = ? AND to_handle = ? + ORDER BY sequence DESC LIMIT ?` + ) + .all(runId, address, rowLimit) as MessageRow[] + ) +} + +export function getUnreadRunMailbox( + this: OrchestrationDb, + runId: string, + limit = 100, + types?: MessageType[] +): MessageRow[] { + const address = `run:${runId}` + const conditions = [ + 'run_id = ?', + 'to_handle = ?', + 'read = 0', + "delivery_contract = 'current_delivery'" + ] + const params: (string | number)[] = [runId, address] + if (types?.length) { + conditions.push(`type IN (${types.map(() => '?').join(',')})`) + params.push(...types) + } + const indexClause = types?.length ? ' INDEXED BY idx_messages_unread_current_run_type' : '' + params.push(Math.max(1, Math.floor(limit))) + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages${indexClause} WHERE ${conditions.join(' AND ')} + ORDER BY sequence ASC LIMIT ?` + ) + .all(...params) as MessageRow[] + ) +} + +export function hasOutstandingRunDelivery(this: OrchestrationDb, runId: string): boolean { + return Boolean( + this.db + .prepare("SELECT 1 FROM deliveries WHERE run_id = ? AND status = 'outstanding' LIMIT 1") + .get(runId) + ) +} + +export type RunDeliveryMethods = { + requireCurrentConsumer: typeof requireCurrentConsumer + getDeliveryRaw: typeof getDeliveryRaw + getDeliveryMessages: typeof getDeliveryMessages + getOrCreateRunDelivery: typeof getOrCreateRunDelivery + acknowledgeRunDelivery: typeof acknowledgeRunDelivery + getRunMailboxHistory: typeof getRunMailboxHistory + getUnreadRunMailbox: typeof getUnreadRunMailbox + hasOutstandingRunDelivery: typeof hasOutstandingRunDelivery +} + +export function attachRunDelivery(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + requireCurrentConsumer, + getDeliveryRaw, + getDeliveryMessages, + getOrCreateRunDelivery, + acknowledgeRunDelivery, + getRunMailboxHistory, + getUnreadRunMailbox, + hasOutstandingRunDelivery + }) +} diff --git a/src/main/runtime/orchestration/db/runs/run-lookup.ts b/src/main/runtime/orchestration/db/runs/run-lookup.ts new file mode 100644 index 00000000000..dd09d1f832b --- /dev/null +++ b/src/main/runtime/orchestration/db/runs/run-lookup.ts @@ -0,0 +1,140 @@ +import type { RunRow } from '../../types' +import { ORCHESTRATION_RUN_PAGE_LIMIT } from '../../../../../shared/orchestration-run-pagination' +import { + isEquivalentPaneKey, + RUN_PANE_KEY_MATCH_SUFFIX_SQL, + paneKeyMatchSuffix +} from '../pane-key-match' +import { exposeRunTimestamps } from '../utc-timestamp' +import { encodeRunListCursor, decodeRunListCursor } from '../run-list-cursor' +import type { RunListPage } from '../run-list-page' +import type { OrchestrationDb } from '../orchestration-db' + +export function getRun(this: OrchestrationDb, id: string): RunRow | undefined { + const run = this.getRunRaw(id) + return run ? exposeRunTimestamps(run) : undefined +} + +export function listRuns( + this: OrchestrationDb, + params: { limit?: number; cursor?: string } = {} +): RunListPage { + if (params.limit === undefined && params.cursor === undefined) { + const rows = this.db + .prepare('SELECT * FROM runs ORDER BY created_at DESC, id DESC') + .all() as RunRow[] + return { runs: rows.map(exposeRunTimestamps), nextCursor: null } + } + const limit = Math.min( + Math.max(1, params.limit ?? ORCHESTRATION_RUN_PAGE_LIMIT), + ORCHESTRATION_RUN_PAGE_LIMIT + ) + const cursor = params.cursor ? decodeRunListCursor(params.cursor) : undefined + const rows = ( + cursor + ? this.db + .prepare( + `SELECT * FROM runs + WHERE created_at < ? OR (created_at = ? AND id < ?) + ORDER BY created_at DESC, id DESC + LIMIT ?` + ) + .all(cursor.createdAt, cursor.createdAt, cursor.id, limit + 1) + : this.db + .prepare('SELECT * FROM runs ORDER BY created_at DESC, id DESC LIMIT ?') + .all(limit + 1) + ) as RunRow[] + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + return { + runs: pageRows.map(exposeRunTimestamps), + nextCursor: hasMore ? encodeRunListCursor(pageRows.at(-1) as RunRow) : null + } +} + +export function getCurrentRunForPane(this: OrchestrationDb, paneKey: string): RunRow | undefined { + const run = this.runsBoundToPane(paneKey)[0] + return run ? exposeRunTimestamps(run) : undefined +} + +// Why: the indexed suffix only narrows candidates; isEquivalentPaneKey still decides, so +// reminted tab halves keep matching and unparseable keys keep requiring an exact match. +export function runsBoundToPane(this: OrchestrationDb, paneKey: string): RunRow[] { + return ( + this.db + .prepare( + `SELECT * FROM runs + WHERE coordinator_pane_key IS NOT NULL AND legacy = 0 + AND ${RUN_PANE_KEY_MATCH_SUFFIX_SQL} = ? + ORDER BY rowid` + ) + .all(paneKeyMatchSuffix(paneKey)) as RunRow[] + ).filter( + (run) => + run.coordinator_pane_key !== null && isEquivalentPaneKey(run.coordinator_pane_key, paneKey) + ) +} + +export function getRunRaw(this: OrchestrationDb, id: string): RunRow | undefined { + return this.db.prepare('SELECT * FROM runs WHERE id = ?').get(id) as RunRow | undefined +} + +export function unbindOtherRunsForPane( + this: OrchestrationDb, + paneKey: string, + exceptRunId?: string +): void { + for (const run of this.runsBoundToPane(paneKey)) { + if (run.id !== exceptRunId) { + if (run.coordinator_handle) { + this.routeAllUnreadDirectMessagesToRunMailbox(run.id, run.coordinator_handle) + } + this.db + .prepare( + `UPDATE runs + SET coordinator_handle = NULL, coordinator_pane_key = NULL, + consumer_generation = consumer_generation + 1, + updated_at = datetime('now') + WHERE id = ?` + ) + .run(run.id) + this.fenceOutstandingDelivery(run.id) + } + } +} + +export function requireRun(this: OrchestrationDb, runId: string): void { + if (!this.getRunRaw(runId)) { + throw new Error(`Run not found: ${runId}`) + } +} + +export function fenceOutstandingDelivery(this: OrchestrationDb, runId: string): void { + this.db + .prepare("UPDATE deliveries SET status = 'fenced' WHERE run_id = ? AND status = 'outstanding'") + .run(runId) +} + +export type RunLookupMethods = { + getRun: typeof getRun + listRuns: typeof listRuns + getCurrentRunForPane: typeof getCurrentRunForPane + runsBoundToPane: typeof runsBoundToPane + getRunRaw: typeof getRunRaw + unbindOtherRunsForPane: typeof unbindOtherRunsForPane + requireRun: typeof requireRun + fenceOutstandingDelivery: typeof fenceOutstandingDelivery +} + +export function attachRunLookup(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + getRun, + listRuns, + getCurrentRunForPane, + runsBoundToPane, + getRunRaw, + unbindOtherRunsForPane, + requireRun, + fenceOutstandingDelivery + }) +} diff --git a/src/main/runtime/orchestration/db/schema/adopt-legacy-run.ts b/src/main/runtime/orchestration/db/schema/adopt-legacy-run.ts new file mode 100644 index 00000000000..a67a6a2f604 --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/adopt-legacy-run.ts @@ -0,0 +1,142 @@ +import type { LegacyAdoptionRow } from '../../types' +import { LEGACY_RUN_ID } from '../contract-constants' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +export function adoptLegacyRunIfNeeded(this: OrchestrationDb): void { + const existing = this.db + .prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?') + .get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined + const hasGraph = this.db + .prepare( + `SELECT 1 + WHERE EXISTS(SELECT 1 FROM tasks WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM dispatch_contexts WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM decision_gates WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM messages WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM question_threads WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM deliveries WHERE run_id = ?)` + ) + .get(LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID) + if (!existing && !hasGraph) { + return + } + + const adoptedRunId = existing?.adopted_run_id ?? generateId('run') + this.db + .prepare( + `INSERT OR IGNORE INTO runs ( + id, objective, home_database, consumer_generation, legacy + ) VALUES (?, ?, 'this_database', 0, 0)` + ) + .run(adoptedRunId, 'Recovered orchestration work from a contract update') + this.db + .prepare( + `INSERT OR IGNORE INTO legacy_adoptions ( + source_run_id, adopted_run_id, scheduler_state_lost + ) VALUES (?, ?, 1)` + ) + .run(LEGACY_RUN_ID, adoptedRunId) + this.db + .prepare( + `UPDATE coordinator_runs + SET status = 'failed', + completed_at = COALESCE( + completed_at, + (SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ?) + ), + scheduler_lost_at = ( + SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ? + ) + WHERE status = 'running' + AND julianday(created_at) <= julianday(( + SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ? + ))` + ) + .run(LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID) + + this.db + .prepare( + `UPDATE deliveries SET status = 'fenced' + WHERE run_id = ? AND status = 'outstanding'` + ) + .run(LEGACY_RUN_ID) + for (const table of [ + 'tasks', + 'dispatch_contexts', + 'decision_gates', + 'messages', + 'question_threads', + 'deliveries' + ]) { + this.db + .prepare(`UPDATE ${table} SET run_id = ? WHERE run_id = ?`) + .run(adoptedRunId, LEGACY_RUN_ID) + } + this.db + .prepare( + `UPDATE runs + SET objective = 'Legacy orchestration state (adopted; inspect only)', + coordinator_handle = NULL, coordinator_pane_key = NULL, + updated_at = datetime('now') + WHERE id = ?` + ) + .run(LEGACY_RUN_ID) + + const mismatch = this.db + .prepare( + `WITH migration_runs(run_id) AS (VALUES (?), (?)) + SELECT 1 + WHERE EXISTS( + SELECT 1 FROM dispatch_contexts d + INNER JOIN tasks t ON t.id = d.task_id + WHERE d.run_id <> t.run_id + AND ( + d.run_id IN (SELECT run_id FROM migration_runs) + OR t.run_id IN (SELECT run_id FROM migration_runs) + ) + ) + OR EXISTS( + SELECT 1 FROM decision_gates g + INNER JOIN tasks t ON t.id = g.task_id + WHERE g.run_id <> t.run_id + AND ( + g.run_id IN (SELECT run_id FROM migration_runs) + OR t.run_id IN (SELECT run_id FROM migration_runs) + ) + ) + OR EXISTS( + SELECT 1 FROM question_threads q + INNER JOIN dispatch_contexts d ON d.id = q.dispatch_id + WHERE q.run_id <> d.run_id + AND ( + q.run_id IN (SELECT run_id FROM migration_runs) + OR d.run_id IN (SELECT run_id FROM migration_runs) + ) + ) + OR EXISTS( + SELECT 1 FROM deliveries d + INNER JOIN json_each(d.message_ids) ids + INNER JOIN messages m ON m.id = ids.value + WHERE d.run_id <> m.run_id + AND ( + d.run_id IN (SELECT run_id FROM migration_runs) + OR m.run_id IN (SELECT run_id FROM migration_runs) + ) + )` + ) + .get(LEGACY_RUN_ID, adoptedRunId) + if (mismatch) { + throw new Error('Legacy orchestration adoption produced inconsistent Run ownership.') + } +} + +export type AdoptLegacyRunMethods = { + adoptLegacyRunIfNeeded: typeof adoptLegacyRunIfNeeded +} + +export function attachAdoptLegacyRun(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + adoptLegacyRunIfNeeded + }) +} diff --git a/src/main/runtime/orchestration/db/schema/backfill-legacy-question-threads.ts b/src/main/runtime/orchestration/db/schema/backfill-legacy-question-threads.ts new file mode 100644 index 00000000000..351f2dc86fe --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/backfill-legacy-question-threads.ts @@ -0,0 +1,129 @@ +import { LEGACY_RUN_ID, LEGACY_CONTRACT_VERSION } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' + +export function backfillLegacyQuestionThreads(this: OrchestrationDb): void { + const messages = this.db + .prepare( + `SELECT id, run_id, from_handle, to_handle, payload, created_at, sequence + FROM messages + WHERE type = 'decision_gate' + AND delivery_contract IN ('legacy_direct', 'current_delivery') + ORDER BY sequence` + ) + .all() as { + id: string + run_id: string + from_handle: string + to_handle: string + payload: string | null + created_at: string + sequence: number + }[] + const getDispatch = this.db.prepare( + 'SELECT id, run_id, task_id FROM dispatch_contexts WHERE id = ? AND contract_version = ?' + ) + const getDispatchesForLegacyQuestion = this.db.prepare( + `SELECT id, run_id, task_id + FROM dispatch_contexts + WHERE contract_version = ? AND assignee_handle = ? + AND (? IS NULL OR task_id = ?) + AND created_at <= ? + AND (completed_at IS NULL OR completed_at >= ?) + ORDER BY rowid + LIMIT 2` + ) + const getAnswer = this.db.prepare( + `SELECT id, body, created_at + FROM messages + WHERE run_id = ? + AND thread_id = ? + AND delivery_contract IN ('legacy_direct', 'current_delivery') + AND from_handle = ? + AND to_handle IN (?, ?) + AND sequence > ? + ORDER BY sequence + LIMIT 1` + ) + const insert = this.db.prepare( + `INSERT OR IGNORE INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle, status, + answer_message_id, answer_body, created_at, answered_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const message of messages) { + let payload: { taskId?: unknown; dispatchId?: unknown } + try { + payload = JSON.parse(message.payload ?? '{}') as { + taskId?: unknown + dispatchId?: unknown + } + } catch { + continue + } + const inferredDispatches = + typeof payload.dispatchId === 'string' + ? [] + : (getDispatchesForLegacyQuestion.all( + LEGACY_CONTRACT_VERSION, + message.from_handle, + typeof payload.taskId === 'string' ? payload.taskId : null, + typeof payload.taskId === 'string' ? payload.taskId : null, + message.created_at, + message.created_at + ) as { id: string; run_id: string; task_id: string }[]) + const dispatch = + typeof payload.dispatchId === 'string' + ? (getDispatch.get(payload.dispatchId, LEGACY_CONTRACT_VERSION) as + | { id: string; run_id: string; task_id: string } + | undefined) + : inferredDispatches.length === 1 + ? inferredDispatches[0] + : undefined + if ( + !dispatch || + (typeof payload.taskId === 'string' && payload.taskId !== dispatch.task_id) || + (message.run_id !== LEGACY_RUN_ID && message.run_id !== dispatch.run_id) + ) { + continue + } + const answer = getAnswer.get( + message.run_id, + message.id, + message.to_handle, + message.from_handle, + `dispatch:${dispatch.id}`, + message.sequence + ) as { id: string; body: string; created_at: string } | undefined + insert.run( + message.id, + dispatch.run_id, + dispatch.id, + message.from_handle, + answer ? 'answered' : 'pending', + answer?.id ?? null, + answer?.body ?? null, + message.created_at, + answer?.created_at ?? null + ) + } + const adoption = this.getLegacyAdoption() + const coordinator = adoption + ? this.getLegacyCoordinatorPrincipal(adoption.adopted_run_id) + : undefined + if (adoption && coordinator?.status === 'revoked') { + this.promoteLegacyCoordinatorMailForTakeover( + adoption.adopted_run_id, + coordinator.terminal_handle + ) + } +} + +export type BackfillLegacyQuestionThreadsMethods = { + backfillLegacyQuestionThreads: typeof backfillLegacyQuestionThreads +} + +export function attachBackfillLegacyQuestionThreads(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + backfillLegacyQuestionThreads + }) +} diff --git a/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts new file mode 100644 index 00000000000..1b3172edf31 --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts @@ -0,0 +1,178 @@ +import { LEGACY_RUN_ID } from '../contract-constants' + +export function createCoreTablesSql(): string { + return ` +CREATE TABLE IF NOT EXISTS runs ( + id TEXT PRIMARY KEY, + objective TEXT NOT NULL, + home_database TEXT NOT NULL DEFAULT 'this_database', + coordinator_handle TEXT, + coordinator_pane_key TEXT, + consumer_generation INTEGER NOT NULL DEFAULT 0, + legacy INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS messages ( + id TEXT NOT NULL, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', + delivery_contract TEXT NOT NULL DEFAULT 'current_delivery' + CHECK(delivery_contract IN ('legacy_direct', 'current_delivery', 'audit_only')), + from_handle TEXT NOT NULL, + to_handle TEXT NOT NULL, + subject TEXT NOT NULL, + body TEXT NOT NULL DEFAULT '', + type TEXT NOT NULL DEFAULT 'status' + CHECK(type IN ( + 'status', 'dispatch', 'worker_done', 'merge_ready', + 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' + )), + priority TEXT NOT NULL DEFAULT 'normal' + CHECK(priority IN ('normal', 'high', 'urgent')), + thread_id TEXT, + payload TEXT, + read INTEGER NOT NULL DEFAULT 0, + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + delivered_at TEXT, + sender_pane_key TEXT +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_id ON messages(id); +CREATE INDEX IF NOT EXISTS idx_inbox ON messages(to_handle, read); +CREATE INDEX IF NOT EXISTS idx_thread ON messages(thread_id); + +CREATE TABLE IF NOT EXISTS run_coordinator_handles ( + run_id TEXT NOT NULL, + terminal_handle TEXT NOT NULL, + first_bound_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (run_id, terminal_handle) +); + +CREATE INDEX IF NOT EXISTS idx_run_coordinator_handles_handle + ON run_coordinator_handles(terminal_handle, run_id); + +CREATE TRIGGER IF NOT EXISTS trg_runs_remember_coordinator_insert +AFTER INSERT ON runs +WHEN NEW.legacy = 0 AND NEW.coordinator_handle IS NOT NULL +BEGIN + INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) + VALUES (NEW.id, NEW.coordinator_handle); +END; + +CREATE TRIGGER IF NOT EXISTS trg_runs_remember_coordinator_update +AFTER UPDATE OF coordinator_handle ON runs +WHEN NEW.legacy = 0 AND NEW.coordinator_handle IS NOT NULL +BEGIN + INSERT OR IGNORE INTO run_coordinator_handles (run_id, terminal_handle) + VALUES (NEW.id, NEW.coordinator_handle); +END; + +CREATE TRIGGER IF NOT EXISTS trg_runs_forget_coordinator_handles +AFTER DELETE ON runs +BEGIN + DELETE FROM run_coordinator_handles WHERE run_id = OLD.id; +END; + +CREATE TABLE IF NOT EXISTS deliveries ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + consumer_generation INTEGER NOT NULL, + message_ids TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'outstanding' + CHECK(status IN ('outstanding', 'acknowledged', 'fenced')), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + acknowledged_at TEXT +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_deliveries_one_outstanding + ON deliveries(run_id) WHERE status = 'outstanding'; +CREATE INDEX IF NOT EXISTS idx_deliveries_run_created + ON deliveries(run_id, created_at); + +CREATE TABLE IF NOT EXISTS mutation_receipts ( + caller_fingerprint TEXT NOT NULL, + request_id TEXT NOT NULL, + method TEXT NOT NULL, + payload_hash TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' + CHECK(state IN ('pending', 'completed')), + receipt TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (caller_fingerprint, request_id) +); + +CREATE TABLE IF NOT EXISTS mutation_caller_identities ( + transport TEXT PRIMARY KEY, + caller_fingerprint TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS worker_dispatches ( + dispatch_id TEXT PRIMARY KEY, + runtime_epoch TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + agent_terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + start_options TEXT NOT NULL DEFAULT '{}', + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS worker_terminal_resources ( + id TEXT PRIMARY KEY, + origin_dispatch_id TEXT NOT NULL, + owner_dispatch_id TEXT NOT NULL, + prior_owner_dispatch_ids TEXT NOT NULL DEFAULT '[]', + worktree_id TEXT, + terminal_handle TEXT NOT NULL, + pane_key TEXT, + process_incarnation TEXT, + host_scope TEXT, + ownership_state TEXT NOT NULL DEFAULT 'owned' + CHECK(ownership_state IN ('owned', 'transferred', 'user_owned', 'external', 'released')), + release_state TEXT NOT NULL DEFAULT 'not_requested' + CHECK(release_state IN ( + 'not_requested', 'retained', 'requested', 'releasing', 'released', 'unknown' + )), + retained_reason TEXT, + release_requested_at TEXT, + release_completed_at TEXT, + release_error TEXT, + archive_source TEXT, + archive_status TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_terminal_resources_owner + ON worker_terminal_resources(owner_dispatch_id); +CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_handle + ON worker_terminal_resources(terminal_handle); +CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_pane + ON worker_terminal_resources(pane_key); +CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_identity + ON worker_terminal_resources(process_incarnation, host_scope); +CREATE INDEX IF NOT EXISTS idx_worker_terminal_resources_release + ON worker_terminal_resources(release_state); + +CREATE TABLE IF NOT EXISTS worker_terminal_archives ( + dispatch_id TEXT PRIMARY KEY, + resource_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK(kind IN ('transcript_pin', 'terminal_tail')), + content TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + + ` +} diff --git a/src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts b/src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts new file mode 100644 index 00000000000..c20f9deb93c --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts @@ -0,0 +1,170 @@ +import { CURRENT_CONTRACT_VERSION, LEGACY_RUN_ID } from '../contract-constants' +import { + REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL, + RUN_PANE_KEY_MATCH_SUFFIX_SQL +} from '../pane-key-match' + +export function createGraphTablesSql(): string { + return ` +CREATE TABLE IF NOT EXISTS federated_dispatches ( + dispatch_id TEXT PRIMARY KEY, + environment_id TEXT NOT NULL, + environment_name TEXT NOT NULL, + peer_fingerprint TEXT NOT NULL, + remote_runtime_epoch TEXT, + protocol_version INTEGER NOT NULL DEFAULT 1, + remote_worktree_id TEXT, + remote_terminal_handle TEXT, + to_home_imported_sequence INTEGER NOT NULL DEFAULT 0, + to_home_acknowledged_sequence INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS remote_dispatch_attachments ( + dispatch_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + home_peer_fingerprint TEXT NOT NULL, + protocol_version INTEGER NOT NULL DEFAULT 1, + runtime_epoch TEXT NOT NULL, + capability_hash TEXT, + pane_key TEXT, + process_incarnation TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_remote_dispatch_attachments_active_pane + ON remote_dispatch_attachments(pane_key) + WHERE state IN ('starting', 'ready'); +CREATE INDEX IF NOT EXISTS idx_remote_dispatch_attachments_active_pane_suffix + ON remote_dispatch_attachments(${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL}) + WHERE state IN ('starting', 'ready') AND pane_key IS NOT NULL; + +CREATE TABLE IF NOT EXISTS federation_relay_items ( + dispatch_id TEXT NOT NULL, + direction TEXT NOT NULL CHECK(direction IN ('to_home', 'to_worker')), + sequence INTEGER NOT NULL, + message_id TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + byte_count INTEGER NOT NULL, + acked_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (dispatch_id, direction, sequence), + UNIQUE (dispatch_id, direction, message_id) +); + +CREATE INDEX IF NOT EXISTS idx_federation_relay_pending + ON federation_relay_items(dispatch_id, direction, acked_at, sequence); + +CREATE TABLE IF NOT EXISTS remote_questions ( + message_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'answered', 'closed')), + answer_message_id TEXT, + answer_body TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + answered_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_remote_questions_dispatch_status + ON remote_questions(dispatch_id, status); + +CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', + parent_id TEXT, + created_by_terminal_handle TEXT, + created_by_pane_key TEXT, + created_by_process_incarnation TEXT, + created_by_run_generation INTEGER, + task_title TEXT, + display_name TEXT, + spec TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ( + 'pending', 'ready', 'dispatched', + 'completed', 'failed', 'blocked' + )), + deps TEXT NOT NULL DEFAULT '[]', + result TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + completed_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status); +CREATE INDEX IF NOT EXISTS idx_tasks_parent ON tasks(parent_id); + +CREATE TABLE IF NOT EXISTS dispatch_contexts ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', + task_id TEXT NOT NULL, + contract_version INTEGER NOT NULL DEFAULT ${CURRENT_CONTRACT_VERSION}, + launch_token_hash TEXT, + assignee_handle TEXT, + assignee_pane_key TEXT, + capability_hash TEXT, + process_incarnation TEXT, + capability_revoked_at TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'dispatched', 'completed', 'failed', 'circuit_broken')), + failure_count INTEGER NOT NULL DEFAULT 0, + last_failure TEXT, + dispatched_at TEXT, + completed_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + last_heartbeat_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_dispatch_task ON dispatch_contexts(task_id); +CREATE INDEX IF NOT EXISTS idx_dispatch_status ON dispatch_contexts(status); +CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_handle ON dispatch_contexts(assignee_handle); + +CREATE TABLE IF NOT EXISTS decision_gates ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', + task_id TEXT NOT NULL, + question TEXT NOT NULL, + options TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'resolved', 'timeout')), + resolution TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + resolved_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_gates_task ON decision_gates(task_id); +CREATE INDEX IF NOT EXISTS idx_gates_status ON decision_gates(status); + +CREATE INDEX IF NOT EXISTS idx_runs_coordinator_pane_leaf + ON runs(${RUN_PANE_KEY_MATCH_SUFFIX_SQL}) + WHERE coordinator_pane_key IS NOT NULL; + +CREATE TABLE IF NOT EXISTS coordinator_runs ( + id TEXT PRIMARY KEY, + spec TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'idle' + CHECK(status IN ('idle', 'running', 'completed', 'failed')), + coordinator_handle TEXT NOT NULL, + poll_interval_ms INTEGER NOT NULL DEFAULT 2000, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + completed_at TEXT, + scheduler_lost_at TEXT +); + ` +} diff --git a/src/main/runtime/orchestration/db/schema/create-tables.ts b/src/main/runtime/orchestration/db/schema/create-tables.ts new file mode 100644 index 00000000000..70baf6eca74 --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/create-tables.ts @@ -0,0 +1,18 @@ +import type { OrchestrationDb } from '../orchestration-db' +import { createCoreTablesSql } from './create-core-tables-sql' +import { createGraphTablesSql } from './create-graph-tables-sql' + +export function createTables(this: OrchestrationDb): void { + this.db.exec(`${createCoreTablesSql()}\n${createGraphTablesSql()}`) + this.createMailboxDeliveryIndexesIfPossible() +} + +export type CreateTablesMethods = { + createTables: typeof createTables +} + +export function attachCreateTables(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createTables + }) +} diff --git a/src/main/runtime/orchestration/db/schema/migrate-legacy-contract-storage.ts b/src/main/runtime/orchestration/db/schema/migrate-legacy-contract-storage.ts new file mode 100644 index 00000000000..844e9bbfb2c --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/migrate-legacy-contract-storage.ts @@ -0,0 +1,150 @@ +import { + LEGACY_RUN_ID, + LEGACY_CONTRACT_VERSION, + CURRENT_CONTRACT_VERSION +} from '../contract-constants' +import { hasLifecycleRejectionMarker } from '../lifecycle-rejection-marker' +import type { OrchestrationDb } from '../orchestration-db' + +export function migrateLegacyContractStorage(this: OrchestrationDb): void { + if (!this.hasColumn('dispatch_contexts', 'contract_version')) { + this.db.exec( + `ALTER TABLE dispatch_contexts + ADD COLUMN contract_version INTEGER NOT NULL DEFAULT ${CURRENT_CONTRACT_VERSION}` + ) + } + if (!this.hasColumn('dispatch_contexts', 'launch_token_hash')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN launch_token_hash TEXT') + } + if (!this.hasColumn('messages', 'delivery_contract')) { + this.db.exec( + `ALTER TABLE messages + ADD COLUMN delivery_contract TEXT NOT NULL DEFAULT 'current_delivery' + CHECK(delivery_contract IN ('legacy_direct', 'current_delivery', 'audit_only'))` + ) + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_messages_delivery_contract + ON messages(run_id, delivery_contract, to_handle, read, sequence); + + CREATE TABLE IF NOT EXISTS legacy_adoptions ( + source_run_id TEXT PRIMARY KEY, + adopted_run_id TEXT UNIQUE NOT NULL, + scheduler_state_lost INTEGER NOT NULL, + adopted_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS legacy_compatibility_principals ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + dispatch_id TEXT, + role TEXT NOT NULL CHECK(role IN ('worker', 'coordinator')), + host_scope TEXT NOT NULL, + terminal_handle TEXT NOT NULL, + pane_key TEXT NOT NULL, + launch_token_hash TEXT NOT NULL, + process_incarnation TEXT, + status TEXT NOT NULL + CHECK(status IN ('committed', 'settled', 'revoked')), + CHECK( + (role = 'worker' AND dispatch_id IS NOT NULL) OR + (role = 'coordinator' AND dispatch_id IS NULL) + ), + UNIQUE(role, run_id, dispatch_id) + ); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_principal_coordinator + ON legacy_compatibility_principals(run_id) + WHERE role = 'coordinator'; + CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_principal_dispatch + ON legacy_compatibility_principals(dispatch_id) + WHERE role = 'worker'; + + CREATE TABLE IF NOT EXISTS legacy_operation_receipts ( + principal_id TEXT NOT NULL, + operation_key TEXT NOT NULL, + method TEXT NOT NULL, + payload_hash TEXT NOT NULL, + effect_id TEXT NOT NULL, + response_json TEXT NOT NULL, + completed_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY(principal_id, operation_key) + ); + + CREATE TABLE IF NOT EXISTS legacy_mail_receipts ( + principal_id TEXT NOT NULL, + message_id TEXT NOT NULL, + acknowledged_at TEXT, + PRIMARY KEY(principal_id, message_id) + ); + `) + + this.db + .prepare( + `UPDATE dispatch_contexts + SET contract_version = ? + WHERE run_id = ? AND capability_hash IS NULL` + ) + .run(LEGACY_CONTRACT_VERSION, LEGACY_RUN_ID) + this.classifyLegacyMessageContracts(LEGACY_RUN_ID, false) + this.ensureLegacySchedulerLossColumn() + this.adoptLegacyRunIfNeeded() +} + +export function classifyLegacyMessageContracts( + this: OrchestrationDb, + runId: string, + adoptedOnly: boolean +): void { + const contractFilter = adoptedOnly + ? " AND delivery_contract IN ('legacy_direct', 'audit_only')" + : '' + this.db + .prepare( + `UPDATE messages SET delivery_contract = 'legacy_direct' + WHERE run_id = ?${contractFilter}` + ) + .run(runId) + const rows = this.db + .prepare(`SELECT id, payload FROM messages WHERE run_id = ?${contractFilter}`) + .all(runId) as { id: string; payload: string | null }[] + const markAuditOnly = this.db.prepare( + "UPDATE messages SET delivery_contract = 'audit_only' WHERE id = ? AND run_id = ?" + ) + for (const row of rows) { + if (hasLifecycleRejectionMarker(row.payload)) { + markAuditOnly.run(row.id, runId) + } + } +} + +export function migrateLegacySchedulerLossProvenance(this: OrchestrationDb): void { + this.ensureLegacySchedulerLossColumn() + this.adoptLegacyRunIfNeeded() + const adoption = this.getLegacyAdoption() + if (adoption) { + this.classifyLegacyMessageContracts(adoption.adopted_run_id, true) + } +} + +export function ensureLegacySchedulerLossColumn(this: OrchestrationDb): void { + if (!this.hasColumn('coordinator_runs', 'scheduler_lost_at')) { + this.db.exec('ALTER TABLE coordinator_runs ADD COLUMN scheduler_lost_at TEXT') + } +} + +export type MigrateLegacyContractStorageMethods = { + migrateLegacyContractStorage: typeof migrateLegacyContractStorage + classifyLegacyMessageContracts: typeof classifyLegacyMessageContracts + migrateLegacySchedulerLossProvenance: typeof migrateLegacySchedulerLossProvenance + ensureLegacySchedulerLossColumn: typeof ensureLegacySchedulerLossColumn +} + +export function attachMigrateLegacyContractStorage(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + migrateLegacyContractStorage, + classifyLegacyMessageContracts, + migrateLegacySchedulerLossProvenance, + ensureLegacySchedulerLossColumn + }) +} diff --git a/src/main/runtime/orchestration/db/schema/migrate-v13-v28.ts b/src/main/runtime/orchestration/db/schema/migrate-v13-v28.ts new file mode 100644 index 00000000000..abac6fede89 --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/migrate-v13-v28.ts @@ -0,0 +1,169 @@ +import { migrateMutationReceiptCapacity } from '../../mutation-receipt-capacity' +import { DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function applySchemaMigrationsV13ToV28(this: OrchestrationDb, current: number): void { + if (current < 13 && !this.hasColumn('worker_dispatches', 'runtime_epoch')) { + this.db.exec('ALTER TABLE worker_dispatches ADD COLUMN runtime_epoch TEXT') + } + if (current < 14) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS federated_dispatches ( + dispatch_id TEXT PRIMARY KEY, + environment_id TEXT NOT NULL, + environment_name TEXT NOT NULL, + peer_fingerprint TEXT NOT NULL, + remote_runtime_epoch TEXT, + protocol_version INTEGER NOT NULL DEFAULT 1, + remote_worktree_id TEXT, + remote_terminal_handle TEXT, + to_home_imported_sequence INTEGER NOT NULL DEFAULT 0, + to_home_acknowledged_sequence INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE IF NOT EXISTS remote_dispatch_attachments ( + dispatch_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + home_peer_fingerprint TEXT NOT NULL, + protocol_version INTEGER NOT NULL DEFAULT 1, + runtime_epoch TEXT NOT NULL, + capability_hash TEXT, + pane_key TEXT, + process_incarnation TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `) + } + if (current < 15) { + if (!this.hasColumn('federated_dispatches', 'to_home_imported_sequence')) { + this.db.exec( + 'ALTER TABLE federated_dispatches ADD COLUMN to_home_imported_sequence INTEGER NOT NULL DEFAULT 0' + ) + } + if (!this.hasColumn('remote_dispatch_attachments', 'to_worker_imported_sequence')) { + this.db.exec( + 'ALTER TABLE remote_dispatch_attachments ADD COLUMN to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0' + ) + } + this.db.exec(` + CREATE TABLE IF NOT EXISTS federation_relay_items ( + dispatch_id TEXT NOT NULL, + direction TEXT NOT NULL CHECK(direction IN ('to_home', 'to_worker')), + sequence INTEGER NOT NULL, + message_id TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + byte_count INTEGER NOT NULL, + acked_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (dispatch_id, direction, sequence), + UNIQUE (dispatch_id, direction, message_id) + ); + CREATE INDEX IF NOT EXISTS idx_federation_relay_pending + ON federation_relay_items(dispatch_id, direction, acked_at, sequence); + `) + } + if (current < 16) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS remote_questions ( + message_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'answered', 'closed')), + answer_message_id TEXT, + answer_body TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + answered_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_remote_questions_dispatch_status + ON remote_questions(dispatch_id, status); + `) + } + if (current < 17 && !this.hasColumn('remote_dispatch_attachments', 'protocol_version')) { + this.db.exec( + 'ALTER TABLE remote_dispatch_attachments ADD COLUMN protocol_version INTEGER NOT NULL DEFAULT 1' + ) + } + if (current < 19) { + this.migrateLegacyContractStorage() + } + if (current < 20) { + this.backfillLegacyQuestionThreads() + } + if (current < 21) { + this.migrateLegacySchedulerLossProvenance() + } + if (current < 22) { + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_handle + ON dispatch_contexts(assignee_handle); + `) + } + if (current < 23) { + this.backfillWorkerTerminalResources() + } + if (current < 24) { + if (!this.hasColumn('tasks', 'created_by_pane_key')) { + this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_pane_key TEXT') + } + if (!this.hasColumn('tasks', 'created_by_process_incarnation')) { + this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_process_incarnation TEXT') + } + if (!this.hasColumn('tasks', 'created_by_run_generation')) { + this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_run_generation INTEGER') + } + } + if (current < 25) { + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_dispatch_active_assignee_handle + ON dispatch_contexts(assignee_handle) + WHERE assignee_handle IS NOT NULL AND status IN ('pending', 'dispatched'); + `) + } + if (current < 26) { + migrateMutationReceiptCapacity(this.db) + } + if (current < 27 && !this.hasColumn('federated_dispatches', 'to_home_acknowledged_sequence')) { + this.db.exec( + 'ALTER TABLE federated_dispatches ADD COLUMN to_home_acknowledged_sequence INTEGER NOT NULL DEFAULT 0' + ) + } + if (current < 28) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS mutation_caller_identities ( + transport TEXT PRIMARY KEY, + caller_fingerprint TEXT NOT NULL UNIQUE + ); + `) + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_pane_leaf + ON dispatch_contexts(${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL}) + WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched'); + CREATE INDEX IF NOT EXISTS idx_dispatch_active_run_assignee_handle + ON dispatch_contexts(run_id, assignee_handle) + WHERE assignee_handle IS NOT NULL AND status IN ('pending', 'dispatched'); + CREATE INDEX IF NOT EXISTS idx_dispatch_active_run_pane_leaf + ON dispatch_contexts(run_id, ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL}) + WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched'); + CREATE INDEX IF NOT EXISTS idx_dispatch_active_assignee_pane_key + ON dispatch_contexts(assignee_pane_key) + WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched'); + `) + this.createMailboxDeliveryIndexesIfPossible() +} diff --git a/src/main/runtime/orchestration/db/schema/migrate-v2-v12.ts b/src/main/runtime/orchestration/db/schema/migrate-v2-v12.ts new file mode 100644 index 00000000000..140a4116030 --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/migrate-v2-v12.ts @@ -0,0 +1,233 @@ +import { LEGACY_RUN_ID } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' + +export function applySchemaMigrationsV2ToV12(this: OrchestrationDb, current: number): void { + // v1 → v2: SQLite can't ALTER a CHECK, so rebuild messages to allow 'heartbeat'; fold in v3's delivered_at to skip a second rebuild. + if (current < 2) { + if (!this.hasColumn('dispatch_contexts', 'last_heartbeat_at')) { + this.db.exec(`ALTER TABLE dispatch_contexts ADD COLUMN last_heartbeat_at TEXT`) + } + + if (!this.messagesTypeCheckAllowsHeartbeat()) { + // Why: recreate indexes here — DROP TABLE drops them; createTables re-runs only next startup, so skipping full-scans until restart. + this.db.exec(` + CREATE TABLE messages_new ( + id TEXT NOT NULL, + from_handle TEXT NOT NULL, + to_handle TEXT NOT NULL, + subject TEXT NOT NULL, + body TEXT NOT NULL DEFAULT '', + type TEXT NOT NULL DEFAULT 'status' + CHECK(type IN ( + 'status', 'dispatch', 'worker_done', 'merge_ready', + 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' + )), + priority TEXT NOT NULL DEFAULT 'normal' + CHECK(priority IN ('normal', 'high', 'urgent')), + thread_id TEXT, + payload TEXT, + read INTEGER NOT NULL DEFAULT 0, + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + delivered_at TEXT + ); + INSERT INTO messages_new ( + id, from_handle, to_handle, subject, body, type, priority, + thread_id, payload, read, sequence, created_at + ) + SELECT + id, from_handle, to_handle, subject, body, type, priority, + thread_id, payload, read, sequence, created_at + FROM messages; + DROP TABLE messages; + ALTER TABLE messages_new RENAME TO messages; + + CREATE UNIQUE INDEX idx_messages_id ON messages(id); + CREATE INDEX idx_inbox ON messages(to_handle, read); + CREATE INDEX idx_messages_undelivered_inbox + ON messages(to_handle, read, delivered_at, sequence); + CREATE INDEX idx_thread ON messages(thread_id); + `) + } + } + + // v2 → v3: add messages.delivered_at. hasColumn probe skips DBs that already got it via the v1→v2 rebuild (else a dup-column error aborts the txn). + if (current < 3) { + if (!this.hasColumn('messages', 'delivered_at')) { + this.db.exec(`ALTER TABLE messages ADD COLUMN delivered_at TEXT`) + } + } + if (current < 4) { + if (!this.hasColumn('tasks', 'created_by_terminal_handle')) { + this.db.exec(`ALTER TABLE tasks ADD COLUMN created_by_terminal_handle TEXT`) + } + } + if (current < 5) { + if (!this.hasColumn('tasks', 'task_title')) { + this.db.exec(`ALTER TABLE tasks ADD COLUMN task_title TEXT`) + } + if (!this.hasColumn('tasks', 'display_name')) { + this.db.exec(`ALTER TABLE tasks ADD COLUMN display_name TEXT`) + } + } + if (current < 6) { + if (!this.hasColumn('dispatch_contexts', 'assignee_pane_key')) { + this.db.exec(`ALTER TABLE dispatch_contexts ADD COLUMN assignee_pane_key TEXT`) + } + if (!this.hasColumn('messages', 'sender_pane_key')) { + this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`) + } + } + if (current < 7) { + this.db + .prepare( + `INSERT OR IGNORE INTO runs ( + id, objective, home_database, consumer_generation, legacy + ) VALUES (?, ?, 'this_database', 0, 1)` + ) + .run(LEGACY_RUN_ID, 'Legacy orchestration state (inspect only)') + for (const table of ['messages', 'tasks', 'dispatch_contexts', 'decision_gates']) { + if (!this.hasColumn(table, 'run_id')) { + this.db.exec( + `ALTER TABLE ${table} ADD COLUMN run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}'` + ) + } + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_messages_run_sequence ON messages(run_id, sequence); + CREATE INDEX IF NOT EXISTS idx_tasks_run_status ON tasks(run_id, status); + CREATE INDEX IF NOT EXISTS idx_dispatch_run_status ON dispatch_contexts(run_id, status); + CREATE INDEX IF NOT EXISTS idx_gates_run_status ON decision_gates(run_id, status); + CREATE INDEX IF NOT EXISTS idx_runs_coordinator_pane ON runs(coordinator_pane_key); + `) + } + if (current < 8) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS deliveries ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + consumer_generation INTEGER NOT NULL, + message_ids TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'outstanding' + CHECK(status IN ('outstanding', 'acknowledged', 'fenced')), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + acknowledged_at TEXT + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_deliveries_one_outstanding + ON deliveries(run_id) WHERE status = 'outstanding'; + CREATE INDEX IF NOT EXISTS idx_deliveries_run_created + ON deliveries(run_id, created_at); + + CREATE TABLE IF NOT EXISTS question_threads ( + message_id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + dispatch_id TEXT NOT NULL, + asker_handle TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'answered', 'closed')), + answer_message_id TEXT, + answer_body TEXT, + answered_by_generation INTEGER, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + answered_at TEXT, + closed_at TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_questions_dispatch_status + ON question_threads(dispatch_id, status); + `) + } + if (current < 9 && !this.messagesTypeCheckAllowsQuestion()) { + this.db.exec(` + CREATE TABLE messages_new ( + id TEXT NOT NULL, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', + from_handle TEXT NOT NULL, + to_handle TEXT NOT NULL, + subject TEXT NOT NULL, + body TEXT NOT NULL DEFAULT '', + type TEXT NOT NULL DEFAULT 'status' + CHECK(type IN ( + 'status', 'dispatch', 'worker_done', 'merge_ready', + 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' + )), + priority TEXT NOT NULL DEFAULT 'normal' + CHECK(priority IN ('normal', 'high', 'urgent')), + thread_id TEXT, + payload TEXT, + read INTEGER NOT NULL DEFAULT 0, + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + delivered_at TEXT, + sender_pane_key TEXT + ); + INSERT INTO messages_new ( + id, run_id, from_handle, to_handle, subject, body, type, priority, + thread_id, payload, read, sequence, created_at, delivered_at, sender_pane_key + ) + SELECT + id, run_id, from_handle, to_handle, subject, body, type, priority, + thread_id, payload, read, sequence, created_at, delivered_at, sender_pane_key + FROM messages; + DROP TABLE messages; + ALTER TABLE messages_new RENAME TO messages; + + CREATE UNIQUE INDEX idx_messages_id ON messages(id); + CREATE INDEX idx_inbox ON messages(to_handle, read); + CREATE INDEX idx_thread ON messages(thread_id); + CREATE INDEX idx_messages_run_sequence ON messages(run_id, sequence); + CREATE INDEX idx_messages_undelivered_inbox + ON messages(to_handle, read, delivered_at, sequence); + `) + } + if (current < 10) { + if (!this.hasColumn('dispatch_contexts', 'capability_hash')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN capability_hash TEXT') + } + if (!this.hasColumn('dispatch_contexts', 'process_incarnation')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN process_incarnation TEXT') + } + if (!this.hasColumn('dispatch_contexts', 'capability_revoked_at')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN capability_revoked_at TEXT') + } + } + if (current < 11) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS mutation_receipts ( + caller_fingerprint TEXT NOT NULL, + request_id TEXT NOT NULL, + method TEXT NOT NULL, + payload_hash TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' + CHECK(state IN ('pending', 'completed')), + receipt TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (caller_fingerprint, request_id) + ); + `) + } + if (current < 12) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS worker_dispatches ( + dispatch_id TEXT PRIMARY KEY, + runtime_epoch TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + agent_terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + start_options TEXT NOT NULL DEFAULT '{}', + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `) + } +} diff --git a/src/main/runtime/orchestration/db/schema/migrate.ts b/src/main/runtime/orchestration/db/schema/migrate.ts new file mode 100644 index 00000000000..4fbd872e5a8 --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/migrate.ts @@ -0,0 +1,35 @@ +import { resolveOrchestrationMigrationStartVersion } from '../../orchestration-schema-version-skew' +import { SCHEMA_VERSION } from '../contract-constants' +import type { OrchestrationDb } from '../orchestration-db' +import { applySchemaMigrationsV13ToV28 } from './migrate-v13-v28' +import { applySchemaMigrationsV2ToV12 } from './migrate-v2-v12' + +// Why: CREATE TABLE IF NOT EXISTS won't alter existing DBs; migrate in a txn that bumps user_version only on success (atomic all-or-nothing). +export function migrate(this: OrchestrationDb): void { + const storedVersion = this.db.pragma('user_version', { simple: true }) as number + const current = resolveOrchestrationMigrationStartVersion(this.db, storedVersion, SCHEMA_VERSION) + if (current >= SCHEMA_VERSION) { + return + } + + this.db.exec('BEGIN IMMEDIATE') + try { + applySchemaMigrationsV2ToV12.call(this, current) + applySchemaMigrationsV13ToV28.call(this, current) + this.db.pragma(`user_version = ${SCHEMA_VERSION}`) + this.db.exec('COMMIT') + } catch (err) { + this.db.exec('ROLLBACK') + throw err + } +} + +export type SchemaMigrateMethods = { + migrate: typeof migrate +} + +export function attachSchemaMigrate(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + migrate + }) +} diff --git a/src/main/runtime/orchestration/db/schema/schema-column-probes.ts b/src/main/runtime/orchestration/db/schema/schema-column-probes.ts new file mode 100644 index 00000000000..a3748c8292c --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/schema-column-probes.ts @@ -0,0 +1,70 @@ +import type { OrchestrationDb } from '../orchestration-db' + +export function hasColumn(this: OrchestrationDb, table: string, column: string): boolean { + const rows = this.db.pragma(`table_info(${table})`) as { name: string }[] + return rows.some((r) => r.name === column) +} + +export function createMailboxDeliveryIndexesIfPossible(this: OrchestrationDb): void { + const hasDeliveredAt = this.hasColumn('messages', 'delivered_at') + if (hasDeliveredAt) { + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_messages_undelivered_inbox + ON messages(to_handle, read, delivered_at, sequence) + `) + } + + if ( + !hasDeliveredAt || + !this.hasColumn('messages', 'run_id') || + !this.hasColumn('messages', 'delivery_contract') + ) { + return + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_messages_undelivered_direct_run + ON messages(run_id, to_handle, sequence) + WHERE read = 0 AND delivered_at IS NULL + AND delivery_contract = 'current_delivery'; + CREATE INDEX IF NOT EXISTS idx_messages_unread_current_inbox + ON messages(to_handle, sequence) + WHERE read = 0 AND delivery_contract = 'current_delivery'; + CREATE INDEX IF NOT EXISTS idx_messages_unread_current_inbox_type + ON messages(to_handle, type, sequence) + WHERE read = 0 AND delivery_contract = 'current_delivery'; + CREATE INDEX IF NOT EXISTS idx_messages_unread_current_run_type + ON messages(run_id, to_handle, type, sequence) + WHERE read = 0 AND delivery_contract = 'current_delivery'; + `) +} + +// Why: sqlite_master holds the table's CREATE SQL incl. the CHECK — cheapest reliable probe for whether it already allows 'heartbeat'. +export function messagesTypeCheckAllowsHeartbeat(this: OrchestrationDb): boolean { + const row = this.db + .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages'") + .get() as { sql: string } | undefined + return !!row && row.sql.includes("'heartbeat'") +} + +export function messagesTypeCheckAllowsQuestion(this: OrchestrationDb): boolean { + const row = this.db + .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages'") + .get() as { sql: string } | undefined + return !!row && row.sql.includes("'question'") +} + +export type SchemaColumnProbesMethods = { + hasColumn: typeof hasColumn + createMailboxDeliveryIndexesIfPossible: typeof createMailboxDeliveryIndexesIfPossible + messagesTypeCheckAllowsHeartbeat: typeof messagesTypeCheckAllowsHeartbeat + messagesTypeCheckAllowsQuestion: typeof messagesTypeCheckAllowsQuestion +} + +export function attachSchemaColumnProbes(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + hasColumn, + createMailboxDeliveryIndexesIfPossible, + messagesTypeCheckAllowsHeartbeat, + messagesTypeCheckAllowsQuestion + }) +} diff --git a/src/main/runtime/orchestration/db/tasks/task-store.ts b/src/main/runtime/orchestration/db/tasks/task-store.ts new file mode 100644 index 00000000000..b06920503c1 --- /dev/null +++ b/src/main/runtime/orchestration/db/tasks/task-store.ts @@ -0,0 +1,255 @@ +import type Database from '../../../../sqlite/sync-database' +import type { TaskStatus, TaskRow } from '../../types' +import { buildOrchestrationTaskDisplayMetadata } from '../../../../../shared/orchestration-task-display' +import { LEGACY_RUN_ID } from '../contract-constants' +import { generateId } from '../generated-id' +import type { TaskRuntimeLineageRow } from '../run-list-page' +import type { OrchestrationDb } from '../orchestration-db' + +// ── Tasks ── + +export function createTask( + this: OrchestrationDb, + task: { + spec: string + taskTitle?: string + displayName?: string + deps?: string[] + parentId?: string + createdByTerminalHandle?: string + createdByPaneKey?: string + createdByProcessIncarnation?: string + createdByRunGeneration?: number + runId?: string + } +): TaskRow { + const runId = task.runId ?? LEGACY_RUN_ID + this.requireRun(runId) + if (task.parentId) { + const parent = this.getTask(task.parentId) + if (!parent || parent.run_id !== runId) { + throw new Error(`Parent task ${task.parentId} must belong to run ${runId}`) + } + } + for (const depId of task.deps ?? []) { + const dependency = this.getTask(depId) + if (!dependency || dependency.run_id !== runId) { + throw new Error(`Dependency task ${depId} must belong to run ${runId}`) + } + } + const id = generateId('task') + const depsJson = JSON.stringify(task.deps ?? []) + const display = buildOrchestrationTaskDisplayMetadata({ + spec: task.spec, + taskTitle: task.taskTitle, + displayName: task.displayName + }) + this.db + .prepare( + `INSERT INTO tasks ( + id, run_id, parent_id, created_by_terminal_handle, created_by_pane_key, + created_by_process_incarnation, created_by_run_generation, + task_title, display_name, spec, status, deps + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + CASE WHEN EXISTS ( + SELECT 1 + FROM json_each(?) requested + LEFT JOIN tasks dependency ON dependency.id = requested.value + WHERE dependency.id IS NULL + OR dependency.run_id <> ? + OR dependency.status <> 'completed' + ) THEN 'pending' ELSE 'ready' END, + ? + )` + ) + .run( + id, + runId, + task.parentId ?? null, + task.createdByTerminalHandle ?? null, + task.createdByPaneKey ?? null, + task.createdByProcessIncarnation ?? null, + task.createdByRunGeneration ?? null, + display.taskTitle || null, + display.displayName || null, + task.spec, + depsJson, + runId, + depsJson + ) + return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow +} + +// Why: return the active creator Dispatch proof with the Task read; runtime still owns pane/process currency. +export function getTask(this: OrchestrationDb, id: string): TaskRow | undefined +export function getTask( + this: OrchestrationDb, + id: string, + dispatchRunId: string +): TaskRuntimeLineageRow | undefined +export function getTask( + this: OrchestrationDb, + id: string, + dispatchRunId?: string +): TaskRow | TaskRuntimeLineageRow | undefined { + if (dispatchRunId === undefined) { + return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow | undefined + } + return this.db + .prepare( + `SELECT t.*, + creator.id AS creator_dispatch_id, + creator.run_id AS creator_dispatch_run_id, + creator.assignee_pane_key AS creator_dispatch_pane_key, + creator.process_incarnation AS creator_dispatch_process_incarnation + FROM tasks t + LEFT JOIN dispatch_contexts creator ON creator.rowid = ( + SELECT candidate.rowid + FROM dispatch_contexts candidate + WHERE candidate.assignee_handle = t.created_by_terminal_handle + AND candidate.run_id = ? + AND candidate.status IN ('pending', 'dispatched') + ORDER BY candidate.rowid DESC + LIMIT 1 + ) + WHERE t.id = ?` + ) + .get(dispatchRunId, id) as TaskRuntimeLineageRow | undefined +} + +export function listTasks( + this: OrchestrationDb, + filter?: { status?: TaskStatus; ready?: boolean; runId?: string } +): TaskRow[] { + const runWhere = filter?.runId ? 'run_id = ? AND ' : '' + const runParams: Database.BindValue[] = filter?.runId ? [filter.runId] : [] + if (filter?.ready) { + return this.db + .prepare(`SELECT * FROM tasks WHERE ${runWhere}status = 'ready' ORDER BY created_at`) + .all(...runParams) as TaskRow[] + } + if (filter?.status) { + return this.db + .prepare(`SELECT * FROM tasks WHERE ${runWhere}status = ? ORDER BY created_at`) + .all(...runParams, filter.status) as TaskRow[] + } + if (filter?.runId) { + return this.db + .prepare('SELECT * FROM tasks WHERE run_id = ? ORDER BY created_at') + .all(filter.runId) as TaskRow[] + } + return this.db.prepare('SELECT * FROM tasks ORDER BY created_at').all() as TaskRow[] +} + +// Why: the correlated indexed lookup avoids materializing every retained Dispatch before filtering Tasks. +export function listTasksWithDispatch( + this: OrchestrationDb, + filter?: { + status?: TaskStatus + ready?: boolean + runId?: string + } +): (TaskRow & { + assignee_handle: string | null + dispatch_id: string | null +})[] { + const whereClauses: string[] = [] + const params: Database.BindValue[] = [] + if (filter?.runId) { + whereClauses.push('t.run_id = ?') + params.push(filter.runId) + } + if (filter?.ready) { + whereClauses.push("t.status = 'ready'") + } else if (filter?.status) { + whereClauses.push('t.status = ?') + params.push(filter.status) + } + const where = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '' + const sql = ` + SELECT + t.*, + d.assignee_handle AS assignee_handle, + d.id AS dispatch_id + FROM tasks t + LEFT JOIN dispatch_contexts d ON d.rowid = ( + SELECT candidate.rowid + FROM dispatch_contexts candidate + WHERE candidate.task_id = t.id + AND candidate.status IN ('pending', 'dispatched') + ORDER BY candidate.rowid DESC + LIMIT 1 + ) + ${where} + ORDER BY t.created_at + ` + return this.db.prepare(sql).all(...params) as (TaskRow & { + assignee_handle: string | null + dispatch_id: string | null + })[] +} + +export function updateTaskStatus( + this: OrchestrationDb, + id: string, + status: TaskStatus, + result?: string +): TaskRow | undefined { + const completedAt = + status === 'completed' || status === 'failed' ? new Date().toISOString() : null + this.db + .prepare( + 'UPDATE tasks SET status = ?, result = COALESCE(?, result), completed_at = COALESCE(?, completed_at) WHERE id = ?' + ) + .run(status, result ?? null, completedAt, id) + + if (status === 'completed') { + this.promoteReadyTasks(id) + this.completeActiveDispatchForTask(id) + } + + return this.getTask(id) +} + +// Why: runs in the status-update transaction, so a completed task never leaves its ready children unpromoted. +export function promoteReadyTasks(this: OrchestrationDb, completedTaskId: string): void { + const candidates = this.db + .prepare("SELECT * FROM tasks WHERE status = 'pending'") + .all() as TaskRow[] + + for (const task of candidates) { + const deps: string[] = JSON.parse(task.deps) + if (!deps.includes(completedTaskId)) { + continue + } + + const allDepsCompleted = deps.every((depId) => { + const dep = this.getTask(depId) + return dep?.status === 'completed' + }) + if (allDepsCompleted) { + this.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id) + } + } +} + +export type TaskStoreMethods = { + createTask: typeof createTask + getTask: typeof getTask + listTasks: typeof listTasks + listTasksWithDispatch: typeof listTasksWithDispatch + updateTaskStatus: typeof updateTaskStatus + promoteReadyTasks: typeof promoteReadyTasks +} + +export function attachTaskStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createTask, + getTask, + listTasks, + listTasksWithDispatch, + updateTaskStatus, + promoteReadyTasks + }) +} diff --git a/src/main/runtime/orchestration/db/utc-timestamp.ts b/src/main/runtime/orchestration/db/utc-timestamp.ts new file mode 100644 index 00000000000..96e1472c94f --- /dev/null +++ b/src/main/runtime/orchestration/db/utc-timestamp.ts @@ -0,0 +1,48 @@ +import type { DeliveryRow, MessageRow, QuestionRow, RunRow } from '../types' + +export const SQLITE_UTC_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/ + +export function exposeUtcTimestamp(timestamp: string | null): string | null { + if (!timestamp || !SQLITE_UTC_TIMESTAMP_RE.test(timestamp)) { + return timestamp + } + return `${timestamp.replace(' ', 'T')}Z` +} + +export function exposeMessageTimestamps(message: MessageRow): MessageRow { + // Why: SQLite stores UTC as timezone-less space format for SQL ordering, but RPC/CLI consumers need an explicit offset. + return { + ...message, + created_at: exposeUtcTimestamp(message.created_at) ?? message.created_at, + delivered_at: exposeUtcTimestamp(message.delivered_at) + } +} + +export function exposeMessageListTimestamps(messages: MessageRow[]): MessageRow[] { + return messages.map(exposeMessageTimestamps) +} + +export function exposeRunTimestamps(run: RunRow): RunRow { + return { + ...run, + created_at: exposeUtcTimestamp(run.created_at) ?? run.created_at, + updated_at: exposeUtcTimestamp(run.updated_at) ?? run.updated_at + } +} + +export function exposeDeliveryTimestamps(delivery: DeliveryRow): DeliveryRow { + return { + ...delivery, + created_at: exposeUtcTimestamp(delivery.created_at) ?? delivery.created_at, + acknowledged_at: exposeUtcTimestamp(delivery.acknowledged_at) + } +} + +export function exposeQuestionTimestamps(question: QuestionRow): QuestionRow { + return { + ...question, + created_at: exposeUtcTimestamp(question.created_at) ?? question.created_at, + answered_at: exposeUtcTimestamp(question.answered_at), + closed_at: exposeUtcTimestamp(question.closed_at) + } +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/federated-worker-start-reconcile.ts b/src/main/runtime/orchestration/db/worker-dispatch/federated-worker-start-reconcile.ts new file mode 100644 index 00000000000..7b36845525e --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/federated-worker-start-reconcile.ts @@ -0,0 +1,113 @@ +import type { WorkerDispatchRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function reconcileFederatedWorkerStart( + this: OrchestrationDb, + params: { + dispatchId: string + state: 'ready' | 'failed' | 'stopped' | 'start_unknown' + stage: string + lastError?: string | null + worktreeId?: string | null + terminalHandle?: string | null + setupState?: string + effects?: unknown[] + residualResources?: unknown[] + } +): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(params.dispatchId) + const worker = this.getWorkerDispatch(params.dispatchId) + if (!dispatch || !worker) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${params.dispatchId} was not found.` + ) + } + if (!['starting', 'start_unknown'].includes(worker.state)) { + this.db.exec('COMMIT') + return worker + } + + if (params.state === 'ready') { + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'ready', stage = ?, worktree_id = COALESCE(?, worktree_id), + agent_terminal_handle = COALESCE(?, agent_terminal_handle), setup_state = ?, + effects = COALESCE(?, effects), + residual_resources = COALESCE(?, residual_resources), last_error = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` + ) + .run( + params.stage, + params.worktreeId ?? null, + params.terminalHandle ?? null, + params.setupState ?? worker.setup_state, + // Why: keep the stored JSON as-is when the peer omits it — re-parsing it here throws on any malformed legacy row. + params.effects ? JSON.stringify(params.effects) : null, + params.residualResources ? JSON.stringify(params.residualResources) : null, + params.dispatchId + ) + this.db + .prepare( + "UPDATE dispatch_contexts SET status = 'dispatched' WHERE id = ? AND status = 'pending'" + ) + .run(params.dispatchId) + this.db + .prepare( + "UPDATE tasks SET status = 'dispatched', completed_at = NULL WHERE id = ? AND status = 'blocked'" + ) + .run(dispatch.task_id) + } else if (params.state === 'start_unknown') { + this.db + .prepare( + `UPDATE worker_dispatches + SET stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` + ) + .run(params.stage, params.lastError ?? worker.last_error, params.dispatchId) + } else { + const reason = params.lastError ?? `The worker server reported ${params.state}.` + this.db + .prepare( + `UPDATE worker_dispatches + SET state = ?, stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` + ) + .run(params.state, params.stage, reason, params.dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', last_failure = ?, completed_at = datetime('now'), + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(reason, params.dispatchId) + this.db + .prepare( + "UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE id = ? AND status IN ('blocked', 'dispatched')" + ) + .run(dispatch.task_id) + this.closeQuestionsForDispatch(params.dispatchId) + } + this.db.exec('COMMIT') + return this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type FederatedWorkerStartReconcileMethods = { + reconcileFederatedWorkerStart: typeof reconcileFederatedWorkerStart +} + +export function attachFederatedWorkerStartReconcile(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + reconcileFederatedWorkerStart + }) +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-abandon.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-abandon.ts new file mode 100644 index 00000000000..93c93a3550c --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-abandon.ts @@ -0,0 +1,89 @@ +import type { WorkerDispatchRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { + releaseContextOnlyDispatch, + type ContextOnlyDispatchReleaseResult +} from '../../context-only-dispatch-release' +import type { OrchestrationDb } from '../orchestration-db' + +export function abandonWorkerDispatch( + this: OrchestrationDb, + dispatchId: string +): + | { + disposition: 'abandoned' | 'already_abandoned' | 'stale' + worker: WorkerDispatchRow + } + | ({ disposition: 'context_only' } & ContextOnlyDispatchReleaseResult) { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!dispatch) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (!worker) { + const released = releaseContextOnlyDispatch( + this.db, + dispatch, + this.getDispatchContext(dispatch.task_id)?.id, + 'abandoned' + ) + if (!released.alreadySettled) { + this.closeQuestionsForDispatch(dispatchId) + } + this.db.exec('COMMIT') + return { disposition: 'context_only', ...released } + } + if (worker.state === 'abandoned') { + this.db.exec('COMMIT') + return { disposition: 'already_abandoned', worker } + } + if (this.getDispatchContext(dispatch.task_id)?.id !== dispatchId) { + this.db.exec('COMMIT') + return { disposition: 'stale', worker } + } + if (worker.state === 'succeeded') { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} already succeeded and cannot be abandoned.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'abandoned', stage = 'abandoned', updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = CASE WHEN status IN ('pending', 'dispatched') THEN 'failed' ELSE status END, + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')), + completed_at = COALESCE(completed_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) + this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return { + disposition: 'abandoned', + worker: this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type WorkerDispatchAbandonMethods = { + abandonWorkerDispatch: typeof abandonWorkerDispatch +} + +export function attachWorkerDispatchAbandon(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + abandonWorkerDispatch + }) +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-authority.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-authority.ts new file mode 100644 index 00000000000..af395f7399f --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-authority.ts @@ -0,0 +1,154 @@ +import { randomBytes } from 'node:crypto' +import { OrchestrationError } from '../../orchestration-error' +import { hashDispatchCapability } from '../dispatch-capability-hash' +import type { OrchestrationDb } from '../orchestration-db' + +export function prepareStartingWorkerAuthority( + this: OrchestrationDb, + params: { + dispatchId: string + handle: string + paneKey: string + processIncarnation: string + launchTokenHash?: string + worktreeId: string + effects: unknown[] + setupState: string + hostScope?: string | null + // 'created': this worker-start operation created the agent terminal (including agent-first + // worktree creation, whose effects receipt says 'reused_agent_terminal'). 'external': an + // explicit --terminal reuse; ownership transfers only from an exact owned settled resource. + terminalOwnership?: 'created' | 'external' + } +): string { + const capability = `dcap_${randomBytes(32).toString('base64url')}` + this.db.exec('BEGIN IMMEDIATE') + try { + // Why: read inside the transaction so the guarded UPDATEs below cannot lose a race with a concurrent state change. + const dispatch = this.getDispatchContextById(params.dispatchId) + const worker = this.getWorkerDispatch(params.dispatchId) + if (!dispatch || dispatch.status !== 'pending' || worker?.state !== 'starting') { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not starting.` + ) + } + if ( + dispatch.launch_token_hash && + params.launchTokenHash && + dispatch.launch_token_hash !== params.launchTokenHash + ) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${params.dispatchId} already has a different launch-token commitment.` + ) + } + const contextUpdate = this.db + .prepare( + `UPDATE dispatch_contexts + SET assignee_handle = ?, assignee_pane_key = ?, process_incarnation = ?, + capability_hash = ?, launch_token_hash = COALESCE(launch_token_hash, ?), + capability_revoked_at = NULL + WHERE id = ? AND status = 'pending'` + ) + .run( + params.handle, + params.paneKey, + params.processIncarnation, + hashDispatchCapability(capability), + params.launchTokenHash ?? null, + params.dispatchId + ) + if (contextUpdate.changes !== 1) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not starting.` + ) + } + const workerUpdate = this.db + .prepare( + `UPDATE worker_dispatches + SET stage = 'authority_attached', worktree_id = ?, agent_terminal_handle = ?, + setup_state = ?, effects = ?, residual_resources = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run( + params.worktreeId, + params.handle, + params.setupState, + JSON.stringify(params.effects), + JSON.stringify( + params.effects.filter((effect) => + Boolean( + effect && + typeof effect === 'object' && + ((effect as { action?: string }).action?.startsWith('created') || + (effect as { action?: string }).action === 'reused_agent_terminal') + ) + ) + ), + params.dispatchId + ) + if (workerUpdate.changes !== 1) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not starting.` + ) + } + if (params.terminalOwnership && !this.getWorkerTerminalResourceByOwner(params.dispatchId)) { + if (params.terminalOwnership === 'created') { + this.createWorkerTerminalResourceStatement({ + dispatchId: params.dispatchId, + worktreeId: params.worktreeId, + terminalHandle: params.handle, + paneKey: params.paneKey, + processIncarnation: params.processIncarnation, + hostScope: params.hostScope, + ownership: 'owned' + }) + } else { + const transferable = this.findTransferableWorkerTerminalResource({ + terminalHandle: params.handle, + paneKey: params.paneKey, + processIncarnation: params.processIncarnation, + hostScope: params.hostScope ?? null + }) + if (transferable) { + this.transferWorkerTerminalResourceStatement({ + resourceId: transferable.id, + toDispatchId: params.dispatchId, + terminalHandle: params.handle, + paneKey: params.paneKey, + processIncarnation: params.processIncarnation, + hostScope: params.hostScope ?? null + }) + } else { + this.createWorkerTerminalResourceStatement({ + dispatchId: params.dispatchId, + worktreeId: params.worktreeId, + terminalHandle: params.handle, + paneKey: params.paneKey, + processIncarnation: params.processIncarnation, + hostScope: params.hostScope, + ownership: 'external' + }) + } + } + } + this.db.exec('COMMIT') + return capability + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type WorkerDispatchAuthorityMethods = { + prepareStartingWorkerAuthority: typeof prepareStartingWorkerAuthority +} + +export function attachWorkerDispatchAuthority(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + prepareStartingWorkerAuthority + }) +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-outcome.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-outcome.ts new file mode 100644 index 00000000000..d4f50dfb2bf --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-outcome.ts @@ -0,0 +1,136 @@ +import type { WorkerDispatchRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function markWorkerDispatchReady( + this: OrchestrationDb, + dispatchId: string, + effects?: unknown[] +): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || dispatch.status !== 'pending' || worker?.state !== 'starting') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) + } + this.db + .prepare("UPDATE dispatch_contexts SET status = 'dispatched' WHERE id = ?") + .run(dispatchId) + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'ready', stage = 'input_accepted', + effects = COALESCE(?, effects), updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(effects ? JSON.stringify(effects) : null, dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function failWorkerStart( + this: OrchestrationDb, + dispatchId: string, + stage: string, + reason: string +): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || !worker || worker.state !== 'starting') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) + } + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', last_failure = ?, completed_at = datetime('now'), + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(reason, dispatchId) + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'failed', stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(stage, reason, dispatchId) + this.db + .prepare("UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE id = ?") + .run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function markWorkerStartUnknown( + this: OrchestrationDb, + dispatchId: string, + stage: string, + reason: string +): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || !worker || worker.state !== 'starting') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'start_unknown', stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(stage, reason, dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) + this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getWorkerDispatch( + this: OrchestrationDb, + dispatchId: string +): WorkerDispatchRow | undefined { + return this.db + .prepare('SELECT * FROM worker_dispatches WHERE dispatch_id = ?') + .get(dispatchId) as WorkerDispatchRow | undefined +} + +export type WorkerDispatchOutcomeMethods = { + markWorkerDispatchReady: typeof markWorkerDispatchReady + failWorkerStart: typeof failWorkerStart + markWorkerStartUnknown: typeof markWorkerStartUnknown + getWorkerDispatch: typeof getWorkerDispatch +} + +export function attachWorkerDispatchOutcome(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + markWorkerDispatchReady, + failWorkerStart, + markWorkerStartUnknown, + getWorkerDispatch + }) +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stage.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stage.ts new file mode 100644 index 00000000000..e00faa458c9 --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stage.ts @@ -0,0 +1,92 @@ +import type { WorkerDispatchRow, WorkerDispatchState } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function recordWorkerStage( + this: OrchestrationDb, + params: { + dispatchId: string + stage: string + worktreeId?: string + terminalHandle?: string + setupState?: string + effects?: unknown[] + residualResources?: unknown[] + lastError?: string + state?: WorkerDispatchState + } +): WorkerDispatchRow { + const current = this.getWorkerDispatch(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Dispatch ${params.dispatchId} was not found.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET stage = ?, state = ?, worktree_id = ?, agent_terminal_handle = ?, + setup_state = ?, effects = ?, residual_resources = ?, last_error = ?, + updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run( + params.stage, + params.state ?? current.state, + params.worktreeId ?? current.worktree_id, + params.terminalHandle ?? current.agent_terminal_handle, + params.setupState ?? current.setup_state, + params.effects ? JSON.stringify(params.effects) : current.effects, + params.residualResources + ? JSON.stringify(params.residualResources) + : current.residual_resources, + params.lastError ?? current.last_error, + params.dispatchId + ) + return this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow +} + +export function updateWorkerSetupEvidence( + this: OrchestrationDb, + params: { + dispatchId: string + setupState: string + effects: unknown[] + } +): { worker: WorkerDispatchRow; changed: boolean } { + const current = this.getWorkerDispatch(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Dispatch ${params.dispatchId} was not found.` + ) + } + const effects = JSON.stringify(params.effects) + if (current.setup_state === params.setupState && current.effects === effects) { + return { worker: current, changed: false } + } + this.db + .prepare( + `UPDATE worker_dispatches + SET setup_state = ?, effects = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(params.setupState, effects, params.dispatchId) + return { + worker: this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow, + changed: true + } +} + +export type WorkerDispatchStageMethods = { + recordWorkerStage: typeof recordWorkerStage + updateWorkerSetupEvidence: typeof updateWorkerSetupEvidence +} + +export function attachWorkerDispatchStage(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + recordWorkerStage, + updateWorkerSetupEvidence + }) +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts new file mode 100644 index 00000000000..91f590895dc --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts @@ -0,0 +1,152 @@ +import type { DispatchContextRow, WorkerDispatchRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { ensureMutationReceiptCapacity } from '../../mutation-receipt-capacity' +import { CURRENT_CONTRACT_VERSION } from '../contract-constants' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +export function createStartingWorkerDispatch( + this: OrchestrationDb, + params: { + taskId: string + startOptions: unknown + launchTokenHash?: string + retryOf?: string + runtimeEpoch?: string + federation?: { + environmentId: string + environmentName: string + peerFingerprint: string + protocolVersion: number + } + mutationReceipt?: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } + } +): { dispatch: DispatchContextRow; worker: WorkerDispatchRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + if (params.mutationReceipt) { + const receipt = params.mutationReceipt + const existing = this.getMutationReceipt(receipt.callerFingerprint, receipt.requestId) + if (existing) { + if (existing.method !== receipt.method || existing.payload_hash !== receipt.payloadHash) { + throw new OrchestrationError( + 'request_mismatch', + `Mutation request ${receipt.requestId} was already used with different input.` + ) + } + throw new OrchestrationError( + 'operation_unknown', + `Mutation ${receipt.requestId} already has a durable acceptance record.` + ) + } + ensureMutationReceiptCapacity(this.db) + this.db + .prepare( + `INSERT INTO mutation_receipts ( + caller_fingerprint, request_id, method, payload_hash, state + ) VALUES (?, ?, ?, ?, 'pending')` + ) + .run(receipt.callerFingerprint, receipt.requestId, receipt.method, receipt.payloadHash) + } + const task = this.getTask(params.taskId) + if (!task) { + throw new OrchestrationError('task_not_found', `Task ${params.taskId} was not found.`) + } + if (params.retryOf) { + const prior = this.getDispatchContextById(params.retryOf) + const priorWorker = this.getWorkerDispatch(params.retryOf) + const latest = this.getDispatchContext(task.id) + if ( + !prior || + prior.task_id !== task.id || + latest?.id !== prior.id || + !priorWorker || + !['failed', 'stopped', 'abandoned'].includes(priorWorker.state) || + !['failed', 'blocked'].includes(task.status) + ) { + throw new OrchestrationError( + 'task_not_startable', + `Task ${task.id} cannot retry from Dispatch ${params.retryOf}.` + ) + } + } else if (task.status !== 'ready') { + throw new OrchestrationError( + 'task_not_startable', + `Task ${task.id} is ${task.status}; only a ready Task can start.` + ) + } + + const id = generateId('ctx') + if (params.mutationReceipt) { + this.db + .prepare( + `UPDATE mutation_receipts + SET receipt = ?, updated_at = datetime('now') + WHERE caller_fingerprint = ? AND request_id = ? AND state = 'pending'` + ) + .run( + JSON.stringify({ accepted: { dispatchId: id } }), + params.mutationReceipt.callerFingerprint, + params.mutationReceipt.requestId + ) + } + this.db + .prepare( + `INSERT INTO dispatch_contexts ( + id, run_id, task_id, contract_version, launch_token_hash, status, dispatched_at + ) VALUES (?, ?, ?, ?, ?, 'pending', datetime('now'))` + ) + .run(id, task.run_id, task.id, CURRENT_CONTRACT_VERSION, params.launchTokenHash ?? null) + this.db + .prepare( + `INSERT INTO worker_dispatches ( + dispatch_id, runtime_epoch, state, stage, start_options + ) VALUES (?, ?, 'starting', 'accepted', ?)` + ) + .run(id, params.runtimeEpoch ?? null, JSON.stringify(params.startOptions)) + if (params.federation) { + this.db + .prepare( + `INSERT INTO federated_dispatches ( + dispatch_id, environment_id, environment_name, peer_fingerprint, protocol_version + ) VALUES (?, ?, ?, ?, ?)` + ) + .run( + id, + params.federation.environmentId, + params.federation.environmentName, + params.federation.peerFingerprint, + params.federation.protocolVersion + ) + } + this.db + .prepare( + "UPDATE tasks SET status = 'dispatched', result = NULL, completed_at = NULL WHERE id = ?" + ) + .run(task.id) + this.db.exec('COMMIT') + this.hasAnyDispatchContextsCache = true + return { + dispatch: this.getDispatchContextById(id) as DispatchContextRow, + worker: this.getWorkerDispatch(id) as WorkerDispatchRow + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type WorkerDispatchStartMethods = { + createStartingWorkerDispatch: typeof createStartingWorkerDispatch +} + +export function attachWorkerDispatchStart(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + createStartingWorkerDispatch + }) +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts new file mode 100644 index 00000000000..d10148c4daf --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts @@ -0,0 +1,238 @@ +import type { DispatchContextRow, WorkerDispatchRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { + releaseContextOnlyDispatch, + type ContextOnlyDispatchReleaseResult +} from '../../context-only-dispatch-release' +import { isEquivalentPaneKey } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function isDispatchProcessCurrent( + this: OrchestrationDb, + params: { + dispatchId: string + paneKey: string | null + processIncarnation: string | null + } +): boolean { + const dispatch = this.getDispatchContextById(params.dispatchId) + return Boolean( + dispatch?.assignee_pane_key && + params.paneKey && + isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) && + dispatch.process_incarnation && + params.processIncarnation === dispatch.process_incarnation + ) +} + +export function beginWorkerStop( + this: OrchestrationDb, + dispatchId: string +): + | { disposition: 'stopping'; worker: WorkerDispatchRow; dispatch: DispatchContextRow } + | { disposition: 'already_settled'; worker: WorkerDispatchRow; dispatch: DispatchContextRow } + | ({ disposition: 'context_only' } & ContextOnlyDispatchReleaseResult) { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (!worker) { + const released = releaseContextOnlyDispatch( + this.db, + dispatch, + this.getDispatchContext(dispatch.task_id)?.id, + 'stopped' + ) + if (!released.alreadySettled) { + this.closeQuestionsForDispatch(dispatchId) + } + this.db.exec('COMMIT') + return { disposition: 'context_only', ...released } + } + if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(worker.state)) { + this.db.exec('COMMIT') + return { disposition: 'already_settled', worker, dispatch } + } + if (!['ready', 'start_unknown'].includes(worker.state)) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} cannot stop from ${worker.state}.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stopping', stage = 'stop_requested', updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) + this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return { + disposition: 'stopping', + worker: this.getWorkerDispatch(dispatchId) as WorkerDispatchRow, + dispatch: this.getDispatchContextById(dispatchId) as DispatchContextRow + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function settleWorkerStop(this: OrchestrationDb, dispatchId: string): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!worker || !dispatch || worker.state !== 'stopping') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not stopping.`) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stopped', stage = 'process_stopped', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', completed_at = datetime('now'), last_failure = 'stopped' + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function reconcileFederatedWorkerStop( + this: OrchestrationDb, + dispatchId: string +): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!worker || !dispatch || !this.getFederatedDispatch(dispatchId)) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${dispatchId} was not found.` + ) + } + if (worker.state === 'stopped') { + this.db.exec('COMMIT') + return worker + } + if (!['stopping', 'stop_unknown'].includes(worker.state)) { + throw new OrchestrationError( + 'dispatch_inactive', + `Federated Dispatch ${dispatchId} cannot reconcile stop from ${worker.state}.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stopped', stage = 'process_stopped', last_error = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('stopping', 'stop_unknown')` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', completed_at = COALESCE(completed_at, datetime('now')), + last_failure = 'stopped' + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function resumeFederatedWorkerForTerminalRelay( + this: OrchestrationDb, + dispatchId: string +): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!worker || !dispatch || worker.state !== 'stopping') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not stopping.`) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'ready', stage = 'remote_report_pending', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(dispatchId) + this.db + .prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ? AND status = 'blocked'") + .run(dispatch.task_id) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function markWorkerStopUnknown( + this: OrchestrationDb, + dispatchId: string, + reason: string +): WorkerDispatchRow { + const worker = this.getWorkerDispatch(dispatchId) + if (!worker || worker.state !== 'stopping') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not stopping.`) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stop_unknown', stage = 'stop_outcome_unknown', last_error = ?, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(reason, dispatchId) + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow +} + +export type WorkerDispatchStopMethods = { + isDispatchProcessCurrent: typeof isDispatchProcessCurrent + beginWorkerStop: typeof beginWorkerStop + settleWorkerStop: typeof settleWorkerStop + reconcileFederatedWorkerStop: typeof reconcileFederatedWorkerStop + resumeFederatedWorkerForTerminalRelay: typeof resumeFederatedWorkerForTerminalRelay + markWorkerStopUnknown: typeof markWorkerStopUnknown +} + +export function attachWorkerDispatchStop(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + isDispatchProcessCurrent, + beginWorkerStop, + settleWorkerStop, + reconcileFederatedWorkerStop, + resumeFederatedWorkerForTerminalRelay, + markWorkerStopUnknown + }) +} diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-recovery.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-recovery.ts new file mode 100644 index 00000000000..c73cb4a74e8 --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-recovery.ts @@ -0,0 +1,98 @@ +import type { + TaskStatus, + DispatchStatus, + WorkerDispatchRow, + LegacyWorkerTerminalRecoveryRow +} from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { DISPATCH_CIRCUIT_BREAK_FAILURES } from '../dispatch-context/dispatch-circuit-breaker' +import type { OrchestrationDb } from '../orchestration-db' + +export function listLegacyWorkerTerminalRecoveryRows( + this: OrchestrationDb +): LegacyWorkerTerminalRecoveryRow[] { + return this.db + .prepare( + `SELECT dc.id AS dispatch_id, dc.task_id, dc.status AS dispatch_status, + dc.contract_version, dc.assignee_handle, dc.assignee_pane_key, + dc.process_incarnation, wd.state AS worker_state, wd.worktree_id, + wd.agent_terminal_handle + FROM dispatch_contexts dc + INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id + WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown') + ORDER BY dc.rowid` + ) + .all() as LegacyWorkerTerminalRecoveryRow[] +} + +export function reconcileMissingWorkerTerminal( + this: OrchestrationDb, + dispatchId: string, + reason: string +): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || !worker) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(worker.state)) { + this.db.exec('COMMIT') + return worker + } + + const activeDispatch = dispatch.status === 'pending' || dispatch.status === 'dispatched' + const stopWasPending = worker.state === 'stopping' || worker.state === 'stop_unknown' + if (activeDispatch) { + const failureCount = dispatch.failure_count + 1 + const dispatchStatus: DispatchStatus = + failureCount >= DISPATCH_CIRCUIT_BREAK_FAILURES ? 'circuit_broken' : 'failed' + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = ?, failure_count = ?, last_failure = ?, + completed_at = datetime('now'), + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(dispatchStatus, failureCount, reason, dispatchId) + if (!stopWasPending) { + const taskStatus: TaskStatus = dispatchStatus === 'circuit_broken' ? 'failed' : 'ready' + this.db + .prepare( + `UPDATE tasks + SET status = ?, completed_at = CASE WHEN ? = 'failed' THEN datetime('now') ELSE NULL END + WHERE id = ? AND status IN ('dispatched', 'blocked')` + ) + .run(taskStatus, taskStatus, dispatch.task_id) + } + this.closeQuestionsForDispatch(dispatchId) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = ?, stage = 'terminal_missing', last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ? + AND state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')` + ) + .run(stopWasPending ? 'stopped' : 'abandoned', reason, dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type WorkerTerminalRecoveryMethods = { + listLegacyWorkerTerminalRecoveryRows: typeof listLegacyWorkerTerminalRecoveryRows + reconcileMissingWorkerTerminal: typeof reconcileMissingWorkerTerminal +} + +export function attachWorkerTerminalRecovery(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + listLegacyWorkerTerminalRecoveryRows, + reconcileMissingWorkerTerminal + }) +} diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts new file mode 100644 index 00000000000..b824f986160 --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts @@ -0,0 +1,204 @@ +import type { + WorkerTerminalResourceRow, + WorkerTerminalArchiveRow, + WorkerTerminalArchiveStatus, + WorkerTerminalRetainedReason +} from '../../worker-terminal-ownership' +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' + +export function storeWorkerTerminalArchive( + this: OrchestrationDb, + params: { + dispatchId: string + resourceId: string + kind: 'transcript_pin' | 'terminal_tail' + content: string + } +): void { + this.db + .prepare( + `INSERT INTO worker_terminal_archives (dispatch_id, resource_id, kind, content) + VALUES (?, ?, ?, ?) + ON CONFLICT(dispatch_id) DO UPDATE SET + resource_id = excluded.resource_id, kind = excluded.kind, content = excluded.content` + ) + .run(params.dispatchId, params.resourceId, params.kind, params.content) +} + +export function commitWorkerTerminalArchiveForRelease( + this: OrchestrationDb, + params: { + dispatchId: string + resourceId: string + kind?: 'transcript_pin' | 'terminal_tail' + content?: string + archiveSource: 'transcript' | 'terminal' + archiveStatus: Extract + } +): WorkerTerminalResourceRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const resource = this.getWorkerTerminalResource(params.resourceId) + if (!resource) { + throw new OrchestrationError( + 'dispatch_not_found', + `Worker terminal resource ${params.resourceId} was not found.` + ) + } + if ( + resource.owner_dispatch_id === params.dispatchId && + resource.ownership_state === 'owned' && + resource.release_state === 'requested' + ) { + if (params.kind && params.content !== undefined) { + this.storeWorkerTerminalArchive({ + dispatchId: params.dispatchId, + resourceId: params.resourceId, + kind: params.kind, + content: params.content + }) + } + const archive = this.getWorkerTerminalArchive(params.dispatchId) + if (!archive || archive.resource_id !== params.resourceId) { + throw new OrchestrationError( + 'archive_failed', + `Output could not be preserved for Dispatch ${params.dispatchId}; the terminal was retained.` + ) + } + this.db + .prepare( + `UPDATE worker_terminal_resources + SET release_state = 'releasing', archive_source = ?, archive_status = ?, + updated_at = datetime('now') + WHERE id = ? AND owner_dispatch_id = ? AND ownership_state = 'owned' + AND release_state = 'requested'` + ) + .run(params.archiveSource, params.archiveStatus, params.resourceId, params.dispatchId) + } + const updated = this.getWorkerTerminalResource(params.resourceId) as WorkerTerminalResourceRow + this.db.exec('COMMIT') + return updated + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function getWorkerTerminalArchive( + this: OrchestrationDb, + dispatchId: string +): WorkerTerminalArchiveRow | undefined { + return this.db + .prepare('SELECT * FROM worker_terminal_archives WHERE dispatch_id = ?') + .get(dispatchId) as WorkerTerminalArchiveRow | undefined +} + +export function settleWorkerTerminalRelease( + this: OrchestrationDb, + resourceId: string +): WorkerTerminalResourceRow { + this.db + .prepare( + `UPDATE worker_terminal_resources + SET release_state = 'released', ownership_state = 'released', + release_completed_at = datetime('now'), release_error = NULL, + updated_at = datetime('now') + WHERE id = ? AND release_state IN ('requested', 'releasing', 'unknown')` + ) + .run(resourceId) + return this.getWorkerTerminalResource(resourceId) as WorkerTerminalResourceRow +} + +export function markWorkerTerminalReleaseUnknown( + this: OrchestrationDb, + resourceId: string, + reason: string +): WorkerTerminalResourceRow { + this.db + .prepare( + `UPDATE worker_terminal_resources + SET release_state = 'unknown', release_error = ?, updated_at = datetime('now') + WHERE id = ? AND release_state IN ('requested', 'releasing')` + ) + .run(reason, resourceId) + return this.getWorkerTerminalResource(resourceId) as WorkerTerminalResourceRow +} + +export function revertWorkerTerminalReleaseToRetained( + this: OrchestrationDb, + resourceId: string, + reason: WorkerTerminalRetainedReason +): WorkerTerminalResourceRow { + this.db + .prepare( + `UPDATE worker_terminal_resources + SET release_state = 'retained', retained_reason = ?, updated_at = datetime('now') + WHERE id = ? AND release_state IN ('requested', 'releasing')` + ) + .run(reason, resourceId) + return this.getWorkerTerminalResource(resourceId) as WorkerTerminalResourceRow +} + +export function retainWorkerTerminalResource( + this: OrchestrationDb, + dispatchId: string +): + | { disposition: 'retained'; resource: WorkerTerminalResourceRow } + | { disposition: 'already_released'; resource: WorkerTerminalResourceRow } + | { disposition: 'release_committed'; resource: WorkerTerminalResourceRow } + | { disposition: 'no_owned_resource'; resource: null } { + this.db.exec('BEGIN IMMEDIATE') + try { + const resource = this.getWorkerTerminalResourceByOwner(dispatchId) + if (!resource) { + this.db.exec('COMMIT') + return { disposition: 'no_owned_resource', resource: null } + } + if (resource.release_state === 'released') { + this.db.exec('COMMIT') + return { disposition: 'already_released', resource } + } + this.db + .prepare( + `UPDATE worker_terminal_resources + SET release_state = 'retained', retained_reason = 'user_requested', + updated_at = datetime('now') + WHERE id = ? AND release_state IN ('not_requested', 'retained', 'requested')` + ) + .run(resource.id) + const updated = this.getWorkerTerminalResource(resource.id) as WorkerTerminalResourceRow + if (updated.release_state !== 'retained') { + this.db.exec('COMMIT') + return { disposition: 'release_committed', resource: updated } + } + this.db.prepare('DELETE FROM worker_terminal_archives WHERE dispatch_id = ?').run(dispatchId) + this.db.exec('COMMIT') + return { disposition: 'retained', resource: updated } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type WorkerTerminalArchiveMethods = { + storeWorkerTerminalArchive: typeof storeWorkerTerminalArchive + commitWorkerTerminalArchiveForRelease: typeof commitWorkerTerminalArchiveForRelease + getWorkerTerminalArchive: typeof getWorkerTerminalArchive + settleWorkerTerminalRelease: typeof settleWorkerTerminalRelease + markWorkerTerminalReleaseUnknown: typeof markWorkerTerminalReleaseUnknown + revertWorkerTerminalReleaseToRetained: typeof revertWorkerTerminalReleaseToRetained + retainWorkerTerminalResource: typeof retainWorkerTerminalResource +} + +export function attachWorkerTerminalArchive(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + storeWorkerTerminalArchive, + commitWorkerTerminalArchiveForRelease, + getWorkerTerminalArchive, + settleWorkerTerminalRelease, + markWorkerTerminalReleaseUnknown, + revertWorkerTerminalReleaseToRetained, + retainWorkerTerminalResource + }) +} diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts new file mode 100644 index 00000000000..93e8c4dc0cb --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts @@ -0,0 +1,143 @@ +import type { DispatchStatus, WorkerDispatchState } from '../../types' +import { deriveWorkerTerminalListState } from '../../worker-terminal-ownership' +import type { + WorkerTerminalResourceRow, + WorkerTerminalListState +} from '../../worker-terminal-ownership' +import { isEquivalentPaneKey } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +// Real user input relinquishes orchestration ownership durably; programmatic prompt delivery, +// query auto-replies, resize, and output never reach this path. +export function markWorkerTerminalUserOwned(this: OrchestrationDb, paneKey: string): number { + this.db.exec('BEGIN IMMEDIATE') + try { + const exact = this.db + .prepare( + `SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources + WHERE pane_key = ? AND ownership_state = 'owned' + AND release_state IN ('not_requested', 'retained', 'requested')` + ) + .all(paneKey) as { id: string; owner_dispatch_id: string; pane_key: string }[] + const candidates = + exact.length > 0 + ? exact + : ( + this.db + .prepare( + `SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources + WHERE ownership_state = 'owned' + AND release_state IN ('not_requested', 'retained', 'requested') + AND pane_key IS NOT NULL` + ) + .all() as { id: string; owner_dispatch_id: string; pane_key: string }[] + ).filter((candidate) => isEquivalentPaneKey(candidate.pane_key, paneKey)) + const update = this.db.prepare( + `UPDATE worker_terminal_resources + SET ownership_state = 'user_owned', release_state = 'retained', + retained_reason = 'user_takeover', updated_at = datetime('now') + WHERE id = ? AND ownership_state = 'owned' + AND release_state IN ('not_requested', 'retained', 'requested')` + ) + let changed = 0 + for (const candidate of candidates) { + const result = Number(update.run(candidate.id).changes) + if (result > 0) { + this.db + .prepare('DELETE FROM worker_terminal_archives WHERE dispatch_id = ?') + .run(candidate.owner_dispatch_id) + changed += result + } + } + this.db.exec('COMMIT') + return changed + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function listWorkerTerminalReleaseBacklog( + this: OrchestrationDb +): WorkerTerminalResourceRow[] { + return this.db + .prepare( + `SELECT * FROM worker_terminal_resources + WHERE release_state IN ('requested', 'releasing') + ORDER BY release_requested_at ASC` + ) + .all() as WorkerTerminalResourceRow[] +} + +export function listWorkerTerminalResources( + this: OrchestrationDb, + params: { runId?: string } = {} +): { + dispatchId: string + taskId: string + runId: string + workerState: WorkerDispatchState + dispatchStatus: DispatchStatus + agentTerminalHandle: string | null + terminalState: WorkerTerminalListState | null + resource: WorkerTerminalResourceRow | null +}[] { + const rows = this.db + .prepare( + `SELECT w.dispatch_id, w.state AS worker_state, w.agent_terminal_handle, + d.task_id, d.run_id, d.status AS dispatch_status + FROM worker_dispatches w + JOIN dispatch_contexts d ON d.id = w.dispatch_id + ${params.runId ? 'WHERE d.run_id = ?' : ''} + ORDER BY w.created_at ASC` + ) + .all(...(params.runId ? [params.runId] : [])) as { + dispatch_id: string + worker_state: WorkerDispatchState + agent_terminal_handle: string | null + task_id: string + run_id: string + dispatch_status: DispatchStatus + }[] + const resources = this.db + .prepare( + `SELECT r.* FROM worker_terminal_resources r + JOIN dispatch_contexts d ON d.id = r.owner_dispatch_id + ${params.runId ? 'WHERE d.run_id = ?' : ''}` + ) + .all(...(params.runId ? [params.runId] : [])) as WorkerTerminalResourceRow[] + const resourceByOwner = new Map( + resources.map((resource) => [resource.owner_dispatch_id, resource]) + ) + return rows.map((row) => { + const resource = resourceByOwner.get(row.dispatch_id) ?? null + return { + dispatchId: row.dispatch_id, + taskId: row.task_id, + runId: row.run_id, + workerState: row.worker_state, + dispatchStatus: row.dispatch_status, + agentTerminalHandle: row.agent_terminal_handle, + terminalState: deriveWorkerTerminalListState({ + workerState: row.worker_state, + agentTerminalHandle: row.agent_terminal_handle, + resource + }), + resource + } + }) +} + +export type WorkerTerminalListingMethods = { + markWorkerTerminalUserOwned: typeof markWorkerTerminalUserOwned + listWorkerTerminalReleaseBacklog: typeof listWorkerTerminalReleaseBacklog + listWorkerTerminalResources: typeof listWorkerTerminalResources +} + +export function attachWorkerTerminalListing(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + markWorkerTerminalUserOwned, + listWorkerTerminalReleaseBacklog, + listWorkerTerminalResources + }) +} diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts new file mode 100644 index 00000000000..01a20948a8d --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts @@ -0,0 +1,168 @@ +import { WORKER_SETTLED_STATES } from '../../worker-terminal-ownership' +import type { + WorkerTerminalResourceRow, + WorkerTerminalRetainedReason +} from '../../worker-terminal-ownership' +import { OrchestrationError } from '../../orchestration-error' +import { parseWorkerTerminalPriorOwnerIds } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +export function requestWorkerTerminalRelease( + this: OrchestrationDb, + dispatchId: string +): + | { disposition: 'requested'; resource: WorkerTerminalResourceRow } + | { disposition: 'already_released'; resource: WorkerTerminalResourceRow } + | { + disposition: 'retained' + resource: WorkerTerminalResourceRow | null + reason: WorkerTerminalRetainedReason + } { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + if (!worker) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (!WORKER_SETTLED_STATES.includes(worker.state)) { + // Why: release is post-completion cleanup only; recording intent for an unsettled or + // uncertain worker would let recovery close a terminal the coordinator never reviewed. + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} is ${worker.state}; only a settled worker can release. Use worker-stop to cancel an active worker.` + ) + } + const resource = this.getWorkerTerminalResourceByOwner(dispatchId) + if (!resource) { + const transferred = this.getWorkerTerminalResourceFormerlyOwnedBy(dispatchId) + this.db.exec('COMMIT') + return transferred + ? { disposition: 'retained', resource: transferred, reason: 'ownership_transferred' } + : { disposition: 'retained', resource: null, reason: 'no_owned_resource' } + } + if (resource.release_state === 'released' || resource.ownership_state === 'released') { + this.db.exec('COMMIT') + return { disposition: 'already_released', resource } + } + if (worker.state === 'stopped' || worker.state === 'abandoned') { + this.db.exec('COMMIT') + return { disposition: 'retained', resource, reason: 'identity_unproven' } + } + if (resource.ownership_state === 'external') { + this.db.exec('COMMIT') + return { + disposition: 'retained', + resource, + reason: (resource.retained_reason as WorkerTerminalRetainedReason) ?? 'external_terminal' + } + } + if (resource.ownership_state === 'user_owned') { + this.db.exec('COMMIT') + return { disposition: 'retained', resource, reason: 'user_takeover' } + } + if (resource.ownership_state === 'transferred') { + this.db.exec('COMMIT') + return { disposition: 'retained', resource, reason: 'ownership_transferred' } + } + if ( + resource.release_state === 'unknown' || + (resource.release_state === 'retained' && resource.retained_reason === 'user_requested') + ) { + this.db.prepare('DELETE FROM worker_terminal_archives WHERE dispatch_id = ?').run(dispatchId) + } + this.db + .prepare( + `UPDATE worker_terminal_resources + SET release_state = CASE + WHEN release_state = 'releasing' THEN 'releasing' + ELSE 'requested' + END, + retained_reason = NULL, + release_requested_at = COALESCE(release_requested_at, datetime('now')), + release_error = NULL, updated_at = datetime('now') + WHERE id = ? AND release_state IN ('not_requested', 'retained', 'requested', 'releasing', 'unknown')` + ) + .run(resource.id) + this.db.exec('COMMIT') + return { + disposition: 'requested', + resource: this.getWorkerTerminalResource(resource.id) as WorkerTerminalResourceRow + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export function settleDeadWorkerTerminalRelease( + this: OrchestrationDb, + params: { + requestingDispatchId: string + resourceId: string + processIncarnation: string + } +): + | { disposition: 'released'; resource: WorkerTerminalResourceRow } + | { disposition: 'retained'; resource: WorkerTerminalResourceRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + const resource = this.getWorkerTerminalResource(params.resourceId) + if (!resource) { + throw new OrchestrationError( + 'dispatch_not_found', + `Worker terminal resource ${params.resourceId} was not found.` + ) + } + const priorOwners = parseWorkerTerminalPriorOwnerIds(resource.prior_owner_dispatch_ids) + const requesterRelated = + resource.owner_dispatch_id === params.requestingDispatchId || + priorOwners?.includes(params.requestingDispatchId) === true + const requester = this.getWorkerDispatch(params.requestingDispatchId) + const owner = this.getWorkerDispatch(resource.owner_dispatch_id) + const requesterSettled = Boolean(requester && WORKER_SETTLED_STATES.includes(requester.state)) + const ownerSettled = Boolean(owner && WORKER_SETTLED_STATES.includes(owner.state)) + if ( + !priorOwners || + !requesterRelated || + !requesterSettled || + !ownerSettled || + resource.process_incarnation !== params.processIncarnation || + resource.ownership_state === 'released' || + !['not_requested', 'retained', 'unknown'].includes(resource.release_state) + ) { + this.db.exec('COMMIT') + return { disposition: 'retained', resource } + } + this.db + .prepare( + `UPDATE worker_terminal_resources + SET release_state = 'released', ownership_state = 'released', retained_reason = NULL, + release_requested_at = COALESCE(release_requested_at, datetime('now')), + release_completed_at = datetime('now'), release_error = NULL, + updated_at = datetime('now') + WHERE id = ? AND process_incarnation = ? AND ownership_state != 'released' + AND release_state IN ('not_requested', 'retained', 'unknown')` + ) + .run(params.resourceId, params.processIncarnation) + const released = this.getWorkerTerminalResource(params.resourceId) as WorkerTerminalResourceRow + this.db.exec('COMMIT') + return released.release_state === 'released' + ? { disposition: 'released', resource: released } + : { disposition: 'retained', resource: released } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } +} + +export type WorkerTerminalReleaseMethods = { + requestWorkerTerminalRelease: typeof requestWorkerTerminalRelease + settleDeadWorkerTerminalRelease: typeof settleDeadWorkerTerminalRelease +} + +export function attachWorkerTerminalRelease(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + requestWorkerTerminalRelease, + settleDeadWorkerTerminalRelease + }) +} diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-resource-store.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-resource-store.ts new file mode 100644 index 00000000000..f6611ea80a6 --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-resource-store.ts @@ -0,0 +1,185 @@ +import type { + WorkerTerminalResourceRow, + WorkerTerminalOwnershipState +} from '../../worker-terminal-ownership' +import { OrchestrationError } from '../../orchestration-error' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' + +// --- Worker terminal resources (schema v23) --------------------------------------------------- + +// Historical renderer input and reuse cannot be proven, so pre-v23 terminals stay external. +export function backfillWorkerTerminalResources(this: OrchestrationDb): void { + const rows = this.db + .prepare( + `SELECT w.dispatch_id, w.worktree_id, w.agent_terminal_handle, + d.assignee_pane_key, d.process_incarnation + FROM worker_dispatches w + JOIN dispatch_contexts d ON d.id = w.dispatch_id + WHERE w.agent_terminal_handle IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM federated_dispatches f WHERE f.dispatch_id = w.dispatch_id) + AND NOT EXISTS ( + SELECT 1 FROM worker_terminal_resources r WHERE r.owner_dispatch_id = w.dispatch_id + )` + ) + .all() as { + dispatch_id: string + worktree_id: string | null + agent_terminal_handle: string + assignee_pane_key: string | null + process_incarnation: string | null + }[] + const insert = this.db.prepare( + `INSERT INTO worker_terminal_resources ( + id, origin_dispatch_id, owner_dispatch_id, worktree_id, terminal_handle, + pane_key, process_incarnation, ownership_state, release_state, retained_reason + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const row of rows) { + insert.run( + generateId('wtr'), + row.dispatch_id, + row.dispatch_id, + row.worktree_id, + row.agent_terminal_handle, + row.assignee_pane_key, + row.process_incarnation, + 'external', + 'retained', + 'legacy_ambiguous' + ) + } +} + +// No transaction: composes inside worker-start's authority transaction. +export function createWorkerTerminalResourceStatement( + this: OrchestrationDb, + params: { + dispatchId: string + worktreeId: string | null + terminalHandle: string + paneKey: string | null + processIncarnation: string | null + hostScope?: string | null + ownership: Extract + } +): WorkerTerminalResourceRow { + const id = generateId('wtr') + this.db + .prepare( + `INSERT INTO worker_terminal_resources ( + id, origin_dispatch_id, owner_dispatch_id, worktree_id, terminal_handle, + pane_key, process_incarnation, host_scope, ownership_state, release_state, + retained_reason + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'not_requested', ?)` + ) + .run( + id, + params.dispatchId, + params.dispatchId, + params.worktreeId, + params.terminalHandle, + params.paneKey, + params.processIncarnation, + params.hostScope ?? null, + params.ownership, + params.ownership === 'external' ? 'external_terminal' : null + ) + return this.getWorkerTerminalResource(id) as WorkerTerminalResourceRow +} + +export function getWorkerTerminalResource( + this: OrchestrationDb, + id: string +): WorkerTerminalResourceRow | undefined { + return this.db.prepare('SELECT * FROM worker_terminal_resources WHERE id = ?').get(id) as + | WorkerTerminalResourceRow + | undefined +} + +export function getWorkerTerminalResourceByOwner( + this: OrchestrationDb, + dispatchId: string +): WorkerTerminalResourceRow | undefined { + return this.db + .prepare('SELECT * FROM worker_terminal_resources WHERE owner_dispatch_id = ?') + .get(dispatchId) as WorkerTerminalResourceRow | undefined +} + +export function getWorkerTerminalResourceFormerlyOwnedBy( + this: OrchestrationDb, + dispatchId: string +): WorkerTerminalResourceRow | undefined { + return this.db + .prepare( + `SELECT * FROM worker_terminal_resources + WHERE prior_owner_dispatch_ids LIKE ? + ORDER BY updated_at DESC LIMIT 1` + ) + .get(`%"${dispatchId}"%`) as WorkerTerminalResourceRow | undefined +} + +// Reusable exact settled terminal: transfers cleanup ownership to the new Dispatch and fences +// release through the old owner. No transaction: composes inside the authority transaction. +export function transferWorkerTerminalResourceStatement( + this: OrchestrationDb, + params: { + resourceId: string + toDispatchId: string + terminalHandle: string + paneKey: string + processIncarnation: string + hostScope: string | null + } +): WorkerTerminalResourceRow { + const resource = this.getWorkerTerminalResource(params.resourceId) + if (!resource) { + throw new OrchestrationError( + 'dispatch_not_found', + `Worker terminal resource ${params.resourceId} was not found.` + ) + } + const priorOwners = JSON.parse(resource.prior_owner_dispatch_ids) as string[] + priorOwners.push(resource.owner_dispatch_id) + this.db + .prepare( + `UPDATE worker_terminal_resources + SET owner_dispatch_id = ?, prior_owner_dispatch_ids = ?, release_state = 'not_requested', + retained_reason = NULL, release_requested_at = NULL, release_completed_at = NULL, + release_error = NULL, terminal_handle = ?, pane_key = ?, process_incarnation = ?, + host_scope = ?, updated_at = datetime('now') + WHERE id = ? AND ownership_state = 'owned'` + ) + .run( + params.toDispatchId, + JSON.stringify(priorOwners), + params.terminalHandle, + params.paneKey, + params.processIncarnation, + params.hostScope, + params.resourceId + ) + return this.getWorkerTerminalResource(params.resourceId) as WorkerTerminalResourceRow +} + +// Finds an owned, settled, exact-match resource for an explicitly reused terminal. + +export type WorkerTerminalResourceStoreMethods = { + backfillWorkerTerminalResources: typeof backfillWorkerTerminalResources + createWorkerTerminalResourceStatement: typeof createWorkerTerminalResourceStatement + getWorkerTerminalResource: typeof getWorkerTerminalResource + getWorkerTerminalResourceByOwner: typeof getWorkerTerminalResourceByOwner + getWorkerTerminalResourceFormerlyOwnedBy: typeof getWorkerTerminalResourceFormerlyOwnedBy + transferWorkerTerminalResourceStatement: typeof transferWorkerTerminalResourceStatement +} + +export function attachWorkerTerminalResourceStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + backfillWorkerTerminalResources, + createWorkerTerminalResourceStatement, + getWorkerTerminalResource, + getWorkerTerminalResourceByOwner, + getWorkerTerminalResourceFormerlyOwnedBy, + transferWorkerTerminalResourceStatement + }) +} diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts new file mode 100644 index 00000000000..a41a8c89352 --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts @@ -0,0 +1,93 @@ +import type { WorkerTerminalResourceRow } from '../../worker-terminal-ownership' +import { OrchestrationError } from '../../orchestration-error' +import { isEquivalentPaneKey } from '../pane-key-match' +import type { OrchestrationDb } from '../orchestration-db' + +// Finds an owned, settled, exact-match resource for an explicitly reused terminal. +export function findTransferableWorkerTerminalResource( + this: OrchestrationDb, + params: { + terminalHandle: string + paneKey: string | null + processIncarnation: string | null + hostScope: string | null + } +): WorkerTerminalResourceRow | undefined { + if (!params.paneKey || !params.processIncarnation) { + return undefined + } + const candidates = this.db + .prepare( + `SELECT r.* FROM worker_terminal_resources r + JOIN worker_dispatches w ON w.dispatch_id = r.owner_dispatch_id + WHERE r.process_incarnation = ? AND r.host_scope IS ? + AND r.ownership_state != 'released'` + ) + .all(params.processIncarnation, params.hostScope) as WorkerTerminalResourceRow[] + const exact = candidates.filter( + (candidate) => + candidate.pane_key && + params.paneKey && + isEquivalentPaneKey(candidate.pane_key, params.paneKey) && + candidate.process_incarnation === params.processIncarnation && + candidate.host_scope === params.hostScope + ) + if ( + exact.some((candidate) => + ['requested', 'releasing', 'unknown'].includes(candidate.release_state) + ) + ) { + throw new OrchestrationError( + 'terminal_release_in_progress', + `Terminal ${params.terminalHandle} has a release in progress; wait for cleanup or use another terminal.` + ) + } + return exact.find( + (candidate) => + candidate.ownership_state === 'owned' && + ['not_requested', 'retained'].includes(candidate.release_state) && + ['succeeded', 'failed', 'stopped', 'abandoned'].includes( + this.getWorkerDispatch(candidate.owner_dispatch_id)?.state ?? '' + ) + ) +} + +export function workerTerminalResourceHasIdentityConflict( + this: OrchestrationDb, + resourceId: string +): boolean { + const resource = this.getWorkerTerminalResource(resourceId) + if (!resource?.pane_key || !resource.process_incarnation) { + return true + } + const candidates = this.db + .prepare( + `SELECT * FROM worker_terminal_resources + WHERE process_incarnation = ? AND host_scope IS ? + AND id != ? AND ownership_state != 'released'` + ) + .all( + resource.process_incarnation, + resource.host_scope, + resource.id + ) as WorkerTerminalResourceRow[] + return candidates.some( + (candidate) => + candidate.pane_key && + isEquivalentPaneKey(candidate.pane_key, resource.pane_key as string) && + candidate.process_incarnation === resource.process_incarnation && + candidate.host_scope === resource.host_scope + ) +} + +export type WorkerTerminalTransferMethods = { + findTransferableWorkerTerminalResource: typeof findTransferableWorkerTerminalResource + workerTerminalResourceHasIdentityConflict: typeof workerTerminalResourceHasIdentityConflict +} + +export function attachWorkerTerminalTransfer(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + findTransferableWorkerTerminalResource, + workerTerminalResourceHasIdentityConflict + }) +} diff --git a/src/main/runtime/orchestration/dispatch-failure-idempotency.test.ts b/src/main/runtime/orchestration/dispatch-failure-idempotency.test.ts new file mode 100644 index 00000000000..9b526c480cc --- /dev/null +++ b/src/main/runtime/orchestration/dispatch-failure-idempotency.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' +import type Database from '../../sqlite/sync-database' +import { OrchestrationDb } from './db' + +describe('dispatch failure idempotency', () => { + it('counts an active dispatch failure only once', () => { + const db = new OrchestrationDb(':memory:') + const task = db.createTask({ spec: 'work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') + + expect(db.failDispatch(dispatch.id, 'exit')?.failure_count).toBe(1) + const duplicate = db.failDispatch(dispatch.id, 'duplicate escalation') + + expect(duplicate?.failure_count).toBe(1) + expect(duplicate?.status).toBe('failed') + db.close() + }) + + it('does not overwrite a completed dispatch', () => { + const db = new OrchestrationDb(':memory:') + const task = db.createTask({ spec: 'work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') + db.completeDispatch(dispatch.id) + + const lateFailure = db.failDispatch(dispatch.id, 'late exit') + + expect(lateFailure?.failure_count).toBe(0) + expect(lateFailure?.status).toBe('completed') + db.close() + }) + + it('rolls back the dispatch when the task update fails', () => { + const db = new OrchestrationDb(':memory:') + const sqlite = (db as unknown as { db: Database.Database }).db + const task = db.createTask({ spec: 'work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') + sqlite.exec(` + CREATE TRIGGER reject_task_failure_update + BEFORE UPDATE ON tasks WHEN OLD.id = '${task.id}' + BEGIN + SELECT RAISE(ABORT, 'blocked'); + END; + `) + + expect(() => db.failDispatch(dispatch.id, 'exit')).toThrow('blocked') + expect(db.getDispatchContextById(dispatch.id)).toMatchObject({ + failure_count: 0, + status: 'dispatched' + }) + expect(db.getTask(task.id)?.status).toBe('dispatched') + + sqlite.exec('DROP TRIGGER reject_task_failure_update') + expect(db.failDispatch(dispatch.id, 'retry')).toMatchObject({ + failure_count: 1, + status: 'failed' + }) + expect(db.getTask(task.id)?.status).toBe('ready') + db.close() + }) +}) diff --git a/src/main/runtime/orchestration/federation-acknowledgment-integrity.test.ts b/src/main/runtime/orchestration/federation-acknowledgment-integrity.test.ts index 3f8f4022393..fd8e32d1a32 100644 --- a/src/main/runtime/orchestration/federation-acknowledgment-integrity.test.ts +++ b/src/main/runtime/orchestration/federation-acknowledgment-integrity.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type Database from '../../sqlite/sync-database' import { OrchestrationDb } from './db' describe('federation acknowledgment integrity', () => { @@ -53,6 +54,17 @@ describe('federation acknowledgment integrity', () => { }) } + function enqueueQuestion(target: OrchestrationDb, dispatchId: string, messageId: string): void { + target.enqueueFederationRelay({ + dispatchId, + direction: 'to_home', + kind: 'question', + payload: '{}', + messageId, + remoteQuestion: true + }) + } + it('rejects a protocol-v3 acknowledgment without a lifecycle verdict atomically', () => { const current = createReadyAttachment(3) const report = enqueueCompletion(current.db, current.dispatchId) @@ -97,4 +109,54 @@ describe('federation acknowledgment integrity', () => { expect(current.db.getRemoteDispatchAttachment(current.dispatchId)?.state).toBe('ready') expect(current.db.listPendingFederationRelay(current.dispatchId, 'to_home')).toHaveLength(0) }) + + it('accepts an identical remote answer replay and rejects a conflicting replay', () => { + const current = createReadyAttachment(3) + enqueueQuestion(current.db, current.dispatchId, 'question_replay') + const answer = { + messageId: 'question_replay', + dispatchId: current.dispatchId, + answerMessageId: 'answer_1', + body: 'Yes' + } + + current.db.answerRemoteQuestion(answer) + + expect(() => current.db.answerRemoteQuestion(answer)).not.toThrow() + expect(() => current.db.answerRemoteQuestion({ ...answer, body: 'No' })).toThrowError( + expect.objectContaining({ code: 'answer_conflict' }) + ) + }) + + it('accepts an identical answer that wins between classification and the guarded update', () => { + const current = createReadyAttachment(3) + enqueueQuestion(current.db, current.dispatchId, 'question_race') + const sqlite = (current.db as unknown as { db: Database.Database }).db + const originalPrepare = sqlite.prepare.bind(sqlite) + let injected = false + const prepare = vi.spyOn(sqlite, 'prepare').mockImplementation((sql) => { + const statement = originalPrepare(sql) + if (!injected && sql.includes('UPDATE remote_questions')) { + injected = true + statement.run('answer_race', 'Yes', 'question_race') + } + return statement + }) + + expect(() => + current.db.answerRemoteQuestion({ + messageId: 'question_race', + dispatchId: current.dispatchId, + answerMessageId: 'answer_race', + body: 'Yes' + }) + ).not.toThrow() + expect(injected).toBe(true) + expect(current.db.getRemoteQuestion('question_race')).toMatchObject({ + status: 'answered', + answer_message_id: 'answer_race', + answer_body: 'Yes' + }) + prepare.mockRestore() + }) }) diff --git a/src/main/runtime/orchestration/message-batch-atomicity.test.ts b/src/main/runtime/orchestration/message-batch-atomicity.test.ts new file mode 100644 index 00000000000..a9bf6237f55 --- /dev/null +++ b/src/main/runtime/orchestration/message-batch-atomicity.test.ts @@ -0,0 +1,94 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type Database from '../../sqlite/sync-database' +import { OrchestrationDb } from './db' + +type MessageMutation = + | 'markAsRead' + | 'markAsDelivered' + | 'markAsUndelivered' + | 'markAsReadAndDelivered' + +const messageIds = Array.from( + { length: 501 }, + (_, index) => `m${index.toString().padStart(3, '0')}` +) + +function seedMessages(sqlite: Database.Database): void { + sqlite.exec(` + WITH RECURSIVE ids(value) AS ( + VALUES(0) + UNION ALL SELECT value + 1 FROM ids WHERE value < 500 + ) + INSERT INTO messages (id, from_handle, to_handle, subject) + SELECT printf('m%03d', value), 'sender', 'recipient', 'subject' FROM ids; + `) +} + +function rejectLastMessageUpdate(sqlite: Database.Database): void { + sqlite.exec(` + CREATE TRIGGER reject_last_message_update + BEFORE UPDATE ON messages WHEN OLD.id = 'm500' + BEGIN + SELECT RAISE(ABORT, 'blocked'); + END; + `) +} + +describe('message batch atomicity', () => { + let db: OrchestrationDb | undefined + + afterEach(() => db?.close()) + + it.each<{ + method: MessageMutation + setup?: string + changedCountSql: string + }>([ + { method: 'markAsRead', changedCountSql: 'SELECT COUNT(*) count FROM messages WHERE read = 1' }, + { + method: 'markAsDelivered', + changedCountSql: 'SELECT COUNT(*) count FROM messages WHERE delivered_at IS NOT NULL' + }, + { + method: 'markAsUndelivered', + setup: "UPDATE messages SET delivered_at = datetime('now')", + changedCountSql: 'SELECT COUNT(*) count FROM messages WHERE delivered_at IS NULL' + }, + { + method: 'markAsReadAndDelivered', + changedCountSql: + 'SELECT COUNT(*) count FROM messages WHERE read = 1 OR delivered_at IS NOT NULL' + } + ])('rolls back $method when a later batch fails', ({ method, setup, changedCountSql }) => { + db = new OrchestrationDb(':memory:') + const sqlite = (db as unknown as { db: Database.Database }).db + seedMessages(sqlite) + if (setup) { + sqlite.exec(setup) + } + rejectLastMessageUpdate(sqlite) + + expect(() => db?.[method](messageIds)).toThrow('blocked') + + const changed = sqlite.prepare(changedCountSql).get() as { count: number } + expect(changed.count).toBe(0) + }) + + it('preserves an outer transaction when an inner batch rolls back', () => { + db = new OrchestrationDb(':memory:') + const sqlite = (db as unknown as { db: Database.Database }).db + seedMessages(sqlite) + rejectLastMessageUpdate(sqlite) + sqlite.exec('BEGIN IMMEDIATE') + sqlite.prepare("UPDATE messages SET subject = 'outer change' WHERE id = 'm000'").run() + + expect(() => db?.markAsRead(messageIds)).toThrow('blocked') + sqlite.exec('COMMIT') + + const first = sqlite.prepare("SELECT subject, read FROM messages WHERE id = 'm000'").get() as { + subject: string + read: number + } + expect(first).toEqual({ subject: 'outer change', read: 0 }) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-reset-db.test.ts b/src/main/runtime/orchestration/orchestration-reset-db.test.ts index 917260e7b2c..0ec8d3bf193 100644 --- a/src/main/runtime/orchestration/orchestration-reset-db.test.ts +++ b/src/main/runtime/orchestration/orchestration-reset-db.test.ts @@ -37,6 +37,12 @@ describe('OrchestrationDb reset scopes', () => { to: `run:${run.id}`, subject: 'status' }) + const localQuestion = db.createQuestion({ + runId: run.id, + dispatchId: started.dispatch.id, + askerHandle: 'worker', + question: 'Continue?' + }) db.enqueueFederationRelay({ dispatchId: started.dispatch.id, direction: 'to_home', @@ -45,7 +51,7 @@ describe('OrchestrationDb reset scopes', () => { messageId: 'question_1', remoteQuestion: true }) - return { run, task, started, message } + return { run, task, started, message, localQuestion } } it('resetAll clears Runs, worker/federation state, and messages', () => { @@ -83,6 +89,19 @@ describe('OrchestrationDb reset scopes', () => { expect(db!.getWorkerDispatch(state.started.dispatch.id)).toBeUndefined() expect(db!.getFederatedDispatch(state.started.dispatch.id)).toBeUndefined() expect(db!.getRemoteQuestion('question_1')).toBeUndefined() + expect(db!.getMessageById(state.localQuestion.message.id)).toBeDefined() + expect(db!.getQuestion(state.localQuestion.message.id)).toMatchObject({ + status: 'closed', + closed_at: expect.any(String) + }) + expect(() => + db!.answerQuestion({ + messageId: state.localQuestion.message.id, + runId: state.run.id, + consumerGeneration: state.run.consumer_generation, + body: 'Yes' + }) + ).toThrowError(expect.objectContaining({ code: 'dispatch_inactive' })) }) it('resetMessages preserves active relay cursors while clearing the Run inbox', () => { From 2fdaa10fd1262ff711c05b6b2e63c8e4f583ad22 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:09:46 -0700 Subject: [PATCH 03/10] fix(terminal): give the composing-chord deferral an owner and a ceiling (#15017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chord held for a live composition waits with `fallbackMs: null`, so nothing but compositionend can end it. `sendTerminalInputAfterComposition` returned void, so nobody could stop it either: the two listeners it puts on the terminal element outlived the pane, and a later composition on that element flushed the stale chord. Return a disposer from the helper and put a sender in front of it that owns every pending chord, so blur and pane teardown drop them the way the Enter path already clears its state. The sender also bounds the wait — generous enough for a conversion candidate window, and it discards rather than sends, because a chord arriving mid-preedit is the corruption the wait exists to prevent. The Enter path needs none of this: its 200 ms fallback always runs, so its listeners cannot outlive it. Pinned so that stays true. Fixes STA-4476 --- ...oard-handlers-ime-composing-chord.test.tsx | 79 ++++++++++++++++++ .../terminal-pane/keyboard-handlers.ts | 14 ++-- .../terminal-ime-deferred-chord.test.ts | 81 +++++++++++++++++++ .../terminal-ime-deferred-chord.ts | 51 ++++++++++++ .../terminal-ime-deferred-newline.test.ts | 40 ++++++++- .../terminal-ime-deferred-newline.ts | 21 ++++- 6 files changed, 275 insertions(+), 11 deletions(-) create mode 100644 src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.test.ts create mode 100644 src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.ts diff --git a/src/renderer/src/components/terminal-pane/keyboard-handlers-ime-composing-chord.test.tsx b/src/renderer/src/components/terminal-pane/keyboard-handlers-ime-composing-chord.test.tsx index 2200f2d5c9e..f13a57115b7 100644 --- a/src/renderer/src/components/terminal-pane/keyboard-handlers-ime-composing-chord.test.tsx +++ b/src/renderer/src/components/terminal-pane/keyboard-handlers-ime-composing-chord.test.tsx @@ -14,6 +14,7 @@ import { XTERM_COMPOSITION_SESSION_END_EVENT, XTERM_COMPOSITION_SESSION_START_EVENT } from './terminal-ime-composition-route' +import { TERMINAL_IME_DEFERRED_CHORD_ABANDON_MS } from './terminal-ime-deferred-chord' import { useTerminalKeyboardShortcuts } from './keyboard-handlers' type KeyboardHandlersDeps = Parameters[0] @@ -30,12 +31,34 @@ function keyboardEvent( return event } +/** Live registrations on the terminal element, so a deferral that never disposes is visible. */ +function trackCompositionListeners(element: HTMLElement): () => number { + const live = new Set() + const watched = new Set(['compositionend', XTERM_COMPOSITION_SESSION_END_EVENT]) + const { addEventListener, removeEventListener } = element + element.addEventListener = function (type, listener, options): void { + if (watched.has(type) && listener) { + live.add(listener) + } + addEventListener.call(this, type, listener, options) + } + element.removeEventListener = function (type, listener, options): void { + if (watched.has(type) && listener) { + live.delete(listener) + } + removeEventListener.call(this, type, listener, options) + } + return () => live.size +} + function createHarness(): { deps: KeyboardHandlersDeps terminalElement: HTMLDivElement terminalInput: HTMLTextAreaElement /** Every byte reaching the pty, in arrival order, whichever route it took. */ wire: string[] + /** Registrations added after the composition route's own, i.e. the deferrals still waiting. */ + deferralListenerCount: () => number startComposition: () => void endComposition: (data: string) => void dispose: () => void @@ -77,6 +100,8 @@ function createHarness(): { capturedTransport: transport, getCurrentTransport: () => transport }) + // Installed after the route so only the deferral's own registrations are counted. + const deferralListenerCount = trackCompositionListeners(terminalElement) const deps: KeyboardHandlersDeps = { tabId: 'tab-1', @@ -110,6 +135,7 @@ function createHarness(): { terminalElement, terminalInput, wire, + deferralListenerCount, startComposition: () => { terminalElement.dispatchEvent( new CustomEvent(XTERM_COMPOSITION_SESSION_START_EVENT, { detail: { id: 1 } }) @@ -220,4 +246,57 @@ describe('a cursor chord pressed during a composition', () => { hook.unmount() harness.dispose() }) + + // STA-4476: an indefinite wait has no exit of its own. Without a disposer the listeners outlive + // the pane and a later composition on the same element flushes the stale chord. + it('drops the held chord and its listeners when the pane tears down', () => { + const harness = createHarness() + const hook = renderHook(() => useTerminalKeyboardShortcuts(harness.deps)) + + harness.startComposition() + pressCmdArrowLeft(harness, true) + expect(harness.deferralListenerCount()).toBeGreaterThan(0) + + hook.unmount() + harness.endComposition('다') + vi.runAllTimers() + + expect(harness.wire).toEqual(['다']) + expect(harness.deferralListenerCount()).toBe(0) + harness.dispose() + }) + + it('drops the held chord and its listeners on blur', () => { + const harness = createHarness() + const hook = renderHook(() => useTerminalKeyboardShortcuts(harness.deps)) + + harness.startComposition() + pressCmdArrowLeft(harness, true) + window.dispatchEvent(new Event('blur')) + harness.endComposition('다') + vi.runAllTimers() + + expect(harness.wire).toEqual(['다']) + expect(harness.deferralListenerCount()).toBe(0) + hook.unmount() + harness.dispose() + }) + + // A composition that never commits must not strand the wait forever. The chord is discarded, + // never sent late — a late send is #12871 again. + it('abandons a chord whose composition never ends', () => { + const harness = createHarness() + const hook = renderHook(() => useTerminalKeyboardShortcuts(harness.deps)) + + harness.startComposition() + pressCmdArrowLeft(harness, true) + vi.advanceTimersByTime(TERMINAL_IME_DEFERRED_CHORD_ABANDON_MS + 1) + harness.endComposition('다') + vi.runAllTimers() + + expect(harness.wire).toEqual(['다']) + expect(harness.deferralListenerCount()).toBe(0) + hook.unmount() + harness.dispose() + }) }) diff --git a/src/renderer/src/components/terminal-pane/keyboard-handlers.ts b/src/renderer/src/components/terminal-pane/keyboard-handlers.ts index 0a5fe0ac5b9..3614af3b888 100644 --- a/src/renderer/src/components/terminal-pane/keyboard-handlers.ts +++ b/src/renderer/src/components/terminal-pane/keyboard-handlers.ts @@ -15,9 +15,9 @@ import { getTerminalImeModifiedEnterKind, isTerminalImeConsumedKey, isTerminalImeEnterKeyUp, - isTerminalImeProcessEnter, - sendTerminalInputAfterComposition + isTerminalImeProcessEnter } from './terminal-ime-deferred-newline' +import { createTerminalImeDeferredChordSender } from './terminal-ime-deferred-chord' import { hasPendingTerminalImeComposition } from './terminal-ime-composition-route' import { requestCapturedTerminalReconfirmation, @@ -290,6 +290,7 @@ export function useTerminalKeyboardShortcuts({ const terminalImeEnterModifierKeydowns = new Set<'shift' | 'ctrl'>() const nativeOnlyShortcutTracker = createTerminalNativeOnlyShortcutTracker() const deferredNewlineSender = createTerminalImeDeferredNewlineSender() + const deferredChordSender = createTerminalImeDeferredChordSender() const modifiedEnterChordOwner = createTerminalImeModifiedEnterChordOwner() const reconcileHeldImeEnterModifiers = ( event: KeyboardEvent, @@ -651,11 +652,10 @@ export function useTerminalKeyboardShortcuts({ // runs after this keydown. Sending now puts a cursor chord ahead of the text it was typed // after — `가나다` then Cmd+Left leaves `다가나` (#12871). Enter is handled above, where a // fallback timer is right because a newline arriving late still arrives; a chord arriving - // mid-preedit is the corruption itself, so this one waits without a deadline. + // mid-preedit is the corruption itself, so this one waits on the composition rather than a + // deadline. The sender owns the wait so blur and teardown can drop it. if (e.isComposing || hasPendingImeComposition) { - sendTerminalInputAfterComposition(pane.terminal.element, sendResolvedInput, { - fallbackMs: null - }) + deferredChordSender.defer(pane.terminal.element, sendResolvedInput) return } sendResolvedInput() @@ -986,6 +986,7 @@ export function useTerminalKeyboardShortcuts({ terminalImeEnterModifierKeydowns.clear() modifiedEnterChordOwner.clear() deferredNewlineSender.clearRedispatchedEnters() + deferredChordSender.cancelPending() observedEnterKeydownTimeStamps.clear() } @@ -1000,6 +1001,7 @@ export function useTerminalKeyboardShortcuts({ optionKittyReleases.clear() modifiedEnterChordOwner.clear() deferredNewlineSender.clearRedispatchedEnters() + deferredChordSender.cancelPending() observedEnterKeydownTimeStamps.clear() window.removeEventListener('keydown', onModifierDown, { capture: true }) window.removeEventListener('keyup', onKeyUp, { capture: true }) diff --git a/src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.test.ts b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.test.ts new file mode 100644 index 00000000000..d54ab6de455 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.test.ts @@ -0,0 +1,81 @@ +// @vitest-environment happy-dom +// STA-4476: the composing-chord deferral waits on the composition, not a deadline (#12871), so it +// is the sender that has to guarantee an exit — otherwise the wait and its listeners outlive the +// pane and a later composition flushes a stale chord. +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + createTerminalImeDeferredChordSender, + TERMINAL_IME_DEFERRED_CHORD_ABANDON_MS +} from './terminal-ime-deferred-chord' +import { XTERM_COMPOSITION_SESSION_END_EVENT } from './terminal-ime-composition-route' + +describe('createTerminalImeDeferredChordSender', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('sends once the composition commits', () => { + const el = document.createElement('div') + const send = vi.fn() + const sender = createTerminalImeDeferredChordSender() + + sender.defer(el, send) + expect(send).not.toHaveBeenCalled() + + el.dispatchEvent(new Event('compositionend')) + vi.advanceTimersByTime(0) + expect(send).toHaveBeenCalledTimes(1) + }) + + it('cancels every pending chord and detaches its listeners', () => { + const el = document.createElement('div') + const removeEventListener = vi.spyOn(el, 'removeEventListener') + const first = vi.fn() + const second = vi.fn() + const sender = createTerminalImeDeferredChordSender() + + sender.defer(el, first) + sender.defer(el, second) + sender.cancelPending() + + expect(removeEventListener).toHaveBeenCalledTimes(4) + + el.dispatchEvent(new Event('compositionend')) + el.dispatchEvent(new CustomEvent(XTERM_COMPOSITION_SESSION_END_EVENT)) + vi.runAllTimers() + expect(first).not.toHaveBeenCalled() + expect(second).not.toHaveBeenCalled() + }) + + it('abandons a chord whose composition never ends rather than sending it late', () => { + const el = document.createElement('div') + const send = vi.fn() + const sender = createTerminalImeDeferredChordSender() + + sender.defer(el, send) + vi.advanceTimersByTime(TERMINAL_IME_DEFERRED_CHORD_ABANDON_MS - 1) + expect(send).not.toHaveBeenCalled() + + vi.advanceTimersByTime(2) + el.dispatchEvent(new Event('compositionend')) + vi.runAllTimers() + expect(send).not.toHaveBeenCalled() + }) + + it('stops tracking a chord that already sent, so a later cancel is inert', () => { + const el = document.createElement('div') + const send = vi.fn() + const sender = createTerminalImeDeferredChordSender() + + sender.defer(el, send) + el.dispatchEvent(new Event('compositionend')) + vi.advanceTimersByTime(0) + sender.cancelPending() + vi.runAllTimers() + + expect(send).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.ts b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.ts new file mode 100644 index 00000000000..8598a0ec1f8 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-chord.ts @@ -0,0 +1,51 @@ +import { sendTerminalInputAfterComposition } from './terminal-ime-deferred-newline' + +/** + * Ceiling on the otherwise indefinite wait. Generous enough for a conversion candidate window, + * which can stay open for seconds; a chord that outlives it is discarded rather than sent late, + * because firing mid-preedit is the corruption the wait exists to prevent (#12871). + */ +export const TERMINAL_IME_DEFERRED_CHORD_ABANDON_MS = 10_000 + +export type TerminalImeDeferredChordSender = { + defer: (terminalElement: HTMLElement | null | undefined, send: () => void) => void + cancelPending: () => void +} + +/** + * Owns every chord held for a live composition so blur and pane teardown can drop them. An + * unowned deferral has no exit when compositionend never arrives: its listeners outlive the pane + * and a later composition flushes the stale chord against a rebound terminal. + */ +export function createTerminalImeDeferredChordSender(): TerminalImeDeferredChordSender { + const pendingStops = new Set<() => void>() + + return { + defer: (terminalElement, send) => { + let abandonTimer: number | undefined + let stopComposingWait: (() => void) | null = null + const stopWaiting = (): void => { + window.clearTimeout(abandonTimer) + pendingStops.delete(stopWaiting) + stopComposingWait?.() + } + abandonTimer = window.setTimeout(stopWaiting, TERMINAL_IME_DEFERRED_CHORD_ABANDON_MS) + pendingStops.add(stopWaiting) + stopComposingWait = sendTerminalInputAfterComposition( + terminalElement, + () => { + stopWaiting() + send() + }, + { fallbackMs: null } + ) + }, + cancelPending: () => { + // Each stop removes itself; deleting the visited entry is safe for a Set iterator. + for (const stopWaiting of pendingStops) { + stopWaiting() + } + pendingStops.clear() + } + } +} diff --git a/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.test.ts b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.test.ts index 5b8613c1f7f..0ac8ce17377 100644 --- a/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.test.ts +++ b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.test.ts @@ -6,7 +6,8 @@ import { createTerminalImeModifiedEnterChordOwner, isTerminalImeEnterKeyUp, isTerminalImeProcessEnter, - sendTerminalInputAfterComposition + sendTerminalInputAfterComposition, + TERMINAL_IME_DEFERRED_NEWLINE_FALLBACK_MS } from './terminal-ime-deferred-newline' import { installTerminalImeCompositionRoute, @@ -53,6 +54,27 @@ describe('sendTerminalInputAfterComposition', () => { expect(send).toHaveBeenCalledTimes(1) }) + // STA-4476: `fallbackMs: null` has no exit of its own, so the disposer is the only thing that + // can detach the listeners when compositionend never arrives. + it('detaches its listeners and never sends once disposed', () => { + const el = document.createElement('div') + const send = vi.fn() + const removeEventListener = vi.spyOn(el, 'removeEventListener') + + const dispose = sendTerminalInputAfterComposition(el, send, { fallbackMs: null }) + dispose() + + expect(removeEventListener).toHaveBeenCalledWith('compositionend', expect.any(Function)) + expect(removeEventListener).toHaveBeenCalledWith( + XTERM_COMPOSITION_SESSION_END_EVENT, + expect.any(Function) + ) + + el.dispatchEvent(new Event('compositionend')) + vi.runAllTimers() + expect(send).not.toHaveBeenCalled() + }) + it('falls back to sending when no compositionend arrives', () => { const el = document.createElement('div') const send = vi.fn() @@ -63,6 +85,22 @@ describe('sendTerminalInputAfterComposition', () => { expect(send).toHaveBeenCalledTimes(1) }) + // STA-4476: why the Enter path needs no disposer of its own — its fallback always runs, so the + // listeners cannot outlive it even when the composition never ends. + it('detaches its listeners on the fallback, not only on compositionend', () => { + const el = document.createElement('div') + const removeEventListener = vi.spyOn(el, 'removeEventListener') + + sendTerminalInputAfterComposition(el, vi.fn()) + vi.advanceTimersByTime(TERMINAL_IME_DEFERRED_NEWLINE_FALLBACK_MS) + + expect(removeEventListener).toHaveBeenCalledWith('compositionend', expect.any(Function)) + expect(removeEventListener).toHaveBeenCalledWith( + XTERM_COMPOSITION_SESSION_END_EVENT, + expect.any(Function) + ) + }) + it('finishes from the captured xterm transaction when deferral starts after compositionend', () => { const el = document.createElement('div') const send = vi.fn() diff --git a/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.ts b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.ts index 6eb2ce0e7e6..b27f25e4742 100644 --- a/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.ts +++ b/src/renderer/src/components/terminal-pane/terminal-ime-deferred-newline.ts @@ -10,15 +10,19 @@ export const TERMINAL_IME_DEFERRED_NEWLINE_FALLBACK_MS = 200 * right for it; a cursor chord arriving mid-preedit reproduces the corruption the wait exists to * prevent, and a conversion can hold its candidate window open for seconds. Dropping the chord * costs one keypress, firing early costs a mangled line. + * + * Returns a disposer that stops waiting without sending. An indefinite wait has no other exit, so + * a caller that can outlive the composition must hold it — otherwise the listeners stay on the + * terminal element and a later composition flushes the stale send. */ export function sendTerminalInputAfterComposition( terminalElement: HTMLElement | null | undefined, send: () => void, options?: { fallbackMs?: number | null } -): void { +): () => void { if (!terminalElement) { - window.setTimeout(send, 0) - return + const immediateTimer = window.setTimeout(send, 0) + return () => window.clearTimeout(immediateTimer) } const fallbackMs = @@ -27,7 +31,7 @@ export function sendTerminalInputAfterComposition( : (options?.fallbackMs ?? TERMINAL_IME_DEFERRED_NEWLINE_FALLBACK_MS) let done = false - const finish = (): void => { + const stopWaiting = (): void => { if (done) { return } @@ -40,6 +44,13 @@ export function sendTerminalInputAfterComposition( if (fallbackTimer !== undefined) { window.clearTimeout(fallbackTimer) } + } + + const finish = (): void => { + if (done) { + return + } + stopWaiting() // xterm flushes the committed glyph after compositionend. window.setTimeout(send, 0) } @@ -54,6 +65,8 @@ export function sendTerminalInputAfterComposition( terminalElement.addEventListener('compositionend', onCompositionEnd) terminalElement.addEventListener(XTERM_COMPOSITION_SESSION_END_EVENT, onCompositionSessionEnd) const fallbackTimer = fallbackMs === null ? undefined : window.setTimeout(finish, fallbackMs) + + return stopWaiting } export type TerminalImeDeferredNewlineSender = { From cc74436b3c61b4b9f6df0b9b872c089a746e2554 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:31:39 -0700 Subject: [PATCH 04/10] fix(skills): keep an unanswered skill root's last known skills instead of reporting none (#15015) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(skills): keep an unanswered skill root's last known skills instead of reporting none An aborted or shed root scan degraded to `{skills: [], unavailable: true}`. Every consumer derives "installed" from the skill list, so an unreadable root read as proof the skill was gone and an already-installed skill offered Install again. - discovery: retain the last answered scan per root (5 min, LRU-bounded, dropped on install/update invalidation and when a root answers as absent) and serve it when a later scan goes unavailable. - source inventory: an unanswered root reports `exists: true`, so check `skippedReason` first rather than presenting it as a successful scan. - useInstalledAgentSkills: when an in-scope root did not answer and the skill was not found, say the status may be incomplete instead of a bare "Not installed". * fix(skills): do not let install verification accept an unanswered root's retained scan Retention makes discovery serve a stalled root's last completed scan, which is the state before an install wrote. Verification read that as proof, so reinstalling a skill into a root that stalled could report success without reading what it wrote — turning a retryable false negative into a false positive. A match now counts only when a root that actually answered reached the skill, so a symlinked placement co-owned by a healthy root still verifies. --- src/main/skills/discovery.test.ts | 80 ++++++++++++++++- src/main/skills/discovery.ts | 58 +++++++++++- ...ill-install-discovery-verification.test.ts | 53 ++++++++++- .../skill-install-discovery-verification.ts | 36 +++++++- .../skills/skill-source-inventory.test.ts | 14 +++ .../skills/skill-source-inventory.ts | 6 ++ .../useInstalledAgentSkills.react.test.tsx | 88 ++++++++++++++++++- .../src/hooks/useInstalledAgentSkills.test.ts | 45 +++++++++- .../src/hooks/useInstalledAgentSkills.ts | 32 ++++++- src/renderer/src/i18n/locales/en.json | 3 + 10 files changed, 403 insertions(+), 12 deletions(-) diff --git a/src/main/skills/discovery.test.ts b/src/main/skills/discovery.test.ts index 720d97f1f66..3d905a01d84 100644 --- a/src/main/skills/discovery.test.ts +++ b/src/main/skills/discovery.test.ts @@ -1,8 +1,13 @@ -import { mkdtemp, mkdir, symlink, writeFile } from 'node:fs/promises' +import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { buildSkillDiscoverySources, clearSkillRootScanCache, discoverSkills } from './discovery' +import { + buildSkillDiscoverySources, + clearSkillRootScanCache, + discoverSkills, + LAST_KNOWN_ROOT_SCAN_RETENTION_MS +} from './discovery' import { TUI_AGENT_CONFIG } from '../../shared/tui-agent-config' import type * as SkillRootFileWalk from './skill-root-file-walk' import type { Repo } from '../../shared/repo-types' @@ -48,6 +53,7 @@ beforeEach(() => { afterEach(() => { unavailableRootPath = null + vi.restoreAllMocks() }) describe('skill discovery', () => { @@ -94,6 +100,76 @@ describe('skill discovery', () => { }) }) + // The impact this exists for: every consumer derives "installed" from the skill + // list, so an empty list for an unanswered root made an installed skill offer + // Install again the moment a mount stalled. + it('serves the last answered skills for a root whose rescan did not answer', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-skills-')) + const home = join(root, 'home') + const stalledRoot = join(home, '.omp', 'agent', 'skills') + await mkdir(join(stalledRoot, 'planning'), { recursive: true }) + await writeFile(join(stalledRoot, 'planning', 'SKILL.md'), '# Planning\n\nPlan work.') + + const first = await discoverSkills({ homeDir: home, cwd: join(root, 'missing-cwd') }) + expect(first.skills.map((skill) => skill.name)).toEqual(['Planning']) + + unavailableRootPath = stalledRoot + const second = await discoverSkills({ + homeDir: home, + cwd: join(root, 'missing-cwd'), + refresh: true + }) + + expect(second.skills.map((skill) => skill.name)).toEqual(['Planning']) + expect(second.sources.find((source) => source.path === stalledRoot)).toMatchObject({ + skippedReason: 'unavailable' + }) + }) + + it('drops the retained copy once a root answers as absent', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-skills-')) + const home = join(root, 'home') + const stalledRoot = join(home, '.omp', 'agent', 'skills') + await mkdir(join(stalledRoot, 'planning'), { recursive: true }) + await writeFile(join(stalledRoot, 'planning', 'SKILL.md'), '# Planning\n\nPlan work.') + await discoverSkills({ homeDir: home, cwd: join(root, 'missing-cwd') }) + await rm(stalledRoot, { recursive: true }) + + await discoverSkills({ homeDir: home, cwd: join(root, 'missing-cwd'), refresh: true }) + unavailableRootPath = stalledRoot + const result = await discoverSkills({ + homeDir: home, + cwd: join(root, 'missing-cwd'), + refresh: true + }) + + // A removed skill must not come back the next time its root stalls. + expect(result.skills.map((skill) => skill.name)).toEqual([]) + }) + + it('stops serving a retained copy once it is older than the retention window', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-skills-')) + const home = join(root, 'home') + const stalledRoot = join(home, '.omp', 'agent', 'skills') + await mkdir(join(stalledRoot, 'planning'), { recursive: true }) + await writeFile(join(stalledRoot, 'planning', 'SKILL.md'), '# Planning\n\nPlan work.') + await discoverSkills({ homeDir: home, cwd: join(root, 'missing-cwd') }) + + const recordedAt = Date.now() + vi.spyOn(Date, 'now').mockReturnValue(recordedAt + LAST_KNOWN_ROOT_SCAN_RETENTION_MS + 1) + unavailableRootPath = stalledRoot + const result = await discoverSkills({ + homeDir: home, + cwd: join(root, 'missing-cwd'), + refresh: true + }) + + expect(result.skills).toEqual([]) + expect(result.sources.find((source) => source.path === stalledRoot)).toMatchObject({ + skippedReason: 'unavailable' + }) + }) + it('discovers home and repo SKILL.md packages with provider metadata', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-skills-')) const home = join(root, 'home') diff --git a/src/main/skills/discovery.ts b/src/main/skills/discovery.ts index 7fa5cd077c5..c5b8515c319 100644 --- a/src/main/skills/discovery.ts +++ b/src/main/skills/discovery.ts @@ -48,14 +48,55 @@ const MAX_CACHED_SKILL_ROOTS = 1_024 // line grow with the install. Root *ids* are safe to log where labels and paths // are not — a repo/plugin id is already a hash. export const MAX_LOGGED_ROOT_IDS = 12 +// Why: a root that did not answer holds unknown skills, not zero, so serving what +// it last held is what stops an installed skill flipping to "Install" while a +// mount is wedged. Bounded rather than forever: it covers a stall +// (MAX_JOINABLE_SCAN_AGE_MS) many times over without pinning a permanently dead +// root to rows that can no longer be checked. +export const LAST_KNOWN_ROOT_SCAN_RETENTION_MS = 5 * 60_000 type RootScan = { exists: boolean; skills: ScannedSkill[]; unavailable?: boolean } const rootScans = new SkillScanCoalescer(MAX_CACHED_SKILL_ROOTS) +/** Last answered scan per root key, read only when a later scan goes unavailable. */ +const lastKnownRootScans = new Map() /** Drop every shared root scan, e.g. after a skill install/update mutates disk. */ export function clearSkillRootScanCache(): void { rootScans.clear() + // A mutation invalidates the retained copy too — it is a pre-mutation answer. + lastKnownRootScans.clear() +} + +function recordLastKnownRootScan(key: string, scan: RootScan): void { + if (!scan.exists) { + // Why: a root that scanned as absent is known-empty, so keeping an older copy + // would resurrect skills the user actually removed the next time it stalls. + lastKnownRootScans.delete(key) + return + } + // Delete first so re-insert refreshes recency under the LRU bound below. + lastKnownRootScans.delete(key) + lastKnownRootScans.set(key, { skills: scan.skills, recordedAt: Date.now() }) + while (lastKnownRootScans.size > MAX_CACHED_SKILL_ROOTS) { + const oldestKey = lastKnownRootScans.keys().next().value + if (oldestKey === undefined) { + break + } + lastKnownRootScans.delete(oldestKey) + } +} + +function readLastKnownRootScan(key: string): ScannedSkill[] { + const retained = lastKnownRootScans.get(key) + if (!retained) { + return [] + } + if (Date.now() - retained.recordedAt > LAST_KNOWN_ROOT_SCAN_RETENTION_MS) { + lastKnownRootScans.delete(key) + return [] + } + return retained.skills } async function pathExists(pathValue: string): Promise { @@ -146,15 +187,18 @@ async function scanRootShared( root: SkillScanRoot, refresh: boolean ): Promise> { + const key = rootScanKey(root) try { - return await rootScans.run( - rootScanKey(root), + const outcome = await rootScans.run( + key, { ttlMs: SKILL_ROOT_SCAN_TTL_MS, refresh }, async (signal) => { const exists = await pathExists(root.path) return { exists, skills: exists ? await scanRoot(root, signal) : [] } } ) + recordLastKnownRootScan(key, outcome.value) + return outcome } catch (error) { if (!isSkillRootUnavailableError(error)) { throw error @@ -166,7 +210,15 @@ async function scanRootShared( // The abort case matters as much as the shed one: callers already waiting on // a scan that is later abandoned for age see it reject, and re-throwing here // would turn one slow root into a failed discovery for every root. - return { value: { exists: true, skills: [], unavailable: true }, cached: false } + // + // Empty skills here is not the same claim as `exists: false`: every consumer + // that derives "installed" does it from the skill list, so shipping an empty + // one turned an unanswered root into proof the skill is gone. Serve what the + // root last held and let `unavailable` carry the uncertainty. + return { + value: { exists: true, skills: readLastKnownRootScan(key), unavailable: true }, + cached: false + } } } diff --git a/src/main/skills/skill-install-discovery-verification.test.ts b/src/main/skills/skill-install-discovery-verification.test.ts index 203228a78f7..88d233c12e7 100644 --- a/src/main/skills/skill-install-discovery-verification.test.ts +++ b/src/main/skills/skill-install-discovery-verification.test.ts @@ -28,7 +28,7 @@ function installResult(paths: string[]): SkillInstallResult { } } -function discovery(rootPaths: string[]): SkillDiscoveryResult { +function discovery(rootPaths: string[], unavailableRoots: string[] = []): SkillDiscoveryResult { return { skills: [ { @@ -46,7 +46,17 @@ function discovery(rootPaths: string[]): SkillDiscoveryResult { updatedAt: null } ], - sources: [], + // A root that did not answer reports `exists` — the host cannot prove otherwise. + sources: unavailableRoots.map((path) => ({ + id: path, + label: 'Agent skills home', + path, + sourceKind: 'home' as const, + providers: ['agent-skills' as const], + owner: null, + exists: true, + skippedReason: 'unavailable' as const + })), scannedAt: 1 } } @@ -122,6 +132,45 @@ describe('WSL install discovery verification', () => { ) }) + // Discovery serves an unanswered root's last completed scan so an installed skill + // does not flip to "Install". Verification must not read that as proof: it is the + // state before this install wrote, so a reinstall would verify without reading disk. + it('does not verify a placement backed only by a root that did not answer', async () => { + discoverSkillsInWslMock.mockResolvedValue( + discovery(['/home/alice/.agents/skills'], ['/home/alice/.agents/skills']) + ) + const result = await verifySkillInstallDiscovery({ + result: installResult(['\\\\wsl.localhost\\Ubuntu\\home\\alice\\.agents\\skills\\review']), + scope: 'global', + homeDirectory: '\\\\wsl.localhost\\Ubuntu\\home\\alice', + wslDistro: 'Ubuntu' + }) + + expect(result).toMatchObject({ + status: 'failed', + errorCategory: 'skill-discovery-canonical-missing' + }) + expect(result.placements[0].failure?.retryable).toBe(true) + }) + + it('still verifies through a co-owning root that did answer', async () => { + discoverSkillsInWslMock.mockResolvedValue( + // The symlinked Claude root answered, so the placement is proven, not retained. + discovery( + ['/home/alice/.agents/skills', '/home/alice/.claude/skills'], + ['/home/alice/.agents/skills'] + ) + ) + const result = await verifySkillInstallDiscovery({ + result: installResult(['\\\\wsl.localhost\\Ubuntu\\home\\alice\\.claude\\skills\\review']), + scope: 'global', + homeDirectory: '\\\\wsl.localhost\\Ubuntu\\home\\alice', + wslDistro: 'Ubuntu' + }) + + expect(result.status).toBe('installed') + }) + it('fails verification without scanning a mismatched UNC distro', async () => { const result = await verifySkillInstallDiscovery({ result: installResult(['\\\\wsl.localhost\\Debian\\home\\alice\\.agents\\skills\\review']), diff --git a/src/main/skills/skill-install-discovery-verification.ts b/src/main/skills/skill-install-discovery-verification.ts index fbbf254f821..c2b092429c2 100644 --- a/src/main/skills/skill-install-discovery-verification.ts +++ b/src/main/skills/skill-install-discovery-verification.ts @@ -1,6 +1,6 @@ import { dirname, posix, resolve } from 'node:path' import type { SkillInstallResult, SkillPlacementResult } from '../../shared/skill-install-contract' -import type { SkillDiscoveryResult } from '../../shared/skills' +import type { DiscoveredSkill, SkillDiscoveryResult } from '../../shared/skills' import { parseWslUncPath } from '../../shared/wsl-paths' import { discoverSkills } from './discovery' import { discoverSkillsInWsl } from './skill-discovery-wsl' @@ -37,6 +37,36 @@ function normalizedPath(path: string, wslDistro?: string): string { : withoutTrailingSlash } +function unreadableRootPaths(discovery: SkillDiscoveryResult, wslDistro?: string): Set { + return new Set( + discovery.sources + .filter((source) => source.skippedReason === 'unavailable') + .map((source) => normalizedPath(source.path, wslDistro)) + ) +} + +/** + * Whether a root that actually answered reached this skill. + * + * Why verification cannot reuse discovery's leniency: a root that did not answer + * is served from the host's last completed scan, which is the state *before* this + * install wrote. Counting that as proof would report a reinstall as verified + * without reading a byte of what it just wrote. A co-owning root that did answer + * is still proof, so a symlinked placement verifies as before. + */ +function isAnsweredSkill( + skill: DiscoveredSkill, + unreadableRoots: ReadonlySet, + wslDistro?: string +): boolean { + if (unreadableRoots.size === 0) { + return true + } + return (skill.rootPaths ?? [skill.rootPath]).some( + (rootPath) => !unreadableRoots.has(normalizedPath(rootPath, wslDistro)) + ) +} + function placementIsDiscovered( discovery: SkillDiscoveryResult, placement: SkillPlacementResult, @@ -48,10 +78,14 @@ function placementIsDiscovered( wslDistro ? posix.dirname(placementPath) : dirname(placement.path), wslDistro ) + const unreadableRoots = unreadableRootPaths(discovery, wslDistro) return discovery.skills.some((skill) => { if (skill.name !== skillName) { return false } + if (!isAnsweredSkill(skill, unreadableRoots, wslDistro)) { + return false + } if (normalizedPath(skill.directoryPath, wslDistro) === placementPath) { return true } diff --git a/src/renderer/src/components/skills/skill-source-inventory.test.ts b/src/renderer/src/components/skills/skill-source-inventory.test.ts index 4aff2e0f8a6..67fd41965e8 100644 --- a/src/renderer/src/components/skills/skill-source-inventory.test.ts +++ b/src/renderer/src/components/skills/skill-source-inventory.test.ts @@ -66,6 +66,20 @@ describe('summarizeSkillSources', () => { expect(scannedSkillSourceCount(entries)).toBe(0) }) + it('reports an unanswered root as unavailable even though it claims to exist', () => { + // Why: the host cannot prove a stalled root is gone, so it reports exists:true + // and carries the uncertainty in skippedReason. Reading exists first counted a + // root nobody walked as scanned, and its retained skills as its full contents. + const entries = summarizeSkillSources( + result({ + sources: [source({ id: 'stalled', exists: true, skippedReason: 'unavailable' })], + skills: [skill()] + }) + ) + expect(entries.map((entry) => entry.status)).toEqual(['unavailable']) + expect(scannedSkillSourceCount(entries)).toBe(0) + }) + it('counts only roots that were actually scanned', () => { const entries = summarizeSkillSources( result({ sources: [source(), source({ id: 'gone', exists: false })] }) diff --git a/src/renderer/src/components/skills/skill-source-inventory.ts b/src/renderer/src/components/skills/skill-source-inventory.ts index df702333c38..1bedd5fa02b 100644 --- a/src/renderer/src/components/skills/skill-source-inventory.ts +++ b/src/renderer/src/components/skills/skill-source-inventory.ts @@ -19,6 +19,12 @@ function ownsSkill(source: SkillDiscoverySource, skill: DiscoveredSkill): boolea } function sourceStatus(source: SkillDiscoverySource): SkillSourceStatus { + // Why before `exists`: an unanswered root reports `exists: true` because the + // host could not prove otherwise. Reading that as `scanned` presented a root + // nobody walked as a successful scan, and its retained skills as its full count. + if (source.skippedReason === 'unavailable') { + return 'unavailable' + } if (source.exists) { return 'scanned' } diff --git a/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx b/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx index 3c379e1c1f2..cfa9028b130 100644 --- a/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx +++ b/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { DiscoveredSkill, SkillDiscoveryResult, + SkillDiscoverySource, SkillDiscoveryTarget } from '../../../shared/skills' import type { ProjectExecutionRuntimeResolution } from '../../../shared/project-execution-runtime' @@ -45,14 +46,32 @@ function skill(overrides: Partial): DiscoveredSkill { } } -function discoveryResult(skills: DiscoveredSkill[] = []): SkillDiscoveryResult { +function discoveryResult( + skills: DiscoveredSkill[] = [], + sources: SkillDiscoverySource[] = [] +): SkillDiscoveryResult { return { skills, - sources: [], + sources, scannedAt: Date.now() } } +/** A root the host could not read: it reports `exists` because it cannot prove otherwise. */ +function unavailableSource(overrides: Partial = {}): SkillDiscoverySource { + return { + id: 'home', + label: 'Agent skills home', + path: '/Users/test/.agents/skills', + sourceKind: 'home', + providers: ['agent-skills'], + owner: null, + exists: true, + skippedReason: 'unavailable', + ...overrides + } +} + function deferred(): { promise: Promise resolve: (value: T) => void @@ -152,6 +171,71 @@ beforeEach(() => { }) describe('useInstalledAgentSkill', () => { + // A root that did not answer holds unknown skills, not zero. Reporting a bare + // "not installed" there is what offered Install for an already-installed skill. + it('says the scan was incomplete when an in-scope root did not answer', async () => { + const scan = deferred() + const discover = vi + .fn<(target?: SkillDiscoveryTarget) => Promise>() + .mockReturnValue(scan.promise) + Object.defineProperty(window, 'api', { + configurable: true, + value: { skills: { discover } } + }) + + await renderProbe() + scan.resolve(discoveryResult([], [unavailableSource()])) + await act(async () => { + await scan.promise + }) + + expect(latestState?.installed).toBe(false) + expect(latestState?.error).toContain('did not respond') + }) + + it('keeps a negative authoritative when the unread root is out of scope', async () => { + const scan = deferred() + const discover = vi + .fn<(target?: SkillDiscoveryTarget) => Promise>() + .mockReturnValue(scan.promise) + Object.defineProperty(window, 'api', { + configurable: true, + value: { skills: { discover } } + }) + + await renderProbe() + // The probe asks only about home roots, so a stalled repo root proves nothing. + scan.resolve( + discoveryResult([], [unavailableSource({ id: 'repo', sourceKind: 'repo', path: '/repo' })]) + ) + await act(async () => { + await scan.promise + }) + + expect(latestState?.installed).toBe(false) + expect(latestState?.error).toBeNull() + }) + + it('does not flag an incomplete scan once the skill is found anyway', async () => { + const scan = deferred() + const discover = vi + .fn<(target?: SkillDiscoveryTarget) => Promise>() + .mockReturnValue(scan.promise) + Object.defineProperty(window, 'api', { + configurable: true, + value: { skills: { discover } } + }) + + await renderProbe() + scan.resolve(discoveryResult([skill({ name: 'orca-linear' })], [unavailableSource()])) + await act(async () => { + await scan.promise + }) + + expect(latestState?.installed).toBe(true) + expect(latestState?.error).toBeNull() + }) + it('ignores stale discovery results after the discovery target changes', async () => { const hostScan = deferred() const wslScan = deferred() diff --git a/src/renderer/src/hooks/useInstalledAgentSkills.test.ts b/src/renderer/src/hooks/useInstalledAgentSkills.test.ts index 7be3dd952c6..d3989662d56 100644 --- a/src/renderer/src/hooks/useInstalledAgentSkills.test.ts +++ b/src/renderer/src/hooks/useInstalledAgentSkills.test.ts @@ -1,11 +1,16 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import type { DiscoveredSkill, SkillDiscoveryResult } from '../../../shared/skills' +import type { + DiscoveredSkill, + SkillDiscoveryResult, + SkillDiscoverySource +} from '../../../shared/skills' import type { ProjectExecutionRuntimeResolution } from '../../../shared/project-execution-runtime' import { GLOBAL_AGENT_SKILL_SOURCE_KINDS, _installedAgentSkillDiscoveryInternalsForTests, hasInstalledAgentSkill, hasInstalledAgentSkillNamed, + hasUnreadableAgentSkillSource, notifyInstalledAgentSkillsRefreshed } from './useInstalledAgentSkills' @@ -157,6 +162,44 @@ describe('hasInstalledAgentSkill', () => { }) }) +describe('hasUnreadableAgentSkillSource', () => { + function source(overrides: Partial): SkillDiscoverySource { + return { + id: 'home', + label: 'Agent skills home', + path: '/Users/test/.agents/skills', + sourceKind: 'home', + providers: ['agent-skills'], + owner: null, + // An unread root reports `exists`: the host could not prove otherwise. + exists: true, + ...overrides + } + } + + it('flags a root that did not answer even though it reports as present', () => { + expect(hasUnreadableAgentSkillSource([source({ skippedReason: 'unavailable' })])).toBe(true) + }) + + it('ignores roots that were scanned or are genuinely absent', () => { + expect( + hasUnreadableAgentSkillSource([ + source({}), + source({ id: 'gone', exists: false, skippedReason: 'missing' }) + ]) + ).toBe(false) + }) + + it('ignores an unread root outside the scopes the caller asked about', () => { + expect( + hasUnreadableAgentSkillSource( + [source({ id: 'repo', sourceKind: 'repo', skippedReason: 'unavailable' })], + GLOBAL_AGENT_SKILL_SOURCE_KINDS + ) + ).toBe(false) + }) +}) + describe('isOrchestrationSkillName', () => { it('matches only the orchestration skill name', () => { expect( diff --git a/src/renderer/src/hooks/useInstalledAgentSkills.ts b/src/renderer/src/hooks/useInstalledAgentSkills.ts index a7d96fe9451..7ce56b4a369 100644 --- a/src/renderer/src/hooks/useInstalledAgentSkills.ts +++ b/src/renderer/src/hooks/useInstalledAgentSkills.ts @@ -8,6 +8,7 @@ import type { } from '../../../shared/skills' import { ORCHESTRATION_SKILL_NAME } from '@/lib/agent-feature-install-commands' import { markOrchestrationSetupComplete } from '@/lib/orchestration-setup-state' +import { translate } from '@/i18n/i18n' import { discoverInstalledAgentSkills, getCachedSkillDiscovery, @@ -93,6 +94,23 @@ export function hasInstalledAgentSkillNamed( }) } +/** + * True when a root this query cares about did not answer, so its skills are + * unknown rather than absent. The host serves such a root's last answer, but a + * root that has never answered has none to serve, and a bare "Not installed" + * there offers Install for a skill that may already be present. + */ +export function hasUnreadableAgentSkillSource( + sources: readonly SkillDiscoverySource[], + sourceKinds?: readonly SkillSourceKind[] +): boolean { + return sources.some( + (source) => + source.skippedReason === 'unavailable' && + (!sourceKinds || sourceKinds.includes(source.sourceKind)) + ) +} + export function notifyInstalledAgentSkillsRefreshed(): void { if (typeof window !== 'undefined') { window.dispatchEvent(new CustomEvent(INSTALLED_AGENT_SKILLS_REFRESHED_EVENT)) @@ -285,6 +303,11 @@ export function useInstalledAgentSkillNames( [candidateSkillNames, enabled, skills, sourceKinds] ) + const incompleteScan = useMemo( + () => enabled && !installed && hasUnreadableAgentSkillSource(sources, sourceKinds), + [enabled, installed, sources, sourceKinds] + ) + useEffect(() => { if (installed && candidateSkillNames.some(isOrchestrationSkillName)) { // Why: older floating-workspace education still keys off this marker; any @@ -299,7 +322,14 @@ export function useInstalledAgentSkillNames( installed, loading: loadingForRender, settled: enabled && resultForRender !== null, - error: errorForRender, + error: + errorForRender ?? + (incompleteScan + ? translate( + 'auto.hooks.useInstalledAgentSkills.unreadableSkillSource', + 'A skill folder did not respond, so this status may be incomplete.' + ) + : null), skills, sources, refresh: forceRefresh diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index a66f2e3b481..518076227d3 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -957,6 +957,9 @@ "useLargeTextControlPaste": { "pasteTooLarge": "Paste is too large." }, + "useInstalledAgentSkills": { + "unreadableSkillSource": "A skill folder did not respond, so this status may be incomplete." + }, "useMacosTccPromptNotice": { "title": "Seeing “Orca would like to access…” prompts?", "description": "Permission messages from macOS may appear when an agent or terminal tool running in Orca attempts to access protected files. Grant Full Disk Access in Settings to reduce these prompts.", From 77ef6bb9eeba7e049e5acaa657f5b00b358f6f30 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:34:18 -0700 Subject: [PATCH 05/10] fix(terminal): verify agent prompt submission (#14962) --- .../agent-prompt-submission-runtime.test.ts | 683 ++++++++++++++++++ ...ent-prompt-submission-verification.test.ts | 155 ++++ .../agent-prompt-submission-verification.ts | 90 +++ src/main/runtime/orca-runtime.test.ts | 18 + src/main/runtime/orca-runtime.ts | 404 ++++++++++- src/main/runtime/rpc/errors.test.ts | 10 + src/main/runtime/rpc/errors.ts | 3 + src/main/runtime/rpc/methods/terminal.ts | 7 +- .../rpc/terminal-agent-prompt-send.test.ts | 3 +- src/shared/agent-status-osc.test.ts | 2 + src/shared/agent-status-osc.ts | 5 +- .../terminal-send-agent-prompt-submit.spec.ts | 106 ++- tests/tools/repro-terminal-send-submit.mjs | 130 +++- 13 files changed, 1557 insertions(+), 59 deletions(-) create mode 100644 src/main/runtime/agent-prompt-submission-runtime.test.ts create mode 100644 src/main/runtime/agent-prompt-submission-verification.test.ts create mode 100644 src/main/runtime/agent-prompt-submission-verification.ts diff --git a/src/main/runtime/agent-prompt-submission-runtime.test.ts b/src/main/runtime/agent-prompt-submission-runtime.test.ts new file mode 100644 index 00000000000..ce19f22a3ca --- /dev/null +++ b/src/main/runtime/agent-prompt-submission-runtime.test.ts @@ -0,0 +1,683 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { AGENT_PROMPT_BRACKETED_PASTE_END } from '../../shared/agent-prompt-injection' +import { OrcaRuntimeService } from './orca-runtime' +import { makeStore } from './runtime-rpc-worktree-store-fixtures' + +const WORKTREE_PATH = '/tmp/worktree-a' + +vi.mock('../git/worktree', () => ({ + listWorktrees: vi.fn().mockResolvedValue([ + { + path: '/tmp/worktree-a', + head: 'abc', + branch: 'feature/prompt-verification', + isBare: false, + isMainWorktree: false + } + ]), + listWorktreesStrict: vi.fn().mockResolvedValue([ + { + path: '/tmp/worktree-a', + head: 'abc', + branch: 'feature/prompt-verification', + isBare: false, + isMainWorktree: false + } + ]) +})) + +async function createPromptRuntime( + onWrite: (runtime: OrcaRuntimeService, data: string, writeIndex: number) => void +): Promise<{ runtime: OrcaRuntimeService; handle: string; writes: string[] }> { + const runtime = new OrcaRuntimeService(makeStore() as never) + const writes: string[] = [] + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: 'pty-prompt' }), + write: (_ptyId, data) => { + writes.push(data) + onWrite(runtime, data, writes.length) + return true + }, + kill: () => true, + getForegroundProcess: async () => null + }) + const terminal = await runtime.createTerminal(`path:${WORKTREE_PATH}`, { + launchAgent: 'aider' + }) + return { runtime, handle: terminal.handle, writes } +} + +describe('agent prompt submission runtime', () => { + afterEach(() => vi.useRealTimers()) + + it('submits exactly once after an observed lifecycle transition', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + }) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + await vi.runAllTimersAsync() + + await expect(submission).resolves.toMatchObject({ accepted: true }) + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('accepts a working-to-idle cycle completed before the first poll', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex idle\x07', Date.now()) + } + }) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + await vi.runAllTimersAsync() + + await expect(submission).resolves.toMatchObject({ accepted: true }) + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('reports redraw-only activity as stalled without retrying Enter', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data.includes(AGENT_PROMPT_BRACKETED_PASTE_END)) { + runtime.onPtyData('pty-prompt', '\x1b[2J\x1b[H› review this', Date.now()) + } else if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b[2J\x1b[H› review this', Date.now()) + } + }) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_stalled') + + await vi.runAllTimersAsync() + + await rejected + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('reports a neutral title transition as stalled without retrying Enter', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;plain shell\x07', Date.now()) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex idle\x07', Date.now()) + } + }) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex idle\x07', Date.now()) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_stalled') + + await vi.runAllTimersAsync() + + await rejected + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('does not send Enter after a permission state appears', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data.includes(AGENT_PROMPT_BRACKETED_PASTE_END)) { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex waiting for permission\x07', Date.now()) + } + }) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_blocked') + + await vi.runAllTimersAsync() + + await rejected + expect(writes).not.toContain('\r') + }) + + it('does not paste into an existing permission prompt', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex waiting for permission\x07', Date.now()) + + await expect(runtime.sendTerminalAgentPrompt(handle, 'review this')).rejects.toThrow( + 'agent_prompt_blocked' + ) + expect(writes).toEqual([]) + }) + + it('does not paste into an output-only permission prompt', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex idle\x07', Date.now()) + vi.setSystemTime(2_000) + runtime.onPtyData( + 'pty-prompt', + 'Permission required\nAllow once\nAllow always\nReject\n', + Date.now() + ) + + await expect(runtime.sendTerminalAgentPrompt(handle, 'review this')).rejects.toThrow( + 'agent_prompt_blocked' + ) + expect(writes).toEqual([]) + }) + + it('does not paste into a coalesced live permission title', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.onPtyData( + 'pty-prompt', + '\x1b]9999;{"state":"working","agentType":"aider"}\x07' + + '\x1b]0;Codex waiting for permission\x07', + Date.now() + ) + + await expect(runtime.sendTerminalAgentPrompt(handle, 'review this')).rejects.toThrow( + 'agent_prompt_blocked' + ) + expect(writes).toEqual([]) + }) + + it('does not paste when split status stripping completes a permission title', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + }) + runtime.onPtyData( + 'pty-prompt', + '\x1b]0;Codex waiting for permission\x1b]9999;{"state":"working","agentType":"aider"', + Date.now() + ) + runtime.onPtyData('pty-prompt', '}\x07\x07', Date.now()) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_blocked') + await vi.runAllTimersAsync() + + await rejected + expect(writes).toEqual([]) + }) + + it('preserves hook permission after an earlier live idle title', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.onPtyData( + 'pty-prompt', + 'Permission required\nAllow once\nAllow always\nReject\n', + Date.now() + ) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex idle\x07', Date.now()) + runtime.onPtyData( + 'pty-prompt', + '\x1b]9999;{"state":"waiting","agentType":"aider"}\x07', + Date.now() + ) + + await expect(runtime.sendTerminalAgentPrompt(handle, 'review this')).rejects.toThrow( + 'agent_prompt_blocked' + ) + expect(writes).toEqual([]) + }) + + it('does not block on permission text restored only as history', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + }) + runtime.seedTerminalRestoreTail('pty-prompt', { + text: 'Permission required\r\nAllow once\r\nAllow always\r\nReject\r\n' + }) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + await vi.runAllTimersAsync() + + await expect(submission).resolves.toMatchObject({ accepted: true }) + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('does not send Enter after output-only permission appears during settlement', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data.includes(AGENT_PROMPT_BRACKETED_PASTE_END)) { + vi.setSystemTime(2_000) + runtime.onPtyData( + 'pty-prompt', + 'Permission required\nAllow once\nAllow always\nReject\n', + Date.now() + ) + } + }) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex idle\x07', Date.now()) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_blocked') + + await vi.runAllTimersAsync() + + await rejected + expect(writes).not.toContain('\r') + }) + + it.each([ + '\x1b]0;Codex waiting for permission\x07\x1b]0;Codex idle\x07', + '\x1b]9999;{"state":"working","agentType":"aider"}\x07' + + '\x1b]0;Codex waiting for permission\x07', + '\x1b]0;Codex waiting for permission\x07' + + '\x1b]9999;{"state":"working","agentType":"aider"}\x07' + ])('does not send Enter after coalesced permission activity', async (output) => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data.includes(AGENT_PROMPT_BRACKETED_PASTE_END)) { + runtime.onPtyData('pty-prompt', output, Date.now()) + } + }) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_blocked') + + await vi.runAllTimersAsync() + + await rejected + expect(writes).not.toContain('\r') + }) + + it('stops a chunked paste when permission appears between chunks', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + let writeChecks = 0 + + const submission = runtime.sendTerminalAgentPrompt(handle, 'x'.repeat(20_000), { + beforeWrite: () => { + writeChecks += 1 + if (writeChecks === 2) { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex waiting for permission\x07', Date.now()) + } + } + }) + + await expect(submission).rejects.toThrow('agent_prompt_blocked') + expect(writes).toHaveLength(2) + expect(writes[1]).toBe(AGENT_PROMPT_BRACKETED_PASTE_END) + expect(writes).not.toContain('\r') + }) + + it('stops a chunked paste after transient output-only permission', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.onPtyData('pty-prompt', 'initial output\n', Date.now()) + let writeChecks = 0 + + const submission = runtime.sendTerminalAgentPrompt(handle, 'x'.repeat(20_000), { + beforeWrite: () => { + writeChecks += 1 + if (writeChecks === 2) { + runtime.onPtyData( + 'pty-prompt', + 'Permission required\nAllow once\nAllow always\nReject\n', + Date.now() + ) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + } + }) + + await expect(submission).rejects.toThrow('agent_prompt_blocked') + expect(writes).toHaveLength(2) + expect(writes[1]).toBe(AGENT_PROMPT_BRACKETED_PASTE_END) + expect(writes).not.toContain('\r') + }) + + it('prefers a later permission title over an earlier explicit idle status', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.onPtyData( + 'pty-prompt', + '\x1b]9999;{"state":"done","agentType":"aider"}\x07', + Date.now() + ) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex waiting for permission\x07', Date.now()) + + await expect(runtime.sendTerminalAgentPrompt(handle, 'review this')).rejects.toThrow( + 'agent_prompt_blocked' + ) + expect(writes).toEqual([]) + }) + + it('prefers later explicit idle evidence over stale permission output', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + let handle = '' + const writes: string[] = [] + const runtime = new OrcaRuntimeService(makeStore() as never, undefined, { + getAgentStatusSnapshot: () => [ + { + paneKey: 'prompt-pane', + terminalHandle: handle, + state: 'done', + prompt: '', + agentType: 'aider', + connectionId: null, + receivedAt: Date.now(), + stateStartedAt: Date.now() + } + ] + }) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: 'pty-prompt' }), + write: (_ptyId, data) => { + writes.push(data) + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + return true + }, + kill: () => true, + getForegroundProcess: async () => null + }) + handle = (await runtime.createTerminal(`path:${WORKTREE_PATH}`, { launchAgent: 'aider' })) + .handle + runtime.onPtyData( + 'pty-prompt', + 'Permission required\nAllow once\nAllow always\nReject\n' + + '\x1b]0;Codex waiting for permission\x07', + Date.now() + ) + vi.setSystemTime(2_000) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + await vi.runAllTimersAsync() + + await expect(submission).resolves.toMatchObject({ accepted: true }) + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('prefers a later working title over an earlier explicit idle status', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + }) + runtime.onPtyData( + 'pty-prompt', + '\x1b]9999;{"state":"done","agentType":"aider"}\x07', + Date.now() + ) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + await vi.runAllTimersAsync() + + await expect(submission).resolves.toMatchObject({ accepted: true }) + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('does not treat an unchanged newer working status as submission evidence', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData( + 'pty-prompt', + '\x1b]9999;{"state":"working","agentType":"aider"}\x07', + Date.now() + ) + } + }) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex waiting for permission\x07', Date.now()) + vi.setSystemTime(2_000) + runtime.onPtyData( + 'pty-prompt', + '\x1b]9999;{"state":"working","agentType":"aider"}\x07', + Date.now() + ) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_stalled') + await vi.runAllTimersAsync() + + await rejected + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('does not write Enter after the PTY generation changes during settlement', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('terminal_handle_stale') + + await vi.advanceTimersByTimeAsync(0) + expect(writes.some((data) => data.includes(AGENT_PROMPT_BRACKETED_PASTE_END))).toBe(true) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'reset' }, + runtime.getPtyOutputSequence('pty-prompt') + ) + await vi.runAllTimersAsync() + + await rejected + expect(writes).not.toContain('\r') + }) + + it('does not reuse explicit permission status across a provider generation reset', async () => { + vi.useFakeTimers() + const controller = new AbortController() + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'continued' }, + 0 + ) + runtime.onPtyData( + 'pty-prompt', + '\x1b]9999;{"state":"waiting","agentType":"aider"}\x07', + Date.now() + ) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'reset' }, + 0 + ) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this', { + signal: controller.signal + }) + const rejected = expect(submission).rejects.toThrow('request_aborted') + await vi.advanceTimersByTimeAsync(0) + + expect(writes.some((data) => data.includes(AGENT_PROMPT_BRACKETED_PASTE_END))).toBe(true) + controller.abort() + await vi.runAllTimersAsync() + await rejected + }) + + it('does not reuse output-only permission across a provider generation reset', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + }) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'continued' }, + 0 + ) + runtime.onPtyData( + 'pty-prompt', + 'Permission required\nAllow once\nAllow always\nReject\n', + Date.now() + ) + const sequenceAtSpawnStart = runtime.getPtyOutputSequence('pty-prompt') + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'reset' }, + sequenceAtSpawnStart + ) + + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + await vi.runAllTimersAsync() + + await expect(submission).resolves.toMatchObject({ accepted: true }) + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('fails closed when new bytes race a reset after old permission output', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'continued' }, + 0 + ) + runtime.onPtyData( + 'pty-prompt', + 'Permission required\nAllow once\nAllow always\nReject\n', + Date.now() + ) + const sequenceAtSpawnStart = runtime.getPtyOutputSequence('pty-prompt') + runtime.onPtyData('pty-prompt', 'replacement startup output\n', Date.now()) + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'reset' }, + sequenceAtSpawnStart + ) + + await expect(runtime.sendTerminalAgentPrompt(handle, 'review this')).rejects.toThrow( + 'agent_prompt_blocked' + ) + expect(writes).toEqual([]) + }) + + it('reports permission reached after the first Enter as blocked', async () => { + vi.useFakeTimers() + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex waiting for permission\x07', Date.now()) + } + }) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this') + const rejected = expect(submission).rejects.toThrow('agent_prompt_blocked') + + await vi.runAllTimersAsync() + + await rejected + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) + + it('serializes concurrent prompt submissions within one PTY generation', async () => { + vi.useFakeTimers() + let enterCount = 0 + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + enterCount += 1 + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + runtime.onPtyData('pty-prompt', '\x1b]0;Codex idle\x07', Date.now()) + } + }) + + const first = runtime.sendTerminalAgentPrompt(handle, 'first prompt') + const second = runtime.sendTerminalAgentPrompt(handle, 'second prompt') + await vi.runAllTimersAsync() + await Promise.all([first, second]) + + const firstPaste = writes.findIndex((data) => data.includes('first prompt')) + const firstEnter = writes.indexOf('\r', firstPaste + 1) + const secondPaste = writes.findIndex((data) => data.includes('second prompt')) + const secondEnter = writes.indexOf('\r', secondPaste + 1) + expect(firstPaste).toBeGreaterThanOrEqual(0) + expect(firstEnter).toBeGreaterThan(firstPaste) + expect(secondPaste).toBeGreaterThan(firstEnter) + expect(secondEnter).toBeGreaterThan(secondPaste) + expect(enterCount).toBe(2) + }) + + it('does not queue a replacement generation behind an obsolete submission', async () => { + vi.useFakeTimers() + let releaseFirst!: () => void + let firstWriteReached!: () => void + const firstWrite = new Promise((resolve) => { + firstWriteReached = resolve + }) + const firstGate = new Promise((resolve) => { + releaseFirst = resolve + }) + const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => { + if (data === '\r') { + runtime.onPtyData('pty-prompt', '\x1b]0;Codex working\x07', Date.now()) + } + }) + + const first = runtime.sendTerminalAgentPrompt(handle, 'obsolete prompt', { + beforeWrite: async () => { + firstWriteReached() + await firstGate + } + }) + await firstWrite + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'reset' }, + 0 + ) + + const replacement = runtime.sendTerminalAgentPrompt(handle, 'replacement prompt') + await vi.runAllTimersAsync() + await expect(replacement).resolves.toMatchObject({ accepted: true }) + expect(writes.some((data) => data.includes('replacement prompt'))).toBe(true) + + releaseFirst() + await expect(first).rejects.toThrow('terminal_handle_stale') + }) + + it('does not close a partial paste after the PTY generation changes', async () => { + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + let writeChecks = 0 + + const submission = runtime.sendTerminalAgentPrompt(handle, 'x'.repeat(20_000), { + beforeWrite: () => { + writeChecks += 1 + if (writeChecks === 2) { + runtime.synchronizePtyOutputSequenceFromProvider( + 'pty-prompt', + { value: 0, generation: 'reset' }, + runtime.getPtyOutputSequence('pty-prompt') + ) + } + } + }) + + await expect(submission).rejects.toThrow('terminal_handle_stale') + expect(writes).toHaveLength(1) + expect(writes[0]).not.toContain(AGENT_PROMPT_BRACKETED_PASTE_END) + }) + + it('does not send delayed Enter after cancellation during settlement', async () => { + vi.useFakeTimers() + const controller = new AbortController() + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this', { + signal: controller.signal + }) + const rejected = expect(submission).rejects.toThrow('request_aborted') + + await vi.advanceTimersByTimeAsync(0) + controller.abort() + await vi.runAllTimersAsync() + + await rejected + expect(writes.filter((data) => data === '\r')).toHaveLength(0) + }) + + it('does not send another Enter after cancellation during verification', async () => { + vi.useFakeTimers() + const controller = new AbortController() + const { runtime, handle, writes } = await createPromptRuntime(() => undefined) + const submission = runtime.sendTerminalAgentPrompt(handle, 'review this', { + signal: controller.signal + }) + const rejected = expect(submission).rejects.toThrow('request_aborted') + + await vi.advanceTimersByTimeAsync(500) + controller.abort() + await vi.runAllTimersAsync() + + await rejected + expect(writes.filter((data) => data === '\r')).toHaveLength(1) + }) +}) diff --git a/src/main/runtime/agent-prompt-submission-verification.test.ts b/src/main/runtime/agent-prompt-submission-verification.test.ts new file mode 100644 index 00000000000..a0f3e1a5213 --- /dev/null +++ b/src/main/runtime/agent-prompt-submission-verification.test.ts @@ -0,0 +1,155 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + AGENT_PROMPT_EFFECT_TIMEOUT_MS, + type AgentPromptActivity, + verifyAgentPromptSubmission +} from './agent-prompt-submission-verification' + +function activity(overrides: Partial = {}): AgentPromptActivity { + return { + generation: 1, + permissionSequence: 2, + workingSequence: 4, + status: 'idle', + ...overrides + } +} + +describe('agent prompt submission verification', () => { + afterEach(() => vi.useRealTimers()) + + it('accepts an observed working transition', async () => { + vi.useFakeTimers() + let current = activity() + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current + }) + + current = activity({ workingSequence: 5, status: 'working' }) + await vi.advanceTimersByTimeAsync(50) + + await expect(verification).resolves.toBeUndefined() + }) + + it('accepts a completed lifecycle transition between polls', async () => { + vi.useFakeTimers() + let current = activity() + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current + }) + + current = activity({ workingSequence: 5 }) + await vi.advanceTimersByTimeAsync(50) + + await expect(verification).resolves.toBeUndefined() + }) + + it('does not accept an unrelated transition to a neutral title', async () => { + vi.useFakeTimers() + let current = activity() + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current + }) + const rejected = expect(verification).rejects.toThrow('agent_prompt_stalled') + + current = activity({ status: null }) + await vi.advanceTimersByTimeAsync(AGENT_PROMPT_EFFECT_TIMEOUT_MS) + + await rejected + }) + + it('reports stalled when no lifecycle transition occurs', async () => { + vi.useFakeTimers() + const current = activity() + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current + }) + const rejected = expect(verification).rejects.toThrow('agent_prompt_stalled') + + await vi.advanceTimersByTimeAsync(AGENT_PROMPT_EFFECT_TIMEOUT_MS) + + await rejected + }) + + it('blocks when permission appears after submit', async () => { + vi.useFakeTimers() + let current = activity() + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current + }) + const rejected = expect(verification).rejects.toThrow('agent_prompt_blocked') + + current = activity({ status: 'permission' }) + await vi.advanceTimersByTimeAsync(50) + + await rejected + }) + + it('blocks when permission appears and clears between polls', async () => { + vi.useFakeTimers() + let current = activity() + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current + }) + const rejected = expect(verification).rejects.toThrow('agent_prompt_blocked') + + current = activity({ permissionSequence: 3 }) + await vi.advanceTimersByTimeAsync(50) + + await rejected + }) + + it('rejects an existing permission state', async () => { + const current = activity({ status: 'permission' }) + + await expect( + verifyAgentPromptSubmission({ baseline: current, readActivity: () => current }) + ).rejects.toThrow('agent_prompt_blocked') + }) + + it('does not accept an unchanged working baseline', async () => { + vi.useFakeTimers() + const current = activity({ status: 'working' }) + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current + }) + const rejected = expect(verification).rejects.toThrow('agent_prompt_stalled') + + await vi.advanceTimersByTimeAsync(AGENT_PROMPT_EFFECT_TIMEOUT_MS) + + await rejected + }) + + it('rejects a replaced terminal generation', async () => { + const baseline = activity() + + await expect( + verifyAgentPromptSubmission({ + baseline, + readActivity: () => activity({ generation: 2 }) + }) + ).rejects.toThrow('terminal_handle_stale') + }) + + it('cancels while waiting for activity', async () => { + vi.useFakeTimers() + const controller = new AbortController() + const current = activity() + const verification = verifyAgentPromptSubmission({ + baseline: current, + readActivity: () => current, + signal: controller.signal + }) + + controller.abort() + + await expect(verification).rejects.toThrow('request_aborted') + }) +}) diff --git a/src/main/runtime/agent-prompt-submission-verification.ts b/src/main/runtime/agent-prompt-submission-verification.ts new file mode 100644 index 00000000000..41b5f5b1732 --- /dev/null +++ b/src/main/runtime/agent-prompt-submission-verification.ts @@ -0,0 +1,90 @@ +export const AGENT_PROMPT_EFFECT_TIMEOUT_MS = 5_000 +const AGENT_PROMPT_EFFECT_POLL_MS = 50 + +export type AgentPromptActivity = Readonly<{ + generation: number + permissionSequence: number + workingSequence: number + status: 'working' | 'permission' | 'idle' | null +}> + +type AgentPromptVerificationOptions = { + baseline: AgentPromptActivity + readActivity: () => AgentPromptActivity + signal?: AbortSignal +} + +export async function verifyAgentPromptSubmission( + options: AgentPromptVerificationOptions +): Promise { + throwIfAgentPromptAborted(options.signal) + assertPromptNotBlocked(options.baseline, options.baseline) + + const deadline = Date.now() + AGENT_PROMPT_EFFECT_TIMEOUT_MS + while (Date.now() < deadline) { + const current = options.readActivity() + assertSamePromptGeneration(options.baseline, current) + assertPromptNotBlocked(options.baseline, current) + if (agentPromptLifecycleChanged(options.baseline, current)) { + return + } + await waitForAgentPromptPoll(options.signal) + } + + const current = options.readActivity() + assertSamePromptGeneration(options.baseline, current) + assertPromptNotBlocked(options.baseline, current) + if (agentPromptLifecycleChanged(options.baseline, current)) { + return + } + throw new Error('agent_prompt_stalled') +} + +function agentPromptLifecycleChanged( + baseline: AgentPromptActivity, + current: AgentPromptActivity +): boolean { + return current.workingSequence > baseline.workingSequence +} + +function assertSamePromptGeneration( + baseline: AgentPromptActivity, + current: AgentPromptActivity +): void { + if (current.generation !== baseline.generation) { + throw new Error('terminal_handle_stale') + } +} + +function assertPromptNotBlocked(baseline: AgentPromptActivity, current: AgentPromptActivity): void { + if (current.status === 'permission' || current.permissionSequence > baseline.permissionSequence) { + throw new Error('agent_prompt_blocked') + } +} + +function throwIfAgentPromptAborted(signal?: AbortSignal): void { + if (signal?.aborted) { + throw new Error('request_aborted') + } +} + +async function waitForAgentPromptPoll(signal?: AbortSignal): Promise { + if (!signal) { + await new Promise((resolve) => setTimeout(resolve, AGENT_PROMPT_EFFECT_POLL_MS)) + return + } + await new Promise((resolve, reject) => { + const onAbort = (): void => { + clearTimeout(timer) + reject(new Error('request_aborted')) + } + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort) + resolve() + }, AGENT_PROMPT_EFFECT_POLL_MS) + signal.addEventListener('abort', onAbort, { once: true }) + if (signal.aborted) { + onAbort() + } + }) +} diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 4c2dd2fbdd7..7bd76f0d316 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -166,6 +166,16 @@ function resetPlatform(): void { } } +function acknowledgeAgentPromptSubmit( + runtime: OrcaRuntimeService, + ptyId: string, + data: string +): void { + if (data === '\r') { + runtime.onPtyData(ptyId, '\x1b]0;Codex working\x07', Date.now()) + } +} + const electronMocks = vi.hoisted(() => { type Listener = (...args: unknown[]) => void const listeners = new Map>() @@ -16762,6 +16772,7 @@ describe('OrcaRuntimeService', () => { spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), write: (_ptyId, data) => { writes.push(data) + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) return true }, kill: () => true, @@ -16828,6 +16839,7 @@ describe('OrcaRuntimeService', () => { } else { prematureEnters += 1 } + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) } return true }, @@ -16876,6 +16888,7 @@ describe('OrcaRuntimeService', () => { spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), write: (_ptyId, data) => { writes.push(data) + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) return true }, kill: () => true, @@ -16913,6 +16926,7 @@ describe('OrcaRuntimeService', () => { runtime.onPtyData('pty-bg', '\x1b[?25hcomposer rendered', Date.now()) }, 1_200) } + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) return true }, kill: () => true, @@ -16945,6 +16959,7 @@ describe('OrcaRuntimeService', () => { spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), write: (_ptyId, data) => { writes.push(data) + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) return true }, kill: () => true, @@ -16983,6 +16998,7 @@ describe('OrcaRuntimeService', () => { runtime.onPtyData('pty-bg', 'final slow composer frame', Date.now()) }, 8_100) } + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) return true }, kill: () => true, @@ -17024,6 +17040,7 @@ describe('OrcaRuntimeService', () => { ) } } + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) return true }, kill: () => true, @@ -17054,6 +17071,7 @@ describe('OrcaRuntimeService', () => { spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), write: (_ptyId, data) => { writes.push(data) + acknowledgeAgentPromptSubmit(runtime, 'pty-bg', data) return true }, kill: () => true, diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index fc6a0fcc715..9cae4ee4798 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -3,6 +3,7 @@ /* eslint-disable no-control-regex -- Why: terminal normalization must strip ANSI and OSC control sequences from PTY output before returning bounded text to agents. */ import { detectAgentStatusFromTitle, + extractLastOscTitle, isClaudeManagementTitle, isCursorNativeAgentTitle, isOpenCodeNativeTitle, @@ -100,6 +101,10 @@ import { AGENT_PROMPT_SUBMIT_DELAY_MS, buildAgentPromptPasteBytes } from '../../shared/agent-prompt-injection' +import { + type AgentPromptActivity, + verifyAgentPromptSubmission +} from './agent-prompt-submission-verification' import { awaitWindowsHostGitEnvironmentReady, gitExecFileAsync, @@ -1483,6 +1488,14 @@ type RuntimePtyWorktreeRecord = { tailWaitState?: TerminalTailWaitState } +type TerminalAgentStatusSnapshot = { + waitText: string + waitBlockedAt: number | null + title: string | null + titleStatus: AgentStatus | null + titleStatusIsLive: boolean +} + type TerminalCreateOptions = { command?: string claudeAgentTeamsSourceCommand?: string @@ -1962,6 +1975,67 @@ const AGENT_PROMPT_RENDER_TIMEOUT_MS = 8000 const AGENT_PROMPT_RENDER_QUIET_MS = 1500 // Why: Claude and Codex emit show-cursor after accepting bracketed paste. const AGENT_PROMPT_RENDER_MARKER = '\x1b[?25h' + +function assertAgentPromptRequestActive(signal?: AbortSignal): void { + if (signal?.aborted) { + throw new Error('request_aborted') + } +} + +async function waitForAgentPromptPromise(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) { + return await promise + } + assertAgentPromptRequestActive(signal) + return await new Promise((resolve, reject) => { + let settled = false + const finish = (result: { value: T } | { error: unknown }): void => { + if (settled) { + return + } + settled = true + signal.removeEventListener('abort', onAbort) + if ('error' in result) { + reject(result.error) + } else { + resolve(result.value) + } + } + const onAbort = (): void => finish({ error: new Error('request_aborted') }) + signal.addEventListener('abort', onAbort, { once: true }) + if (signal.aborted) { + onAbort() + return + } + promise.then( + (value) => finish({ value }), + (error: unknown) => finish({ error }) + ) + }) +} + +async function waitForAgentPromptDelay(delayMs: number, signal?: AbortSignal): Promise { + if (!signal) { + await new Promise((resolve) => setTimeout(resolve, delayMs)) + return + } + assertAgentPromptRequestActive(signal) + await new Promise((resolve, reject) => { + const onAbort = (): void => { + clearTimeout(timer) + reject(new Error('request_aborted')) + } + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort) + resolve() + }, delayMs) + signal.addEventListener('abort', onAbort, { once: true }) + if (signal.aborted) { + onAbort() + } + }) +} + const MOBILE_TERMINAL_SURFACE_TIMEOUT_MS = 10_000 // Why: the split already failed; the caller waits on this teardown only to learn whether the // fallback kill is needed, so keep it short — an unreachable host must not stall the rejection. @@ -3136,6 +3210,13 @@ export class OrcaRuntimeService { private titleObservationSequence = 0 private headlessTerminals = new Map() private ptyOutputSequenceById = new Map() + private agentPromptLifecycleByPtyId = new Map< + string, + { status: AgentStatus | null; workingSequence: number; updatedAt: number } + >() + private agentPromptPermissionSequenceByPtyId = new Map() + private agentPromptExplicitStatusFloorByPtyId = new Map() + private agentPromptSubmissionTailByPtyId = new Map>() private providerSequenceInitializedPtys = new Set() private providerSequenceOffsetByPtyId = new Map() private providerSnapshotPreferredPtys = new Set() @@ -10957,6 +11038,11 @@ export class OrcaRuntimeService { // bell — the chunk's agentStatus:set events must reach the renderer // before its pty:sideEffect batch. retainedAgentStatusChanged = this.emitTerminalAgentStatusEvents(ptyId, agentStatusChunk) + const lastPayloadTitleOffset = + agentStatusChunk.lastPayloadCleanOffset === null + ? null + : (previousTitleScanTail?.length ?? 0) + agentStatusChunk.lastPayloadCleanOffset + this.restoreAgentPromptLifecycleByteOrder(ptyId, titleInput, lastPayloadTitleOffset) } finally { // Why: flushed in the finally so a throwing tracker callback cannot // strand this chunk's facts to be emitted under the next chunk's seq. @@ -11052,6 +11138,7 @@ export class OrcaRuntimeService { } if (tailGainedNewerBlockedReason(previousWaitState, nextWaitState, state.appended)) { pty.waitBlockedAt = at + this.recordAgentPromptPermissionObservation(ptyId) } state.lastAt = at state.lastWaitState = nextWaitState @@ -11518,6 +11605,7 @@ export class OrcaRuntimeService { const identityOnlyTitle = this.isLiveCursorNativeTitle(rawTitle, meta) const recordedTitle = identityOnlyTitle ? null : normalizedTitle const agentStatus = identityOnlyTitle ? null : detectAgentStatusFromTitle(rawTitle) + this.recordAgentPromptLifecycleState(ptyId, agentStatus) let ptyRecordChanged = false const pty = this.ptysById.get(ptyId) if (pty) { @@ -11633,6 +11721,9 @@ export class OrcaRuntimeService { this.oscTitleScanTailByPtyId.delete(ptyId) this.osc7ScanTailByPtyId.delete(ptyId) this.agentStatusOscProcessorsByPtyId.delete(ptyId) + this.agentPromptLifecycleByPtyId.delete(ptyId) + this.agentPromptPermissionSequenceByPtyId.delete(ptyId) + this.clearWaitBlockedCheckState(ptyId) const pty = this.ptysById.get(ptyId) if (pty) { pty.lastOscTitle = null @@ -11646,13 +11737,18 @@ export class OrcaRuntimeService { pty.lastAgentStatusRichInvalidatedAtEpochMs = Date.now() pty.managementTitle = null pty.managementTitleAt = null + pty.waitBlockedAt = null + pty.tailWaitState = undefined } for (const leaf of this.getLeavesForPty(ptyId)) { leaf.lastOscTitle = null leaf.lastOscTitleAt = null leaf.lastAgentStatus = null leaf.lastAgentStatusObservedLive = false + leaf.waitBlockedAt = null + leaf.tailWaitState = undefined } + this.primeWaitBlockedBaselineFromSeededTail(ptyId) this.clearAgentRowSnapshotsForPty(ptyId) } @@ -11791,6 +11887,10 @@ export class OrcaRuntimeService { } let retainedChanged = false for (const payload of chunk.payloads) { + this.recordAgentPromptLifecycleState( + ptyId, + mapExplicitAgentStateToRuntimeTerminalStatus(payload.state) + ) for (const target of targets.values()) { retainedChanged = this.retainAgentRowSnapshot( @@ -11876,6 +11976,64 @@ export class OrcaRuntimeService { return this.ptyOutputSequenceById.get(ptyId) ?? 0 } + private recordAgentPromptLifecycleState(ptyId: string, status: AgentStatus | null): void { + if (status === 'permission') { + this.recordAgentPromptPermissionObservation(ptyId) + } + const current = this.agentPromptLifecycleByPtyId.get(ptyId) + const updatedAt = Date.now() + if (!current) { + this.agentPromptLifecycleByPtyId.set(ptyId, { + status, + workingSequence: status === 'working' ? 1 : 0, + updatedAt + }) + return + } + this.agentPromptLifecycleByPtyId.set(ptyId, { + status, + workingSequence: + current.workingSequence + (status === 'working' && current.status !== 'working' ? 1 : 0), + updatedAt + }) + } + + private recordAgentPromptPermissionObservation(ptyId: string): void { + this.agentPromptPermissionSequenceByPtyId.set( + ptyId, + (this.agentPromptPermissionSequenceByPtyId.get(ptyId) ?? 0) + 1 + ) + } + + private restoreAgentPromptLifecycleByteOrder( + ptyId: string, + titleInput: string, + lastPayloadTitleOffset: number | null + ): void { + if (lastPayloadTitleOffset === null) { + return + } + const titleRange = findLastCompleteOscTitleRange(titleInput) + if (!titleRange || titleRange.end <= lastPayloadTitleOffset) { + return + } + const title = extractLastOscTitle(titleInput) + if (title === null) { + return + } + const status = detectAgentStatusFromTitle(title) + const current = this.agentPromptLifecycleByPtyId.get(ptyId) + if (!current || current.status === status) { + return + } + this.agentPromptLifecycleByPtyId.set(ptyId, { + status, + workingSequence: + current.workingSequence + (status === 'working' && current.status !== 'working' ? 1 : 0), + updatedAt: Date.now() + }) + } + private getPtyLifecycleGeneration(ptyId: string): number { const existing = this.ptyLifecycleGenerationById.get(ptyId) if (existing !== undefined) { @@ -11888,6 +12046,9 @@ export class OrcaRuntimeService { private advancePtyLifecycleGeneration(ptyId: string): void { this.ptyLifecycleGenerationById.set(ptyId, this.nextPtyLifecycleGeneration++) + this.agentPromptLifecycleByPtyId.delete(ptyId) + this.agentPromptPermissionSequenceByPtyId.delete(ptyId) + this.agentPromptExplicitStatusFloorByPtyId.set(ptyId, Date.now()) this.legacyWorkerRecoveredPtys.delete(ptyId) // Why: a respawn under the same session id needs its own subscriber-driven attach. this.subscriberDrivenProviderAttachesByPtyId.delete(ptyId) @@ -11940,6 +12101,7 @@ export class OrcaRuntimeService { if (pty) { pty.wslDistro = null } + // Why: raced post-spawn bytes may already contain the replacement's permission state. if (replacesExistingRuntimeGeneration && postSpawnSequence === 0) { this.resetTrackedTerminalStateForProviderGeneration(ptyId) } @@ -14819,6 +14981,7 @@ export class OrcaRuntimeService { this.providerBufferAcquisitionsByPtyId.delete(ptyId) this.providerVisibleStateByPtyId.delete(ptyId) this.providerVisibleRetryAtByPtyId.delete(ptyId) + this.agentPromptExplicitStatusFloorByPtyId.delete(ptyId) this.agentStatusOscProcessorsByPtyId.delete(ptyId) this.terminalSpawnCommandsByPtyId.delete(ptyId) this.disposePtyTitleTracker(ptyId) @@ -17950,17 +18113,33 @@ export class OrcaRuntimeService { options: { beforeWrite?: (ptyId: string) => void | Promise suffixFailureError?: string + signal?: AbortSignal } = {} ): Promise { const payload = buildAgentPromptPasteBytes(prompt) - const bytesWritten = Buffer.byteLength(`${payload}${AGENT_PROMPT_SUBMIT}`, 'utf8') const pty = this.getLivePtyForHandle(handle) if (pty) { if (!pty.pty.connected) { throw new Error('terminal_not_writable') } await assertTerminalInputWithinLimitWithYield(payload) - await this.writeTerminalAgentPrompt(pty.pty.ptyId, payload, options) + const generation = this.getPtyLifecycleGeneration(pty.pty.ptyId) + const submits = await this.serializeAgentPromptSubmission( + pty.pty.ptyId, + generation, + async () => { + this.assertLiveTerminalHandleTargetsPty(handle, pty.pty.ptyId) + this.assertAgentPromptGeneration(pty.pty.ptyId, generation) + return await this.writeTerminalAgentPrompt( + handle, + pty.pty.ptyId, + generation, + payload, + options + ) + } + ) + const bytesWritten = Buffer.byteLength(payload, 'utf8') + submits return { handle, accepted: true, bytesWritten } } @@ -17974,7 +18153,13 @@ export class OrcaRuntimeService { if (await this.isLeafPtyProvenAbsent(leaf.ptyId)) { throw new Error('terminal_not_writable') } - await this.writeTerminalAgentPrompt(leaf.ptyId, payload, options) + const generation = this.getPtyLifecycleGeneration(leaf.ptyId) + const submits = await this.serializeAgentPromptSubmission(leaf.ptyId, generation, async () => { + this.assertLiveTerminalHandleTargetsPty(handle, leaf.ptyId!) + this.assertAgentPromptGeneration(leaf.ptyId!, generation) + return await this.writeTerminalAgentPrompt(handle, leaf.ptyId!, generation, payload, options) + }) + const bytesWritten = Buffer.byteLength(payload, 'utf8') + submits return { handle, accepted: true, bytesWritten } } @@ -17982,21 +18167,10 @@ export class OrcaRuntimeService { const ptyId = this.getTerminalAgentStatusPtyId(handle) const terminal = this.getTerminalAgentStatusSnapshot(handle, ptyId) const explicitStatus = this.getFreshExplicitAgentStatusForHandle(handle) - const blockedByWaitText = detectTerminalWaitBlockedReason(terminal.waitText) - const liveTitleClearsBlockedText = - terminal.titleStatusIsLive && - terminal.titleStatus !== null && - terminal.titleStatus !== 'permission' && - !isOpenCodeNativeTitle(terminal.title) - if (terminal.titleStatus === 'permission' && terminal.titleStatusIsLive) { - return { handle, isRunningAgent: true, status: 'permission' } - } + const lifecycle = this.agentPromptLifecycleByPtyId.get(ptyId) if ( - blockedByWaitText && - !liveTitleClearsBlockedText && - (!explicitStatus || - explicitStatus.status === 'permission' || - (terminal.waitBlockedAt !== null && terminal.waitBlockedAt >= explicitStatus.updatedAt)) + (terminal.titleStatus === 'permission' && terminal.titleStatusIsLive) || + this.hasAuthoritativeTerminalWaitPermission(terminal, explicitStatus, lifecycle) ) { return { handle, isRunningAgent: true, status: 'permission' } } @@ -18066,13 +18240,7 @@ export class OrcaRuntimeService { private getTerminalAgentStatusSnapshot( handle: string, expectedPtyId: string - ): { - waitText: string - waitBlockedAt: number | null - title: string | null - titleStatus: AgentStatus | null - titleStatusIsLive: boolean - } { + ): TerminalAgentStatusSnapshot { const pty = this.getLivePtyForHandle(handle) if (pty) { if (!pty.pty.connected || pty.pty.ptyId !== expectedPtyId) { @@ -18131,6 +18299,36 @@ export class OrcaRuntimeService { } } + private hasAuthoritativeTerminalWaitPermission( + terminal: TerminalAgentStatusSnapshot, + explicitStatus: { status: AgentStatus; updatedAt: number } | null, + lifecycle: { status: AgentStatus | null; updatedAt: number } | null | undefined + ): boolean { + const blockedByWaitText = detectTerminalWaitBlockedReason(terminal.waitText) + if (!blockedByWaitText) { + return false + } + const liveTitleClearsBlockedText = + terminal.titleStatusIsLive && + terminal.titleStatus !== null && + terminal.titleStatus !== 'permission' && + !isOpenCodeNativeTitle(terminal.title) + if (liveTitleClearsBlockedText && lifecycle?.status !== terminal.titleStatus) { + return false + } + const newestPermissionAt = Math.max( + explicitStatus?.status === 'permission' ? explicitStatus.updatedAt : -1, + lifecycle?.status === 'permission' ? lifecycle.updatedAt : -1, + terminal.waitBlockedAt ?? -1 + ) + const newestClearAt = Math.max( + explicitStatus && explicitStatus.status !== 'permission' ? explicitStatus.updatedAt : -1, + lifecycle?.status && lifecycle.status !== 'permission' ? lifecycle.updatedAt : -1 + ) + // Equal wall-clock observations fail closed because their raw intra-chunk order is unknown. + return newestPermissionAt >= 0 && newestPermissionAt >= newestClearAt + } + private async terminalHasShellForegroundProcess(handle: string, ptyId: string): Promise { if (!this.ptyController) { return false @@ -18530,13 +18728,20 @@ export class OrcaRuntimeService { } private async writeTerminalAgentPrompt( + handle: string, ptyId: string, + generation: number, pastePayload: string, options: { beforeWrite?: (ptyId: string) => void | Promise suffixFailureError?: string + signal?: AbortSignal } = {} - ): Promise { + ): Promise { + assertAgentPromptRequestActive(options.signal) + this.assertAgentPromptGeneration(ptyId, generation) + const permissionBaseline = this.getAgentPromptActivity(handle, ptyId) + this.assertAgentPromptPermissionSafe(permissionBaseline, permissionBaseline) const renderGate = this.createAgentPromptRenderGate(ptyId) let wrotePasteBytes = false let completedPaste = false @@ -18545,7 +18750,15 @@ export class OrcaRuntimeService { let chunk = chunks.next() while (!chunk.done) { const nextChunk = chunks.next() + assertAgentPromptRequestActive(options.signal) + this.assertAgentPromptGeneration(ptyId, generation) await options.beforeWrite?.(ptyId) + assertAgentPromptRequestActive(options.signal) + this.assertAgentPromptGeneration(ptyId, generation) + this.assertAgentPromptPermissionSafe( + permissionBaseline, + this.getAgentPromptActivity(handle, ptyId) + ) if (nextChunk.done) { renderGate?.arm() } @@ -18561,7 +18774,11 @@ export class OrcaRuntimeService { } completedPaste = true } catch (error) { - if (wrotePasteBytes && !completedPaste) { + if ( + wrotePasteBytes && + !completedPaste && + this.getPtyLifecycleGeneration(ptyId) === generation + ) { this.ptyController?.write(ptyId, AGENT_PROMPT_BRACKETED_PASTE_END) } renderGate?.dispose() @@ -18569,11 +18786,16 @@ export class OrcaRuntimeService { } if (renderGate) { - await renderGate.wait() - renderGate.dispose() + try { + await waitForAgentPromptPromise(renderGate.wait(), options.signal) + } finally { + renderGate.dispose() + } } else { - await new Promise((resolve) => setTimeout(resolve, AGENT_PROMPT_SUBMIT_DELAY_MS)) + await waitForAgentPromptDelay(AGENT_PROMPT_SUBMIT_DELAY_MS, options.signal) } + assertAgentPromptRequestActive(options.signal) + this.assertAgentPromptGeneration(ptyId, generation) try { await options.beforeWrite?.(ptyId) } catch (error) { @@ -18582,10 +18804,93 @@ export class OrcaRuntimeService { } throw error } + assertAgentPromptRequestActive(options.signal) + this.assertAgentPromptGeneration(ptyId, generation) + const baseline = this.getAgentPromptActivity(handle, ptyId) + this.assertAgentPromptPermissionSafe(permissionBaseline, baseline) const suffixWrote = this.ptyController?.write(ptyId, AGENT_PROMPT_SUBMIT) ?? false if (!suffixWrote) { throw new Error(options.suffixFailureError ?? 'terminal_not_writable') } + await verifyAgentPromptSubmission({ + baseline, + readActivity: () => this.getAgentPromptActivity(handle, ptyId), + signal: options.signal + }) + return 1 + } + + private async serializeAgentPromptSubmission( + ptyId: string, + generation: number, + submit: () => Promise + ): Promise { + const queueKey = `${ptyId}\u0000${generation}` + const previous = this.agentPromptSubmissionTailByPtyId.get(queueKey) ?? Promise.resolve() + const submission = previous.catch(() => undefined).then(submit) + const tail = submission.then( + () => undefined, + () => undefined + ) + this.agentPromptSubmissionTailByPtyId.set(queueKey, tail) + try { + return await submission + } finally { + if (this.agentPromptSubmissionTailByPtyId.get(queueKey) === tail) { + this.agentPromptSubmissionTailByPtyId.delete(queueKey) + } + } + } + + private getAgentPromptActivity(handle: string, ptyId: string): AgentPromptActivity { + this.assertLiveTerminalHandleTargetsPty(handle, ptyId) + const explicitCandidate = this.getFreshExplicitAgentStatusForHandle(handle) + const explicitFloor = this.agentPromptExplicitStatusFloorByPtyId.get(ptyId) + const explicit = + explicitCandidate && + (explicitFloor === undefined || explicitCandidate.updatedAt > explicitFloor) + ? explicitCandidate + : null + const lifecycle = this.agentPromptLifecycleByPtyId.get(ptyId) + const ptyStatus = + lifecycle || explicitFloor === undefined + ? (this.ptysById.get(ptyId)?.lastAgentStatus ?? null) + : null + const lifecycleIsNewer = + lifecycle && + (!explicit || + lifecycle.updatedAt > explicit.updatedAt || + (lifecycle.updatedAt === explicit.updatedAt && lifecycle.status === 'permission')) + const terminal = this.getTerminalAgentStatusSnapshot(handle, ptyId) + const status = this.hasAuthoritativeTerminalWaitPermission(terminal, explicit, lifecycle) + ? 'permission' + : lifecycleIsNewer + ? lifecycle.status + : (explicit?.status ?? ptyStatus ?? null) + return { + generation: this.getPtyLifecycleGeneration(ptyId), + permissionSequence: this.agentPromptPermissionSequenceByPtyId.get(ptyId) ?? 0, + workingSequence: lifecycle?.workingSequence ?? 0, + status + } + } + + private assertAgentPromptPermissionSafe( + baseline: AgentPromptActivity, + current: AgentPromptActivity + ): void { + if ( + current.status === 'permission' || + current.permissionSequence > baseline.permissionSequence + ) { + throw new Error('agent_prompt_blocked') + } + } + + private assertAgentPromptGeneration(ptyId: string, expected: number): void { + if (this.getPtyLifecycleGeneration(ptyId) !== expected) { + throw new Error('terminal_handle_stale') + } } private createAgentPromptRenderGate(ptyId: string): { @@ -37277,7 +37582,7 @@ export class OrcaRuntimeService { const WAIT_BLOCKED_CHECK_MIN_INTERVAL_MS = 50 // Why: chunks that could complete an actionable prompt bypass the throttle so blocked stamps stay immediate; scanned over the new chunk + short carry, never the whole window. const WAIT_BLOCKED_KEYWORD_PATTERN = - /press enter|press t to trust|do you trust|trust this|trusted workspace|update available|choose working directory|codex just got an upgrade|hooks need review/ + /press enter|press t to trust|do you trust|trust this|trusted workspace|permission required|requires permission|allow once|allow always|update available|choose working directory|codex just got an upgrade|hooks need review/ const WAIT_BLOCKED_KEYWORD_CARRY_CHARS = 31 const MAX_TAIL_LINES = 2000 const MAX_TAIL_CHARS = 256 * 1024 @@ -39360,6 +39665,45 @@ function isTerminalSendSettlementAgent( return agent === 'claude' || agent === 'codex' } +function findLastCompleteOscTitleRange(data: string): { start: number; end: number } | null { + // Why: one forward cursor keeps hostile unterminated OSC output linear-time. + let last: { start: number; end: number } | null = null + let searchFrom = 0 + while (searchFrom < data.length) { + const start = data.indexOf('\x1b]', searchFrom) + if (start === -1) { + break + } + const command = data[start + 2] + if ((command !== '0' && command !== '1' && command !== '2') || data[start + 3] !== ';') { + searchFrom = start + 2 + continue + } + let cursor = start + 4 + for (; cursor < data.length; cursor += 1) { + if (data[cursor] === '\x07') { + last = { start, end: cursor + 1 } + searchFrom = cursor + 1 + break + } + if (data[cursor] !== '\x1b') { + continue + } + if (data[cursor + 1] === '\\') { + last = { start, end: cursor + 2 } + searchFrom = cursor + 2 + } else { + searchFrom = cursor + } + break + } + if (cursor === data.length) { + break + } + } + return last +} + function terminalTitleBlocksExplicitAgentStatus(title: string | null): boolean { if (!title) { return false diff --git a/src/main/runtime/rpc/errors.test.ts b/src/main/runtime/rpc/errors.test.ts index 9d0888846d1..190a5b88be5 100644 --- a/src/main/runtime/rpc/errors.test.ts +++ b/src/main/runtime/rpc/errors.test.ts @@ -45,6 +45,16 @@ describe('mapRuntimeError', () => { } ) + it.each(['agent_prompt_blocked', 'agent_prompt_stalled', 'request_aborted'])( + 'preserves the agent prompt failure %s', + (code) => { + expect(mapRuntimeError('req_1', { runtimeId: 'runtime-1' }, new Error(code))).toMatchObject({ + ok: false, + error: { code, message: code } + }) + } + ) + it.each([ 'remote_update_manual_required', 'remote_update_not_available', diff --git a/src/main/runtime/rpc/errors.ts b/src/main/runtime/rpc/errors.ts index 532b28ffa34..5d07ace973d 100644 --- a/src/main/runtime/rpc/errors.ts +++ b/src/main/runtime/rpc/errors.ts @@ -61,10 +61,13 @@ const RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([ 'terminal_tab_close_timeout', 'terminal_tab_not_found', 'terminal_tab_pinned', + 'agent_prompt_blocked', + 'agent_prompt_stalled', 'no_active_terminal', 'repo_not_found', 'timeout', 'invalid_limit', + 'request_aborted', 'remote_update_manual_required', 'remote_update_not_available', 'remote_update_not_downloaded', diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index 4f1925683d2..1b88b4afc73 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -1226,7 +1226,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ defineMethod({ name: 'terminal.send', params: TerminalSend, - handler: async (params, { runtime, clientId }) => { + handler: async (params, { runtime, clientId, signal }) => { await assertTerminalSendTextWithinLimit(params.text) await assertTerminalSendTextWithinLimit(params.resolvedLaunchDraft?.text) const queryReplyClientId = clientId ?? params.client?.id @@ -1374,7 +1374,10 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ let result try { result = useSettledAgentPrompt - ? await runtime.sendTerminalAgentPrompt(params.terminal, params.text!, { beforeWrite }) + ? await runtime.sendTerminalAgentPrompt(params.terminal, params.text!, { + beforeWrite, + signal + }) : await runtime.sendTerminal( params.terminal, { diff --git a/src/main/runtime/rpc/terminal-agent-prompt-send.test.ts b/src/main/runtime/rpc/terminal-agent-prompt-send.test.ts index a5d71278bb4..eb7624c95e4 100644 --- a/src/main/runtime/rpc/terminal-agent-prompt-send.test.ts +++ b/src/main/runtime/rpc/terminal-agent-prompt-send.test.ts @@ -45,7 +45,8 @@ describe('terminal agent prompt send RPC', () => { expect(response.ok).toBe(true) expect(runtime.isTerminalRunningSettledPromptAgent).toHaveBeenCalledWith('terminal-1') expect(sendTerminalAgentPrompt).toHaveBeenCalledWith('terminal-1', 'review this change', { - beforeWrite: undefined + beforeWrite: undefined, + signal: undefined }) expect(sendTerminal).not.toHaveBeenCalled() }) diff --git a/src/shared/agent-status-osc.test.ts b/src/shared/agent-status-osc.test.ts index ac14c495786..4a0711cc426 100644 --- a/src/shared/agent-status-osc.test.ts +++ b/src/shared/agent-status-osc.test.ts @@ -17,6 +17,7 @@ describe('createAgentStatusOscProcessor', () => { agentType: 'codex' } ]) + expect(result.lastPayloadCleanOffset).toBe('before'.length) }) it('preserves parser state across split OSC 9999 chunks', () => { @@ -32,5 +33,6 @@ describe('createAgentStatusOscProcessor', () => { prompt: 'ok' } ]) + expect(result.lastPayloadCleanOffset).toBe(0) }) }) diff --git a/src/shared/agent-status-osc.ts b/src/shared/agent-status-osc.ts index 95908805907..39f9703bf85 100644 --- a/src/shared/agent-status-osc.ts +++ b/src/shared/agent-status-osc.ts @@ -6,6 +6,7 @@ const OSC_AGENT_STATUS_PREFIX = '\x1b]9999;' export type ProcessedAgentStatusChunk = { cleanData: string payloads: ParsedAgentStatusPayload[] + lastPayloadCleanOffset: number | null } function findAgentStatusTerminator( @@ -40,6 +41,7 @@ export function createAgentStatusOscProcessor(): (data: string) => ProcessedAgen pending = '' const payloads: ParsedAgentStatusPayload[] = [] + let lastPayloadCleanOffset: number | null = null let cleanData = '' let cursor = 0 @@ -77,10 +79,11 @@ export function createAgentStatusOscProcessor(): (data: string) => ProcessedAgen const parsed = parseAgentStatusPayload(combined.slice(payloadStart, terminator.index)) if (parsed) { payloads.push(parsed) + lastPayloadCleanOffset = cleanData.length } cursor = terminator.index + terminator.length } - return { cleanData, payloads } + return { cleanData, payloads, lastPayloadCleanOffset } } } diff --git a/tests/e2e/terminal-send-agent-prompt-submit.spec.ts b/tests/e2e/terminal-send-agent-prompt-submit.spec.ts index dcff9d73352..1ae525d29da 100644 --- a/tests/e2e/terminal-send-agent-prompt-submit.spec.ts +++ b/tests/e2e/terminal-send-agent-prompt-submit.spec.ts @@ -18,8 +18,8 @@ mkdirSync(fixtureBin) writeFileSync( fakeCodex, process.platform === 'win32' - ? `@echo off\r\n"${process.execPath}" "${fixtureScript}" --fake-agent --report "%ORCA_FAKE_AGENT_REPORT%" --marker "%ORCA_FAKE_AGENT_MARKER%" --allow-unframed-paste\r\n` - : `#!/usr/bin/env sh\nexec "${process.execPath}" "${fixtureScript}" --fake-agent --report "$ORCA_FAKE_AGENT_REPORT" --marker "$ORCA_FAKE_AGENT_MARKER"\n`, + ? `@echo off\r\n"${process.execPath}" "${fixtureScript}" --fake-agent --report "%ORCA_FAKE_AGENT_REPORT%" --marker "%ORCA_FAKE_AGENT_MARKER%" --allow-unframed-paste %*\r\n` + : `#!/usr/bin/env sh\nexec "${process.execPath}" "${fixtureScript}" --fake-agent --report "$ORCA_FAKE_AGENT_REPORT" --marker "$ORCA_FAKE_AGENT_MARKER" "$@"\n`, 'utf8' ) if (process.platform !== 'win32') { @@ -27,6 +27,7 @@ if (process.platform !== 'win32') { } test.use({ + seedTestRepo: false, orcaAppExtraEnv: { PATH: `${fixtureBin}${path.delimiter}${process.env.PATH ?? ''}`, ORCA_FAKE_AGENT_REPORT: fixtureReport, @@ -87,3 +88,104 @@ test('CLI text plus Enter waits for a slow agent composer before submitting', as markerReceived: true }) }) + +test('CLI reports a swallowed Enter without submitting a second Enter', async ({ + electronApp, + orcaPage, + testRepoPath +}) => { + test.setTimeout(90_000) + await waitForSessionReady(orcaPage) + const userDataDir = await electronApp.evaluate(({ app }) => app.getPath('userData')) + const repoRoot = process.cwd() + let stdout = '' + try { + const result = await execFileAsync( + process.execPath, + [ + path.join(repoRoot, 'tests', 'tools', 'repro-terminal-send-submit.mjs'), + '--cli', + path.join(repoRoot, 'config', 'scripts', 'orca-dev.mjs'), + '--worktree', + testRepoPath, + '--agent-command', + 'codex --swallow-first-enter', + '--expect-stalled', + '--report', + fixtureReport, + '--marker', + fixtureMarker, + '--discard-report' + ], + { + cwd: repoRoot, + env: { ...process.env, ORCA_DEV_USER_DATA_PATH: userDataDir }, + timeout: 60_000 + } + ) + stdout = result.stdout + } catch (error) { + const failed = error as Error & { stdout?: string; stderr?: string } + throw new Error([failed.message, failed.stdout, failed.stderr].filter(Boolean).join('\n')) + } + + expect(JSON.parse(stdout)).toMatchObject({ + rescueSent: false, + sendErrorCode: 'agent_prompt_stalled', + contractOk: true, + submitted: false, + prematureEnters: 0, + receivedEnters: 1, + swallowedEnters: 1, + markerReceived: true + }) +}) + +test('CLI does not write prompt bytes into an active permission dialog', async ({ + electronApp, + orcaPage, + testRepoPath +}) => { + test.setTimeout(90_000) + await waitForSessionReady(orcaPage) + const userDataDir = await electronApp.evaluate(({ app }) => app.getPath('userData')) + const repoRoot = process.cwd() + let stdout = '' + try { + const result = await execFileAsync( + process.execPath, + [ + path.join(repoRoot, 'tests', 'tools', 'repro-terminal-send-submit.mjs'), + '--cli', + path.join(repoRoot, 'config', 'scripts', 'orca-dev.mjs'), + '--worktree', + testRepoPath, + '--expect-blocked', + '--report', + fixtureReport, + '--marker', + fixtureMarker, + '--discard-report' + ], + { + cwd: repoRoot, + env: { ...process.env, ORCA_DEV_USER_DATA_PATH: userDataDir }, + timeout: 60_000 + } + ) + stdout = result.stdout + } catch (error) { + const failed = error as Error & { stdout?: string; stderr?: string } + throw new Error([failed.message, failed.stdout, failed.stderr].filter(Boolean).join('\n')) + } + + expect(JSON.parse(stdout)).toMatchObject({ + rescueSent: false, + sendErrorCode: 'agent_prompt_blocked', + contractOk: true, + submitted: false, + receivedBytes: 0, + receivedEnters: 0, + markerReceived: false + }) +}) diff --git a/tests/tools/repro-terminal-send-submit.mjs b/tests/tools/repro-terminal-send-submit.mjs index bed37d9add4..201e51c9364 100644 --- a/tests/tools/repro-terminal-send-submit.mjs +++ b/tests/tools/repro-terminal-send-submit.mjs @@ -66,7 +66,7 @@ function runCommand(command, args, options = {}) { reject( Object.assign( new Error(`${command} ${args.join(' ')} exited ${code}${detail ? `: ${detail}` : ''}`), - { stderr } + { stdout, stderr } ) ) }) @@ -76,10 +76,24 @@ function runCommand(command, args, options = {}) { async function callOrca(cli, args, cwd) { const command = cli.endsWith('.mjs') ? process.execPath : cli const prefixArgs = cli.endsWith('.mjs') ? [cli] : [] - const { stdout } = await runCommand(command, [...prefixArgs, ...args, '--json'], { cwd }) + let stdout + try { + const result = await runCommand(command, [...prefixArgs, ...args, '--json'], { cwd }) + stdout = result.stdout + } catch (error) { + const parsed = JSON.parse(error?.stdout?.trim() ?? 'null') + if (parsed?.ok === false) { + throw Object.assign(new Error(parsed.error?.message ?? JSON.stringify(parsed.error)), { + code: parsed.error?.code + }) + } + throw error + } const parsed = JSON.parse(stdout.trim()) if (parsed.ok === false) { - throw new Error(parsed.error?.message ?? JSON.stringify(parsed.error)) + throw Object.assign(new Error(parsed.error?.message ?? JSON.stringify(parsed.error)), { + code: parsed.error?.code + }) } return parsed.result ?? parsed } @@ -146,7 +160,10 @@ async function parentMain() { const reportPath = path.resolve(argValue('report', path.join(tempDir, 'report.json'))) const marker = argValue('marker', `ORCA_TERMINAL_SEND_${process.pid}_${Date.now()}`) const prompt = `${marker} ${'slow composer payload '.repeat(24)}` + const expectStalled = hasFlag('expect-stalled') + const expectBlocked = hasFlag('expect-blocked') await mkdir(tempDir, { recursive: true }) + await rm(reportPath, { force: true }) const command = argValue('agent-command') ?? @@ -158,6 +175,7 @@ async function parentMain() { shellQuote(marker), '--timeout-ms', String(timeoutMs), + ...(expectBlocked ? ['--permission-before-send'] : []), ...(process.platform === 'win32' ? ['--allow-unframed-paste'] : []) ])) const added = await callOrca(cli, ['repo', 'add', '--path', cwd], cwd) @@ -186,19 +204,37 @@ async function parentMain() { } try { - await callOrca( - cli, - ['terminal', 'wait', '--terminal', handle, '--for', 'tui-idle', '--timeout-ms', '10000'], - cwd - ) - await callOrca( - cli, - ['terminal', 'send', '--terminal', handle, '--text', prompt, '--enter'], - cwd - ) + if (expectBlocked) { + const setupReport = await readReport(reportPath, 10_000) + if (!setupReport) { + throw new Error('terminal permission prompt did not materialize') + } + } else { + await callOrca( + cli, + ['terminal', 'wait', '--terminal', handle, '--for', 'tui-idle', '--timeout-ms', '10000'], + cwd + ) + } + let sendErrorCode = null + try { + await callOrca( + cli, + ['terminal', 'send', '--terminal', handle, '--text', prompt, '--enter'], + cwd + ) + } catch (error) { + const expectedError = + (expectStalled && error?.code === 'agent_prompt_stalled') || + (expectBlocked && error?.code === 'agent_prompt_blocked') + if (!expectedError) { + throw error + } + sendErrorCode = error.code + } let report = await readReport(reportPath, 1_000) let rescueSent = false - if (!report) { + if (!report && !expectStalled && !expectBlocked) { rescueSent = true await callOrca(cli, ['terminal', 'send', '--terminal', handle, '--enter'], cwd) report = await readReport(reportPath, timeoutMs) @@ -210,10 +246,25 @@ async function parentMain() { handle, promptBytes: Buffer.byteLength(prompt, 'utf8'), rescueSent, + sendErrorCode, ...report } console.log(JSON.stringify(summary, null, 2)) - if (!report.contractOk || rescueSent) { + const expectedStallObserved = + sendErrorCode === 'agent_prompt_stalled' && + report.submitted === false && + report.receivedEnters === 1 && + report.swallowedEnters === 1 + const expectedBlockObserved = + sendErrorCode === 'agent_prompt_blocked' && + report.receivedBytes === 0 && + report.receivedEnters === 0 + if ( + !report.contractOk || + rescueSent || + (expectStalled && !expectedStallObserved) || + (expectBlocked && !expectedBlockObserved) + ) { process.exitCode = 1 } } finally { @@ -231,6 +282,8 @@ async function fakeAgentMain() { const marker = argValue('marker') const timeoutMs = parsePositiveInteger('timeout-ms', DEFAULT_TIMEOUT_MS) const pasteFramingRequired = !hasFlag('allow-unframed-paste') + const swallowFirstEnter = hasFlag('swallow-first-enter') + const permissionBeforeSend = hasFlag('permission-before-send') if (!reportPath || !marker) { throw new Error('--fake-agent requires --report and --marker') } @@ -239,49 +292,80 @@ async function fakeAgentMain() { } process.stdin.resume() process.stdout.write('OpenAI Codex\nmodel: fake\ndirectory: fixture\n> ') + if (permissionBeforeSend) { + process.stdout.write('\nPermission required\nAllow once\nAllow always\nReject\n') + } let input = '' let countedCarriages = 0 let prematureEnters = 0 + let receivedEnters = 0 + let swallowedEnters = 0 let composerReady = false let renderScheduled = false let finished = false - const finish = async () => { - if (finished) { - return - } - finished = true + const writeReport = async (submitted) => { const hasBracketedPasteFrame = input.includes(BEGIN) && input.includes(END) const report = { - contractOk: prematureEnters === 0 && (!pasteFramingRequired || hasBracketedPasteFrame), - submitted: true, + contractOk: + prematureEnters === 0 && + (permissionBeforeSend || !pasteFramingRequired || hasBracketedPasteFrame), + submitted, prematureEnters, + receivedEnters, + swallowedEnters, pasteFramingRequired, hasBracketedPasteFrame, markerReceived: input.includes(marker), receivedBytes: Buffer.byteLength(input, 'utf8') } await writeFile(reportPath, JSON.stringify(report, null, 2)) + return report + } + + const finish = async () => { + if (finished) { + return + } + finished = true + const report = await writeReport(true) process.stdout.write(`\nORCA_TERMINAL_SEND_REPORT ${report.contractOk ? 'ok' : 'rescued'}\n`) process.exit(report.contractOk ? 0 : 7) } const timeout = setTimeout(() => process.exit(8), timeoutMs) + if (permissionBeforeSend) { + setTimeout(() => void writeReport(false), 250) + } process.stdin.on('data', (chunk) => { input += chunk.toString('utf8') if (!renderScheduled && input.includes(marker)) { renderScheduled = true setTimeout(() => { composerReady = true - process.stdout.write('\x1b[?25hcomposer rendered') + const pasteStart = input.indexOf(BEGIN) + const pasteEnd = input.indexOf(END, pasteStart + BEGIN.length) + const composer = + pasteStart !== -1 && pasteEnd !== -1 + ? input.slice(pasteStart + BEGIN.length, pasteEnd) + : input + process.stdout.write(`\x1b[?25h\x1b[2J\x1b[H› ${composer}`) }, COMPOSER_RENDER_MS) } let nextCarriage = input.indexOf('\r', countedCarriages) while (nextCarriage !== -1) { countedCarriages = nextCarriage + 1 if (composerReady) { + receivedEnters += 1 + if (swallowFirstEnter && swallowedEnters === 0) { + swallowedEnters += 1 + void writeReport(false) + nextCarriage = input.indexOf('\r', countedCarriages) + continue + } clearTimeout(timeout) + process.stdout.write('\x1b]0;Codex working\x07') void finish() return } From 3bb87ff93ba1b6a2eab7644d72da789602c40585 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:51:53 -0700 Subject: [PATCH 06/10] reland(shell): one portable Unix startup dialect, with both revert causes fixed (#15018) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * reland: portable startup-shell dialect, with the two revert causes fixed Relands #14863 (reverted by #14975) with fixes for both regressions the revert cited. 1. History GC deleted folder-workspace shell history. The live set was built from `getAllWorktreeMeta()` alone, but a folder workspace's PTY carries `folder:` as its worktree id, so every live folder workspace looked orphaned. `getKnownWorktreeIdsForHistoryGc` now unions in `getFolderWorkspaces()`. Both consumers — the history-directory prune and the fish-history sweep — read that one set, so the fix covers bash, zsh and fish history alike. The directory prune had this gap since #1524; #14863 only widened its blast radius to fish files. 2. A copied Codex resume command aborted under `set -u`. Its leading clear statement has to test `$fish_pid`, and that unbound expansion takes the whole line — including the agent launch — down with it. Copied text runs in a shell Orca never spawned, so nothing can seed that variable first. The removal now rides on the agent itself as `env -u`, which needs no shell syntax and no expansion. Verified byte-identical under `set -u` in sh, bash, zsh, dash, ksh and fish. `env` cannot run the `cd` builtin, and a child `cd` would not move the agent, so the prefix is placed on the agent rather than on the whole `cd … && agent` chain. cmd and PowerShell have no nounset hazard and keep their clear ahead of the `cd`, which preserves `cd … && agent` — a failed `cd` still cannot launch the agent in the wrong directory. * fix(history-gc): stop three more paths from deleting live shell history Found by adversarial review of the reland. All three are the same class as the bug that caused the revert: a live set that is missing a category of real workspace, so the GC reads it as orphaned. 1. Profiles. The history root is `userData/terminal-history`, which has no profile segment, but the Store the GC consults is per-profile. So after a profile switch the live set condemned every other profile's history — and fish history, which lands in the user's own fish data dir, is shared by every profile on the machine. The live set now unions in the inactive profiles' worktrees and folder workspaces, read from their data files. A profile whose ids cannot be read reports the empty set rather than one that condemns real history. 2. No empty-set guard on the tree scan. `sweepOrphanedFishHistoryFiles` refuses an empty live set because it cannot be told apart from a store that failed to hydrate; the directory scan, which deletes more, had no such guard. A store that fell back to default state would have taken every worktree's bash and zsh history with it, across all roots including WSL. Four existing tests passed `new Set()` and relied on "empty means everything is orphaned" — exactly the behavior being removed — so they now pass a real live set. 3. Relay fish history. The relay isolates its history tree under its own root but wrote fish history into the shared fish data dir under the desktop naming, keyed by the CLIENT's worktree ids. On a machine running both Orca and a relay host, the desktop sweep deleted remote sessions' history once it went stale. Relay files are now `orca_relay_`, which the sweep's pattern deliberately does not match; the relay still deletes them by exact name when the worktree goes away. * fix(resume): enforce the env-removal invariants instead of documenting them Both found by adversarial review; both were unreachable from today's callers and silent if reached, which is exactly how they would survive to a caller that does reach them. - A pinned CODEX_HOME and the removal named the same variable, and `env -u` strips what the assignment just set — so the agent would have resumed against the real home and not found the session. The removal list now excludes any name the prefix pins, keeping the assignment authoritative as the old `clear…; CODEX_HOME=x agent` ordering did. Same fix in the git-bash twin. The PowerShell branch already clears before it assigns, so it was never affected. - Placement was keyed on the platform while the grammar it selects is keyed on the shell, so `platform: 'linux'` with `shell: 'powershell'` emitted POSIX `env -u` into a PowerShell line. PowerShell now routes to the PowerShell builder whatever the host, and the POSIX/cmd split below asks the shell rather than the platform. --- .github/workflows/pr.yml | 6 + .../scripts/pr-workflow-parallelism.test.mjs | 1 + src/main/daemon/daemon-pty-adapter.ts | 48 ++- .../daemon/daemon-zsh-shell-ready-rcfile.ts | 7 +- src/main/daemon/shell-ready.ts | 4 +- src/main/fish-history-session.test.ts | 238 ++++++++++++ src/main/fish-history-session.ts | 139 +++++++ src/main/ipc/pty.ts | 6 +- src/main/ipc/worktrees.ts | 28 +- src/main/providers/local-pty-provider.ts | 19 +- ...ocal-pty-shell-ready-wrapper-generation.ts | 5 +- src/main/providers/pty-provider-contract.ts | 4 + src/main/providers/ssh-pty-provider.test.ts | 10 +- src/main/providers/ssh-pty-provider.ts | 49 +-- .../providers/ssh-pty-session-reattach.ts | 36 ++ src/main/providers/ssh-pty-spawn-request.ts | 4 + .../remote-worktree-history-cleanup.test.ts | 30 ++ src/main/remote-worktree-history-cleanup.ts | 15 + src/main/runtime/orca-runtime.test.ts | 149 +++++++- src/main/runtime/orca-runtime.ts | 17 + src/main/shell-templates.ts | 35 +- src/main/terminal-history-deletion.ts | 77 +++- ...inal-history-fish-session.node-pty.test.ts | 145 +++++++ src/main/terminal-history-gc.ts | 65 +++- src/main/terminal-history-id.ts | 6 + src/main/terminal-history-paths.ts | 6 +- .../terminal-history-tombstone-retry.test.ts | 137 ++++++- src/main/terminal-history.test.ts | 353 +++++++++++++++++- src/main/terminal-history.ts | 201 ++++++++-- .../history-gc-profile-worktree-ids.test.ts | 117 ++++++ .../window/history-gc-profile-worktree-ids.ts | 80 ++++ .../window/history-gc-worktree-ids.test.ts | 76 +++- src/main/window/history-gc-worktree-ids.ts | 36 +- src/main/wsl-fish-history-cleanup.test.ts | 89 +++++ src/main/wsl-fish-history-cleanup.ts | 81 ++++ .../zsh-scoped-histfile.live-shell.test.ts | 106 ++++++ src/relay/pty-handler.ts | 49 ++- src/relay/pty-shell-launch.ts | 201 +--------- src/relay/pty-shell-overlay-wrappers.ts | 207 ++++++++++ src/relay/terminal-history.test.ts | 55 +++ src/relay/terminal-history.ts | 134 +++++++ .../settings/CliSkillRuntimeSetup.tsx | 5 +- .../folder-workspace-composer-submit.test.ts | 2 +- ...nnection-cold-restore-agent-resume.test.ts | 3 +- .../src/lib/ai-vault-resume-command.test.ts | 4 +- .../src/lib/ai-vault-resume-command.ts | 59 ++- .../src/lib/ai-vault-resume-shell.test.ts | 78 ++-- src/renderer/src/lib/ai-vault-resume-shell.ts | 17 +- .../launch-agent-background-session.test.ts | 2 +- ...h-agent-in-new-tab-windows-quoting.test.ts | 6 +- .../src/lib/launch-work-item-direct.test.ts | 8 +- .../lib/resume-sleeping-agent-session.test.ts | 2 +- .../src/lib/tui-agent-startup.test.ts | 2 +- src/shared/agent-resume-launch-command.ts | 7 +- src/shared/ai-vault-resume-command.test.ts | 77 +++- src/shared/ai-vault-resume-command.ts | 100 ++++- src/shared/global-settings-types.ts | 3 +- src/shared/hermes-startup-query.ts | 11 +- ...artup-shell-portability.live-shell.test.ts | 266 +++++++++++++ .../tui-agent-startup-session-options.test.ts | 2 +- src/shared/tui-agent-startup-shell.test.ts | 92 +++-- src/shared/tui-agent-startup-shell.ts | 205 ++++++++-- src/shared/tui-agent-startup.test.ts | 33 +- 63 files changed, 3506 insertions(+), 549 deletions(-) create mode 100644 src/main/fish-history-session.test.ts create mode 100644 src/main/fish-history-session.ts create mode 100644 src/main/remote-worktree-history-cleanup.test.ts create mode 100644 src/main/remote-worktree-history-cleanup.ts create mode 100644 src/main/terminal-history-fish-session.node-pty.test.ts create mode 100644 src/main/terminal-history-id.ts create mode 100644 src/main/window/history-gc-profile-worktree-ids.test.ts create mode 100644 src/main/window/history-gc-profile-worktree-ids.ts create mode 100644 src/main/wsl-fish-history-cleanup.test.ts create mode 100644 src/main/wsl-fish-history-cleanup.ts create mode 100644 src/main/zsh-scoped-histfile.live-shell.test.ts create mode 100644 src/relay/pty-shell-overlay-wrappers.ts create mode 100644 src/relay/terminal-history.test.ts create mode 100644 src/relay/terminal-history.ts create mode 100644 src/shared/startup-shell-portability.live-shell.test.ts diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6826ecfb731..c44b2abaa56 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -270,7 +270,10 @@ jobs: src/main/providers/local-pty-shell-ready-zsh-zdotdir-normalization.test.ts \ src/main/providers/__tests__/shell-ready-framework-example.test.ts \ src/main/pty/omp-shell-wrapper.node-pty.test.ts \ + src/main/terminal-history-fish-session.node-pty.test.ts \ + src/main/zsh-scoped-histfile.live-shell.test.ts \ src/renderer/src/components/terminal-pane/fish-color-scheme-child-stdin.node-pty.test.ts \ + src/shared/startup-shell-portability.live-shell.test.ts \ src/shared/posix-command-path-lookup.test.ts test: @@ -309,7 +312,10 @@ jobs: --exclude=src/main/providers/local-pty-shell-ready-zsh-zdotdir-normalization.test.ts \ --exclude=src/main/providers/__tests__/shell-ready-framework-example.test.ts \ --exclude=src/main/pty/omp-shell-wrapper.node-pty.test.ts \ + --exclude=src/main/terminal-history-fish-session.node-pty.test.ts \ + --exclude=src/main/zsh-scoped-histfile.live-shell.test.ts \ --exclude=src/renderer/src/components/terminal-pane/fish-color-scheme-child-stdin.node-pty.test.ts \ + --exclude=src/shared/startup-shell-portability.live-shell.test.ts \ --exclude=src/shared/posix-command-path-lookup.test.ts \ --exclude=tests/e2e/cross-version-wire/** \ --shard=${{ matrix.shard }}/${{ matrix.shard_total }} diff --git a/config/scripts/pr-workflow-parallelism.test.mjs b/config/scripts/pr-workflow-parallelism.test.mjs index e934d1d836d..536059fe4cc 100644 --- a/config/scripts/pr-workflow-parallelism.test.mjs +++ b/config/scripts/pr-workflow-parallelism.test.mjs @@ -15,6 +15,7 @@ const shellContractFiles = [ 'src/main/providers/local-pty-shell-ready-zsh-zdotdir-discovery.test.ts', 'src/main/providers/local-pty-shell-ready-zsh-zdotdir-normalization.test.ts', 'src/main/providers/__tests__/shell-ready-framework-example.test.ts', + 'src/main/zsh-scoped-histfile.live-shell.test.ts', 'src/shared/posix-command-path-lookup.test.ts' ] const patchedNodePtyContractFiles = [ diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index aad29987053..7d2063e2a7d 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -66,6 +66,9 @@ import { recognizeAgentProcessFromCommandLine } from '../../shared/agent-process import { shouldUseShellReadyStartupDelivery } from '../../shared/codex-startup-delivery' import type { PtyIncarnationId } from '../../shared/pty-incarnation' import { resolveSafePtyDefaultCwd } from '../providers/pty-default-cwd' +import { resolveUnixShellPath } from '../providers/local-pty-utils' +import { injectHistoryEnv, injectWslFishHistoryEnv, logHistoryInjection } from '../terminal-history' +import { addWslEnvKeys } from '../wsl-env' import { PtyWriteUnavailableError } from '../providers/pty-write-unavailable-error' import { ColdRestorePayloadCache, type ColdRestorePayload } from './cold-restore-payload-cache' import { CheckpointSessionQueue } from './daemon-checkpoint-session-queue' @@ -425,7 +428,8 @@ export class DaemonPtyAdapter implements IPtyProvider { } async spawn(opts: PtySpawnOptions): Promise { - const sessionId = opts.sessionId ?? mintPtySessionId(opts.worktreeId) + const spawnOpts = this.withHistoryIsolation(opts) + const sessionId = spawnOpts.sessionId ?? mintPtySessionId(spawnOpts.worktreeId) const operation = { exitsBySessionId: new Map(), ignoredExitIncarnationIds: new Set(), @@ -444,7 +448,9 @@ export class DaemonPtyAdapter implements IPtyProvider { } try { return await this.withHistorySpawnLock(sessionId, () => - this.withDaemonRetry(() => this.doSpawn({ ...opts, sessionId }, operation, historyRecovery)) + this.withDaemonRetry(() => + this.doSpawn({ ...spawnOpts, sessionId }, operation, historyRecovery) + ) ) } finally { if (historyRecovery.freeze) { @@ -458,6 +464,44 @@ export class DaemonPtyAdapter implements IPtyProvider { } } + private withHistoryIsolation(opts: PtySpawnOptions): PtySpawnOptions { + const wslContext = resolveWslSessionContext({ + cwd: opts.cwd, + sessionId: opts.sessionId, + shellOverride: opts.shellOverride, + terminalWindowsWslDistro: opts.terminalWindowsWslDistro + }) + if ( + opts.attachOnly === true || + (opts.sessionId !== undefined && opts.isNewSession !== true) || + !opts.worktreeId || + opts.historyIsolationEnabled !== true || + (process.platform === 'win32' && !wslContext) + ) { + return opts + } + const env = { ...opts.env } + const preferredShell = wslContext + ? 'bash' + : opts.shellOverride || env.SHELL || process.env.SHELL || '/bin/zsh' + const shellPath = resolveUnixShellPath(preferredShell) + const historyArgs = [ + env, + opts.worktreeId, + shellPath, + opts.cwd ?? resolveSafePtyDefaultCwd() + ] as const + const result = wslContext + ? injectHistoryEnv(...historyArgs, { wslDistro: wslContext.distro }) + : injectHistoryEnv(...historyArgs) + if (wslContext) { + injectWslFishHistoryEnv(env, opts.worktreeId, wslContext.distro) + addWslEnvKeys(env, ['HISTFILE', 'fish_history']) + } + logHistoryInjection(opts.worktreeId, result) + return { ...opts, env } + } + private async doSpawn( opts: PtySpawnOptions, operation: PendingDaemonSpawnOperation, diff --git a/src/main/daemon/daemon-zsh-shell-ready-rcfile.ts b/src/main/daemon/daemon-zsh-shell-ready-rcfile.ts index 4e5d4242d81..1b436bb1ea5 100644 --- a/src/main/daemon/daemon-zsh-shell-ready-rcfile.ts +++ b/src/main/daemon/daemon-zsh-shell-ready-rcfile.ts @@ -1,6 +1,10 @@ import { getPosixOmpShellWrapper } from '../pty/omp-shell-wrapper' import { getPosixCodexShellLaunchPreflight } from '../pty/codex-shell-launch-preflight' -import { getZshFinalZdotdirRestoreBlock, getZshStartupFileSourceBlock } from '../shell-templates' +import { + getZshFinalZdotdirRestoreBlock, + getZshStartupFileSourceBlock, + ZSH_HISTFILE_RESTORE_BLOCK +} from '../shell-templates' export function getDaemonZshShellReadyRcfileContent(): string { return `# Orca daemon zsh shell-ready wrapper @@ -23,6 +27,7 @@ if [[ ! -o login ]]; then [[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" ${getPosixOmpShellWrapper()} [[ -n "\${ORCA_CODEX_HOME:-}" ]] && export CODEX_HOME="\${ORCA_CODEX_HOME}" +${ZSH_HISTFILE_RESTORE_BLOCK} ${getPosixCodexShellLaunchPreflight()} fi __orca_osc133_precmd() { diff --git a/src/main/daemon/shell-ready.ts b/src/main/daemon/shell-ready.ts index 7989401e4d0..0f9b2d65885 100644 --- a/src/main/daemon/shell-ready.ts +++ b/src/main/daemon/shell-ready.ts @@ -16,7 +16,8 @@ import { getZshEnvTemplate, getZshFinalZdotdirRestoreBlock, getZshShellReadyMarkerRegistrationBlock, - getZshStartupFileSourceBlock + getZshStartupFileSourceBlock, + ZSH_HISTFILE_RESTORE_BLOCK } from '../shell-templates' import { SHELL_READY_MARKER } from './daemon-shell-ready-marker' import { getDaemonBashShellReadyRcfileContent } from './daemon-bash-shell-ready-rcfile' @@ -121,6 +122,7 @@ __orca_restore_agent_teams_path [[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" ${getPosixOmpShellWrapper()} [[ -n "\${ORCA_CODEX_HOME:-}" ]] && export CODEX_HOME="\${ORCA_CODEX_HOME}" +${ZSH_HISTFILE_RESTORE_BLOCK} ${getPosixCodexShellLaunchPreflight()} ${getZshShellReadyMarkerRegistrationBlock(SHELL_READY_MARKER)} ${getZshFinalZdotdirRestoreBlock()} diff --git a/src/main/fish-history-session.test.ts b/src/main/fish-history-session.test.ts new file mode 100644 index 00000000000..060c1f2e57b --- /dev/null +++ b/src/main/fish-history-session.test.ts @@ -0,0 +1,238 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + deleteFishHistoryFile, + fishHistorySessionName, + isSafeFishHistorySession, + relayFishHistorySessionName, + resolveFishHistoryDir, + sweepOrphanedFishHistoryFiles +} from './fish-history-session' + +const HASH_A = 'a1b2c3d4e5f60718' +const HASH_B = '00112233445566ff' +const SESSION_A = fishHistorySessionName(HASH_A) + +describe('fish history session naming', () => { + it('mints a session name that is a valid fish variable value', () => { + // fish falls back to the shared default for anything that is not a valid + // variable name, which would silently un-isolate the worktree. + expect(SESSION_A).toBe('orca_a1b2c3d4e5f60718') + expect(SESSION_A).toMatch(/^[A-Za-z_][A-Za-z0-9_]*$/) + }) + + it.each([ + ['orca_deadbeef', true], + ['orca_', false], + ['fish_history', false], + ['orca_../../etc/passwd', false], + ['orca_DEADBEEF', false], + ['', false], + [undefined, false], + [42, false] + ])('accepts %s as a safe session name: %s', (value, expected) => { + expect(isSafeFishHistorySession(value)).toBe(expected) + }) +}) + +describe('fish history directory resolution', () => { + // Why join rather than literals: the separator is platform-specific, and on + // Windows `isAbsolute('/data')` is also true, so a hardcoded '/data/fish' + // fails there for a reason that has nothing to do with the behavior tested. + it('follows XDG_DATA_HOME when it is absolute', () => { + expect(resolveFishHistoryDir({ XDG_DATA_HOME: '/data', HOME: '/home/me' })).toBe( + join('/data', 'fish') + ) + }) + + it('ignores a relative XDG_DATA_HOME the way fish does', () => { + expect(resolveFishHistoryDir({ XDG_DATA_HOME: 'relative', HOME: '/home/me' })).toBe( + join('/home/me', '.local', 'share', 'fish') + ) + }) + + it('falls back to HOME when XDG_DATA_HOME is unset', () => { + expect(resolveFishHistoryDir({ HOME: '/home/me' })).toBe( + join('/home/me', '.local', 'share', 'fish') + ) + }) +}) + +describe('fish history deletion', () => { + let root: string + let fishDir: string + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-fish-history-')) + fishDir = join(root, 'fish') + mkdirSync(fishDir, { recursive: true }) + }) + + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + }) + + const historyFile = (session: string): string => join(fishDir, `${session}_history`) + + it('deletes the session file rather than truncating it', () => { + writeFileSync(historyFile(SESSION_A), '- cmd: ls\n') + + expect(deleteFishHistoryFile(SESSION_A, [fishDir])).toBe(true) + expect(existsSync(historyFile(SESSION_A))).toBe(false) + }) + + it('tries every candidate directory, since the PTY may not share this env', () => { + const other = join(root, 'other-xdg', 'fish') + mkdirSync(other, { recursive: true }) + writeFileSync(join(other, `${SESSION_A}_history`), '- cmd: ls\n') + + expect(deleteFishHistoryFile(SESSION_A, [fishDir, other])).toBe(true) + expect(existsSync(join(other, `${SESSION_A}_history`))).toBe(false) + }) + + it('refuses a session name it did not mint', () => { + const foreign = join(fishDir, 'fish_history') + writeFileSync(foreign, 'the user’s real history\n') + + expect(deleteFishHistoryFile('fish_history', [fishDir])).toBe(false) + expect(existsSync(foreign)).toBe(true) + }) + + // Why skipped on Windows: symlinkSync needs Developer Mode or elevation there. + it.skipIf(process.platform === 'win32')( + 'refuses to follow a symlink out of the fish data dir', + () => { + const outside = join(root, 'precious.txt') + writeFileSync(outside, 'keep me\n') + symlinkSync(outside, historyFile(SESSION_A)) + + expect(deleteFishHistoryFile(SESSION_A, [fishDir])).toBe(false) + expect(existsSync(outside)).toBe(true) + } + ) + + it('reports false when there is nothing to delete', () => { + expect(deleteFishHistoryFile(SESSION_A, [fishDir])).toBe(false) + }) +}) + +describe('orphaned fish history sweep', () => { + let root: string + let fishDir: string + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-fish-sweep-')) + fishDir = join(root, 'fish') + mkdirSync(fishDir, { recursive: true }) + }) + + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + }) + + it('removes files for dead worktrees and keeps live ones', () => { + writeFileSync(join(fishDir, `${fishHistorySessionName(HASH_A)}_history`), 'live\n') + writeFileSync(join(fishDir, `${fishHistorySessionName(HASH_B)}_history`), 'dead\n') + + expect(sweepOrphanedFishHistoryFiles(new Set([HASH_A]), [fishDir])).toBe(1) + expect(existsSync(join(fishDir, `${fishHistorySessionName(HASH_A)}_history`))).toBe(true) + expect(existsSync(join(fishDir, `${fishHistorySessionName(HASH_B)}_history`))).toBe(false) + }) + + it("never touches the user's own history files", () => { + // The whole safety of the sweep rests on the orca__ prefix. + const untouched = ['fish_history', 'work_history', 'orca_history', 'orca_nothex_history'] + for (const name of untouched) { + writeFileSync(join(fishDir, name), 'mine\n') + } + + expect(sweepOrphanedFishHistoryFiles(new Set([HASH_A]), [fishDir])).toBe(0) + for (const name of untouched) { + expect(existsSync(join(fishDir, name))).toBe(true) + } + }) + + it('refuses to sweep on an empty live set, which cannot be told from a cold store', () => { + writeFileSync(join(fishDir, `${fishHistorySessionName(HASH_B)}_history`), 'dead\n') + + expect(sweepOrphanedFishHistoryFiles(new Set(), [fishDir])).toBe(0) + expect(existsSync(join(fishDir, `${fishHistorySessionName(HASH_B)}_history`))).toBe(true) + }) + + it('spares a file younger than the age guard, since the live set is a snapshot', () => { + // The reachable race: a worktree created after the snapshot but before the + // sweep looks dead while it is actively writing the history it owns. + const fresh = join(fishDir, `${fishHistorySessionName(HASH_B)}_history`) + writeFileSync(fresh, 'just created\n') + + expect(sweepOrphanedFishHistoryFiles(new Set([HASH_A]), [fishDir], 5 * 60 * 1000)).toBe(0) + expect(existsSync(fresh)).toBe(true) + }) + + it('sweeps a file older than the age guard', () => { + const stale = join(fishDir, `${fishHistorySessionName(HASH_B)}_history`) + writeFileSync(stale, 'long dead\n') + const wellPast = Date.now() + 60 * 60 * 1000 + + expect( + sweepOrphanedFishHistoryFiles(new Set([HASH_A]), [fishDir], 5 * 60 * 1000, wellPast) + ).toBe(1) + expect(existsSync(stale)).toBe(false) + }) + + it('tolerates a directory that does not exist', () => { + expect(sweepOrphanedFishHistoryFiles(new Set([HASH_A]), [join(root, 'absent')])).toBe(0) + }) + + it('sweeps every candidate directory once', () => { + const other = join(root, 'other', 'fish') + mkdirSync(other, { recursive: true }) + for (const dir of [fishDir, other]) { + writeFileSync(join(dir, `${fishHistorySessionName(HASH_B)}_history`), 'dead\n') + } + + expect(sweepOrphanedFishHistoryFiles(new Set([HASH_A]), [fishDir, other, fishDir])).toBe(2) + }) +}) + +/** + * A relay host keyed by its CLIENT's worktree ids shares one fish data dir with + * any desktop Orca on the same machine, whose live set knows nothing of those + * ids. The name is the only thing that keeps that sweep off remote history. + */ +describe('relay fish history naming', () => { + let relayRoot: string + + beforeEach(() => { + relayRoot = mkdtempSync(join(tmpdir(), 'orca-fish-relay-')) + }) + + afterEach(() => { + rmSync(relayRoot, { recursive: true, force: true }) + }) + + it('is namespaced apart from desktop session names', () => { + expect(relayFishHistorySessionName(HASH_A)).not.toBe(fishHistorySessionName(HASH_A)) + }) + + it('is still a safe session name, so the relay can delete it by name', () => { + expect(isSafeFishHistorySession(relayFishHistorySessionName(HASH_A))).toBe(true) + }) + + it('survives a desktop sweep that knows none of the relay worktree ids', () => { + const dir = relayRoot + const relayFile = join(dir, `${relayFishHistorySessionName(HASH_A)}_history`) + const desktopFile = join(dir, `${fishHistorySessionName(HASH_B)}_history`) + writeFileSync(relayFile, 'relay') + writeFileSync(desktopFile, 'desktop') + + // A live set with neither hash in it: everything attributable is orphaned. + const removed = sweepOrphanedFishHistoryFiles(new Set(['deadbeef']), [dir]) + + expect(removed).toBe(1) + expect(existsSync(desktopFile)).toBe(false) + expect(existsSync(relayFile)).toBe(true) + }) +}) diff --git a/src/main/fish-history-session.ts b/src/main/fish-history-session.ts new file mode 100644 index 00000000000..6e76f64dd33 --- /dev/null +++ b/src/main/fish-history-session.ts @@ -0,0 +1,139 @@ +import { lstatSync, readdirSync, rmSync } from 'node:fs' +import { homedir } from 'node:os' +import { isAbsolute, join } from 'node:path' + +/** + * Per-worktree fish history. + * + * fish ignores HISTFILE entirely — history lives at `/_history` + * where `` comes only from the `fish_history` variable, and there is no + * custom-directory knob. So the session NAME is the only isolation lever, and + * the file necessarily lands in the user's fish data dir rather than inside + * Orca's history tree. That is why deletion needs its own path here instead of + * riding the tree tombstone the way bash/zsh history does. + * + * `XDG_DATA_HOME` is deliberately not redirected: that would move every other + * tool's data for the session, not just fish's history. + */ +const SESSION_PREFIX = 'orca_' +const RELAY_SESSION_PREFIX = 'orca_relay_' +const SAFE_SESSION_NAME = /^orca_(?:relay_)?[0-9a-f]{1,64}$/ +/** Deliberately does NOT match the relay prefix. A relay host keyed by its + * CLIENT's worktree ids shares the one fish data dir with any desktop Orca on + * the same machine, whose live set knows nothing of those ids — so an + * attributable name is the only thing keeping that sweep off remote history. + * Relay files are removed by exact name when their worktree goes away. */ +const ORCA_HISTORY_FILE = /^orca_([0-9a-f]{1,64})_history$/ + +export function isSafeFishHistorySession(session: unknown): session is string { + return typeof session === 'string' && SAFE_SESSION_NAME.test(session) +} + +export function fishHistorySessionName(worktreeHash: string): string { + return `${SESSION_PREFIX}${worktreeHash}` +} + +/** Relay-owned counterpart, namespaced out of the desktop sweep's reach. */ +export function relayFishHistorySessionName(worktreeHash: string): string { + return `${RELAY_SESSION_PREFIX}${worktreeHash}` +} + +/** The directory fish will write history into, for a given environment. + * Resolve this from the PTY's SPAWN env: Orca can be launched with a different + * `XDG_DATA_HOME`/`HOME` than the shells it spawns, and fish follows its own. */ +export function resolveFishHistoryDir(env: NodeJS.ProcessEnv = process.env): string { + const xdg = env.XDG_DATA_HOME?.trim() + const dataHome = + xdg && isAbsolute(xdg) ? xdg : join(env.HOME?.trim() || homedir(), '.local', 'share') + return join(dataHome, 'fish') +} + +/** Removes one history file, refusing anything that is not a regular file. + * Why lstat: a symlink here would take `rmSync` out of the fish data dir. */ +function removeHistoryFile(path: string): boolean { + try { + if (!lstatSync(path).isFile()) { + return false + } + rmSync(path) + return true + } catch { + return false + } +} + +/** Delete one worktree's fish history file, trying every directory it could be in. + * Callers pass the directory recorded at spawn time plus this process's own, since + * the two disagree whenever Orca's environment differs from the PTY's. */ +export function deleteFishHistoryFile(session: string, dirs: Iterable): boolean { + if (!isSafeFishHistorySession(session)) { + return false + } + let removed = false + for (const dir of new Set(dirs)) { + removed = removeHistoryFile(join(dir, `${session}_history`)) || removed + } + return removed +} + +/** + * Delete every Orca-minted fish history file whose worktree is gone. + * + * Why a sweep and not just per-worktree deletion: the history file outlives the + * directory that names it. A crash between tombstone and removal, a hand-deleted + * history dir, or a file written by an older build all orphan one, and only a + * scan of the data dir can find those. Matching on Orca's own `orca__` + * prefix means the user's real `fish_history` is never a candidate. + * + * `minAgeMs` mirrors the age guard the history-tree GC already applies: the + * live-worktree set is a snapshot, so a worktree created between the snapshot + * and this sweep would otherwise look dead and lose the history it is actively + * writing. A live session's file keeps a recent mtime, so this doubles as a + * second check on the set itself. + * + * Refuses to run at all on an empty live set: that cannot be told apart from a + * store that failed to hydrate, and unlike the tree GC this deletes files + * outside Orca's own directory. + */ +export function sweepOrphanedFishHistoryFiles( + liveWorktreeHashes: ReadonlySet, + dirs: Iterable, + minAgeMs = 0, + now = Date.now() +): number { + if (liveWorktreeHashes.size === 0) { + return 0 + } + let removed = 0 + for (const dir of new Set(dirs)) { + let entries: string[] + try { + entries = readdirSync(dir) + } catch { + continue + } + for (const entry of entries) { + const hash = ORCA_HISTORY_FILE.exec(entry)?.[1] + if (!hash || liveWorktreeHashes.has(hash)) { + continue + } + const path = join(dir, entry) + if (minAgeMs > 0 && !isOlderThan(path, minAgeMs, now)) { + continue + } + if (removeHistoryFile(path)) { + removed += 1 + } + } + } + return removed +} + +/** Treats an unreadable file as too young: never delete on a failed stat. */ +function isOlderThan(path: string, minAgeMs: number, now: number): boolean { + try { + return now - lstatSync(path).mtimeMs >= minAgeMs + } catch { + return false + } +} diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index ba1255c058e..376b705ef05 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -4762,7 +4762,8 @@ export function registerPtyHandlers( rows: args.rows, cwd, env, - ...(isNewDaemonSession ? { isNewSession: true } : {}) + ...(isNewDaemonSession ? { isNewSession: true } : {}), + historyIsolationEnabled: getSettings?.()?.terminalScopeHistoryByWorktree ?? true } if (!args.connectionId && !isDaemonHostSpawn) { spawnOptions.codexHomePathOverride = { value: selectedCodexHomePath } @@ -6479,7 +6480,8 @@ export function registerPtyHandlers( cwd, ...(prevalidatedCwd && !isDaemonHostSpawn ? { prevalidatedCwd } : {}), env: spawnEnv, - ...(isMintedSessionId ? { isNewSession: true } : {}) + ...(isMintedSessionId ? { isNewSession: true } : {}), + historyIsolationEnabled: getSettings?.()?.terminalScopeHistoryByWorktree ?? true } if (!args.connectionId && !isDaemonHostSpawn) { spawnOptions.codexHomePathOverride = { value: selectedCodexHomePath } diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index beebaf70058..c7ec109e2e6 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -265,6 +265,7 @@ import { removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval, recoverLocalWindowsWorktreeRemoval } from '../local-worktree-removal-recovery' +import { deleteRemoteWorktreeHistory } from '../remote-worktree-history-cleanup' const NullableWorkspaceLinkedItemSchema = WorkspaceLinkedItemSchema.nullable() const NullableTaskSourceContextSchema = TaskSourceContextSchema.nullable() @@ -2493,15 +2494,15 @@ export function registerWorktreeHandlers( 'Cannot delete the project root workspace. Remove the folder project instead.' ) } + const ownerHost = parseExecutionHostId(removalHostId) + const sshPtyProvider = + ownerHost?.kind === 'ssh' ? getSshPtyProvider(ownerHost.targetId) : undefined // Why: folder workspaces share one root, so there's no Git remove step to close shells; sweep PTYs before dropping metadata. await withWorktreeRemoveStageSpan('pty_sweep', 'folder', async () => { // Folder projects can be SSH-backed, so fence the sweep to the owning host exactly // like the git paths — the local inventory must never reach a remote workspace's id. // The resolved repo is authoritative here: path-derived metadata is shared by // same-id host copies and can describe a different owner's workspace. - const ownerHost = parseExecutionHostId(removalHostId) - const sshPtyProvider = - ownerHost?.kind === 'ssh' ? getSshPtyProvider(ownerHost.targetId) : undefined const externalHost = ownerHost?.kind === 'ssh' || ownerHost?.kind === 'runtime' await killAllProcessesForWorktree(args.worktreeId, { runtime, @@ -2524,6 +2525,7 @@ export function registerWorktreeHandlers( }) }) await withWorktreeRemoveStageSpan('metadata_purge', 'folder', async () => { + await deleteRemoteWorktreeHistory(sshPtyProvider, args.worktreeId) removeWorktreeMetadataAndTransientState( store, args.worktreeId, @@ -2612,6 +2614,14 @@ export function registerWorktreeHandlers( } finally { await removalGate.finish(removalCompleted) } + // Why history first: the worktree is already gone from git and + // disk by here, so a rejecting push-target cleanup must not be + // able to skip history removal and leave the user's commands on + // the remote host. + await deleteRemoteWorktreeHistory( + getSshPtyProvider(repo.connectionId), + args.worktreeId + ) await cleanupUnusedWorktreePushTargetRemoteSsh( provider!, repo.path, @@ -2720,6 +2730,14 @@ export function registerWorktreeHandlers( } // Why: a manually deleted worktree is already gone; persisted metadata proves it was an Orca-known row, so no force is needed. if (repo.connectionId) { + // Why history first: the worktree is already gone from git and + // disk by here, so a rejecting push-target cleanup must not be + // able to skip history removal and leave the user's commands on + // the remote host. + await deleteRemoteWorktreeHistory( + getSshPtyProvider(repo.connectionId), + args.worktreeId + ) await cleanupUnusedWorktreePushTargetRemoteSsh( provider!, repo.path, @@ -2898,6 +2916,10 @@ export function registerWorktreeHandlers( removedPushTarget, store ) + await deleteRemoteWorktreeHistory( + getSshPtyProvider(remoteConnectionId), + args.worktreeId + ) rememberPreservedBranchCleanupTarget( args.worktreeId, removalHostId, diff --git a/src/main/providers/local-pty-provider.ts b/src/main/providers/local-pty-provider.ts index af0e17656bd..cf98ca7cf0b 100644 --- a/src/main/providers/local-pty-provider.ts +++ b/src/main/providers/local-pty-provider.ts @@ -19,8 +19,10 @@ import { splitWorktreeIdForFilesystem } from '../../shared/worktree/id' import { isBracketedPasteSafeShell } from '../../shared/startup-command-submission' import { injectHistoryEnv, - updateHistFileForFallback, - logHistoryInjection + injectWslFishHistoryEnv, + updateHistoryEnvForFallback, + logHistoryInjection, + type HistoryInjectionResult } from '../terminal-history' import type { IPtyProvider, PtyProcessInfo, PtySpawnOptions, PtySpawnResult } from './types' import { @@ -872,7 +874,15 @@ export class LocalPtyProvider implements IPtyProvider { historyResult = injectHistoryEnv(finalEnv, worktreeId, effectiveShellPath, cwd, { wslDistro: launchWslDistro }) + if (isWslTerminal && launchWslDistro) { + injectWslFishHistoryEnv(finalEnv, worktreeId, launchWslDistro) + addWslEnvKeys(finalEnv, ['HISTFILE', 'fish_history']) + } logHistoryInjection(worktreeId, historyResult) + } else { + // Why: injectHistoryEnv is what normally clears it, so when history is off + // an inherited ORCA_HISTFILE would still reach the wrapper. Credit: #11146. + delete finalEnv.ORCA_HISTFILE } await prepareLocalPtySpawn(id) @@ -895,8 +905,9 @@ export class LocalPtyProvider implements IPtyProvider { ptySpawn: pty.spawn, getShellReadyConfig: getFallbackShellReadyConfig, // Why: on zsh→bash fallback HISTFILE still points to zsh_history; update before spawn so the child inherits it (design doc §8). - onBeforeFallbackSpawn: historyResult?.histFile - ? (env, fallbackShell) => updateHistFileForFallback(env, fallbackShell) + onBeforeFallbackSpawn: historyResult?.historyDir + ? (env, fallbackShell) => + updateHistoryEnvForFallback(env, fallbackShell, historyResult as HistoryInjectionResult) : undefined, windowsFallbackAttempts }) diff --git a/src/main/providers/local-pty-shell-ready-wrapper-generation.ts b/src/main/providers/local-pty-shell-ready-wrapper-generation.ts index 29863d0a169..7c32a283aa2 100644 --- a/src/main/providers/local-pty-shell-ready-wrapper-generation.ts +++ b/src/main/providers/local-pty-shell-ready-wrapper-generation.ts @@ -11,7 +11,8 @@ import { getZshEnvTemplate, getZshFinalZdotdirRestoreBlock, getZshShellReadyMarkerRegistrationBlock, - getZshStartupFileSourceBlock + getZshStartupFileSourceBlock, + ZSH_HISTFILE_RESTORE_BLOCK } from '../shell-templates' import { getBashShellReadyRcfileContent } from './local-pty-shell-ready-bash-rcfile' import { @@ -44,6 +45,7 @@ if [[ ! -o login ]]; then ${getPosixOmpShellWrapper()} # Why: Codex must keep using Orca's runtime CODEX_HOME after rc files. [[ -n "\${ORCA_CODEX_HOME:-}" ]] && export CODEX_HOME="\${ORCA_CODEX_HOME}" +${ZSH_HISTFILE_RESTORE_BLOCK} ${getPosixCodexShellLaunchPreflight()} fi __orca_osc133_precmd() { @@ -96,6 +98,7 @@ __orca_restore_agent_teams_path [[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" ${getPosixOmpShellWrapper()} [[ -n "\${ORCA_CODEX_HOME:-}" ]] && export CODEX_HOME="\${ORCA_CODEX_HOME}" +${ZSH_HISTFILE_RESTORE_BLOCK} ${getPosixCodexShellLaunchPreflight()} ${getZshShellReadyMarkerRegistrationBlock(SHELL_READY_MARKER_ESCAPED)} ${getZshFinalZdotdirRestoreBlock()} diff --git a/src/main/providers/pty-provider-contract.ts b/src/main/providers/pty-provider-contract.ts index 43bbb15a125..3ebdb6827fb 100644 --- a/src/main/providers/pty-provider-contract.ts +++ b/src/main/providers/pty-provider-contract.ts @@ -70,6 +70,8 @@ export type PtySpawnOptions = { * Existing-session attach paths must stay false so recovery checks do not * replace the daemon out from under a still-live PTY. */ isNewSession?: boolean + /** Host setting forwarded additively to the process owner; old owners ignore it. */ + historyIsolationEnabled?: boolean /** Attach the named session atomically or fail without creating a process. */ attachOnly?: boolean /** Exact persisted owner expected by an attach-only routing decision. */ @@ -119,6 +121,8 @@ export type IPtyProvider = { /** Re-probes a degraded durable host before main commits to fallback spawn semantics. */ recoverFreshSpawnRouting?: () => Promise spawn(opts: PtySpawnOptions): Promise + /** Process-owner cleanup for history stored outside the workspace tree. */ + deleteWorktreeHistory?: (worktreeId: string) => Promise /** Whether this spawn target can append the Git guard after its final env merge. */ supportsGitCredentialGuardHost?: (sessionId?: string) => boolean /** Explicit false selects pre-claim legacy spawn for a preserved old daemon. */ diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index 40301906f52..b3b64be7a71 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -369,14 +369,16 @@ describe('SshPtyProvider', () => { }) }) - it('forwards explicit shellOverride and terminalWindowsWslDistro to the relay mux', async () => { + it('forwards shell, WSL, and scoped-history options to the relay mux', async () => { mux.request.mockResolvedValue({ id: 'pty-2' }) await provider.spawn({ cols: 120, rows: 40, shellOverride: 'powershell.exe', - terminalWindowsWslDistro: 'Ubuntu' + terminalWindowsWslDistro: 'Ubuntu', + worktreeId: 'repo-1::/remote/wt', + historyIsolationEnabled: true }) expectRequest(mux.request, 'pty.spawn', { @@ -385,7 +387,9 @@ describe('SshPtyProvider', () => { cwd: undefined, env: { [POWERLEVEL10K_WIZARD_DISABLE_ENV]: 'true' }, shellOverride: 'powershell.exe', - terminalWindowsWslDistro: 'Ubuntu' + terminalWindowsWslDistro: 'Ubuntu', + worktreeId: 'repo-1::/remote/wt', + historyIsolationEnabled: true }) }) diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index eb24ffdbb8b..99df854f4d0 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -14,7 +14,7 @@ import { spawnFreshSshPty } from './ssh-agent-session-create-operation' import { mapSshPtyProcessList } from './ssh-agent-session-process-list' import { requestSshPtyAttach, - reattachSshPtySessionWithExitFence, + reattachSshPtySessionForSpawn, type PtySourceRecoveryRequest, type SshPtyAttachResult } from './ssh-pty-session-reattach' @@ -22,7 +22,6 @@ import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request' import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities' import type { PtyProcessInspection } from './pty-process-inspection' -import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' import { writeToSshPty, writeToSshPtyWithSettlement } from './ssh-pty-write' // Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch. @@ -91,36 +90,18 @@ export class SshPtyProvider implements IPtyProvider { } } if (opts.sessionId) { - let result: Awaited> | undefined - try { - result = await reattachSshPtySessionWithExitFence({ - mux: this.mux, - connectionId: this.connectionId, - sessionId: opts.sessionId, - options: opts, - exitRaceTracker: this.spawnExitRaces, - installSourceActivation: (relayPtyId, activation) => - this.outputState.installReceivingActivation(relayPtyId, activation), - rememberPtyIncarnation: (relayPtyId, incarnationId) => - this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId) - }) - if (result.sourceRecovery?.status === 'restoreRequired') { - throw new Error( - `${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}` - ) - } - this.livePtyIds.add(result.id) - result.sourceActivationLease?.commit() - const { - sourceActivationLease: _lease, - sourceRecovery: _sourceRecovery, - ...spawnResult - } = result - return spawnResult - } catch (error) { - result?.sourceActivationLease?.rollback() - throw error - } + return await reattachSshPtySessionForSpawn({ + mux: this.mux, + connectionId: this.connectionId, + sessionId: opts.sessionId, + options: opts, + exitRaceTracker: this.spawnExitRaces, + installSourceActivation: (relayPtyId, activation) => + this.outputState.installReceivingActivation(relayPtyId, activation), + rememberPtyIncarnation: (relayPtyId, incarnationId) => + this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId), + acceptLivePty: (relayPtyId) => this.livePtyIds.add(relayPtyId) + }) } const supportsCreateOperation = opts.agentSessionCreateOperationId @@ -151,6 +132,10 @@ export class SshPtyProvider implements IPtyProvider { }) } + async deleteWorktreeHistory(worktreeId: string): Promise { + await this.mux.request('pty.deleteWorktreeHistory', { worktreeId }) + } + async supportsAgentSessionClaims(options: { signal?: AbortSignal } = {}): Promise { return await this.agentSessionCapabilities.supportsClaims(options) } diff --git a/src/main/providers/ssh-pty-session-reattach.ts b/src/main/providers/ssh-pty-session-reattach.ts index 34974940caf..06e4c9ae7c5 100644 --- a/src/main/providers/ssh-pty-session-reattach.ts +++ b/src/main/providers/ssh-pty-session-reattach.ts @@ -255,3 +255,39 @@ export async function reattachSshPtySessionWithExitFence( args.exitRaceTracker.finish(operation) } } + +/** + * The full reattach path a spawn takes when it carries a sessionId: fence the + * exit race, reject a session the relay can no longer restore, and commit or + * roll back the source-activation lease. + * + * Lives here rather than in SshPtyProvider.spawn so the lease's commit and + * rollback stay in one place — a caller that only wrapped the fence could + * return without committing and silently leak the activation. + */ +export async function reattachSshPtySessionForSpawn( + args: Parameters[0] & { + acceptLivePty: (relayPtyId: string) => void + } +): Promise { + let result: SshPtyReattachResult | undefined + try { + result = await reattachSshPtySessionWithExitFence(args) + if (result.sourceRecovery?.status === 'restoreRequired') { + throw new Error( + `${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(args.connectionId, result.id)}` + ) + } + args.acceptLivePty(result.id) + result.sourceActivationLease?.commit() + const { + sourceActivationLease: _lease, + sourceRecovery: _sourceRecovery, + ...spawnResult + } = result + return spawnResult + } catch (error) { + result?.sourceActivationLease?.rollback() + throw error + } +} diff --git a/src/main/providers/ssh-pty-spawn-request.ts b/src/main/providers/ssh-pty-spawn-request.ts index 889c0813ee9..9f493d9e778 100644 --- a/src/main/providers/ssh-pty-spawn-request.ts +++ b/src/main/providers/ssh-pty-spawn-request.ts @@ -22,6 +22,10 @@ export function buildSshPtySpawnRequest(args: { // Why: the relay needs launch identity for plugin env overlays and provider-side delivery. ...(options.command ? { command: options.command } : {}), ...(options.launchAgent ? { launchAgent: options.launchAgent } : {}), + ...(options.worktreeId ? { worktreeId: options.worktreeId } : {}), + ...(options.historyIsolationEnabled !== undefined + ? { historyIsolationEnabled: options.historyIsolationEnabled } + : {}), ...(options.shellOverride !== undefined ? { shellOverride: options.shellOverride } : {}), ...(options.terminalWindowsWslDistro !== undefined ? { terminalWindowsWslDistro: options.terminalWindowsWslDistro } diff --git a/src/main/remote-worktree-history-cleanup.test.ts b/src/main/remote-worktree-history-cleanup.test.ts new file mode 100644 index 00000000000..ce0f23a60f9 --- /dev/null +++ b/src/main/remote-worktree-history-cleanup.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from 'vitest' +import { deleteRemoteWorktreeHistory } from './remote-worktree-history-cleanup' + +describe('deleteRemoteWorktreeHistory', () => { + it('repeats idempotent cleanup through the PTY owner', async () => { + const deleteWorktreeHistory = vi.fn().mockResolvedValue(undefined) + const provider = { deleteWorktreeHistory } as never + + await deleteRemoteWorktreeHistory(provider, 'repo-1::/remote/wt') + await deleteRemoteWorktreeHistory(provider, 'repo-1::/remote/wt') + + expect(deleteWorktreeHistory).toHaveBeenCalledTimes(2) + }) + + it('degrades safely when an old relay does not expose cleanup', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const provider = { + deleteWorktreeHistory: vi.fn().mockRejectedValue(new Error('unknown request method')) + } as never + + try { + await expect( + deleteRemoteWorktreeHistory(provider, 'repo-1::/remote/wt') + ).resolves.toBeUndefined() + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Remote cleanup unavailable')) + } finally { + warn.mockRestore() + } + }) +}) diff --git a/src/main/remote-worktree-history-cleanup.ts b/src/main/remote-worktree-history-cleanup.ts new file mode 100644 index 00000000000..8842758d15e --- /dev/null +++ b/src/main/remote-worktree-history-cleanup.ts @@ -0,0 +1,15 @@ +import type { IPtyProvider } from './providers/pty-provider-contract' + +/** Best-effort cleanup through the PTY owner; older relays may not expose the method. */ +export async function deleteRemoteWorktreeHistory( + provider: IPtyProvider | undefined, + worktreeId: string +): Promise { + try { + await provider?.deleteWorktreeHistory?.(worktreeId) + } catch (error) { + console.warn( + `[pty:history] Remote cleanup unavailable: ${error instanceof Error ? error.message : String(error)}` + ) + } +} diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 7bd76f0d316..17117479f80 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -6465,7 +6465,8 @@ describe('OrcaRuntimeService', () => { worktreeId: `${TEST_REPO_ID}::/remote/feature` } ]), - shutdown: vi.fn().mockResolvedValue(undefined) + shutdown: vi.fn().mockResolvedValue(undefined), + deleteWorktreeHistory: vi.fn().mockResolvedValue(undefined) } const runtime = new OrcaRuntimeService(remoteStore as never, undefined, { getSshProvider: () => ptyProvider as never @@ -6482,6 +6483,9 @@ describe('OrcaRuntimeService', () => { 'pty-remote', expect.objectContaining({ immediate: true }) ) + expect(ptyProvider.deleteWorktreeHistory).toHaveBeenCalledWith( + `${TEST_REPO_ID}::/remote/feature` + ) expect(ptyProvider.shutdown.mock.invocationCallOrder[0]).toBeLessThan( gitProvider.removeWorktree.mock.invocationCallOrder[0] ) @@ -33158,7 +33162,7 @@ describe('OrcaRuntimeService', () => { expect(spawn).not.toHaveBeenCalled() }) - it('uses POSIX quoting for mobile agent launch commands in WSL project runtimes', async () => { + it('uses portable Unix quoting for mobile agent launch commands in WSL project runtimes', async () => { await withPlatform('win32', async () => { const spawn = vi.fn().mockResolvedValue({ id: 'pty-agent' }) const runtime = new OrcaRuntimeService({ @@ -33196,7 +33200,7 @@ describe('OrcaRuntimeService', () => { expect(spawn).toHaveBeenCalledWith( expect.objectContaining({ - command: "command-code --profile mobile '--note' 'can'\\''t'", + command: `command-code --profile mobile '--note' 'can'"'"'t'`, cwd: TEST_WORKTREE_PATH, worktreeId: TEST_WORKTREE_ID }) @@ -48435,6 +48439,145 @@ describe('OrcaRuntimeService', () => { } }) + it('routes already-missing SSH runtime history cleanup through the PTY owner', async () => { + const repo = { + id: 'repo-runtime-ssh', + path: '/remote/repo', + displayName: 'ssh', + badgeColor: 'blue', + addedAt: 1, + connectionId: 'ssh-1' + } + const worktreeId = `${repo.id}::/remote/already-deleted` + const metaById: Record = { + [worktreeId]: makeWorktreeMeta({ hostId: 'ssh:ssh-1', orcaCreationSource: 'ssh' }) + } + const removeWorktreeMeta = vi.fn((id: string) => { + delete metaById[id] + }) + const runtimeStore = { + ...store, + getRepos: () => [repo], + getRepo: (id: string) => (id === repo.id ? repo : undefined), + getAllWorktreeMeta: () => metaById, + getWorktreeMeta: (id: string) => metaById[id], + removeWorktreeMeta + } + const gitProvider = { + listWorktrees: vi.fn().mockResolvedValue([ + { + path: repo.path, + head: 'main', + branch: 'refs/heads/main', + isBare: false, + isMainWorktree: true + } + ]) + } + const fsProvider = { + stat: vi.fn().mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })) + } + const deleteWorktreeHistory = vi.fn().mockResolvedValue(undefined) + const ptyProvider = { deleteWorktreeHistory } as never + registerSshGitProvider(repo.connectionId, gitProvider as never) + registerSshFilesystemProvider(repo.connectionId, fsProvider as never) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getSshProvider: () => ptyProvider + }) + + try { + await expect(runtime.removeManagedWorktree(`id:${worktreeId}`)).resolves.toEqual({}) + } finally { + unregisterSshGitProvider(repo.connectionId) + unregisterSshFilesystemProvider(repo.connectionId) + } + + expect(deleteWorktreeHistory).toHaveBeenCalledWith(worktreeId) + expect(deleteWorktreeHistory.mock.invocationCallOrder[0]).toBeLessThan( + removeWorktreeMeta.mock.invocationCallOrder[0] + ) + expect(removeWorktreeMeta).toHaveBeenCalledWith(worktreeId, 'ssh:ssh-1') + }) + + it('routes SSH runtime orphan-directory history cleanup through the PTY owner', async () => { + const repo = { + id: 'repo-runtime-ssh', + path: '/remote/repo', + displayName: 'ssh', + badgeColor: 'blue', + addedAt: 1, + connectionId: 'ssh-1' + } + const worktreePath = '/remote/orphan' + const worktreeId = `${repo.id}::${worktreePath}` + const metaById: Record = { + [worktreeId]: makeWorktreeMeta({ + hostId: 'ssh:ssh-1', + orcaCreatedAt: Date.now(), + orcaCreationSource: 'ssh' + }) + } + const removeWorktreeMeta = vi.fn((id: string) => { + delete metaById[id] + }) + const runtimeStore = { + ...store, + getRepos: () => [repo], + getRepo: (id: string) => (id === repo.id ? repo : undefined), + getAllWorktreeMeta: () => metaById, + getWorktreeMeta: (id: string) => metaById[id], + removeWorktreeMeta + } + const gitProvider = { + listWorktrees: vi.fn().mockResolvedValue([ + { + path: repo.path, + head: 'main', + branch: 'refs/heads/main', + isBare: false, + isMainWorktree: true + } + ]) + } + const fsProvider = { + lstat: vi.fn(async (path: string) => ({ + type: path === `${worktreePath}/.git` ? 'file' : 'directory' + })), + readFile: vi.fn(async (path: string) => ({ + isBinary: false, + content: + path === `${worktreePath}/.git` + ? `gitdir: ${repo.path}/.git/worktrees/orphan\n` + : `${worktreePath}/.git\n` + })), + deletePath: vi.fn().mockResolvedValue(undefined) + } + const deleteWorktreeHistory = vi.fn().mockResolvedValue(undefined) + const ptyProvider = { + listProcesses: vi.fn().mockResolvedValue([]), + shutdown: vi.fn().mockResolvedValue(undefined), + deleteWorktreeHistory + } + registerSshGitProvider(repo.connectionId, gitProvider as never) + registerSshFilesystemProvider(repo.connectionId, fsProvider as never) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getSshProvider: () => ptyProvider as never + }) + + try { + await expect(runtime.removeManagedWorktree(`id:${worktreeId}`, true)).resolves.toEqual({}) + } finally { + unregisterSshGitProvider(repo.connectionId) + unregisterSshFilesystemProvider(repo.connectionId) + } + + expect(fsProvider.deletePath).toHaveBeenCalledWith(worktreePath, true) + expect(deleteWorktreeHistory).toHaveBeenCalledWith(worktreeId) + expect(deleteWorktreeHistory.mock.invocationCallOrder[0]).toBeLessThan( + removeWorktreeMeta.mock.invocationCallOrder[0] + ) + }) + it('force-removes a legacy Orca-created runtime orphaned worktree directory after Git tracking is gone', async () => { const parentDir = await mkdtemp(join(tmpdir(), 'orca-runtime-orphan-')) const repoPath = join(parentDir, 'repo') diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 9cae4ee4798..fb490109950 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -1039,6 +1039,7 @@ import { resolveWorktreeSharedDirectories } from '../git/worktree-shared-directories' import { deleteWorktreeHistoryDir } from '../terminal-history-deletion' +import { deleteRemoteWorktreeHistory } from '../remote-worktree-history-cleanup' import { cleanupUnusedWorktreePushTargetRemote, cleanupUnusedWorktreePushTargetRemoteSsh, @@ -26204,6 +26205,7 @@ export class OrcaRuntimeService { .then((gate) => gate.finish(false)) .catch(() => {}) } + await deleteRemoteWorktreeHistory(sshPtyProvider, removalTarget.id) this.clearOptimisticReconcileToken(removalTarget.id) this.removeWorktreeMetadataAndHistory(store, removalTarget.id) this.preservedBranchCleanupByScope.delete(cleanupScopeKey) @@ -26248,6 +26250,7 @@ export class OrcaRuntimeService { console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) }) } + await deleteRemoteWorktreeHistory(folderSshPtyProvider, removalTarget.id) this.removeWorktreeMetadataAndHistory(store, removalTarget.id) this.preservedBranchCleanupByScope.delete(cleanupScopeKey) this.invalidateResolvedWorktreeCache() @@ -26332,6 +26335,10 @@ export class OrcaRuntimeService { removedPushTarget, store ) + await deleteRemoteWorktreeHistory( + this.getSshProviderFn?.(repo.connectionId), + removalTarget.id + ) } else { const removalGate = await this.acquireFileWatcherRemoval(removalTarget.path) let removalCompleted = false @@ -26437,6 +26444,12 @@ export class OrcaRuntimeService { store, localWorktreeGitOptions )) + if (repo.connectionId) { + await deleteRemoteWorktreeHistory( + this.getSshProviderFn?.(repo.connectionId), + removalTarget.id + ) + } this.clearOptimisticReconcileToken(removalTarget.id) this.removeWorktreeMetadataAndHistory(store, removalTarget.id) this.preservedBranchCleanupByScope.delete(cleanupScopeKey) @@ -26530,6 +26543,10 @@ export class OrcaRuntimeService { removedPushTarget, store ) + await deleteRemoteWorktreeHistory( + this.getSshProviderFn?.(repo.connectionId), + removalTarget.id + ) this.rememberPreservedBranchCleanupTarget( removalTarget.id, cleanupHostId, diff --git a/src/main/shell-templates.ts b/src/main/shell-templates.ts index 97288a2b971..8748303c5c6 100644 --- a/src/main/shell-templates.ts +++ b/src/main/shell-templates.ts @@ -103,6 +103,29 @@ unset _orca_spawn_orig_zdotdir _orca_user_zdotdir _orca_zshenv_source_dir _orca_ ` } +/** + * Restores the worktree-scoped HISTFILE that macOS `/etc/zshrc` destroys. + * + * That file assigns `HISTFILE=${ZDOTDIR:-$HOME}/.zsh_history` with no + * check-before-set, and it runs before any wrapper file Orca controls — so the + * injected value is already gone, and because ZDOTDIR still points at Orca's + * wrapper dir the replacement lands INSIDE it. Per-worktree history was a + * silent no-op on the primary platform as a result (#11044). + * + * Emitted after the user's own startup file has been sourced, so it is the last + * word, exactly like the CODEX_HOME/OPENCODE_CONFIG_DIR restores beside it. + */ +export const ZSH_HISTFILE_RESTORE_BLOCK = `if [[ -n "\${ORCA_HISTFILE:-}" ]]; then + HISTFILE="$ORCA_HISTFILE" +elif [[ "\${HISTFILE:-}" == "$ZDOTDIR/.zsh_history" ]]; then + # Why also when Orca injected nothing: /etc/zshrc derived this from Orca's + # wrapper ZDOTDIR, so history would accumulate INSIDE the wrapper dir and the + # user's real history would be invisible — the plain #11044 bug, with no + # per-worktree scoping involved. Matching the exact clobbered value means a + # HISTFILE the user set deliberately is never touched. + HISTFILE="\${ORCA_ORIG_ZDOTDIR:-$HOME}/.zsh_history" +fi` + export function getZshStartupFileSourceBlock(options: { fileName: '.zprofile' | '.zshrc' | '.zlogin' homeExpression?: string @@ -168,11 +191,13 @@ fi ` } -// Why: fish has no ZDOTDIR-style wrapper dir, so the marker rides `--init-command` -// and fires on fish_prompt — the earliest event fish exposes (STA-3417). Unlike zsh's -// zle-line-init this lands just *before* fish arms `?2004h`, which PostReadyFlushGate -// absorbs. `builtin printf` so a user-defined printf can't silently swallow the marker -// and send every launch to the ready timeout. +// Why: fish has no ZDOTDIR-style wrapper dir, so the marker rides `--init-command`, +// which fish runs AFTER config.fish (verified on 4.7.1) — the same last-word +// guarantee the zsh/bash wrapper files rely on. It fires on fish_prompt, the +// earliest event fish exposes (STA-3417). Unlike zsh's zle-line-init this lands +// just *before* fish arms `?2004h`, which PostReadyFlushGate absorbs. `builtin +// printf` so a user-defined printf can't silently swallow the marker and send +// every launch to the ready timeout. export function getFishShellReadyInitCommand(escapedMarker: string): string { return `if test "$ORCA_SHELL_READY_MARKER" = 1 function __orca_shell_ready_marker --on-event fish_prompt diff --git a/src/main/terminal-history-deletion.ts b/src/main/terminal-history-deletion.ts index b17eaa7ba70..1fea1128ebe 100644 --- a/src/main/terminal-history-deletion.ts +++ b/src/main/terminal-history-deletion.ts @@ -1,24 +1,39 @@ -import { basename, join } from 'node:path' +import { basename, dirname, join } from 'node:path' import { existsSync, mkdirSync, readdirSync, renameSync } from 'node:fs' import { removeHostTree } from './host-tree-removal' +import { deleteFishHistoryFile, resolveFishHistoryDir } from './fish-history-session' +import { readHistoryMeta } from './terminal-history' import { getHistoryRoot, - hashWorktreeId, listWslHistoryRoots, PENDING_DELETE_DIR_NAME } from './terminal-history-paths' +import { hashWorktreeId } from './terminal-history-id' +import { deleteWslFishHistoryFile } from './wsl-fish-history-cleanup' const pendingHistoryTreeRemovals = new Map>() +export const MAX_PENDING_HISTORY_TREE_REMOVALS = 64 // Why: a tombstone that fails once (Windows EBUSY under AV) would otherwise sit on disk for the whole // desktop session — only the next launch re-queues it. Bounded so a genuinely stuck tree stops retrying. export const HISTORY_TREE_REMOVAL_RETRY_DELAYS_MS = [30_000, 120_000] const historyTreeRemovalAttempts = new Map() const historyTreeRemovalRetryTimers = new Map>() +const wslDistroByTombstone = new Map() + +function wslDistroForHistoryRoot(historyRoot: string): string | undefined { + return basename(dirname(historyRoot)) === 'terminal-history-wsl' + ? basename(historyRoot) + : undefined +} function getPendingDeleteRoot(historyRoot: string): string { return join(historyRoot, PENDING_DELETE_DIR_NAME) } +function historyRootForTombstone(dir: string): string { + return dirname(dirname(dir)) +} + /** Move a history tree to a pending-delete tombstone (metadata-only) so the critical path never walks it. */ function tombstoneHistoryTree(dir: string, historyRoot: string): string | null { if (!existsSync(dir)) { @@ -51,6 +66,7 @@ function scheduleHistoryTreeRemovalRetry(dir: string): void { if (retryDelayMs === undefined) { // Out of in-process attempts: the tombstone stays on disk and the next startup drain re-queues it. historyTreeRemovalAttempts.delete(dir) + wslDistroByTombstone.delete(dir) return } historyTreeRemovalAttempts.set(dir, attempt + 1) @@ -62,19 +78,46 @@ function scheduleHistoryTreeRemovalRetry(dir: string): void { historyTreeRemovalRetryTimers.set(dir, timer) } -function scheduleHistoryTreeRemoval(dir: string): void { +function scheduleHistoryTreeRemoval(dir: string, wslDistro?: string): void { if (pendingHistoryTreeRemovals.has(dir)) { return } - // A rescan (GC / startup drain) hitting the same tombstone supersedes its pending retry. + // Leave excess tombstones on disk; admission is intentionally bounded. + if ( + pendingHistoryTreeRemovals.size + historyTreeRemovalRetryTimers.size >= + MAX_PENDING_HISTORY_TREE_REMOVALS + ) { + return + } + // A rescan must not cancel a delayed retry for a real failure. const pendingRetry = historyTreeRemovalRetryTimers.get(dir) if (pendingRetry) { - clearTimeout(pendingRetry) - historyTreeRemovalRetryTimers.delete(dir) + return } - const removal = removeHostTree(dir) + if (wslDistro) { + wslDistroByTombstone.set(dir, wslDistro) + } + let removalSucceeded = false + const cleanupDistro = wslDistroByTombstone.get(dir) + const meta = cleanupDistro ? readHistoryMeta(dir) : null + const cleanup = + cleanupDistro && meta?.fishSession + ? // Why swallow rather than rethrow: fish history cleanup is best effort + // and runs `wsl.exe --exec fish`, which fails outright on a distro that + // has no fish — which is most of them. Rethrowing chained that failure + // into removeHostTree below, so those users' history trees were never + // reclaimed at all. + deleteWslFishHistoryFile(cleanupDistro, meta.fishSession).catch((err: unknown) => { + console.warn( + `[pty:history] Failed to delete WSL fish history: ${err instanceof Error ? err.message : String(err)}` + ) + }) + : null + const removal = (cleanup ? cleanup.then(() => removeHostTree(dir)) : removeHostTree(dir)) .then(() => { + removalSucceeded = true historyTreeRemovalAttempts.delete(dir) + wslDistroByTombstone.delete(dir) }) .catch((err: unknown) => { console.warn( @@ -86,6 +129,9 @@ function scheduleHistoryTreeRemoval(dir: string): void { if (pendingHistoryTreeRemovals.get(dir) === removal) { pendingHistoryTreeRemovals.delete(dir) } + if (removalSucceeded) { + schedulePendingHistoryTreeRemovals(historyRootForTombstone(dir)) + } }) pendingHistoryTreeRemovals.set(dir, removal) } @@ -93,11 +139,23 @@ function scheduleHistoryTreeRemoval(dir: string): void { /** Tombstone one history tree and queue its recursive removal off the caller's critical path. * Returns false when the rename failed, leaving the tree for a later GC pass to reclaim. */ export function scheduleWorktreeHistoryTreeDeletion(dir: string, historyRoot: string): boolean { + // Why first: fish keeps its history in the user's fish data dir, outside this tree, + // so the meta.json naming the session must still be readable when we look it up. + const meta = readHistoryMeta(dir) + const wslDistro = wslDistroForHistoryRoot(historyRoot) + if (meta?.fishSession && !wslDistro) { + // Why both directories: the recorded one is what the PTY's fish saw, this + // process's own is the fallback when meta.json predates that field. + deleteFishHistoryFile(meta.fishSession, [ + ...(meta.fishHistoryDir ? [meta.fishHistoryDir] : []), + resolveFishHistoryDir() + ]) + } const tombstone = tombstoneHistoryTree(dir, historyRoot) if (!tombstone) { return false } - scheduleHistoryTreeRemoval(tombstone) + scheduleHistoryTreeRemoval(tombstone, wslDistro) return true } @@ -109,7 +167,7 @@ export function schedulePendingHistoryTreeRemovals(historyRoot: string): void { } try { for (const entry of readdirSync(pendingRoot)) { - scheduleHistoryTreeRemoval(join(pendingRoot, entry)) + scheduleHistoryTreeRemoval(join(pendingRoot, entry), wslDistroForHistoryRoot(historyRoot)) } } catch { // Non-fatal. @@ -131,6 +189,7 @@ export function cancelPendingHistoryTreeRemovalRetries(): void { } historyTreeRemovalRetryTimers.clear() historyTreeRemovalAttempts.clear() + wslDistroByTombstone.clear() } /** Drain every history root's tombstones and await the in-flight removals. Tests only: production diff --git a/src/main/terminal-history-fish-session.node-pty.test.ts b/src/main/terminal-history-fish-session.node-pty.test.ts new file mode 100644 index 00000000000..f4d7b3cf404 --- /dev/null +++ b/src/main/terminal-history-fish-session.node-pty.test.ts @@ -0,0 +1,145 @@ +/** + * Real-fish proof for worktree-scoped fish history. + * + * fish IGNORES HISTFILE, so the directory+filename mechanism bash and zsh use does + * not transfer: history lives at `$XDG_DATA_HOME/fish/${fish_history}_history` and + * the only isolation knob is the session NAME. This suite pins the two facts + * `injectHistoryEnv` bets on — that fish picks up `fish_history` from the spawn + * environment (it imports env vars as global variables at startup), and that the + * file it then writes is the one `resolveFishHistoryDir` points at. + * + * Interactive is mandatory: fish writes no history in non-interactive mode, so the + * PTY and the typed line are the test, not scaffolding. DA1/CPR/OSC-11 probes are + * answered here because no real xterm is attached — without them fish stalls ~10s + * on its DA1 read sentinel before painting a prompt. + */ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { fishRequirementViolation, resolveFishBinary } from '../shared/fish-binary-requirement' +import { fishHistorySessionName, resolveFishHistoryDir } from './fish-history-session' + +const FISH = resolveFishBinary(4) +const itWithFish = FISH.available ? it : it.skip + +const PROMPT_MARK = 'ORCAHIST> ' +const WORKTREE_HASH = 'deadbeefdeadbeef' +const MARKER = 'echo orca-worktree-scoped-history' + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +async function waitUntil(predicate: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (predicate()) { + return true + } + await sleep(20) + } + return false +} + +describe('fish keeps per-worktree history under the session Orca names', () => { + let home: string | null = null + + // Always runs, so the CI lane cannot report green with the regression below skipped. + it('has the fish this suite needs when CI requires one', () => { + expect(fishRequirementViolation(FISH)).toBeNull() + }) + + afterEach(() => { + if (home) { + rmSync(home, { recursive: true, force: true }) + home = null + } + }) + + itWithFish( + 'writes an interactive command to $XDG_DATA_HOME/fish/_history, not the shared file', + async () => { + const nodePty = await import('node-pty') + + home = mkdtempSync(path.join(tmpdir(), 'orca-fish-history-')) + const dataHome = path.join(home, 'data') + mkdirSync(path.join(home, 'fish'), { recursive: true }) + writeFileSync( + path.join(home, 'fish/config.fish'), + [ + 'set -g fish_greeting ""', + `function fish_prompt; printf '${PROMPT_MARK}'; end`, + 'function fish_right_prompt; end', + '' + ].join('\n') + ) + + const session = fishHistorySessionName(WORKTREE_HASH) + const term = nodePty.spawn(FISH.path as string, ['-l', '-i'], { + name: 'xterm-256color', + cols: 120, + rows: 30, + cwd: home, + // Fully pinned: no ambient HOME/XDG_* reaches fish, so this cannot pass + // only on a machine whose real fish config happens to cooperate. + env: { + PATH: process.env.PATH ?? '/usr/bin:/bin', + HOME: home, + TERM: 'xterm-256color', + // LC_ALL wins over any LANG/LC_* a host might contribute, pinning fish's locale. + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US.UTF-8', + XDG_CONFIG_HOME: home, + XDG_DATA_HOME: dataHome, + // The production injection under test (terminal-history.ts). + fish_history: session + } + }) + + let rendered = '' + term.onData((chunk) => { + rendered += chunk + if (chunk.includes('\x1b[0c') || chunk.includes('\x1b[c')) { + term.write('\x1b[?62;4;6;22c') + } + if (chunk.includes('\x1b[6n')) { + term.write('\x1b[1;1R') + } + if (chunk.includes('\x1b]10;?') || chunk.includes('\x1b]11;?')) { + term.write('\x1b]11;rgb:1e1e/1e1e/1e1e\x1b\\') + } + }) + let exited = false + term.onExit(() => { + exited = true + }) + + expect(await waitUntil(() => rendered.includes(PROMPT_MARK), 15_000)).toBe(true) + term.write(`${MARKER}\r`) + expect(await waitUntil(() => rendered.includes('orca-worktree-scoped-history'), 5_000)).toBe( + true + ) + // fish flushes history on exit, so the read must wait for the process to go. + term.write('exit\r') + expect(await waitUntil(() => exited, 10_000)).toBe(true) + try { + term.kill() + } catch { + // already gone + } + + const scopedPath = path.join( + resolveFishHistoryDir({ XDG_DATA_HOME: dataHome }), + `${session}_history` + ) + expect(scopedPath).toBe(path.join(dataHome, 'fish', `${session}_history`)) + const scoped = readFileSync(scopedPath as string, 'utf8') + // YAML-ish records, not one line per command — any reader must handle this shape. + expect(scoped).toContain(`- cmd: ${MARKER}`) + expect(scoped).toMatch(/^ {2}when: \d+$/m) + + // Isolation: the default session file fish would otherwise have used is absent. + expect(() => readFileSync(path.join(dataHome, 'fish', 'fish_history'), 'utf8')).toThrow() + }, + 40_000 + ) +}) diff --git a/src/main/terminal-history-gc.ts b/src/main/terminal-history-gc.ts index 2a6d2280945..d67e3a7dd5c 100644 --- a/src/main/terminal-history-gc.ts +++ b/src/main/terminal-history-gc.ts @@ -1,5 +1,5 @@ import { join } from 'node:path' -import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs' +import { existsSync, readdirSync, statSync } from 'node:fs' import { getHistoryRoot, listWslHistoryRoots, @@ -9,6 +9,9 @@ import { schedulePendingHistoryTreeRemovals, scheduleWorktreeHistoryTreeDeletion } from './terminal-history-deletion' +import { readHistoryMeta } from './terminal-history' +import { resolveFishHistoryDir, sweepOrphanedFishHistoryFiles } from './fish-history-session' +import { hashWorktreeId } from './terminal-history-id' // Why 5 minutes: GC runs ~10s after startup, and the live-worktree snapshot is // taken just before. A worktree created between the snapshot and GC execution @@ -25,8 +28,21 @@ let historyGcRunning = false function gcScanRoot( root: string, liveWorktreeIds: Set -): { totalDirs: number; orphaned: number; pruned: number; totalSizeKB: number } { - const result = { totalDirs: 0, orphaned: 0, pruned: 0, totalSizeKB: 0 } +): { + totalDirs: number + orphaned: number + pruned: number + totalSizeKB: number + /** Every fish data dir a meta.json in this root names, for the orphan sweep. */ + fishHistoryDirs: Set +} { + const result = { + totalDirs: 0, + orphaned: 0, + pruned: 0, + totalSizeKB: 0, + fishHistoryDirs: new Set() + } if (!existsSync(root)) { return result } @@ -61,11 +77,11 @@ function gcScanRoot( continue } - const meta = JSON.parse(readFileSync(metaPath, 'utf-8')) as { - worktreeId?: string - createdAt?: string + const meta = readHistoryMeta(entryPath) + if (meta?.fishHistoryDir) { + result.fishHistoryDirs.add(meta.fishHistoryDir) } - if (!meta.worktreeId) { + if (!meta?.worktreeId) { continue } @@ -100,11 +116,25 @@ function gcScanRoot( export function runHistoryGc(liveWorktreeIds: Set): void { try { // Why: finish tombstones left by quit mid-rm before scanning live worktree hashes. + // Safe ahead of the guard below: these entries were already condemned by a + // completed GC, and leaving them renamed-but-present strands disk forever. schedulePendingHistoryTreeRemovals(getHistoryRoot()) + // Why refuse rather than treat every entry as orphaned: an empty live set is + // what a store that fell back to default state looks like, and it cannot be + // told apart from a user who genuinely has no worktrees — who also has no + // history to collect. So refusing costs nothing, and it is the difference + // between a recoverable bad load and every worktree's shell history being + // deleted. `sweepOrphanedFishHistoryFiles` refuses it for the same reason; + // this is the path that deletes more. + if (liveWorktreeIds.size === 0) { + console.log('[pty:history:gc] Skipped: live worktree set is empty') + return + } const main = gcScanRoot(getHistoryRoot(), liveWorktreeIds) // Also scan WSL history directories (each distro has its own subdirectory). const wslTotals = { totalDirs: 0, orphaned: 0, pruned: 0, totalSizeKB: 0 } + const liveFishHistoryDirs = new Set(main.fishHistoryDirs) for (const distroRoot of listWslHistoryRoots()) { schedulePendingHistoryTreeRemovals(distroRoot) const r = gcScanRoot(distroRoot, liveWorktreeIds) @@ -112,6 +142,27 @@ export function runHistoryGc(liveWorktreeIds: Set): void { wslTotals.orphaned += r.orphaned wslTotals.pruned += r.pruned wslTotals.totalSizeKB += r.totalSizeKB + for (const dir of r.fishHistoryDirs) { + liveFishHistoryDirs.add(dir) + } + } + + // Why a sweep on top of per-worktree deletion: a fish history file lives in + // the user's fish data dir, so it outlives the directory that names it. A + // crash between tombstone and removal, or a hand-deleted history dir, leaves + // one with nothing left to point at it. Collecting the dirs the live meta + // files name covers a machine whose XDG_DATA_HOME changed between runs. + const fishDirs = new Set([resolveFishHistoryDir()]) + for (const dir of liveFishHistoryDirs) { + fishDirs.add(dir) + } + const fishOrphans = sweepOrphanedFishHistoryFiles( + new Set([...liveWorktreeIds].map(hashWorktreeId)), + fishDirs, + GC_MIN_AGE_MS + ) + if (fishOrphans > 0) { + console.log(`[pty:history:gc] Swept ${fishOrphans} orphaned fish history file(s)`) } const totalDirs = main.totalDirs + wslTotals.totalDirs diff --git a/src/main/terminal-history-id.ts b/src/main/terminal-history-id.ts new file mode 100644 index 00000000000..38454baa29e --- /dev/null +++ b/src/main/terminal-history-id.ts @@ -0,0 +1,6 @@ +import { createHash } from 'node:crypto' + +/** First 16 hex chars of SHA-256 of the worktreeId. */ +export function hashWorktreeId(worktreeId: string): string { + return createHash('sha256').update(worktreeId).digest('hex').slice(0, 16) +} diff --git a/src/main/terminal-history-paths.ts b/src/main/terminal-history-paths.ts index e942dd8bc3f..263f39c8c35 100644 --- a/src/main/terminal-history-paths.ts +++ b/src/main/terminal-history-paths.ts @@ -1,4 +1,3 @@ -import { createHash } from 'node:crypto' import { existsSync, readdirSync } from 'node:fs' import { join } from 'node:path' import { app } from 'electron' @@ -8,10 +7,7 @@ const HISTORY_DIR_NAME_WSL = 'terminal-history-wsl' // Why: rename live history out of the way first so a quit mid-rm still leaves a durable tombstone GC can finish. export const PENDING_DELETE_DIR_NAME = '.pending-delete' -/** First 16 hex chars of SHA-256 of the worktreeId. */ -export function hashWorktreeId(worktreeId: string): string { - return createHash('sha256').update(worktreeId).digest('hex').slice(0, 16) -} +export { hashWorktreeId } from './terminal-history-id' export function getHistoryRoot(): string { return join(app.getPath('userData'), HISTORY_DIR_NAME) diff --git a/src/main/terminal-history-tombstone-retry.test.ts b/src/main/terminal-history-tombstone-retry.test.ts index 195ee82d9f3..67cb9926322 100644 --- a/src/main/terminal-history-tombstone-retry.test.ts +++ b/src/main/terminal-history-tombstone-retry.test.ts @@ -1,12 +1,13 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' let userDataDir: string -const { removeHostTreeMock } = vi.hoisted(() => ({ - removeHostTreeMock: vi.fn<(dir: string) => Promise>() +const { removeHostTreeMock, deleteWslFishHistoryFileMock } = vi.hoisted(() => ({ + removeHostTreeMock: vi.fn<(dir: string) => Promise>(), + deleteWslFishHistoryFileMock: vi.fn<(distro: string, session: string) => Promise>() })) vi.mock('electron', () => ({ @@ -19,11 +20,19 @@ vi.mock('./host-tree-removal', () => ({ removeHostTree: removeHostTreeMock })) +vi.mock('./wsl-fish-history-cleanup', () => ({ + deleteWslFishHistoryFile: deleteWslFishHistoryFileMock +})) + import { hashWorktreeId } from './terminal-history-paths' +import { fishHistorySessionName } from './fish-history-session' import { cancelPendingHistoryTreeRemovalRetries, deleteWorktreeHistoryDir, - HISTORY_TREE_REMOVAL_RETRY_DELAYS_MS + flushPendingWorktreeHistoryDeletions, + MAX_PENDING_HISTORY_TREE_REMOVALS, + HISTORY_TREE_REMOVAL_RETRY_DELAYS_MS, + schedulePendingHistoryTreeRemovals } from './terminal-history-deletion' /** A tombstone whose rm fails once used to sit on disk for the rest of the session — only the next @@ -32,6 +41,11 @@ describe('tombstoned history removal retries', () => { beforeEach(() => { userDataDir = mkdtempSync(join(tmpdir(), 'orca-history-retry-')) removeHostTreeMock.mockReset() + deleteWslFishHistoryFileMock.mockReset() + deleteWslFishHistoryFileMock.mockResolvedValue(undefined) + removeHostTreeMock.mockImplementation(async (dir) => { + rmSync(dir, { recursive: true, force: true }) + }) vi.useFakeTimers() }) @@ -71,7 +85,6 @@ describe('tombstoned history removal retries', () => { it('does not re-arm a retry after the removal succeeds', async () => { seedWorktreeHistory('repo-1::/path/clean-wt') - removeHostTreeMock.mockResolvedValue(undefined) deleteWorktreeHistoryDir('repo-1::/path/clean-wt') await vi.advanceTimersByTimeAsync(0) @@ -79,4 +92,118 @@ describe('tombstoned history removal retries', () => { expect(removeHostTreeMock).toHaveBeenCalledTimes(1) }) + + it('admits a bounded tombstone batch and drains the rest from disk', async () => { + const distroRoot = join(userDataDir, 'terminal-history-wsl', 'Ubuntu') + mkdirSync(distroRoot, { recursive: true }) + const releases: (() => void)[] = [] + removeHostTreeMock.mockImplementation( + (dir) => + new Promise((resolve) => + releases.push(() => { + rmSync(dir, { recursive: true, force: true }) + resolve() + }) + ) + ) + + for (let index = 0; index < 1_000; index++) { + const worktreeId = `repo-1::/path/wsl-${index}` + const dir = join(distroRoot, hashWorktreeId(worktreeId)) + mkdirSync(dir, { recursive: true }) + writeFileSync( + join(dir, 'meta.json'), + JSON.stringify({ + worktreeId, + fishSession: fishHistorySessionName(hashWorktreeId(worktreeId)) + }) + ) + } + + for (let index = 0; index < 1_000; index++) { + deleteWorktreeHistoryDir(`repo-1::/path/wsl-${index}`) + } + await vi.advanceTimersByTimeAsync(0) + expect(removeHostTreeMock).toHaveBeenCalledTimes(64) + expect(deleteWslFishHistoryFileMock).toHaveBeenCalledTimes(64) + expect(releases).toHaveLength(64) + + while (releases.length > 0) { + releases.splice(0).forEach((release) => release()) + await vi.advanceTimersByTimeAsync(0) + } + expect(removeHostTreeMock).toHaveBeenCalledTimes(1_000) + expect(deleteWslFishHistoryFileMock).toHaveBeenCalledTimes(1_000) + }) + + it('caps persistent failures and leaves excess tombstones for a later disk batch', async () => { + const distroRoot = join(userDataDir, 'terminal-history-wsl', 'Ubuntu') + mkdirSync(distroRoot, { recursive: true }) + // Why removeHostTree and not the fish cleanup: a failing fish cleanup no + // longer blocks tree removal (most distros have no fish), so the tree + // removal itself has to be the thing that fails to exercise the cap. + removeHostTreeMock.mockRejectedValue(new Error('EBUSY')) + for (let index = 0; index < 1_000; index++) { + const worktreeId = `repo-1::/path/fail-${index}` + const dir = join(distroRoot, hashWorktreeId(worktreeId)) + mkdirSync(dir, { recursive: true }) + writeFileSync( + join(dir, 'meta.json'), + JSON.stringify({ + worktreeId, + fishSession: fishHistorySessionName(hashWorktreeId(worktreeId)) + }) + ) + deleteWorktreeHistoryDir(worktreeId) + } + await vi.advanceTimersByTimeAsync(0) + expect(vi.getTimerCount()).toBe(64) + expect(readdirSync(join(distroRoot, '.pending-delete'))).toHaveLength(1_000) + + for (const delay of HISTORY_TREE_REMOVAL_RETRY_DELAYS_MS) { + await vi.advanceTimersByTimeAsync(delay) + expect(vi.getTimerCount()).toBeLessThanOrEqual(64) + } + await vi.advanceTimersByTimeAsync(0) + removeHostTreeMock.mockReset() + removeHostTreeMock.mockImplementation(async (dir) => { + rmSync(dir, { recursive: true, force: true }) + }) + schedulePendingHistoryTreeRemovals(distroRoot) + await flushPendingWorktreeHistoryDeletions() + expect(removeHostTreeMock.mock.calls.length).toBeGreaterThan( + MAX_PENDING_HISTORY_TREE_REMOVALS * (HISTORY_TREE_REMOVAL_RETRY_DELAYS_MS.length + 1) + ) + expect(readdirSync(join(distroRoot, '.pending-delete'))).toHaveLength(0) + }) + + it('still removes the history tree when WSL fish cleanup fails', async () => { + // Why this matters: `injectWslFishHistoryEnv` records a fishSession for EVERY + // WSL terminal, and cleanup shells out to `wsl.exe --exec fish`. On a distro + // with no fish that always fails — which used to chain into removeHostTree and + // leak the bash/zsh history tree for those users permanently. + const distroRoot = join(userDataDir, 'terminal-history-wsl', 'Ubuntu') + mkdirSync(distroRoot, { recursive: true }) + deleteWslFishHistoryFileMock.mockRejectedValue(new Error('fish: command not found')) + removeHostTreeMock.mockImplementation(async (dir) => { + rmSync(dir, { recursive: true, force: true }) + }) + const worktreeId = 'repo-1::/path/no-fish' + const dir = join(distroRoot, hashWorktreeId(worktreeId)) + mkdirSync(dir, { recursive: true }) + writeFileSync( + join(dir, 'meta.json'), + JSON.stringify({ + worktreeId, + fishSession: fishHistorySessionName(hashWorktreeId(worktreeId)) + }) + ) + + deleteWorktreeHistoryDir(worktreeId) + await flushPendingWorktreeHistoryDeletions() + + expect(deleteWslFishHistoryFileMock).toHaveBeenCalled() + expect(removeHostTreeMock).toHaveBeenCalled() + expect(readdirSync(join(distroRoot, '.pending-delete'))).toHaveLength(0) + }) }) diff --git a/src/main/terminal-history.test.ts b/src/main/terminal-history.test.ts index c1900e3d0b5..9b9a3a78ea8 100644 --- a/src/main/terminal-history.test.ts +++ b/src/main/terminal-history.test.ts @@ -7,23 +7,39 @@ const { mkdirSyncMock, writeFileSyncMock, readFileSyncMock, + lstatSyncMock, rmSyncMock, renameSyncMock, readdirSyncMock, statSyncMock, + openSyncMock, + fstatSyncMock, + closeSyncMock, getPathMock, - rmAsyncMock + rmAsyncMock, + deleteWslFishHistoryFileMock } = vi.hoisted(() => ({ existsSyncMock: vi.fn(), mkdirSyncMock: vi.fn(), writeFileSyncMock: vi.fn(), readFileSyncMock: vi.fn(), + lstatSyncMock: vi.fn(), rmSyncMock: vi.fn(), renameSyncMock: vi.fn(), readdirSyncMock: vi.fn(), statSyncMock: vi.fn(), + openSyncMock: vi.fn((_path?: string | Buffer) => 1), + fstatSyncMock: vi.fn(() => ({ + dev: 1n, + ino: 2n, + birthtimeNs: 3n, + isFile: () => true, + isDirectory: () => true + })), + closeSyncMock: vi.fn(), getPathMock: vi.fn(), - rmAsyncMock: vi.fn(async () => undefined) + rmAsyncMock: vi.fn(async () => undefined), + deleteWslFishHistoryFileMock: vi.fn(async () => undefined) })) vi.mock('fs', () => ({ @@ -31,10 +47,14 @@ vi.mock('fs', () => ({ mkdirSync: mkdirSyncMock, writeFileSync: writeFileSyncMock, readFileSync: readFileSyncMock, + lstatSync: lstatSyncMock, rmSync: rmSyncMock, renameSync: renameSyncMock, readdirSync: readdirSyncMock, - statSync: statSyncMock + statSync: statSyncMock, + openSync: openSyncMock, + fstatSync: fstatSyncMock, + closeSync: closeSyncMock })) // Spread the real module: this factory replaces node:fs/promises for the whole import graph, so a @@ -60,21 +80,31 @@ vi.mock('./wsl', () => ({ toLinuxPath: toLinuxPathMock })) +vi.mock('./wsl-fish-history-cleanup', () => ({ + deleteWslFishHistoryFile: deleteWslFishHistoryFileMock +})) + import { resolveShellKind, ensureHistoryDir, injectHistoryEnv, - updateHistFileForFallback + MAX_HISTORY_META_BYTES, + updateHistoryEnvForFallback, + type HistoryInjectionResult } from './terminal-history' +import { fishHistorySessionName } from './fish-history-session' import { hashWorktreeId } from './terminal-history-paths' import { + cancelPendingHistoryTreeRemovalRetries, deleteWorktreeHistoryDir, flushPendingWorktreeHistoryDeletions } from './terminal-history-deletion' + import { runHistoryGc, scheduleHistoryGc } from './terminal-history-gc' describe('terminal-history', () => { afterEach(() => { + cancelPendingHistoryTreeRemovalRetries() vi.useRealTimers() }) @@ -82,9 +112,30 @@ describe('terminal-history', () => { vi.clearAllMocks() // Why: clearAllMocks keeps implementations, so a throwing rename from one test would leak forward. renameSyncMock.mockReset() + readFileSyncMock.mockReset() + readdirSyncMock.mockReset() + rmAsyncMock.mockReset() + rmAsyncMock.mockResolvedValue(undefined) getPathMock.mockReturnValue('/fake/userData') existsSyncMock.mockReturnValue(true) statSyncMock.mockReturnValue({ isDirectory: () => true, size: 100 }) + openSyncMock.mockImplementation(() => 1) + fstatSyncMock.mockReturnValue({ + dev: 1n, + ino: 2n, + birthtimeNs: 3n, + isFile: () => true, + isDirectory: () => true + }) + lstatSyncMock.mockReturnValue({ + dev: 1n, + ino: 2n, + birthtimeNs: 3n, + size: 100, + isDirectory: () => true, + isFile: () => true, + isSymbolicLink: () => false + }) }) describe('resolveShellKind', () => { @@ -228,12 +279,103 @@ describe('terminal-history', () => { expect(result.histFile).toBeNull() }) - it('does not inject HISTFILE for fish (Phase 2)', () => { + it('scopes fish by session name instead of HISTFILE, which fish ignores', () => { const env: Record = {} const result = injectHistoryEnv(env, 'repo-1::/path/wt', '/usr/bin/fish', '/path/wt') expect(env.HISTFILE).toBeUndefined() expect(result.shell).toBe('fish') + expect(result.fishSession).toBe(fishHistorySessionName(hashWorktreeId('repo-1::/path/wt'))) + expect(env.fish_history).toBe(result.fishSession) + // Deletion can only find the out-of-tree fish file through this record. + expect(writeFileSyncMock).toHaveBeenCalledWith( + expect.stringContaining('meta.json'), + expect.stringContaining(`"fishSession":"${result.fishSession}"`), + expect.anything() + ) + }) + + it('records the history DIRECTORY from the SPAWN env, not this process env', () => { + // The file is written by the PTY's fish, and the two envs disagree whenever + // Orca was launched with a different XDG_DATA_HOME than the shells it spawns. + // Pinned so the assertion cannot accidentally read the developer's own value. + const originalDataHome = process.env.XDG_DATA_HOME + process.env.XDG_DATA_HOME = ['', 'main', 'process', 'data'].join(sep) + try { + const env: Record = { XDG_DATA_HOME: ['', 'spawn', 'data'].join(sep) } + injectHistoryEnv(env, 'repo-1::/path/wt', '/usr/bin/fish', '/path/wt') + const meta = JSON.parse(writeFileSyncMock.mock.calls.at(-1)?.[1] as string) as Record< + string, + string + > + + expect(meta.fishHistoryDir).toBe(['', 'spawn', 'data', 'fish'].join(sep)) + } finally { + if (originalDataHome === undefined) { + delete process.env.XDG_DATA_HOME + } else { + process.env.XDG_DATA_HOME = originalDataHome + } + } + }) + + it('replaces oversized metadata without parsing it', () => { + statSyncMock.mockReturnValue({ isDirectory: () => true, size: MAX_HISTORY_META_BYTES + 1 }) + const env = { XDG_DATA_HOME: ['', 'active'].join(sep) } + + injectHistoryEnv(env, 'repo-1::/path/wt', '/usr/bin/fish', '/path/wt') + + expect(readFileSyncMock).not.toHaveBeenCalled() + const meta = JSON.parse(writeFileSyncMock.mock.calls.at(-1)?.[1] as string) as Record< + string, + string + > + expect(meta.fishHistoryDir).toBe(['', 'active', 'fish'].join(sep)) + }) + + it('gives two fish worktrees different history sessions', () => { + const envA: Record = {} + injectHistoryEnv(envA, 'repo-1::/path/wt-a', '/usr/bin/fish', '/path/wt-a') + const envB: Record = {} + injectHistoryEnv(envB, 'repo-1::/path/wt-b', '/usr/bin/fish', '/path/wt-b') + + expect(envA.fish_history).not.toBe(envB.fish_history) + }) + + it('drops an ORCA_HISTFILE inherited from a parent Orca PTY', () => { + // Why: an Orca terminal opened from inside another Orca terminal inherits + // it, and the zsh wrapper would then re-export the PARENT worktree's + // history path here. Credit: caught by @innocarpe in #11146. + const env: Record = { + ORCA_HISTFILE: ['', 'other', 'wt', 'zsh_history'].join(sep) + } + + injectHistoryEnv(env, 'repo-1::/path/wt', '/bin/zsh', '/path/wt') + + expect(env.ORCA_HISTFILE).toBe(env.HISTFILE) + expect(env.ORCA_HISTFILE).not.toContain('other') + }) + + it('drops an inherited ORCA_HISTFILE even when it injects nothing', () => { + // The dangerous variant: the early return would otherwise leave the stale + // value pointing the wrapper at another worktree, overriding HISTFILE. + const env: Record = { + HISTFILE: ['', 'mine', 'zsh_history'].join(sep), + ORCA_HISTFILE: ['', 'other', 'wt', 'zsh_history'].join(sep) + } + + injectHistoryEnv(env, 'repo-1::/path/wt', '/bin/zsh', '/path/wt') + + expect(env.HISTFILE).toBe(['', 'mine', 'zsh_history'].join(sep)) + expect(env.ORCA_HISTFILE).toBeUndefined() + }) + + it('preserves a caller-supplied fish_history', () => { + const env: Record = { fish_history: 'mine' } + const result = injectHistoryEnv(env, 'repo-1::/path/wt', '/usr/bin/fish', '/path/wt') + + expect(env.fish_history).toBe('mine') + expect(result.fishSession).toBeNull() }) it('degrades gracefully when directory creation fails', () => { @@ -249,12 +391,19 @@ describe('terminal-history', () => { }) }) - describe('updateHistFileForFallback', () => { + describe('updateHistoryEnvForFallback', () => { + const zshInjection = (): HistoryInjectionResult => ({ + shell: 'zsh', + histFile: '/fake/userData/terminal-history/abc123/zsh_history', + fishSession: null, + historyDir: '/fake/userData/terminal-history/abc123' + }) + it('updates HISTFILE to match fallback shell', () => { const env: Record = { HISTFILE: '/fake/userData/terminal-history/abc123/zsh_history' } - updateHistFileForFallback(env, '/bin/bash') + updateHistoryEnvForFallback(env, '/bin/bash', zshInjection()) expect(env.HISTFILE).toBe('/fake/userData/terminal-history/abc123/bash_history') }) @@ -262,18 +411,45 @@ describe('terminal-history', () => { const env: Record = { HISTFILE: '/fake/userData/terminal-history/abc123/zsh_history' } - updateHistFileForFallback(env, '/bin/sh') + updateHistoryEnvForFallback(env, '/bin/sh', zshInjection()) expect(env.HISTFILE).toBeUndefined() }) - it('is a no-op when HISTFILE is not set', () => { + it('is a no-op when nothing was injected', () => { const env: Record = {} - updateHistFileForFallback(env, '/bin/bash') + updateHistoryEnvForFallback(env, '/bin/bash', { + shell: 'unknown', + histFile: null, + fishSession: null, + historyDir: null + }) expect(env.HISTFILE).toBeUndefined() }) + + it('swaps an injected fish session for the fallback shell HISTFILE', () => { + const env: Record = { fish_history: 'orca_abc123' } + updateHistoryEnvForFallback(env, '/bin/bash', { + shell: 'fish', + histFile: null, + fishSession: 'orca_abc123', + historyDir: '/fake/userData/terminal-history/abc123' + }) + expect(env.fish_history).toBeUndefined() + expect(env.HISTFILE).toBe('/fake/userData/terminal-history/abc123/bash_history') + }) + + it('keeps a caller-supplied fish_history the injection never claimed', () => { + const env: Record = { fish_history: 'mine' } + updateHistoryEnvForFallback(env, '/bin/bash', zshInjection()) + expect(env.fish_history).toBe('mine') + }) }) describe('deleteWorktreeHistoryDir', () => { + const worktreeId = 'repo-1::/path/wt' + const session = fishHistorySessionName(hashWorktreeId(worktreeId)) + const historyFilename = `${session}_history` + it('tombstones then async-removes the history directory without recursive rmSync', async () => { existsSyncMock.mockReturnValue(true) deleteWorktreeHistoryDir('repo-1::/path/wt') @@ -286,6 +462,84 @@ describe('terminal-history', () => { await flushPendingWorktreeHistoryDeletions() }) + it('deletes the fish history file in the directory meta.json recorded', async () => { + const recordedDir = ['', 'spawn', 'data', 'fish'].join(sep) + existsSyncMock.mockReturnValue(true) + lstatSyncMock.mockReturnValue({ isFile: () => true }) + readFileSyncMock.mockReturnValue( + JSON.stringify({ worktreeId, fishSession: session, fishHistoryDir: recordedDir }) + ) + + deleteWorktreeHistoryDir('repo-1::/path/wt') + + expect(rmSyncMock).toHaveBeenCalledWith([recordedDir, historyFilename].join(sep)) + await flushPendingWorktreeHistoryDeletions() + }) + + it('also tries this process fish dir, for meta written before that field', async () => { + const originalDataHome = process.env.XDG_DATA_HOME + process.env.XDG_DATA_HOME = ['', 'main', 'data'].join(sep) + try { + existsSyncMock.mockReturnValue(true) + lstatSyncMock.mockReturnValue({ isFile: () => true }) + readFileSyncMock.mockReturnValue(JSON.stringify({ worktreeId, fishSession: session })) + + deleteWorktreeHistoryDir('repo-1::/path/wt') + + expect(rmSyncMock).toHaveBeenCalledWith( + ['', 'main', 'data', 'fish', historyFilename].join(sep) + ) + } finally { + if (originalDataHome === undefined) { + delete process.env.XDG_DATA_HOME + } else { + process.env.XDG_DATA_HOME = originalDataHome + } + } + await flushPendingWorktreeHistoryDeletions() + }) + + it('ignores a fishSession that does not belong to this history directory', async () => { + // Why: the session name is re-derived from the directory's own hash, so a + // tampered meta.json naming someone else's session cannot steer the delete. + existsSyncMock.mockReturnValue(true) + lstatSyncMock.mockReturnValue({ isFile: () => true }) + readFileSyncMock.mockReturnValue( + JSON.stringify({ worktreeId, fishSession: 'orca_deadbeefdeadbeef' }) + ) + + deleteWorktreeHistoryDir('repo-1::/path/wt') + + expect(rmSyncMock).not.toHaveBeenCalledWith(expect.stringContaining('orca_deadbeefdeadbeef')) + await flushPendingWorktreeHistoryDeletions() + }) + + it('refuses to unlink a fish history path that is not a regular file', async () => { + existsSyncMock.mockReturnValue(true) + lstatSyncMock.mockReturnValue({ isFile: () => false }) + readFileSyncMock.mockReturnValue(JSON.stringify({ worktreeId, fishSession: session })) + + deleteWorktreeHistoryDir('repo-1::/path/wt') + + expect(rmSyncMock).not.toHaveBeenCalled() + await flushPendingWorktreeHistoryDeletions() + }) + + it('cleans WSL Fish history through the owning distro before removing metadata', async () => { + readFileSyncMock.mockReturnValue(JSON.stringify({ worktreeId, fishSession: session })) + readdirSyncMock.mockImplementation((path: string) => { + if (path.endsWith('terminal-history-wsl')) { + return ['Ubuntu'] + } + return [] + }) + + deleteWorktreeHistoryDir(worktreeId) + await flushPendingWorktreeHistoryDeletions() + + expect(deleteWslFishHistoryFileMock).toHaveBeenCalledWith('Ubuntu', session) + }) + it('leaves the live directory alone when the tombstone rename fails', async () => { existsSyncMock.mockReturnValue(true) renameSyncMock.mockImplementation(() => { @@ -309,6 +563,7 @@ describe('terminal-history', () => { }) it('retries pending tombstones on flush after a failed async rm (app-quit durability)', async () => { + let leftoverTombstonePresent = true existsSyncMock.mockImplementation((p: string) => { const path = String(p) if (path.includes('.pending-delete') && !path.endsWith('.pending-delete')) { @@ -321,11 +576,13 @@ describe('terminal-history', () => { }) readdirSyncMock.mockImplementation((p: string) => { if (String(p).endsWith('.pending-delete')) { - return ['leftover-tombstone'] + return leftoverTombstonePresent ? ['leftover-tombstone'] : [] } return [] }) - rmAsyncMock.mockResolvedValue(undefined) + rmAsyncMock.mockImplementation(async () => { + leftoverTombstonePresent = false + }) await flushPendingWorktreeHistoryDeletions() expect(rmAsyncMock).toHaveBeenCalledWith( expect.stringContaining('leftover-tombstone'), @@ -388,6 +645,64 @@ describe('terminal-history', () => { ) }) + // Why: an empty live set is what a store that fell back to default state + // looks like, and it is indistinguishable from a user with no worktrees — + // who has no history to collect either. Treating it as "everything is + // orphaned" turns a recoverable bad load into deleted shell history. + it('refuses to prune anything when the live set is empty', () => { + existsSyncMock.mockImplementation((p: string) => !p.includes('terminal-history-wsl')) + readdirSyncMock.mockImplementation((dir: string) => { + if (dir.endsWith('.pending-delete')) { + return [] + } + if (dir.endsWith('terminal-history')) { + return ['dir1', 'dir2'] + } + return ['meta.json'] + }) + statSyncMock.mockReturnValue({ isDirectory: () => true, size: 100 }) + readFileSyncMock.mockReturnValue( + JSON.stringify({ + worktreeId: 'some-wt', + createdAt: new Date(Date.now() - 10 * 60 * 1000).toISOString() + }) + ) + + runHistoryGc(new Set()) + + expect(renameSyncMock).not.toHaveBeenCalled() + expect(rmSyncMock).not.toHaveBeenCalled() + expect(rmAsyncMock).not.toHaveBeenCalled() + }) + + it('continues GC after one orphan tombstone fails', async () => { + existsSyncMock.mockImplementation((path: string) => !path.includes('terminal-history-wsl')) + readdirSyncMock.mockImplementation((dir: string) => { + if (dir.endsWith('.pending-delete')) { + return [] + } + if (dir.endsWith('terminal-history')) { + return ['broken', 'healthy'] + } + return ['meta.json'] + }) + statSyncMock.mockReturnValue({ isDirectory: () => true, size: 100 }) + readFileSyncMock.mockReturnValue( + JSON.stringify({ + worktreeId: 'orphan', + createdAt: new Date(Date.now() - 10 * 60 * 1000).toISOString() + }) + ) + renameSyncMock.mockImplementationOnce(() => { + throw new Error('busy') + }) + + expect(() => runHistoryGc(new Set(['live-wt']))).not.toThrow() + expect(renameSyncMock).toHaveBeenCalledTimes(2) + expect(rmAsyncMock).toHaveBeenCalledTimes(1) + await flushPendingWorktreeHistoryDeletions() + }) + it('skips recently-created directories to avoid TOCTOU race', () => { existsSyncMock.mockImplementation((p: string) => { if (p.includes('terminal-history-wsl')) { @@ -407,7 +722,7 @@ describe('terminal-history', () => { JSON.stringify({ worktreeId: 'unknown-wt', createdAt: new Date().toISOString() }) ) - runHistoryGc(new Set()) + runHistoryGc(new Set(['live-wt'])) // Should NOT prune because the directory is too young expect(rmSyncMock).not.toHaveBeenCalled() @@ -416,15 +731,16 @@ describe('terminal-history', () => { it('does not throw when history root does not exist', () => { existsSyncMock.mockReturnValue(false) - expect(() => runHistoryGc(new Set())).not.toThrow() - expect(readdirSyncMock).not.toHaveBeenCalled() + expect(() => runHistoryGc(new Set(['live-wt']))).not.toThrow() + expect(readdirSyncMock).not.toHaveBeenCalledWith('/fake/userData/terminal-history') }) it('drains delete tombstones asynchronously instead of scanning them as worktrees', async () => { + let tombstonePresent = true existsSyncMock.mockImplementation((p: string) => !String(p).includes('terminal-history-wsl')) readdirSyncMock.mockImplementation((dir: string) => { if (String(dir).endsWith('.pending-delete')) { - return ['abc123.1700000000000.deadbeef'] + return tombstonePresent ? ['abc123.1700000000000.deadbeef'] : [] } if (String(dir).endsWith('terminal-history')) { return ['.pending-delete'] @@ -432,8 +748,11 @@ describe('terminal-history', () => { return ['meta.json'] }) statSyncMock.mockReturnValue({ isDirectory: () => true, size: 100 }) + rmAsyncMock.mockImplementation(async () => { + tombstonePresent = false + }) - runHistoryGc(new Set()) + runHistoryGc(new Set(['live-wt'])) // The tombstone queue is drained off-thread; GC must never rmSync it or count it as a worktree. expect(rmSyncMock).not.toHaveBeenCalled() diff --git a/src/main/terminal-history.ts b/src/main/terminal-history.ts index 38dd7968264..c15f903c2de 100644 --- a/src/main/terminal-history.ts +++ b/src/main/terminal-history.ts @@ -1,10 +1,18 @@ import { join, basename } from 'node:path' -import { mkdirSync, existsSync, writeFileSync } from 'node:fs' +import { mkdirSync, existsSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { + fishHistorySessionName, + isSafeFishHistorySession, + resolveFishHistoryDir +} from './fish-history-session' import { parseWslPath, toLinuxPath } from './wsl' -import { getHistoryRoot, getHistoryRootWsl, hashWorktreeId } from './terminal-history-paths' +import { getHistoryRoot, getHistoryRootWsl } from './terminal-history-paths' +import { hashWorktreeId } from './terminal-history-id' type ShellKind = 'zsh' | 'bash' | 'fish' | 'pwsh' | 'powershell' | 'cmd' | 'unknown' +export const MAX_HISTORY_META_BYTES = 32 * 1024 + // ─── Shell Detection ─────────────────────────────────────────────── /** Resolve the shell kind from a shell binary path. @@ -33,14 +41,16 @@ export function resolveShellKind(shellPath: string): ShellKind { return 'unknown' } -/** Map shell kind to the filename used inside the history directory. */ +/** Map shell kind to the filename used inside the history directory. + * fish is absent on purpose: it ignores HISTFILE and keeps history in its own + * data dir keyed by session name (see fish-history-session.ts). */ function historyFilename(shell: ShellKind): string | null { switch (shell) { case 'zsh': return 'zsh_history' case 'bash': return 'bash_history' - // Phase 2: fish and PowerShell use different mechanisms + // Phase 2: PowerShell and cmd use different mechanisms case 'fish': case 'pwsh': case 'powershell': @@ -68,17 +78,81 @@ export function ensureHistoryDir(worktreeHash: string, wslDistro?: string): stri } } -/** Write meta.json alongside history files for debuggability. */ -function writeMetaFile(dir: string, worktreeId: string): void { +/** Write meta.json alongside history files, for debuggability and for GC. + * `fishSession` is load-bearing, not diagnostic: fish history lives outside + * this directory, so deletion can only find it by the name recorded here. + * `fishHistoryDir` is resolved from the SPAWN env, which is the one fish + * follows — this process's own may differ. */ +function writeMetaFile( + dir: string, + worktreeId: string, + fish?: { session: string; historyDir?: string } +): void { try { const metaPath = join(dir, 'meta.json') - if (!existsSync(metaPath)) { - writeFileSync(metaPath, JSON.stringify({ worktreeId, createdAt: new Date().toISOString() }), { - mode: 0o600 - }) + const existing = existsSync(metaPath) ? readHistoryMeta(dir) : null + if ( + existing && + (!fish || + (existing.fishSession === fish.session && existing.fishHistoryDir === fish.historyDir)) + ) { + return + } + writeFileSync( + metaPath, + JSON.stringify({ + worktreeId, + createdAt: existing?.createdAt ?? new Date().toISOString(), + ...(fish ? { fishSession: fish.session } : {}), + ...(fish?.historyDir ? { fishHistoryDir: fish.historyDir } : {}) + }), + { mode: 0o600 } + ) + } catch { + // Non-fatal — a missing meta.json only costs GC attribution. + } +} + +export type HistoryDirMeta = { + worktreeId?: string + createdAt?: string + /** fish session name whose history file lives in the user's fish data dir. */ + fishSession?: string + /** Directory that session's history file was written to, as the PTY saw it. */ + fishHistoryDir?: string +} + +/** Read one history directory's meta.json, or null when it is absent or unparseable. */ +export function readHistoryMeta(dir: string): HistoryDirMeta | null { + try { + const metaPath = join(dir, 'meta.json') + if (statSync(metaPath).size > MAX_HISTORY_META_BYTES) { + return null + } + const raw: unknown = JSON.parse(readFileSync(metaPath, 'utf-8')) + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + return null + } + const record = raw as Record + // Why re-derive: the session name is a pure function of this directory's own + // hash, so a meta.json naming someone else's session cannot steer deletion. + const expectedFishSession = fishHistorySessionName(basename(dir).split('.')[0]) + const fishSession = + isSafeFishHistorySession(record.fishSession) && record.fishSession === expectedFishSession + ? record.fishSession + : undefined + const fishHistoryDir = + fishSession && typeof record.fishHistoryDir === 'string' && record.fishHistoryDir + ? record.fishHistoryDir + : undefined + return { + ...(typeof record.worktreeId === 'string' ? { worktreeId: record.worktreeId } : {}), + ...(typeof record.createdAt === 'string' ? { createdAt: record.createdAt } : {}), + ...(fishSession ? { fishSession } : {}), + ...(fishHistoryDir ? { fishHistoryDir } : {}) } } catch { - // Non-fatal — meta.json is purely for diagnostics. + return null } } @@ -87,6 +161,10 @@ function writeMetaFile(dir: string, worktreeId: string): void { export type HistoryInjectionResult = { shell: ShellKind histFile: string | null + /** fish session name exported as `fish_history`; null when fish is not the shell. */ + fishSession: string | null + /** Worktree history dir as the spawned shell sees it (Linux-visible under WSL). */ + historyDir: string | null } /** Build shell-specific history env overrides for a PTY spawn. @@ -103,18 +181,31 @@ export function injectHistoryEnv( cwd: string, options: { wslDistro?: string | null } = {} ): HistoryInjectionResult { + // Why unconditionally first: ORCA_HISTFILE is Orca-owned, and an Orca PTY + // launched from inside another Orca PTY inherits the parent's. Left in place, + // the zsh wrapper would re-export a PREVIOUS worktree's history path into this + // shell — the cross-worktree leak this feature exists to prevent — and it would + // also override a caller-supplied HISTFILE on the early return below. + // Credit: caught by @innocarpe in #11146. + delete spawnEnv.ORCA_HISTFILE + const shell = resolveShellKind(shellPath) - const result: HistoryInjectionResult = { shell, histFile: null } + const result: HistoryInjectionResult = { + shell, + histFile: null, + fishSession: null, + historyDir: null + } const filename = historyFilename(shell) - if (!filename) { - // Unknown shell or Phase 2 shell (fish, pwsh, cmd) — leave unchanged. + if (!filename && shell !== 'fish') { + // Unknown shell or Phase 2 shell (pwsh, cmd) — leave unchanged. return result } - // Check-before-set: if the caller already provided HISTFILE, preserve it. - // This follows the pattern used by Ghostty, Kitty, and VS Code (§6). - if (spawnEnv.HISTFILE) { + // Check-before-set: if the caller already provided the shell's history knob, + // preserve it. Same pattern Ghostty, Kitty, and VS Code use for HISTFILE (§6). + if (shell === 'fish' ? spawnEnv.fish_history : spawnEnv.HISTFILE) { return result } @@ -130,46 +221,92 @@ export function injectHistoryEnv( return result } + if (!filename) { + // fish: the directory holds no history, only the meta.json that lets deletion + // find the session file fish keeps in its own data dir. fish never runs as the + // inner WSL shell, so histDir needs no /mnt conversion here. + const session = fishHistorySessionName(worktreeHash) + // Resolve from the SPAWN env: that is the XDG_DATA_HOME/HOME fish will see, + // which need not match the one this process was launched with. + writeMetaFile(histDir, worktreeId, { session, historyDir: resolveFishHistoryDir(spawnEnv) }) + spawnEnv.fish_history = session + result.fishSession = session + result.historyDir = histDir + return result + } + writeMetaFile(histDir, worktreeId) const histFilePath = join(histDir, filename) // For WSL, convert the Windows path to a Linux-visible path. spawnEnv.HISTFILE = wslDistro ? toLinuxPath(histFilePath) : histFilePath + // Why a second variable: macOS `/etc/zshrc` assigns HISTFILE unconditionally + // (`HISTFILE=${ZDOTDIR:-$HOME}/.zsh_history`) and runs before Orca's wrapper + // .zshrc, so by then the injected value is gone from HISTFILE itself. The + // wrapper restores it from here once the user's own config has loaded (#11044). + spawnEnv.ORCA_HISTFILE = spawnEnv.HISTFILE result.histFile = spawnEnv.HISTFILE + result.historyDir = spawnEnv.HISTFILE.replace(/[/\\][^/\\]+$/, '') return result } -/** Update HISTFILE in spawnEnv when shell fallback changes the shell kind. - * For example, if zsh fails and bash takes over, the HISTFILE should point - * to bash_history instead of zsh_history. */ -export function updateHistFileForFallback( +/** WSL's outer executable hides the login shell, so carry both history knobs. */ +export function injectWslFishHistoryEnv( spawnEnv: Record, - fallbackShellPath: string + worktreeId: string, + wslDistro: string +): string | null { + if (spawnEnv.fish_history) { + return null + } + const worktreeHash = hashWorktreeId(worktreeId) + const historyDir = ensureHistoryDir(worktreeHash, wslDistro) + if (!historyDir) { + return null + } + const session = fishHistorySessionName(worktreeHash) + // Why no historyDir: this session's file lives inside the WSL distro, so a + // path resolved from THIS process's Windows environment names an unrelated + // host directory — which the host GC sweep would then scan. WSL cleanup goes + // through `deleteWslFishHistoryFile`, which resolves the path in the distro. + writeMetaFile(historyDir, worktreeId, { session }) + spawnEnv.fish_history = session + return session +} + +/** Re-point the history env when shell fallback changes the shell kind — e.g. zsh + * fails and bash takes over, so HISTFILE must name bash_history. A fish primary + * injected `fish_history` instead, which the fallback shell cannot use. */ +export function updateHistoryEnvForFallback( + spawnEnv: Record, + fallbackShellPath: string, + injected: HistoryInjectionResult ): void { - if (!spawnEnv.HISTFILE) { + // Only ever undo what this spawn injected; a caller-supplied value stays. + if (injected.fishSession && spawnEnv.fish_history === injected.fishSession) { + delete spawnEnv.fish_history + } + if (!injected.historyDir) { return } - const newShell = resolveShellKind(fallbackShellPath) - const newFilename = historyFilename(newShell) + const newFilename = historyFilename(resolveShellKind(fallbackShellPath)) if (!newFilename) { - // Fallback to an unknown shell — remove HISTFILE override entirely - // so the shell uses its own default. + // Fallback to an unknown shell — drop the override so it uses its own default. delete spawnEnv.HISTFILE + delete spawnEnv.ORCA_HISTFILE return } - - // Replace the filename portion of the HISTFILE path. - const dir = spawnEnv.HISTFILE.replace(/[/\\][^/\\]+$/, '') - spawnEnv.HISTFILE = `${dir}/${newFilename}` + spawnEnv.HISTFILE = `${injected.historyDir}/${newFilename}` + spawnEnv.ORCA_HISTFILE = spawnEnv.HISTFILE } /** Log the history injection result for diagnostics. */ export function logHistoryInjection(worktreeId: string, result: HistoryInjectionResult): void { const truncatedId = worktreeId.length > 60 ? `${worktreeId.slice(0, 60)}...` : worktreeId console.log( - `[pty:history] worktreeId=${truncatedId} shell=${result.shell} histFile=${result.histFile ?? 'none'}` + `[pty:history] worktreeId=${truncatedId} shell=${result.shell} histFile=${result.histFile ?? 'none'} fishSession=${result.fishSession ?? 'none'}` ) } diff --git a/src/main/window/history-gc-profile-worktree-ids.test.ts b/src/main/window/history-gc-profile-worktree-ids.test.ts new file mode 100644 index 00000000000..b3f0960eab9 --- /dev/null +++ b/src/main/window/history-gc-profile-worktree-ids.test.ts @@ -0,0 +1,117 @@ +/** + * Terminal history is keyed by worktree id under a root with no profile + * segment, while the Store the GC consults holds one profile's ids. Without + * these, switching profiles makes every other profile's history look orphaned. + */ +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { folderWorkspaceKey } from '../../shared/workspace-scope' +import { getOtherProfileWorktreeIdsForHistoryGc } from './history-gc-profile-worktree-ids' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function userDataWithProfiles( + activeProfileId: string, + profiles: { id: string; state?: unknown; raw?: string }[] +): string { + const root = mkdtempSync(join(tmpdir(), 'orca-gc-profiles-')) + roots.push(root) + mkdirSync(join(root, 'profiles'), { recursive: true }) + writeFileSync( + join(root, 'orca-profile-index.json'), + JSON.stringify({ + activeProfileId, + profiles: profiles.map(({ id }) => ({ + id, + name: id, + kind: 'local', + createdAt: 0, + updatedAt: 0, + lastOpenedAt: 0, + avatar: { kind: 'initials', initials: id.slice(0, 2), color: 'neutral' } + })) + }) + ) + for (const profile of profiles) { + mkdirSync(join(root, 'profiles', profile.id), { recursive: true }) + if (profile.raw !== undefined) { + writeFileSync(join(root, 'profiles', profile.id, 'orca-data.json'), profile.raw) + } else if (profile.state !== undefined) { + writeFileSync( + join(root, 'profiles', profile.id, 'orca-data.json'), + JSON.stringify(profile.state) + ) + } + } + return root +} + +describe('getOtherProfileWorktreeIdsForHistoryGc', () => { + it('collects worktrees and folder workspaces from the inactive profiles', () => { + const root = userDataWithProfiles('active', [ + { id: 'active', state: { worktreeMeta: { 'repo::/active': {} }, folderWorkspaces: [] } }, + { + id: 'other', + state: { + worktreeMeta: { 'repo::/other': {}, 'repo::/other-2': {} }, + folderWorkspaces: [{ id: 'fw-other' }] + } + } + ]) + + const result = getOtherProfileWorktreeIdsForHistoryGc(root) + + expect(result.unreadableProfiles).toBe(0) + expect(result.ids).toEqual( + new Set(['repo::/other', 'repo::/other-2', folderWorkspaceKey('fw-other')]) + ) + }) + + // The active profile's ids come from the live Store, which is authoritative; + // re-reading its file would only race a write in progress. + it('skips the active profile', () => { + const root = userDataWithProfiles('active', [ + { id: 'active', state: { worktreeMeta: { 'repo::/active': {} } } } + ]) + + expect(getOtherProfileWorktreeIdsForHistoryGc(root).ids.size).toBe(0) + }) + + it('reports a corrupt profile data file as unreadable rather than as no worktrees', () => { + const root = userDataWithProfiles('active', [ + { id: 'active', state: {} }, + { id: 'other', raw: '{ this is not json' } + ]) + + const result = getOtherProfileWorktreeIdsForHistoryGc(root) + + expect(result.unreadableProfiles).toBe(1) + expect(result.ids.size).toBe(0) + }) + + it('reports a missing profile data file as unreadable', () => { + const root = userDataWithProfiles('active', [{ id: 'active', state: {} }, { id: 'other' }]) + + expect(getOtherProfileWorktreeIdsForHistoryGc(root).unreadableProfiles).toBe(1) + }) + + // A single-profile install must not pay for this, and no index at all is the + // pre-profiles layout rather than an error. + it('is empty and complete when there is no profile index', () => { + const root = mkdtempSync(join(tmpdir(), 'orca-gc-profiles-')) + roots.push(root) + + expect(getOtherProfileWorktreeIdsForHistoryGc(root)).toEqual({ + ids: new Set(), + unreadableProfiles: 0 + }) + }) +}) diff --git a/src/main/window/history-gc-profile-worktree-ids.ts b/src/main/window/history-gc-profile-worktree-ids.ts new file mode 100644 index 00000000000..6f82b96fc37 --- /dev/null +++ b/src/main/window/history-gc-profile-worktree-ids.ts @@ -0,0 +1,80 @@ +import { readFileSync } from 'node:fs' +import { folderWorkspaceKey } from '../../shared/workspace-scope' +import { + getOrcaProfileDataFile, + getProfileUserDataPath +} from '../orca-profiles/profile-storage-paths' +import { getOrcaProfileIndexPath, readProfileIndex } from '../orca-profiles/profile-index-store' + +/** + * Worktree ids owned by Orca profiles OTHER than the running one. + * + * Why the history GC needs these: terminal history is keyed by worktree id + * under `userData/terminal-history`, which has no profile segment, and fish + * history lands in the user's own fish data dir — but the Store the GC asks for + * live ids only ever reads the ACTIVE profile's data file. So after a profile + * switch every other profile's history looks orphaned, and the GC deletes shell + * history those profiles are still using. + * + * Reading their data files directly is deliberate: a Store per profile would + * run migrations and normalization against state another profile owns. Only the + * two id-bearing collections are read, and any unreadable profile is skipped — + * a profile whose ids cannot be established must widen the live set's + * uncertainty, never narrow it, so failure here is handled by the caller + * refusing to prune rather than by pruning more. + */ +export function getOtherProfileWorktreeIdsForHistoryGc(userDataPath = getProfileUserDataPath()): { + ids: Set + unreadableProfiles: number +} { + const ids = new Set() + const index = readProfileIndex(getOrcaProfileIndexPath(userDataPath)) + if (!index) { + return { ids, unreadableProfiles: 0 } + } + let unreadableProfiles = 0 + for (const profile of index.profiles) { + if (profile.id === index.activeProfileId) { + continue + } + const collected = readProfileWorktreeIds(getOrcaProfileDataFile(profile.id, userDataPath)) + if (!collected) { + unreadableProfiles += 1 + continue + } + for (const id of collected) { + ids.add(id) + } + } + return { ids, unreadableProfiles } +} + +function readProfileWorktreeIds(dataFile: string): Set | null { + let parsed: unknown + try { + parsed = JSON.parse(readFileSync(dataFile, 'utf8')) + } catch { + // Missing is indistinguishable from corrupt here, and both mean the same + // thing to the caller: this profile's ids are unknown. + return null + } + if (!parsed || typeof parsed !== 'object') { + return null + } + const state = parsed as { worktreeMeta?: unknown; folderWorkspaces?: unknown } + const ids = new Set() + if (state.worktreeMeta && typeof state.worktreeMeta === 'object') { + for (const id of Object.keys(state.worktreeMeta)) { + ids.add(id) + } + } + if (Array.isArray(state.folderWorkspaces)) { + for (const workspace of state.folderWorkspaces) { + const id = (workspace as { id?: unknown } | null)?.id + if (typeof id === 'string' && id) { + ids.add(folderWorkspaceKey(id)) + } + } + } + return ids +} diff --git a/src/main/window/history-gc-worktree-ids.test.ts b/src/main/window/history-gc-worktree-ids.test.ts index 1aca7d585b0..b61d2aaca3f 100644 --- a/src/main/window/history-gc-worktree-ids.test.ts +++ b/src/main/window/history-gc-worktree-ids.test.ts @@ -1,18 +1,70 @@ -import { describe, expect, it, vi } from 'vitest' +/** + * The GC live set decides what history is "orphaned". Anything missing from it + * gets deleted, so a workspace kind that owns history and is absent here is a + * data-loss bug — that is what reverted #14863 (#14975). + */ +import { describe, expect, it } from 'vitest' +import { folderWorkspaceKey } from '../../shared/workspace-scope' import { getKnownWorktreeIdsForHistoryGc } from './history-gc-worktree-ids' -describe('getKnownWorktreeIdsForHistoryGc', () => { - it('uses persisted metadata keys without probing repo paths', () => { - const store = { - getAllWorktreeMeta: vi.fn(() => ({ - 'repo-1::/worktree-a': {}, - 'repo-2::/worktree-b': {} - })) - } +const noOtherProfiles = () => ({ ids: new Set(), unreadableProfiles: 0 }) - expect(getKnownWorktreeIdsForHistoryGc(store as never)).toEqual( - new Set(['repo-1::/worktree-a', 'repo-2::/worktree-b']) +const store = (worktreeIds: string[], folderIds: string[]) => + ({ + getAllWorktreeMeta: () => Object.fromEntries(worktreeIds.map((id) => [id, {}])), + getFolderWorkspaces: () => folderIds.map((id) => ({ id })) + }) as never + +describe('getKnownWorktreeIdsForHistoryGc', () => { + it('includes git worktrees', () => { + expect( + getKnownWorktreeIdsForHistoryGc(store(['repo-1::/path/wt'], []), noOtherProfiles) + ).toEqual(new Set(['repo-1::/path/wt'])) + }) + + // Why: a folder workspace's PTY carries `folder:` as its worktree id, so + // injectHistoryEnv mints history under that key. They live in a separate store + // collection, so a set built only from worktree metadata makes every live + // folder workspace look orphaned and its history gets swept on a later start. + it('includes folder workspaces under their PTY workspace key', () => { + const live = getKnownWorktreeIdsForHistoryGc( + store(['repo-1::/path/wt'], ['fw-1', 'fw-2']), + noOtherProfiles ) - expect(store.getAllWorktreeMeta).toHaveBeenCalledTimes(1) + + expect(live.has(folderWorkspaceKey('fw-1'))).toBe(true) + expect(live.has(folderWorkspaceKey('fw-2'))).toBe(true) + expect(live.size).toBe(3) + }) + + it('is empty only when there is genuinely nothing live', () => { + expect(getKnownWorktreeIdsForHistoryGc(store([], []), noOtherProfiles).size).toBe(0) + }) + + // Why: the history root has no profile segment but the store does, so the + // other profiles' worktrees own history this set would otherwise condemn the + // first time the user switches profiles. + it('includes worktrees owned by other profiles', () => { + const live = getKnownWorktreeIdsForHistoryGc(store(['repo-1::/b'], ['fw-b']), () => ({ + ids: new Set(['repo-1::/a', folderWorkspaceKey('fw-a')]), + unreadableProfiles: 0 + })) + + expect(live).toEqual( + new Set(['repo-1::/b', folderWorkspaceKey('fw-b'), 'repo-1::/a', folderWorkspaceKey('fw-a')]) + ) + }) + + // Why empty and not "just the ids we did read": an unreadable profile's + // worktrees are indistinguishable from deleted ones, and runHistoryGc refuses + // to prune on an empty set — so this reports "do not collect" rather than + // handing back a set that condemns real history. + it('reports nothing live when a profile could not be read', () => { + const live = getKnownWorktreeIdsForHistoryGc(store(['repo-1::/b'], ['fw-b']), () => ({ + ids: new Set(['repo-1::/a']), + unreadableProfiles: 1 + })) + + expect(live.size).toBe(0) }) }) diff --git a/src/main/window/history-gc-worktree-ids.ts b/src/main/window/history-gc-worktree-ids.ts index beec2f9cfc1..df614967192 100644 --- a/src/main/window/history-gc-worktree-ids.ts +++ b/src/main/window/history-gc-worktree-ids.ts @@ -1,7 +1,39 @@ import type { Store } from '../persistence' +import { folderWorkspaceKey } from '../../shared/workspace-scope' +import { getOtherProfileWorktreeIdsForHistoryGc } from './history-gc-profile-worktree-ids' +/** + * Every workspace key that owns shell history, for the history GC's live set. + * + * Why folder workspaces are included: a folder workspace's PTY carries + * `folder:` as its worktree id (see folder-workspace-composer-submit.ts), + * so `injectHistoryEnv` mints history under that key exactly as it does for a + * git worktree. They live in a separate store collection, so a set built only + * from `getAllWorktreeMeta()` makes every live folder workspace look orphaned — + * and the GC then deletes history the user is still accumulating. + */ export function getKnownWorktreeIdsForHistoryGc( - store: Pick + store: Pick, + readOtherProfiles = getOtherProfileWorktreeIdsForHistoryGc ): Set { - return new Set(Object.keys(store.getAllWorktreeMeta())) + const live = new Set(Object.keys(store.getAllWorktreeMeta())) + for (const workspace of store.getFolderWorkspaces()) { + live.add(folderWorkspaceKey(workspace.id)) + } + // Why the other profiles too: the history root is not profile-scoped but this + // store is, so on its own the live set condemns every other profile's history + // the moment the user switches. An unreadable profile means those ids are + // unknown, and an incomplete live set is exactly what deletes real history — + // so report the empty set, which runHistoryGc treats as "prune nothing". + const others = readOtherProfiles() + if (others.unreadableProfiles > 0) { + console.warn( + `[pty:history:gc] Skipping GC: ${others.unreadableProfiles} profile(s) could not be read` + ) + return new Set() + } + for (const id of others.ids) { + live.add(id) + } + return live } diff --git a/src/main/wsl-fish-history-cleanup.test.ts b/src/main/wsl-fish-history-cleanup.test.ts new file mode 100644 index 00000000000..fd8db40f01d --- /dev/null +++ b/src/main/wsl-fish-history-cleanup.test.ts @@ -0,0 +1,89 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + __resetWslFishHistoryCleanups, + deleteWslFishHistoryFile, + flushWslFishHistoryCleanups +} from './wsl-fish-history-cleanup' + +// Why reset on BOTH sides: the cleanup queue is module state, so one test's +// pending work would otherwise serialize behind — or deadlock — the next. +beforeEach(() => { + __resetWslFishHistoryCleanups() +}) + +afterEach(async () => { + await flushWslFishHistoryCleanups() + __resetWslFishHistoryCleanups() +}) + +describe('deleteWslFishHistoryFile', () => { + it('uses direct argv and bounds a distro cleanup subprocess', async () => { + const run = vi.fn().mockResolvedValue({ stdout: '', stderr: '' }) + + await deleteWslFishHistoryFile('Ubuntu Test', 'orca_0123456789abcdef', run) + + expect(run).toHaveBeenCalledWith( + 'wsl.exe', + [ + '--distribution', + 'Ubuntu Test', + '--exec', + 'fish', + '--command', + expect.stringContaining('orca_0123456789abcdef_history') + ], + { timeout: 5_000, windowsHide: true } + ) + }) + + it('rejects an unsafe session before spawning', async () => { + const run = vi.fn() + + await deleteWslFishHistoryFile('Ubuntu', '../../user-history', run) + + expect(run).not.toHaveBeenCalled() + }) + + it('coalesces concurrent requests for the same distro and session', async () => { + // Why: the GC sweep and an explicit worktree delete can queue the same + // tombstone, and one wsl.exe launch per cleanup is enough. + let settle!: () => void + const run = vi.fn().mockReturnValue( + new Promise((resolve) => { + settle = () => resolve({ stdout: '', stderr: '' }) + }) + ) + const session = 'orca_0123456789abcdef' + + const first = deleteWslFishHistoryFile('Ubuntu', session, run) + const second = deleteWslFishHistoryFile('Ubuntu', session, run) + + expect(second).toBe(first) + expect(run).toHaveBeenCalledTimes(1) + settle() + await Promise.all([first, second]) + }) + + it('keeps distinct distros independent of each other', async () => { + const run = vi.fn().mockResolvedValue({ stdout: '', stderr: '' }) + + await Promise.all([ + deleteWslFishHistoryFile('Ubuntu', 'orca_0123456789abcdef', run), + deleteWslFishHistoryFile('Debian', 'orca_0123456789abcdef', run) + ]) + + expect(run).toHaveBeenCalledTimes(2) + }) + + it('permits a retry once a failed cleanup settles', async () => { + const run = vi.fn().mockRejectedValueOnce(new Error('distro offline')).mockResolvedValue({ + stdout: '', + stderr: '' + }) + const session = 'orca_0123456789abcdef' + + await expect(deleteWslFishHistoryFile('Ubuntu', session, run)).rejects.toThrow('distro offline') + await expect(deleteWslFishHistoryFile('Ubuntu', session, run)).resolves.toBeUndefined() + expect(run).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/wsl-fish-history-cleanup.ts b/src/main/wsl-fish-history-cleanup.ts new file mode 100644 index 00000000000..18aebb650a2 --- /dev/null +++ b/src/main/wsl-fish-history-cleanup.ts @@ -0,0 +1,81 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { isSafeFishHistorySession } from './fish-history-session' + +const execFileAsync = promisify(execFile) + +/** Deduped by distro+session, so a rescan that re-queues the same tombstone + * joins the running call instead of spawning a second `wsl.exe`. + * + * Not otherwise rate-limited: the caller admits at most + * MAX_PENDING_HISTORY_TREE_REMOVALS tombstones at a time, so fanout is bounded + * but can still be dozens of `wsl.exe` launches during a startup GC drain on a + * machine with many dead WSL fish worktrees. Narrowing that belongs with the + * tombstone scheduler that owns the admission, not here. */ +const cleanupsInFlight = new Map>() + +/** Why run this INSIDE the distro: the history file lives on the distro's own + * filesystem under its `$XDG_DATA_HOME`/`$HOME`, which only a shell in there + * can resolve. `string match -qr '^/'` rejects a relative XDG_DATA_HOME the + * same way fish itself does before falling back. */ +function fishCleanupScript(session: string): string { + return [ + 'set -l data_home $XDG_DATA_HOME', + 'string match -qr "^/" -- $data_home; or set data_home "$HOME/.local/share"', + `command rm -f -- "$data_home/fish/${session}_history"` + ].join('; ') +} + +export function deleteWslFishHistoryFile( + distro: string, + session: string, + run: typeof execFileAsync = execFileAsync +): Promise { + if (!distro.trim() || !isSafeFishHistorySession(session)) { + return Promise.resolve() + } + const key = `${distro}\0${session}` + const existing = cleanupsInFlight.get(key) + if (existing) { + return existing + } + const cleanup = runCleanup(distro, session, run).finally(() => { + cleanupsInFlight.delete(key) + }) + cleanupsInFlight.set(key, cleanup) + return cleanup +} + +async function runCleanup( + distro: string, + session: string, + run: typeof execFileAsync +): Promise { + await run( + 'wsl.exe', + ['--distribution', distro, '--exec', 'fish', '--command', fishCleanupScript(session)], + { timeout: 5_000, windowsHide: true } + ) +} + +/** Test-only: drop in-flight state so one test's pending work cannot reach the next. */ +export function __resetWslFishHistoryCleanups(): void { + cleanupsInFlight.clear() +} + +/** Await every in-flight cleanup; for deterministic shutdown and tests. + * Why snapshot-and-drain rather than re-reading the map in the loop condition: + * an entry that has settled but whose `.finally` has not yet removed it would + * make `while (size > 0)` re-await a resolved promise forever, starving the + * event loop instead of returning. */ +export async function flushWslFishHistoryCleanups(): Promise { + const awaited = new Set>() + let pending = [...cleanupsInFlight.values()] + while (pending.length > 0) { + for (const cleanup of pending) { + awaited.add(cleanup) + } + await Promise.allSettled(pending) + pending = [...cleanupsInFlight.values()].filter((cleanup) => !awaited.has(cleanup)) + } +} diff --git a/src/main/zsh-scoped-histfile.live-shell.test.ts b/src/main/zsh-scoped-histfile.live-shell.test.ts new file mode 100644 index 00000000000..98af8914ec6 --- /dev/null +++ b/src/main/zsh-scoped-histfile.live-shell.test.ts @@ -0,0 +1,106 @@ +/** + * Real-zsh proof that a worktree-scoped HISTFILE survives shell startup. + * + * macOS `/etc/zshrc` assigns `HISTFILE=${ZDOTDIR:-$HOME}/.zsh_history` with no + * check-before-set, and it runs before every wrapper file Orca controls. So the + * value `injectHistoryEnv` put in the spawn env is already gone by the time the + * user reaches a prompt — and because ZDOTDIR still points at Orca's wrapper + * dir, the replacement lands inside it. Per-worktree history was therefore a + * silent no-op on the primary platform (#11044). + * + * Only a real zsh can show this: the string the wrapper emits looks correct + * either way, and the whole bug lives in what /etc/zshrc does between the spawn + * env and the first prompt. + */ +import { execFileSync, spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { getZshShellReadyRcfileContent } from './providers/local-pty-shell-ready-wrapper-generation' +import { getZshEnvTemplate, ZSH_HISTFILE_RESTORE_BLOCK } from './shell-templates' + +// Why probe and execute the same binary: guarding on `zsh` from PATH but then +// running a hardcoded `/bin/zsh` lets the guard pass on a host that installs zsh +// elsewhere, and the test fails for a missing binary rather than a wrapper +// defect. The absolute path is resolved once so the sandboxed PATH below cannot +// lose it. +const hasZsh = process.platform !== 'win32' && spawnSync('zsh', ['--version']).status === 0 +const ZSH_PATH = hasZsh + ? (spawnSync('sh', ['-c', 'command -v zsh'], { encoding: 'utf8' }).stdout || '').trim() + : '' +const itWithZsh = hasZsh ? it : it.skip + +function runLoginZsh(home: string, zdotdir: string, env: Record): string { + // -o noglobalrcs is deliberately NOT passed: /etc/zshrc is the thing under test. + return execFileSync(ZSH_PATH, ['-li', '-c', 'echo "RESULT=$HISTFILE"'], { + encoding: 'utf8', + timeout: 20_000, + env: { + PATH: '/usr/bin:/bin', + HOME: home, + ZDOTDIR: zdotdir, + ORCA_ORIG_ZDOTDIR: home, + ORCA_ZSHENV_SOURCE_DIR: home, + ...env + } + }) +} + +describe('worktree-scoped HISTFILE survives zsh startup', () => { + const withWrapper = (run: (home: string, zdotdir: string) => void): void => { + const home = mkdtempSync(join(tmpdir(), 'orca-scoped-histfile-')) + const zdotdir = join(home, 'shell-ready', 'zsh') + mkdirSync(zdotdir, { recursive: true }) + writeFileSync(join(zdotdir, '.zshenv'), getZshEnvTemplate(zdotdir)) + writeFileSync(join(zdotdir, '.zshrc'), getZshShellReadyRcfileContent()) + writeFileSync(join(zdotdir, '.zlogin'), `${ZSH_HISTFILE_RESTORE_BLOCK}\n`) + try { + run(home, zdotdir) + } finally { + rmSync(home, { recursive: true, force: true }) + } + } + + itWithZsh('keeps the injected path that a system zshrc would otherwise clobber', () => { + withWrapper((home, zdotdir) => { + const scoped = join(home, 'orca-history', 'zsh_history') + + const output = runLoginZsh(home, zdotdir, { HISTFILE: scoped, ORCA_HISTFILE: scoped }) + + expect(output).toContain(`RESULT=${scoped}`) + }) + }) + + itWithZsh('never leaves history inside Orca’s own wrapper directory', () => { + withWrapper((home, zdotdir) => { + const scoped = join(home, 'orca-history', 'zsh_history') + + const output = runLoginZsh(home, zdotdir, { HISTFILE: scoped, ORCA_HISTFILE: scoped }) + + // The exact failure mode of #11044: history written into shell-ready/zsh. + expect(output).not.toContain(zdotdir) + }) + }) + + itWithZsh('leaves HISTFILE exactly as an unwrapped zsh would when Orca injects nothing', () => { + // Why compared against an unwrapped run rather than asserted non-empty: what + // zsh defaults to is platform-specific. macOS `/etc/zshrc` assigns HISTFILE, + // so it is always set there; a stock Ubuntu zsh has no such file and leaves + // it EMPTY. The contract is that Orca's wrapper does not change it either + // way, which is the same assertion on both. + withWrapper((home, zdotdir) => { + const wrapped = runLoginZsh(home, zdotdir, {}) + const unwrapped = execFileSync(ZSH_PATH, ['-li', '-c', 'echo "RESULT=$HISTFILE"'], { + encoding: 'utf8', + timeout: 20_000, + env: { PATH: '/usr/bin:/bin', HOME: home } + }) + + const histfileOf = (output: string): string => + /^RESULT=(.*)$/m.exec(output)?.[1]?.trim() ?? '' + expect(histfileOf(wrapped)).toBe(histfileOf(unwrapped)) + expect(wrapped).not.toContain('ORCA_HISTFILE') + }) + }) +}) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index c077378ebda..16d256e72a9 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -2,7 +2,7 @@ import type { IPty } from 'node-pty' import type * as NodePty from 'node-pty' import { existsSync } from 'node:fs' -import { join } from 'node:path' +import { basename, join } from 'node:path' import { randomUUID } from 'node:crypto' import { resolveWindowsGitBashShellPath } from '../main/git-bash' import { WINDOWS_GIT_BASH_SHELL } from '../shared/windows-terminal-shell' @@ -79,6 +79,12 @@ import { type AgentSessionOwnerBinding } from '../shared/agent-session-host-authority' import { createPtySlaveEchoProbe, readPtySlavePath } from '../shared/pty-slave-line-discipline-echo' +import { + deleteRelayFishHistory, + deleteRelayHistory, + injectRelayFishHistoryEnv, + injectRelayHistoryEnv +} from './terminal-history' // Why: only Linux compiles node-pty (no prebuilt), so the build-tools remedy is a closable setup gap // there and wrong advice anywhere node-pty ships one. The relay only sees an unloadable binding, never @@ -158,6 +164,7 @@ type ManagedPty = { shellPathEnv?: string envToDelete: string[] gitCredentialPromptGuarded: boolean + historyIsolationEnabled?: boolean startupCommand?: ManagedStartupCommand physicalExit?: PhysicalExitTracker forceKillSent?: boolean @@ -327,6 +334,8 @@ type SerializedPtyEntry = { envToDelete?: string[] /** Optional for state serialized by relays predating the credential guard. */ gitCredentialPromptGuarded?: boolean + /** Optional for state serialized by relays predating scoped history. */ + historyIsolationEnabled?: boolean agentSessionOwners?: AgentSessionOwnerBinding[] } @@ -860,6 +869,13 @@ export class PtyHandler { this.dispatcher.onRequest('pty.serialize', (p) => this.serialize(p)) this.dispatcher.onRequest('pty.revive', (p) => this.revive(p)) this.dispatcher.onRequest('pty.getProfiles', async () => listShellProfiles()) + this.dispatcher.onRequest('pty.deleteWorktreeHistory', async (p) => { + if (typeof p.worktreeId === 'string') { + deleteRelayFishHistory(p.worktreeId) + deleteRelayHistory(p.worktreeId) + } + return { ok: true } + }) this.dispatcher.onRequest('pty.closeStartupQueryAuthority', (p) => this.closeStartupQueryAuthority(p) ) @@ -1387,7 +1403,8 @@ export class PtyHandler { context?: RequestContext ): Promise { const env = params.env as Record | undefined - const worktreeId = env?.ORCA_WORKTREE_ID + const worktreeId = + typeof params.worktreeId === 'string' ? params.worktreeId : env?.ORCA_WORKTREE_ID const worktreePath = worktreeId ? splitWorktreeId(worktreeId)?.worktreePath : undefined const cwd = typeof params.cwd === 'string' ? params.cwd : resolveDefaultCwd() const finishCreation = this.beginPtyCreation([worktreePath, cwd]) @@ -1503,7 +1520,9 @@ export class PtyHandler { const shellOverride = typeof params.shellOverride === 'string' ? params.shellOverride.trim() : '' const resolvedShellOverride = resolvePtyShellOverride(shellOverride) - const shell = resolvedShellOverride || resolveDefaultShell() + const requestedEnvShell = + process.platform !== 'win32' && typeof env?.SHELL === 'string' ? env.SHELL.trim() : '' + const shell = resolvedShellOverride || requestedEnvShell || resolveDefaultShell() let id: string do { id = `pty-${this.nextId++}` @@ -1525,6 +1544,15 @@ export class PtyHandler { { id, paneKey, shell, command, launchAgent }, envToDelete ) + const worktreeId = + typeof params.worktreeId === 'string' ? params.worktreeId : env?.ORCA_WORKTREE_ID + const historyIsolationEnabled = params.historyIsolationEnabled === true + if (historyIsolationEnabled && worktreeId && basename(shell).toLowerCase().startsWith('fish')) { + injectRelayFishHistoryEnv(spawnEnv, worktreeId) + } + if (historyIsolationEnabled && worktreeId) { + injectRelayHistoryEnv(spawnEnv, worktreeId, shell) + } const launchCommandHint = resolveSetupAgentSequenceLaunchCommand(spawnEnv, command) // Why: SSH PTYs bypass main's host-env builder, so apply the guard after the relay merges its authoritative env. const gitCredentialPromptGuarded = applyTerminalGitCredentialPromptGuard(spawnEnv, { @@ -1591,7 +1619,6 @@ export class PtyHandler { paneKey: typeof params.paneKey === 'string' ? params.paneKey : paneKey, tabId: typeof params.tabId === 'string' ? params.tabId : tabId } - const worktreeId = typeof env?.ORCA_WORKTREE_ID === 'string' ? env.ORCA_WORKTREE_ID : undefined const startupIngressIntent = params.startupIngressVersion === PTY_STARTUP_INGRESS_VERSION ? parsePtyStartupIngressIntent(params.startupIngress) @@ -1612,6 +1639,7 @@ export class PtyHandler { ...(explicitTerm !== undefined ? { explicitTerm } : {}), envToDelete, gitCredentialPromptGuarded, + ...(historyIsolationEnabled ? { historyIsolationEnabled: true } : {}), shellPath: shell, shellCwd: cwd, shellPathEnv: spawnEnv.PATH, @@ -2034,6 +2062,7 @@ export class PtyHandler { ...(managed.explicitTerm !== undefined ? { explicitTerm: managed.explicitTerm } : {}), envToDelete: managed.envToDelete, gitCredentialPromptGuarded: managed.gitCredentialPromptGuarded, + ...(managed.historyIsolationEnabled ? { historyIsolationEnabled: true } : {}), ...(managed.terminalHandle ? { terminalHandle: managed.terminalHandle } : {}) }) } @@ -2097,11 +2126,22 @@ export class PtyHandler { // Why: serialized state may come from an older/untrusted client; reapply fresh-spawn bounds. const envToDelete = sanitizeEnvToDelete(entry.envToDelete) const shell = resolveDefaultShell() + const historyIsolationEnabled = entry.historyIsolationEnabled === true const spawnEnv = this.buildSpawnEnv( revivedEnv, { id: entry.id, paneKey: entry.paneKey, shell }, envToDelete ) + if ( + historyIsolationEnabled && + entry.worktreeId && + basename(shell).toLowerCase().startsWith('fish') + ) { + injectRelayFishHistoryEnv(spawnEnv, entry.worktreeId) + } + if (historyIsolationEnabled && entry.worktreeId) { + injectRelayHistoryEnv(spawnEnv, entry.worktreeId, shell) + } // Why: revive lacks the original launch command, so reuse the fresh-spawn guard decision (legacy defaults to unguarded). const gitCredentialPromptGuarded = entry.gitCredentialPromptGuarded === true if (gitCredentialPromptGuarded) { @@ -2137,6 +2177,7 @@ export class PtyHandler { ...(explicitTerm !== undefined ? { explicitTerm } : {}), envToDelete, gitCredentialPromptGuarded, + ...(historyIsolationEnabled ? { historyIsolationEnabled: true } : {}), ownerBackend: resolvePtyOwnerBackend({ platform: process.platform, shellPath: shell diff --git a/src/relay/pty-shell-launch.ts b/src/relay/pty-shell-launch.ts index eaa9ac15e4c..1af77e87b7c 100644 --- a/src/relay/pty-shell-launch.ts +++ b/src/relay/pty-shell-launch.ts @@ -1,28 +1,14 @@ -import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' -import { dirname, join } from 'node:path' -import { getPosixOmpShellWrapper } from '../main/pty/omp-shell-wrapper' -import { - BASH_PROMPT_COMMAND_COMPOSITION_BLOCK, - getZshFinalZdotdirRestoreBlock, - getZshShellReadyMarkerRegistrationBlock, - SHELL_STARTUP_IDENTITY_MARKER_BLOCK, - getZshStartupFileSourceBlock -} from '../main/shell-templates' - +import { join } from 'node:path' +import { ensureOverlayRestoreWrappers } from './pty-shell-overlay-wrappers' const RELAY_SHELL_READY_DIR = '.orca-relay/shell-ready' const POSIX_LOGIN_ARGS = ['-l'] -const SHELL_READY_MARKER_ESCAPED = '\\033]777;orca-shell-ready\\007' export type RelayShellLaunchConfig = { args: string[] env: Record } -function quotePosixSingle(value: string): string { - return `'${value.replace(/'/g, `'\\''`)}'` -} - function shellBasename(shellPath: string): string { return shellPath.replace(/\\/g, '/').split('/').pop()?.toLowerCase() ?? '' } @@ -81,189 +67,6 @@ function resolveOriginalZdotdir(env: Record): string { ) } -function ensureOverlayRestoreWrappers(root: string): void { - const zshDir = join(root, 'zsh') - const bashDir = join(root, 'bash') - - const zshEnv = `# Orca relay zsh overlay wrapper -${SHELL_STARTUP_IDENTITY_MARKER_BLOCK} -export ORCA_ORIG_ZDOTDIR="\${ORCA_ORIG_ZDOTDIR:-$HOME}" -case "\${ORCA_ORIG_ZDOTDIR%/}" in - */shell-ready/zsh) export ORCA_ORIG_ZDOTDIR="$HOME" ;; -esac -[[ -f "$ORCA_ORIG_ZDOTDIR/.zshenv" ]] && source "$ORCA_ORIG_ZDOTDIR/.zshenv" -export ORCA_USER_ZDOTDIR="\${ZDOTDIR:-\${ORCA_ORIG_ZDOTDIR:-$HOME}}" -case "\${ORCA_USER_ZDOTDIR%/}" in - */shell-ready/zsh) export ORCA_USER_ZDOTDIR="$HOME" ;; -esac -export ZDOTDIR=${quotePosixSingle(zshDir)} -` - const zshProfile = `# Orca relay zsh overlay wrapper -${getZshStartupFileSourceBlock({ - fileName: '.zprofile', - homeExpression: '"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"' -})} -` - const zshRc = `# Orca relay zsh overlay wrapper -${getZshStartupFileSourceBlock({ - fileName: '.zshrc', - homeExpression: '"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"', - interactiveOnly: true -})} -if [[ ! -o login ]]; then - # Why: remote startup files can re-export user defaults after relay spawn. - [[ -n "\${ORCA_OPENCODE_CONFIG_DIR:-}" ]] && export OPENCODE_CONFIG_DIR="\${ORCA_OPENCODE_CONFIG_DIR}" - [[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" - [[ -n "\${ORCA_REMOTE_CLI_BIN_DIR:-}" ]] && case ":$PATH:" in *:"\${ORCA_REMOTE_CLI_BIN_DIR}":*) ;; *) export PATH="\${ORCA_REMOTE_CLI_BIN_DIR}:$PATH" ;; esac - ${getPosixOmpShellWrapper()} -fi -if [[ ! -o login ]]; then -${getZshFinalZdotdirRestoreBlock('"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"')} -fi -` - const zshLogin = `# Orca relay zsh overlay wrapper -${getZshStartupFileSourceBlock({ - fileName: '.zlogin', - homeExpression: '"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"', - interactiveOnly: true -})} -# Why: .zlogin is the final zsh login startup file before the prompt. -[[ -n "\${ORCA_OPENCODE_CONFIG_DIR:-}" ]] && export OPENCODE_CONFIG_DIR="\${ORCA_OPENCODE_CONFIG_DIR}" -[[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" -[[ -n "\${ORCA_REMOTE_CLI_BIN_DIR:-}" ]] && case ":$PATH:" in *:"\${ORCA_REMOTE_CLI_BIN_DIR}":*) ;; *) export PATH="\${ORCA_REMOTE_CLI_BIN_DIR}:$PATH" ;; esac -${getPosixOmpShellWrapper()} -${getZshFinalZdotdirRestoreBlock('"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"')} -${getZshShellReadyMarkerRegistrationBlock(SHELL_READY_MARKER_ESCAPED)} -` - const bashRc = `# Orca relay bash overlay wrapper -${SHELL_STARTUP_IDENTITY_MARKER_BLOCK} -[[ -f /etc/profile ]] && source /etc/profile -if [[ -f "$HOME/.bash_profile" ]]; then - source "$HOME/.bash_profile" -elif [[ -f "$HOME/.bash_login" ]]; then - source "$HOME/.bash_login" -elif [[ -f "$HOME/.profile" ]]; then - source "$HOME/.profile" -fi -# Why: enable bracketed paste so Orca can deliver a multiline startup prompt as -# a single literal paste (ESC[200~…ESC[201~); without it, older readline builds -# treat each embedded newline as Enter and mangle the prompt into PS2 -# continuation. Modern readline defaults this on; force it for the rest. -[[ $- == *i* ]] && bind 'set enable-bracketed-paste on' 2>/dev/null -# Why: remote startup files can re-export user defaults after relay spawn. -[[ -n "\${ORCA_OPENCODE_CONFIG_DIR:-}" ]] && export OPENCODE_CONFIG_DIR="\${ORCA_OPENCODE_CONFIG_DIR}" -[[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" -[[ -n "\${ORCA_REMOTE_CLI_BIN_DIR:-}" ]] && case ":$PATH:" in *:"\${ORCA_REMOTE_CLI_BIN_DIR}":*) ;; *) export PATH="\${ORCA_REMOTE_CLI_BIN_DIR}:$PATH" ;; esac -${getPosixOmpShellWrapper()} -# Why: SSH bash sessions need the same command lifecycle markers as local -# bash so agent rows stop showing "working" when the foreground command exits. -__orca_initializing_wrapper=1 -__orca_osc133_precmd() { - local exit_code=$? - __orca_in_prompt_command=1 - if [[ -n "\${__orca_in_command:-}" ]]; then - printf "\\033]133;D;%s\\007" "$exit_code" - unset __orca_in_command - fi - printf "\\033]133;A\\007" - return "$exit_code" -} -__orca_osc133_prompt_done() { - unset __orca_in_prompt_command; __orca_adopt_outer_debug_trap - trap '__orca_osc133_preexec' DEBUG -} -__orca_osc133_preexec() { - if [[ -n "\${__orca_prompt_status_capture_command:-}" && "$BASH_COMMAND" == "$__orca_prompt_status_capture_command" ]]; then - unset __orca_initial_prompt - __orca_in_legacy_prompt_wrapper=1 - return 0 - fi - if [[ -n "\${__orca_initializing_wrapper:-}\${__orca_in_debug_capture:-}\${__orca_initial_prompt:-}\${__orca_in_prompt_dispatch:-}\${__orca_in_legacy_prompt_wrapper:-}\${__orca_in_prompt_command:-}" ]]; then - [[ -z "\${__orca_initializing_wrapper:-}\${__orca_in_debug_capture:-}" ]] || return 0 - if [[ -n "\${__orca_initial_prompt:-}" && "$BASH_COMMAND" == "__orca_osc133_precmd" ]]; then - unset __orca_initial_prompt; return 0 - fi - if [[ -n "\${__orca_in_prompt_dispatch:-}" ]]; then - [[ -n "\${__orca_dispatching_user_prompt_command:-}" ]] || return 0 - if [[ "\${FUNCNAME[1]:-}" == "__orca_run_prompt_command_array" ]]; then - case "$BASH_COMMAND" in - '(( __orca_exit_code == 0 ))'|'__orca_restore_prompt_status "$__orca_exit_code"'|'eval "$__orca_prompt_part"'|'eval "$__orca_final_prompt_command"'|__orca_dispatching_user_prompt_command=*|__orca_osc133_precmd|__orca_osc133_prompt_done|__orca_prompt_mark) return 0 ;; - esac - fi - elif [[ "\${FUNCNAME[1]:-}" == "__orca_run_prompt_command_array" || "$BASH_COMMAND" == "__orca_run_prompt_command_array" ]]; then - return 0 - fi - [[ -z "\${__orca_in_legacy_prompt_wrapper:-}" || -n "\${__orca_dispatching_user_prompt_command:-}" ]] || return 0 - if [[ -n "\${__orca_in_prompt_command:-}" && "$BASH_COMMAND" == "__orca_in_debug_capture=1" ]]; then - return 0 - fi - fi - case "\${FUNCNAME[1]:-}" in __orca_osc133_*|__orca_prompt_mark|__orca_restore_prompt_status) return 0 ;; esac - case "$BASH_COMMAND" in __orca_osc133_precmd|__orca_osc133_prompt_done|__orca_prompt_mark) return 0 ;; esac - __orca_run_user_debug_trap - [[ -z "\${__orca_in_prompt_command:-}" ]] || return 0 - [[ -z "\${__orca_in_command:-}" ]] || return 0 - printf "\\033]133;C\\007" - __orca_in_command=1 -} -${BASH_PROMPT_COMMAND_COMPOSITION_BLOCK} -__orca_prepend_prompt_command "__orca_osc133_precmd" -# Why: SSH startup commands are renderer-delivered; emit the same internal -# readiness marker as local shells only when that delivery mode asks for it. -if [[ "\${ORCA_SHELL_READY_MARKER:-0}" == "1" ]]; then - __orca_prompt_mark() { - printf "${SHELL_READY_MARKER_ESCAPED}" - } - __orca_append_prompt_command "__orca_prompt_mark" -fi -__orca_append_prompt_command '__orca_in_debug_capture=1; __orca_prompt_had_functrace=""; if [[ -o functrace ]]; then __orca_prompt_had_functrace=1; set +T; fi; __orca_outer_debug_trap_spec="$(trap -p DEBUG)"; [[ -z "$__orca_prompt_had_functrace" ]] || set -T; unset __orca_prompt_had_functrace __orca_in_debug_capture' -__orca_append_prompt_command "__orca_osc133_prompt_done" -__orca_had_functrace="" -[[ -o functrace ]] && __orca_had_functrace=1 -set +T -__orca_debug_trap_spec="$(trap -p DEBUG)" -[[ -z "$__orca_had_functrace" ]] || set -T -if [[ -n "$__orca_debug_trap_spec" && "$__orca_debug_trap_spec" != "trap -- '__orca_osc133_preexec' DEBUG" ]]; then - __orca_debug_trap_command="\${__orca_debug_trap_spec#trap -- }" - __orca_debug_trap_command="\${__orca_debug_trap_command% DEBUG}" - eval "__orca_user_debug_trap=$__orca_debug_trap_command" -fi -unset __orca_debug_trap_spec __orca_debug_trap_command __orca_had_functrace -unset -f __orca_normalize_prompt_command_part __orca_normalize_prompt_command __orca_prepend_prompt_command __orca_append_prompt_command -unset __orca_prompt_command_normalized -# Why: arm DEBUG after wrapper setup so the relay rcfile itself does not emit -# fake command-start/end markers before the first prompt. -__orca_initial_prompt=1 -trap '__orca_osc133_preexec' DEBUG -unset __orca_initializing_wrapper -` - - const files = [ - [join(zshDir, '.zshenv'), zshEnv], - [join(zshDir, '.zprofile'), zshProfile], - [join(zshDir, '.zshrc'), zshRc], - [join(zshDir, '.zlogin'), zshLogin], - [join(bashDir, 'rcfile'), bashRc] - ] as const - - for (const [path, content] of files) { - mkdirSync(dirname(path), { recursive: true }) - let existing: string | null = null - try { - existing = readFileSync(path, 'utf8') - } catch { - existing = null - } - // Why: relay wrapper files persist under ~/.orca-relay across app - // upgrades. Existence alone is not enough; stale wrappers would miss - // later fixes such as preserving post-.zshenv ZDOTDIR. - if (existing !== content) { - writeFileSync(path, content, 'utf8') - } - chmodSync(path, 0o644) - } -} - export function getRelayShellLaunchConfig( shellPath: string, env: Record, diff --git a/src/relay/pty-shell-overlay-wrappers.ts b/src/relay/pty-shell-overlay-wrappers.ts new file mode 100644 index 00000000000..1fa2e9b821a --- /dev/null +++ b/src/relay/pty-shell-overlay-wrappers.ts @@ -0,0 +1,207 @@ +import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { getPosixOmpShellWrapper } from '../main/pty/omp-shell-wrapper' +import { + BASH_PROMPT_COMMAND_COMPOSITION_BLOCK, + getZshFinalZdotdirRestoreBlock, + getZshShellReadyMarkerRegistrationBlock, + SHELL_STARTUP_IDENTITY_MARKER_BLOCK, + ZSH_HISTFILE_RESTORE_BLOCK, + getZshStartupFileSourceBlock +} from '../main/shell-templates' + +/** Writes the zsh/bash overlay wrapper files a relay-spawned shell sources. + * Split from pty-shell-launch.ts so the launch-config decisions stay readable + * next to each other rather than buried under ~150 lines of shell templates. */ + +const SHELL_READY_MARKER_ESCAPED = '\\033]777;orca-shell-ready\\007' + +function quotePosixSingle(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'` +} + +export function ensureOverlayRestoreWrappers(root: string): void { + const zshDir = join(root, 'zsh') + const bashDir = join(root, 'bash') + + const zshEnv = `# Orca relay zsh overlay wrapper +${SHELL_STARTUP_IDENTITY_MARKER_BLOCK} +export ORCA_ORIG_ZDOTDIR="\${ORCA_ORIG_ZDOTDIR:-$HOME}" +case "\${ORCA_ORIG_ZDOTDIR%/}" in + */shell-ready/zsh) export ORCA_ORIG_ZDOTDIR="$HOME" ;; +esac +[[ -f "$ORCA_ORIG_ZDOTDIR/.zshenv" ]] && source "$ORCA_ORIG_ZDOTDIR/.zshenv" +export ORCA_USER_ZDOTDIR="\${ZDOTDIR:-\${ORCA_ORIG_ZDOTDIR:-$HOME}}" +case "\${ORCA_USER_ZDOTDIR%/}" in + */shell-ready/zsh) export ORCA_USER_ZDOTDIR="$HOME" ;; +esac +export ZDOTDIR=${quotePosixSingle(zshDir)} +` + const zshProfile = `# Orca relay zsh overlay wrapper +${getZshStartupFileSourceBlock({ + fileName: '.zprofile', + homeExpression: '"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"' +})} +` + const zshRc = `# Orca relay zsh overlay wrapper +${getZshStartupFileSourceBlock({ + fileName: '.zshrc', + homeExpression: '"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"', + interactiveOnly: true +})} +if [[ ! -o login ]]; then + # Why: remote startup files can re-export user defaults after relay spawn. + [[ -n "\${ORCA_OPENCODE_CONFIG_DIR:-}" ]] && export OPENCODE_CONFIG_DIR="\${ORCA_OPENCODE_CONFIG_DIR}" + [[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" + [[ -n "\${ORCA_REMOTE_CLI_BIN_DIR:-}" ]] && case ":$PATH:" in *:"\${ORCA_REMOTE_CLI_BIN_DIR}":*) ;; *) export PATH="\${ORCA_REMOTE_CLI_BIN_DIR}:$PATH" ;; esac + ${getPosixOmpShellWrapper()} +${ZSH_HISTFILE_RESTORE_BLOCK} +fi +if [[ ! -o login ]]; then +${getZshFinalZdotdirRestoreBlock('"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"')} +fi +` + const zshLogin = `# Orca relay zsh overlay wrapper +${getZshStartupFileSourceBlock({ + fileName: '.zlogin', + homeExpression: '"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"', + interactiveOnly: true +})} +# Why: .zlogin is the final zsh login startup file before the prompt. +[[ -n "\${ORCA_OPENCODE_CONFIG_DIR:-}" ]] && export OPENCODE_CONFIG_DIR="\${ORCA_OPENCODE_CONFIG_DIR}" +[[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" +[[ -n "\${ORCA_REMOTE_CLI_BIN_DIR:-}" ]] && case ":$PATH:" in *:"\${ORCA_REMOTE_CLI_BIN_DIR}":*) ;; *) export PATH="\${ORCA_REMOTE_CLI_BIN_DIR}:$PATH" ;; esac +${getPosixOmpShellWrapper()} +${ZSH_HISTFILE_RESTORE_BLOCK} +${getZshFinalZdotdirRestoreBlock('"${ORCA_USER_ZDOTDIR:-${ORCA_ORIG_ZDOTDIR:-$HOME}}"')} +${getZshShellReadyMarkerRegistrationBlock(SHELL_READY_MARKER_ESCAPED)} +` + const bashRc = `# Orca relay bash overlay wrapper +${SHELL_STARTUP_IDENTITY_MARKER_BLOCK} +[[ -f /etc/profile ]] && source /etc/profile +if [[ -f "$HOME/.bash_profile" ]]; then + source "$HOME/.bash_profile" +elif [[ -f "$HOME/.bash_login" ]]; then + source "$HOME/.bash_login" +elif [[ -f "$HOME/.profile" ]]; then + source "$HOME/.profile" +fi +# Why: enable bracketed paste so Orca can deliver a multiline startup prompt as +# a single literal paste (ESC[200~…ESC[201~); without it, older readline builds +# treat each embedded newline as Enter and mangle the prompt into PS2 +# continuation. Modern readline defaults this on; force it for the rest. +[[ $- == *i* ]] && bind 'set enable-bracketed-paste on' 2>/dev/null +# Why: remote startup files can re-export user defaults after relay spawn. +[[ -n "\${ORCA_OPENCODE_CONFIG_DIR:-}" ]] && export OPENCODE_CONFIG_DIR="\${ORCA_OPENCODE_CONFIG_DIR}" +[[ -n "\${ORCA_MIMOCODE_HOME:-}" ]] && export MIMOCODE_HOME="\${ORCA_MIMOCODE_HOME}" +[[ -n "\${ORCA_REMOTE_CLI_BIN_DIR:-}" ]] && case ":$PATH:" in *:"\${ORCA_REMOTE_CLI_BIN_DIR}":*) ;; *) export PATH="\${ORCA_REMOTE_CLI_BIN_DIR}:$PATH" ;; esac +${getPosixOmpShellWrapper()} +${ZSH_HISTFILE_RESTORE_BLOCK} +# Why: SSH bash sessions need the same command lifecycle markers as local +# bash so agent rows stop showing "working" when the foreground command exits. +__orca_initializing_wrapper=1 +__orca_osc133_precmd() { + local exit_code=$? + __orca_in_prompt_command=1 + if [[ -n "\${__orca_in_command:-}" ]]; then + printf "\\033]133;D;%s\\007" "$exit_code" + unset __orca_in_command + fi + printf "\\033]133;A\\007" + return "$exit_code" +} +__orca_osc133_prompt_done() { + unset __orca_in_prompt_command; __orca_adopt_outer_debug_trap + trap '__orca_osc133_preexec' DEBUG +} +__orca_osc133_preexec() { + if [[ -n "\${__orca_prompt_status_capture_command:-}" && "$BASH_COMMAND" == "$__orca_prompt_status_capture_command" ]]; then + unset __orca_initial_prompt + __orca_in_legacy_prompt_wrapper=1 + return 0 + fi + if [[ -n "\${__orca_initializing_wrapper:-}\${__orca_in_debug_capture:-}\${__orca_initial_prompt:-}\${__orca_in_prompt_dispatch:-}\${__orca_in_legacy_prompt_wrapper:-}\${__orca_in_prompt_command:-}" ]]; then + [[ -z "\${__orca_initializing_wrapper:-}\${__orca_in_debug_capture:-}" ]] || return 0 + if [[ -n "\${__orca_initial_prompt:-}" && "$BASH_COMMAND" == "__orca_osc133_precmd" ]]; then + unset __orca_initial_prompt; return 0 + fi + if [[ -n "\${__orca_in_prompt_dispatch:-}" ]]; then + [[ -n "\${__orca_dispatching_user_prompt_command:-}" ]] || return 0 + if [[ "\${FUNCNAME[1]:-}" == "__orca_run_prompt_command_array" ]]; then + case "$BASH_COMMAND" in + '(( __orca_exit_code == 0 ))'|'__orca_restore_prompt_status "$__orca_exit_code"'|'eval "$__orca_prompt_part"'|'eval "$__orca_final_prompt_command"'|__orca_dispatching_user_prompt_command=*|__orca_osc133_precmd|__orca_osc133_prompt_done|__orca_prompt_mark) return 0 ;; + esac + fi + elif [[ "\${FUNCNAME[1]:-}" == "__orca_run_prompt_command_array" || "$BASH_COMMAND" == "__orca_run_prompt_command_array" ]]; then + return 0 + fi + [[ -z "\${__orca_in_legacy_prompt_wrapper:-}" || -n "\${__orca_dispatching_user_prompt_command:-}" ]] || return 0 + if [[ -n "\${__orca_in_prompt_command:-}" && "$BASH_COMMAND" == "__orca_in_debug_capture=1" ]]; then + return 0 + fi + fi + case "\${FUNCNAME[1]:-}" in __orca_osc133_*|__orca_prompt_mark|__orca_restore_prompt_status) return 0 ;; esac + case "$BASH_COMMAND" in __orca_osc133_precmd|__orca_osc133_prompt_done|__orca_prompt_mark) return 0 ;; esac + __orca_run_user_debug_trap + [[ -z "\${__orca_in_prompt_command:-}" ]] || return 0 + [[ -z "\${__orca_in_command:-}" ]] || return 0 + printf "\\033]133;C\\007" + __orca_in_command=1 +} +${BASH_PROMPT_COMMAND_COMPOSITION_BLOCK} +__orca_prepend_prompt_command "__orca_osc133_precmd" +# Why: SSH startup commands are renderer-delivered; emit the same internal +# readiness marker as local shells only when that delivery mode asks for it. +if [[ "\${ORCA_SHELL_READY_MARKER:-0}" == "1" ]]; then + __orca_prompt_mark() { + printf "${SHELL_READY_MARKER_ESCAPED}" + } + __orca_append_prompt_command "__orca_prompt_mark" +fi +__orca_append_prompt_command '__orca_in_debug_capture=1; __orca_prompt_had_functrace=""; if [[ -o functrace ]]; then __orca_prompt_had_functrace=1; set +T; fi; __orca_outer_debug_trap_spec="$(trap -p DEBUG)"; [[ -z "$__orca_prompt_had_functrace" ]] || set -T; unset __orca_prompt_had_functrace __orca_in_debug_capture' +__orca_append_prompt_command "__orca_osc133_prompt_done" +__orca_had_functrace="" +[[ -o functrace ]] && __orca_had_functrace=1 +set +T +__orca_debug_trap_spec="$(trap -p DEBUG)" +[[ -z "$__orca_had_functrace" ]] || set -T +if [[ -n "$__orca_debug_trap_spec" && "$__orca_debug_trap_spec" != "trap -- '__orca_osc133_preexec' DEBUG" ]]; then + __orca_debug_trap_command="\${__orca_debug_trap_spec#trap -- }" + __orca_debug_trap_command="\${__orca_debug_trap_command% DEBUG}" + eval "__orca_user_debug_trap=$__orca_debug_trap_command" +fi +unset __orca_debug_trap_spec __orca_debug_trap_command __orca_had_functrace +unset -f __orca_normalize_prompt_command_part __orca_normalize_prompt_command __orca_prepend_prompt_command __orca_append_prompt_command +unset __orca_prompt_command_normalized +# Why: arm DEBUG after wrapper setup so the relay rcfile itself does not emit +# fake command-start/end markers before the first prompt. +__orca_initial_prompt=1 +trap '__orca_osc133_preexec' DEBUG +unset __orca_initializing_wrapper +` + + const files = [ + [join(zshDir, '.zshenv'), zshEnv], + [join(zshDir, '.zprofile'), zshProfile], + [join(zshDir, '.zshrc'), zshRc], + [join(zshDir, '.zlogin'), zshLogin], + [join(bashDir, 'rcfile'), bashRc] + ] as const + + for (const [path, content] of files) { + mkdirSync(dirname(path), { recursive: true }) + let existing: string | null = null + try { + existing = readFileSync(path, 'utf8') + } catch { + existing = null + } + // Why: relay wrapper files persist under ~/.orca-relay across app + // upgrades. Existence alone is not enough; stale wrappers would miss + // later fixes such as preserving post-.zshenv ZDOTDIR. + if (existing !== content) { + writeFileSync(path, content, 'utf8') + } + chmodSync(path, 0o644) + } +} diff --git a/src/relay/terminal-history.test.ts b/src/relay/terminal-history.test.ts new file mode 100644 index 00000000000..b6c59e332d3 --- /dev/null +++ b/src/relay/terminal-history.test.ts @@ -0,0 +1,55 @@ +import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { hashWorktreeId } from '../main/terminal-history-id' +import { deleteRelayHistory, injectRelayHistoryEnv } from './terminal-history' + +const worktreeId = 'relay-test::/remote/worktree' +const historyDir = join(homedir(), '.orca-remote', 'terminal-history') +const historyPrefix = hashWorktreeId(worktreeId) + +afterEach(() => { + for (const filename of ['bash_history', 'zsh_history']) { + rmSync(join(historyDir, `${historyPrefix}-${filename}`), { force: true }) + } +}) + +describe('relay shell history', () => { + it.each(['/bin/bash', '/usr/bin/zsh'])('scopes %s without replacing caller HISTFILE', (shell) => { + const env: Record = {} + const dir = injectRelayHistoryEnv(env, worktreeId, shell) + expect(dir).toBe(historyDir) + expect(env.HISTFILE).toBe( + join( + historyDir, + `${historyPrefix}-${shell.endsWith('bash') ? 'bash_history' : 'zsh_history'}` + ) + ) + + const custom = { HISTFILE: '/custom/history' } + expect(injectRelayHistoryEnv(custom, worktreeId, shell)).toBeNull() + expect(custom.HISTFILE).toBe('/custom/history') + }) + + it('does not scope unsupported shells and cleans up idempotently', () => { + const env: Record = {} + expect(injectRelayHistoryEnv(env, worktreeId, '/bin/fish')).toBeNull() + deleteRelayHistory(worktreeId) + deleteRelayHistory(worktreeId) + expect(existsSync(join(historyDir, `${historyPrefix}-bash_history`))).toBe(false) + }) + + it.skipIf(process.platform === 'win32')('refuses a pre-existing final symlink', () => { + mkdirSync(historyDir, { recursive: true }) + const target = join(historyDir, 'relay-unrelated-history') + const path = join(historyDir, `${historyPrefix}-bash_history`) + writeFileSync(target, 'unrelated') + symlinkSync(target, path) + + expect(injectRelayHistoryEnv({}, worktreeId, '/bin/bash')).toBeNull() + expect(readFileSync(target, 'utf8')).toBe('unrelated') + rmSync(path) + rmSync(target) + }) +}) diff --git a/src/relay/terminal-history.ts b/src/relay/terminal-history.ts new file mode 100644 index 00000000000..37335644df1 --- /dev/null +++ b/src/relay/terminal-history.ts @@ -0,0 +1,134 @@ +import { + closeSync, + constants as fsConstants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + unlinkSync +} from 'node:fs' +import { homedir } from 'node:os' +import { basename, join } from 'node:path' +import { hashWorktreeId } from '../main/terminal-history-id' +import { + deleteFishHistoryFile, + relayFishHistorySessionName, + resolveFishHistoryDir +} from '../main/fish-history-session' + +const HISTORY_ROOT = join(homedir(), '.orca-remote', 'terminal-history') + +function historyFilename(shell: string): string | null { + const name = basename(shell).toLowerCase() + if (name.startsWith('bash')) { + return 'bash_history' + } + if (name.startsWith('zsh')) { + return 'zsh_history' + } + return null +} + +export function injectRelayHistoryEnv( + env: Record, + worktreeId: string, + shell: string +): string | null { + // Why first: same reason as the desktop path — an inherited ORCA_HISTFILE + // would otherwise survive every early return below and let the remote wrapper + // re-export another worktree's history path. + delete env.ORCA_HISTFILE + if (env.HISTFILE) { + return null + } + const filename = historyFilename(shell) + if (!filename) { + return null + } + try { + mkdirSync(HISTORY_ROOT, { recursive: true, mode: 0o700 }) + const rootStat = lstatSync(HISTORY_ROOT) + if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { + return null + } + const path = join(HISTORY_ROOT, `${hashWorktreeId(worktreeId)}-${filename}`) + let existing: ReturnType | null = null + try { + const stat = lstatSync(path, { bigint: true }) + if (stat.isSymbolicLink() || !stat.isFile()) { + return null + } + existing = stat + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + return null + } + } + const fd = openSync( + path, + fsConstants.O_RDWR | + fsConstants.O_CREAT | + (process.platform === 'win32' ? 0 : fsConstants.O_NOFOLLOW), + 0o600 + ) + const actual = fstatSync(fd, { bigint: true }) + if ( + !actual.isFile() || + (existing && (actual.dev !== existing.dev || actual.ino !== existing.ino)) + ) { + closeSync(fd) + return null + } + closeSync(fd) + env.HISTFILE = path + // Why a second variable: a remote macOS `/etc/zshrc` assigns HISTFILE + // unconditionally before the wrapper runs, so the injected value is gone by + // the first prompt. The wrapper restores it from here (#11044) — the same + // contract the desktop PTY path uses. + env.ORCA_HISTFILE = path + return HISTORY_ROOT + } catch { + return null + } +} + +export function deleteRelayHistory(worktreeId: string): void { + try { + const rootStat = lstatSync(HISTORY_ROOT) + if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { + return + } + for (const filename of ['bash_history', 'zsh_history']) { + const path = join(HISTORY_ROOT, `${hashWorktreeId(worktreeId)}-${filename}`) + try { + if (!lstatSync(path).isSymbolicLink()) { + unlinkSync(path) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } + } + } + } catch (error) { + console.warn( + `[pty:history] Failed to delete relay shell history: ${error instanceof Error ? error.message : String(error)}` + ) + } +} + +/** fish keeps history in its own data dir under a session NAME, so the relay + * isolates it the same way the desktop app does and deletes by that name. + * No metadata file is needed: the name is a pure function of the worktree id. */ +export function injectRelayFishHistoryEnv(env: Record, worktreeId: string): void { + if (env.fish_history) { + return + } + env.fish_history = relayFishHistorySessionName(hashWorktreeId(worktreeId)) +} + +export function deleteRelayFishHistory(worktreeId: string, env?: NodeJS.ProcessEnv): void { + deleteFishHistoryFile(relayFishHistorySessionName(hashWorktreeId(worktreeId)), [ + resolveFishHistoryDir(env) + ]) +} diff --git a/src/renderer/src/components/settings/CliSkillRuntimeSetup.tsx b/src/renderer/src/components/settings/CliSkillRuntimeSetup.tsx index 913b6356322..4206fb46c87 100644 --- a/src/renderer/src/components/settings/CliSkillRuntimeSetup.tsx +++ b/src/renderer/src/components/settings/CliSkillRuntimeSetup.tsx @@ -229,9 +229,8 @@ function wrapWindowsSkillCommandWithNpxPrerequisite( } function isPosixFamilyWindowsShellConfigured(): boolean { - return ( - resolveWindowsShellStartupFamily(useAppStore.getState().settings?.terminalWindowsShell) === - 'posix' + return ['posix', 'unix'].includes( + resolveWindowsShellStartupFamily(useAppStore.getState().settings?.terminalWindowsShell) ) } diff --git a/src/renderer/src/components/sidebar/folder-workspace-composer-submit.test.ts b/src/renderer/src/components/sidebar/folder-workspace-composer-submit.test.ts index 749038df3e4..a96564b6652 100644 --- a/src/renderer/src/components/sidebar/folder-workspace-composer-submit.test.ts +++ b/src/renderer/src/components/sidebar/folder-workspace-composer-submit.test.ts @@ -588,7 +588,7 @@ describe('submitFolderWorkspaceCreate', () => { 'folder-workspace-1', expect.objectContaining({ startup: expect.objectContaining({ - command: "claude 'Use Bob'\\''s POSIX startup'" + command: `claude 'Use Bob'"'"'s POSIX startup'` }) }) ) diff --git a/src/renderer/src/components/terminal-pane/pty-connection-cold-restore-agent-resume.test.ts b/src/renderer/src/components/terminal-pane/pty-connection-cold-restore-agent-resume.test.ts index 8e29afb8763..35b37855238 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection-cold-restore-agent-resume.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection-cold-restore-agent-resume.test.ts @@ -298,8 +298,7 @@ describe('connectPanePty', () => { expect(transport.connect).toHaveBeenCalledWith( expect.objectContaining({ sessionId: 'lost-pty', - command: - "codex '--dangerously-bypass-approvals-and-sandbox' 'resume' 'codex-session-1'\\''s'", + command: `codex '--dangerously-bypass-approvals-and-sandbox' 'resume' 'codex-session-1'"'"'s'`, env: expect.objectContaining({ ORCA_PANE_KEY: paneKey, ORCA_TAB_ID: 'tab-1', diff --git a/src/renderer/src/lib/ai-vault-resume-command.test.ts b/src/renderer/src/lib/ai-vault-resume-command.test.ts index f89f4054bf4..57dd3499bc5 100644 --- a/src/renderer/src/lib/ai-vault-resume-command.test.ts +++ b/src/renderer/src/lib/ai-vault-resume-command.test.ts @@ -306,7 +306,7 @@ describe('ai vault resume command runtime', () => { } }) ).toBe( - "unset CODEX_HOME; unset ORCA_CODEX_HOME; cd '/home/alice/repo' && codex 'resume' 'session one'" + `cd '/home/alice/repo' && env -u CODEX_HOME -u ORCA_CODEX_HOME codex 'resume' 'session one'` ) }) @@ -598,7 +598,7 @@ describe('ai vault resume command runtime', () => { }) expect(command).toBe( - "unset CODEX_HOME; unset ORCA_CODEX_HOME; cd '/home/alice/repo' && codex 'resume' 'session one'" + `cd '/home/alice/repo' && env -u CODEX_HOME -u ORCA_CODEX_HOME codex 'resume' 'session one'` ) expect(command).not.toContain('/retired/shared-home') }) diff --git a/src/renderer/src/lib/ai-vault-resume-command.ts b/src/renderer/src/lib/ai-vault-resume-command.ts index 92e826f83a9..45943e8f9fa 100644 --- a/src/renderer/src/lib/ai-vault-resume-command.ts +++ b/src/renderer/src/lib/ai-vault-resume-command.ts @@ -16,7 +16,6 @@ import { } from '../../../shared/tui-agent-launch-defaults' import { parseWslUncPath } from '../../../shared/wsl-paths' import type { AgentStartupShell } from '../../../shared/tui-agent-startup-shell' -import { clearEnvCommand, commandSeparator } from '../../../shared/tui-agent-startup-shell' import type { AppState } from '@/store/types' import type { AiVaultSessionDragPayload } from '@/lib/ai-vault-session-drag' import { getLocalProjectExecutionRuntimeContext } from '@/lib/local-preflight-context' @@ -64,16 +63,16 @@ type AiVaultResumeWorktreeArgs = { } export function buildAiVaultResumeCopyCommandForWorktree(args: AiVaultResumeWorktreeArgs): string { - const command = buildAiVaultResumeForWorktree(args, true).command - if (args.session.agent !== 'codex' || args.session.codexHome !== null) { - return command - } - const shell = resolveAiVaultResumeShell(args) - const separator = commandSeparator(shell) - const clearHomes = ['CODEX_HOME', 'ORCA_CODEX_HOME'] - .map((name) => clearEnvCommand(name, shell)) - .join(separator) - return `${clearHomes}${separator}${command}` + // Why an `env -u` prefix on the agent rather than a preceding clear statement: + // this text is COPIED, so it runs in a shell Orca never spawned and cannot + // seed. A clear statement has to test `$fish_pid`, an unbound expansion that + // aborts the line under `set -u` — and because the clear came first, it took + // the agent launch down with it (the regression that reverted #14863). + const clearEnvNames = + args.session.agent === 'codex' && args.session.codexHome === null + ? (['CODEX_HOME', 'ORCA_CODEX_HOME'] as const) + : undefined + return buildAiVaultResumeForWorktree(args, true, clearEnvNames).command } export function buildAiVaultResumeStartupForWorktree( @@ -121,7 +120,10 @@ export function buildAiVaultDropRepinStartup(args: { function buildAiVaultResumeForWorktree( args: AiVaultResumeWorktreeArgs, - embedCwd: boolean + embedCwd: boolean, + /** Copy-path only: names the pasted line must strip off the agent itself. + * Spawned startups drop them through `envToDelete` instead. */ + clearEnvNames?: readonly string[] ): AiVaultResumeStartup { const providerSession = getAiVaultAgentProviderSession(args.session) if ( @@ -189,14 +191,16 @@ function buildAiVaultResumeForWorktree( platform, commandOverride: startupPlan.launchConfig.agentCommand, codexHome, - shell: liveShell + shell: liveShell, + clearEnvNames }) : buildAiVaultResumeShellCommand({ resumeCommand: startupPlan.launchCommand, cwd, platform, codexHome, - shell: liveShell + shell: liveShell, + clearEnvNames }), ...(startupPlan.env ? { env: startupPlan.env } : {}), ...realHomeCodexResumeEnvDeletion(args.session), @@ -221,7 +225,8 @@ function buildAiVaultResumeForWorktree( codexHome, // Why: non-resumable agents queue through this fallback too, so it must // quote for the live Windows shell like the startup-plan branch above. - shell: liveShell + shell: liveShell, + clearEnvNames }), ...startupCwd, ...realHomeCodexResumeEnvDeletion(args.session) @@ -241,32 +246,10 @@ function resolveAiVaultResumeShell(args: AiVaultResumeWorktreeArgs): AgentStartu state: args.state, worktreeId: args.worktreeId, platform, - isLocalSession, - parsedByClientLoginShell: isLocalSession && runsOnClientLoginShell(args, platform) + isLocalSession }) } -/** - * Whether the resume line is handed to THIS machine's login shell. - * - * Why not `isLocalSession`: that only says the session file was scanned locally - * (no executionHostId), which is still true when the worktree lives on an SSH - * or runtime host — the command then goes to that host's shell. The WSL case is - * caught by the platform mismatch (a WSL worktree resolves to 'linux' on win32). - */ -function runsOnClientLoginShell( - args: AiVaultResumeWorktreeArgs, - platform: NodeJS.Platform -): boolean { - const executionHost = parseExecutionHostId( - getExecutionHostIdForWorktree(args.state, args.worktreeId ?? args.state.activeWorktreeId) - ) - if (executionHost?.kind === 'ssh' || executionHost?.kind === 'runtime') { - return false - } - return platform === CLIENT_PLATFORM -} - export function getAiVaultAgentProviderSession( session: Pick & { filePath?: string } ): AgentProviderSessionMetadata | null { diff --git a/src/renderer/src/lib/ai-vault-resume-shell.test.ts b/src/renderer/src/lib/ai-vault-resume-shell.test.ts index e2ab4b9a173..2fff99a44dc 100644 --- a/src/renderer/src/lib/ai-vault-resume-shell.test.ts +++ b/src/renderer/src/lib/ai-vault-resume-shell.test.ts @@ -51,45 +51,33 @@ function withLoginShell(shell: string, run: () => T): T { } describe('resolveAiVaultResumeStartupShell', () => { - it('reports the fish dialect for a local session under a fish login shell', () => { - expect( - withLoginShell('/opt/homebrew/bin/fish', () => - resolveAiVaultResumeStartupShell({ - state: makeState(), - worktreeId: 'repo-1::worktree-1', - platform: 'darwin', - isLocalSession: true, - parsedByClientLoginShell: true - }) - ) - ).toBe('fish') - }) + // Why no login-shell cases: the Unix branch emits quoting and env clearing that + // are correct in sh and fish alike, so it no longer probes $SHELL at all — see + // startup-shell-portability.live-shell.test.ts for the proof it holds. + it.each(['/opt/homebrew/bin/fish', '/bin/zsh', '/bin/bash'])( + 'reports one Unix dialect regardless of the login shell (%s)', + (loginShell) => { + expect( + withLoginShell(loginShell, () => + resolveAiVaultResumeStartupShell({ + state: makeState(), + worktreeId: 'repo-1::worktree-1', + platform: 'darwin', + isLocalSession: true + }) + ) + ).toBe('posix') + } + ) - it('stays on sh for zsh users', () => { - expect( - withLoginShell('/bin/zsh', () => - resolveAiVaultResumeStartupShell({ - state: makeState(), - worktreeId: 'repo-1::worktree-1', - platform: 'darwin', - isLocalSession: true, - parsedByClientLoginShell: true - }) - ) - ).toBe('posix') - }) - - it('stays on sh for a LOCAL session whose command a remote host parses', () => { - // The reachable case: a locally scanned session has no executionHostId, so - // isLocalSession stays true while the command is bound for an SSH host. + it('stays on the Unix dialect for a LOCAL session whose command a remote host parses', () => { expect( withLoginShell('/opt/homebrew/bin/fish', () => resolveAiVaultResumeStartupShell({ state: makeState(), worktreeId: 'repo-1::worktree-1', platform: 'linux', - isLocalSession: true, - parsedByClientLoginShell: false + isLocalSession: true }) ) ).toBe('posix') @@ -104,23 +92,7 @@ describe('copied real-home Codex resume command', () => { codexHome: null } - it('clears inherited Codex homes with fish syntax under a fish login shell', () => { - expect( - withLoginShell('/opt/homebrew/bin/fish', () => - buildAiVaultResumeCopyCommandForWorktree({ - state: makeState(), - worktreeId: 'repo-1::worktree-1', - session - }) - ) - ).toBe( - "set -e CODEX_HOME; set -e ORCA_CODEX_HOME; cd '/home/alice/repo' && codex 'resume' 'session one'" - ) - }) - - it('keeps `unset` when a fish client targets an SSH worktree', () => { - // The session was scanned locally (no executionHostId), but the worktree lives - // on an SSH host: `set -e CODEX_HOME` would enable errexit there, not clear it. + it('clears inherited Codex homes for a worktree on an SSH host', () => { expect( withLoginShell('/opt/homebrew/bin/fish', () => buildAiVaultResumeCopyCommandForWorktree({ @@ -130,11 +102,13 @@ describe('copied real-home Codex resume command', () => { }) ) ).toBe( - "unset CODEX_HOME; unset ORCA_CODEX_HOME; cd '/home/alice/repo' && codex 'resume' 'session one'" + `cd '/home/alice/repo' && env -u CODEX_HOME -u ORCA_CODEX_HOME codex 'resume' 'session one'` ) }) - it('keeps `unset` for sh-family login shells', () => { + it('emits the same self-contained teardown under an sh-family login shell', () => { + // Why identical to the fish case: this text is COPIED, so it may be pasted + // into any shell — it carries its own fish/sh branch instead of guessing. expect( withLoginShell('/bin/bash', () => buildAiVaultResumeCopyCommandForWorktree({ @@ -144,7 +118,7 @@ describe('copied real-home Codex resume command', () => { }) ) ).toBe( - "unset CODEX_HOME; unset ORCA_CODEX_HOME; cd '/home/alice/repo' && codex 'resume' 'session one'" + `cd '/home/alice/repo' && env -u CODEX_HOME -u ORCA_CODEX_HOME codex 'resume' 'session one'` ) }) }) diff --git a/src/renderer/src/lib/ai-vault-resume-shell.ts b/src/renderer/src/lib/ai-vault-resume-shell.ts index 9b6214fb247..3d36b28c07f 100644 --- a/src/renderer/src/lib/ai-vault-resume-shell.ts +++ b/src/renderer/src/lib/ai-vault-resume-shell.ts @@ -4,11 +4,9 @@ import { CLIENT_PLATFORM } from '@/lib/new-workspace' import { resolveLocalWindowsTerminalShellOverrideForTab } from '../../../shared/local-windows-terminal-runtime' import { resolveWindowsShellStartupFamily } from '../../../shared/windows-terminal-shell' import { - resolveLoginShellStartupDialect, resolveStartupShell, type AgentStartupShell } from '../../../shared/tui-agent-startup-shell' -import { getClientLoginShell } from '@/lib/client-login-shell' import { parseWorkspaceKey } from '../../../shared/workspace-scope' import { parseWslUncPath } from '../../../shared/wsl-paths' @@ -28,19 +26,12 @@ export function resolveAiVaultResumeStartupShell(args: { worktreeId?: string | null platform: NodeJS.Platform isLocalSession: boolean - /** - * True only when this machine's own login shell parses the command. A local - * session is NOT enough: a locally scanned session carries no executionHostId - * yet can target an SSH/runtime/WSL worktree, whose shell is unrelated. - */ - parsedByClientLoginShell?: boolean }): AgentStartupShell { - // Why: fish rejects `unset`, so the client's login shell decides the dialect — - // but only when it is the shell that reads the line; otherwise it stays sh. + // Why no login-shell probe: everything this command is built from — quoting + // and env clearing — is emitted in a form that is correct in sh and fish + // alike, so the Unix branch never has to know which one reads the line. if (args.platform !== 'win32') { - return args.parsedByClientLoginShell - ? resolveLoginShellStartupDialect(getClientLoginShell()) - : 'posix' + return 'posix' } const projectRuntime = args.isLocalSession ? getLocalProjectExecutionRuntimeContext(args.state, args.worktreeId, CLIENT_PLATFORM) diff --git a/src/renderer/src/lib/launch-agent-background-session.test.ts b/src/renderer/src/lib/launch-agent-background-session.test.ts index 78fd3a91f6e..5dc076afc1b 100644 --- a/src/renderer/src/lib/launch-agent-background-session.test.ts +++ b/src/renderer/src/lib/launch-agent-background-session.test.ts @@ -332,7 +332,7 @@ describe('launchAgentBackgroundSession', () => { expect(mockSpawn).toHaveBeenCalledWith( expect.objectContaining({ cwd: 'C:\\Users\\jinwo\\repo\\feature', - command: "claude '--dangerously-skip-permissions' 'don'\\''t use powershell quoting'", + command: `claude '--dangerously-skip-permissions' 'don'"'"'t use powershell quoting'`, connectionId: null, worktreeId: 'wt-1', tabId: expect.stringMatching(UUID_RE) diff --git a/src/renderer/src/lib/launch-agent-in-new-tab-windows-quoting.test.ts b/src/renderer/src/lib/launch-agent-in-new-tab-windows-quoting.test.ts index c4034ba5bee..4d91f86a632 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab-windows-quoting.test.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab-windows-quoting.test.ts @@ -238,7 +238,7 @@ describe('launchAgentInNewTab Windows shell quoting', () => { expect(mockQueueTabStartupCommand).toHaveBeenCalledWith( 'tab-1', expect.objectContaining({ - command: "claude '--dangerously-skip-permissions' --prefill 'review Bob'\\''s change'" + command: `claude '--dangerously-skip-permissions' --prefill 'review Bob'"'"'s change'` }) ) }) @@ -293,7 +293,7 @@ describe('launchAgentInNewTab Windows shell quoting', () => { expect(mockQueueTabStartupCommand).toHaveBeenCalledWith( 'tab-1', expect.objectContaining({ - command: "claude '--dangerously-skip-permissions' --prefill 'review Bob'\\''s change'" + command: `claude '--dangerously-skip-permissions' --prefill 'review Bob'"'"'s change'` }) ) }) @@ -326,7 +326,7 @@ describe('launchAgentInNewTab Windows shell quoting', () => { launchAgentInNewTab({ agent: 'codex', worktreeId: 'wt-1' }) const queued = mockQueueTabStartupCommand.mock.calls.at(-1)?.[1] as { command: string } - expect(queued.command).toContain("'don'\\''t'") + expect(queued.command).toContain(`'don'"'"'t'`) expect(queued.command).not.toContain("'don''t'") }) }) diff --git a/src/renderer/src/lib/launch-work-item-direct.test.ts b/src/renderer/src/lib/launch-work-item-direct.test.ts index 52243fcc679..e70526aa3a7 100644 --- a/src/renderer/src/lib/launch-work-item-direct.test.ts +++ b/src/renderer/src/lib/launch-work-item-direct.test.ts @@ -688,7 +688,9 @@ describe('launchWorkItemDirect', () => { expect(mocks.activateAndRevealWorktree).toHaveBeenCalled() const activationOptions = mocks.activateAndRevealWorktree.mock.calls.at(-1)?.[1] - expect(activationOptions.startup.command).toContain('unset ORCA_PI_PREFILL') + expect(activationOptions.startup.command).toContain( + `command test -n "$fish_pid" && set --erase -g ORCA_PI_PREFILL; command test -z "$fish_pid" && unset ORCA_PI_PREFILL; true` + ) expect(activationOptions.startup.command).not.toContain('Remove-Item Env:ORCA_PI_PREFILL') }) @@ -729,7 +731,9 @@ describe('launchWorkItemDirect', () => { expect(mocks.ensureRemoteDetectedAgents).toHaveBeenCalledWith('ssh-1') expect(mocks.ensureDetectedAgents).not.toHaveBeenCalled() const activationOptions = mocks.activateAndRevealWorktree.mock.calls.at(-1)?.[1] - expect(activationOptions.startup.command).toContain('unset ORCA_PI_PREFILL') + expect(activationOptions.startup.command).toContain( + `command test -n "$fish_pid" && set --erase -g ORCA_PI_PREFILL; command test -z "$fish_pid" && unset ORCA_PI_PREFILL; true` + ) }) it('plans direct local Windows-path launches with POSIX startup for WSL project runtime', async () => { diff --git a/src/renderer/src/lib/resume-sleeping-agent-session.test.ts b/src/renderer/src/lib/resume-sleeping-agent-session.test.ts index c4f219f1524..c90aff730c4 100644 --- a/src/renderer/src/lib/resume-sleeping-agent-session.test.ts +++ b/src/renderer/src/lib/resume-sleeping-agent-session.test.ts @@ -885,7 +885,7 @@ describe('resumeSleepingAgentSessionsForWorktree', () => { const resumedTab = state.tabsByWorktree['wt-1']?.[0] expect(resumedTab?.launchAgent).toBe('claude') expect(state.pendingStartupByTabId[resumedTab!.id]?.command).toContain( - "'--resume' 'sess-1'\\''s'" + `'--resume' 'sess-1'"'"'s'` ) }) }) diff --git a/src/renderer/src/lib/tui-agent-startup.test.ts b/src/renderer/src/lib/tui-agent-startup.test.ts index 74dba64f02c..768a5363c85 100644 --- a/src/renderer/src/lib/tui-agent-startup.test.ts +++ b/src/renderer/src/lib/tui-agent-startup.test.ts @@ -352,7 +352,7 @@ describe('buildAgentDraftLaunchPlan', () => { }) ).toEqual({ agent: 'pi', - launchCommand: 'pi; unset ORCA_PI_PREFILL', + launchCommand: `pi; command test -n "$fish_pid" && set --erase -g ORCA_PI_PREFILL; command test -z "$fish_pid" && unset ORCA_PI_PREFILL; true`, expectedProcess: 'pi', env: { ORCA_PI_PREFILL: 'https://github.com/acme/repo/issues/42' }, launchConfig: emptyLaunchConfig('pi') diff --git a/src/shared/agent-resume-launch-command.ts b/src/shared/agent-resume-launch-command.ts index 1093735d266..cda40878c28 100644 --- a/src/shared/agent-resume-launch-command.ts +++ b/src/shared/agent-resume-launch-command.ts @@ -1,5 +1,6 @@ import type { ResumableTuiAgent } from './agent-session-resume' import { + isPosixStartupShell, quoteStartupArg, tokenizeStartupCommand, type AgentStartupShell @@ -37,9 +38,9 @@ function findClaudeExecutableIndex(tokens: readonly string[], shell: AgentStartu return i } if ( - // Why: `NAME=value cmd` is posix-only syntax; on cmd/PowerShell such a - // token is just a bogus executable name, not a prefix to skip. - (shell === 'posix' && /^[A-Za-z_][A-Za-z0-9_]*=/.test(token)) || + // Why: `NAME=value cmd` is sh-family syntax (fish included, 3.1+); on + // cmd/PowerShell such a token is a bogus executable name, not a prefix. + (isPosixStartupShell(shell) && /^[A-Za-z_][A-Za-z0-9_]*=/.test(token)) || (shell === 'powershell' && token === '&' && i === 0) ) { continue diff --git a/src/shared/ai-vault-resume-command.test.ts b/src/shared/ai-vault-resume-command.test.ts index a111b9eed35..2ebb5d59794 100644 --- a/src/shared/ai-vault-resume-command.test.ts +++ b/src/shared/ai-vault-resume-command.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest' -import { buildAiVaultResumeCommand } from './ai-vault-resume-command' +import { + buildAiVaultResumeCommand, + buildAiVaultResumeShellCommand +} from './ai-vault-resume-command' describe('buildAiVaultResumeCommand', () => { it('uses Antigravity conversation ids instead of Gemini resume flags', () => { @@ -150,3 +153,75 @@ describe('buildAiVaultResumeCommand', () => { ).toBe("cd '/Users/ada/repo' && prime-agent --resume 'dddddddd-eeee-4fff-8aaa-111111111111'") }) }) + +describe('buildAiVaultResumeShellCommand env removal', () => { + const base = { + resumeCommand: "codex 'resume' 'sid'", + cwd: '/repo', + clearEnvNames: ['CODEX_HOME', 'ORCA_CODEX_HOME'] + } + + it('carries the removal on the agent under a POSIX shell', () => { + expect(buildAiVaultResumeShellCommand({ ...base, platform: 'darwin' })).toBe( + "cd '/repo' && env -u CODEX_HOME -u ORCA_CODEX_HOME codex 'resume' 'sid'" + ) + }) + + // Why: `env -u` strips what the assignment just set, so an unfiltered list + // would silently resume against the real home instead of the pinned one. + it('keeps a pinned CODEX_HOME authoritative instead of stripping it', () => { + const command = buildAiVaultResumeShellCommand({ + ...base, + platform: 'darwin', + codexHome: '/home/a/.codex-work' + }) + + expect(command).toBe( + "cd '/repo' && CODEX_HOME='/home/a/.codex-work' env -u ORCA_CODEX_HOME codex 'resume' 'sid'" + ) + expect(command).not.toContain('-u CODEX_HOME') + }) + + it('keeps a pinned CODEX_HOME authoritative under a git-bash shell too', () => { + expect( + buildAiVaultResumeShellCommand({ + ...base, + platform: 'win32', + shell: 'posix', + codexHome: '/c/users/a/.codex-work' + }) + ).toBe( + "cd '/repo' && CODEX_HOME='/c/users/a/.codex-work' env -u ORCA_CODEX_HOME codex 'resume' 'sid'" + ) + }) + + // Why: the shell decides the grammar, not the host. Keying placement on the + // platform emitted POSIX `env -u` into a PowerShell line. + it('uses PowerShell grammar for a PowerShell shell on a non-Windows host', () => { + const command = buildAiVaultResumeShellCommand({ + ...base, + platform: 'linux', + shell: 'powershell' + }) + + expect(command).not.toContain('env -u') + expect(command).toBe( + 'Remove-Item Env:CODEX_HOME -ErrorAction SilentlyContinue; ' + + 'Remove-Item Env:ORCA_CODEX_HOME -ErrorAction SilentlyContinue; ' + + "Set-Location -LiteralPath '/repo'; codex 'resume' 'sid'" + ) + }) + + it('keeps the cmd clear ahead of the cd so a failed cd cannot launch the agent', () => { + expect( + buildAiVaultResumeShellCommand({ + ...base, + cwd: 'C:\\repo', + platform: 'win32', + shell: 'cmd' + }) + ).toBe( + 'set "CODEX_HOME=" & set "ORCA_CODEX_HOME=" & cd /d "C:\\repo" && codex \'resume\' \'sid\'' + ) + }) +}) diff --git a/src/shared/ai-vault-resume-command.ts b/src/shared/ai-vault-resume-command.ts index 13c33b8145e..26c5b637e82 100644 --- a/src/shared/ai-vault-resume-command.ts +++ b/src/shared/ai-vault-resume-command.ts @@ -3,10 +3,12 @@ // and (when known) the live tab's shell. import { TUI_AGENT_CONFIG } from './tui-agent-config' import { + clearEnvCommand, commandSeparator, isPosixStartupShell, quoteStartupArg, - type AgentStartupShell + type AgentStartupShell, + withoutEnvCommand } from './tui-agent-startup-shell' import type { AiVaultAgent, AiVaultSession } from './ai-vault-types' @@ -19,6 +21,7 @@ export function buildAiVaultResumeCommand(args: { codexHome?: string | null resumeFilePath?: string | null shell?: AgentStartupShell + clearEnvNames?: readonly string[] }): string { const { agent, sessionId, cwd, platform, commandOverride, codexHome, resumeFilePath, shell } = args @@ -45,7 +48,8 @@ export function buildAiVaultResumeCommand(args: { cwd, platform, codexHome, - shell + shell, + clearEnvNames: args.clearEnvNames }) } @@ -54,42 +58,69 @@ export function buildAiVaultResumeShellCommand(args: { cwd: string | null platform: NodeJS.Platform codexHome?: string | null + /** Env names the agent must not inherit. Applied as a prefix on the agent + * itself, never on the whole `cd … && agent` chain — `cd` is a shell builtin + * that `env` cannot run, and a child `cd` would not move the agent anyway. */ + clearEnvNames?: readonly string[] // Why: the QUEUED resume command is typed into the live tab shell, so its // cd/env prefix must match that shell. Shell-less persisted commands keep the // legacy self-contained `cmd /d /s /c` wrapper. shell?: AgentStartupShell }): string { - const { cwd, platform, codexHome, shell } = args + const { cwd, platform, codexHome, shell, clearEnvNames } = args // Why: shell-aware commands are parsed by a known running shell, while // shell-less persisted commands keep the legacy self-contained cmd wrapper. - if (platform === 'win32' && shell && shell !== 'cmd') { + // PowerShell routes here whatever the platform: it is the shell, not the + // host, that decides which grammar the line has to be written in. + if (shell === 'powershell' || (platform === 'win32' && shell && shell !== 'cmd')) { return buildResumeShellCommandForShell({ resumeCommand: args.resumeCommand, cwd, codexHome: codexHome?.trim() || null, - shell + shell, + clearEnvNames }) } - const resumeCommand = `${codexHomeEnvPrefix(codexHome?.trim() || null, platform)}${ - args.resumeCommand + const resolvedCodexHome = codexHome?.trim() || null + // Why filter: the prefix and the removal name the same variable, and `env -u` + // strips what the assignment just set, so an unfiltered list would silently + // resume against the real home. Keeping the assignment authoritative matches + // the old `clear…; CODEX_HOME=x agent` ordering. + const clearNames = resolvedCodexHome + ? clearEnvNames?.filter((name) => name !== 'CODEX_HOME') + : clearEnvNames + // Why the two placements differ: `set -u` aborts on the unbound `$fish_pid` + // the POSIX clear statement has to test, so there it must not precede the + // agent — `env -u` carries the removal on the agent itself. cmd has no such + // hazard, and keeping its clear ahead of the `cd` preserves `cd … && agent`, + // so a failed `cd` still cannot run the agent in the wrong directory. + // Keyed on the shell, not the platform: the shell is what picks the grammar. + const dialect = shell ?? (platform === 'win32' ? 'cmd' : 'posix') + const clearsOnAgent = clearNames?.length && isPosixStartupShell(dialect) + const resumeCommand = `${codexHomeEnvPrefix(resolvedCodexHome, platform, shell)}${ + clearsOnAgent ? withoutEnvCommand(clearNames, args.resumeCommand, dialect) : args.resumeCommand }` + const clearPrefix = + clearNames?.length && !clearsOnAgent + ? `${clearEnvCommand(clearNames, dialect)}${commandSeparator(dialect)}` + : '' if (platform === 'win32' && shell === 'cmd') { // Why: an interactive cmd splits the doubled quotes required by a nested // `cmd /s /c` wrapper, so queued commands must use direct cmd syntax. - return cwd ? `cd /d ${quoteWindowsCmdArg(cwd)} && ${resumeCommand}` : resumeCommand + return `${clearPrefix}${cwd ? `cd /d ${quoteWindowsCmdArg(cwd)} && ${resumeCommand}` : resumeCommand}` } if (!cwd) { - return resumeCommand + return `${clearPrefix}${resumeCommand}` } if (platform === 'win32') { - const inner = `cd /d ${quoteWindowsCmdArg(cwd)} && ${resumeCommand}` + const inner = `${clearPrefix}cd /d ${quoteWindowsCmdArg(cwd)} && ${resumeCommand}` return `cmd /d /s /c ${quoteWindowsCmdArg(inner)}` } - return `cd ${quoteShellArg(cwd, platform)} && ${resumeCommand}` + return `cd ${quoteResumeArg(cwd, platform, shell)} && ${resumeCommand}` } function buildResumeShellCommandForShell(args: { @@ -97,18 +128,33 @@ function buildResumeShellCommandForShell(args: { cwd: string | null codexHome: string | null shell: Exclude + clearEnvNames?: readonly string[] }): string { - const { cwd, codexHome, shell } = args + const { cwd, codexHome, shell, clearEnvNames } = args if (isPosixStartupShell(shell)) { // Why: git-bash on a Windows host runs a POSIX shell, so reuse the same // inline-env + `cd ''` prefix as the non-Windows path. const envPrefix = codexHome ? `CODEX_HOME=${quoteStartupArg(codexHome, shell)} ` : '' - const command = `${envPrefix}${args.resumeCommand}` + // Why filter: see the twin in buildAiVaultResumeShellCommand — `env -u` + // would strip the home the prefix just set. + const clearNames = codexHome + ? clearEnvNames?.filter((name) => name !== 'CODEX_HOME') + : clearEnvNames + const command = `${envPrefix}${ + clearNames?.length + ? withoutEnvCommand(clearNames, args.resumeCommand, shell) + : args.resumeCommand + }` return cwd ? `cd ${quoteStartupArg(cwd, shell)} && ${command}` : command } const separator = commandSeparator(shell) const segments: string[] = [] + // Why ahead of Set-Location: PowerShell has no `set -u` expansion hazard, so + // the removal keeps its original leading position. + if (clearEnvNames?.length) { + segments.push(clearEnvCommand(clearEnvNames, shell)) + } if (cwd) { segments.push(`Set-Location -LiteralPath ${quoteStartupArg(cwd, shell)}`) } @@ -184,21 +230,37 @@ function buildAgentResumeInvocation( } } -function codexHomeEnvPrefix(codexHome: string | null, platform: NodeJS.Platform): string { +function codexHomeEnvPrefix( + codexHome: string | null, + platform: NodeJS.Platform, + shell?: AgentStartupShell +): string { if (!codexHome) { return '' } if (platform === 'win32') { return `set ${quoteWindowsCmdArg(`CODEX_HOME=${codexHome}`)} && ` } - return `CODEX_HOME=${quoteShellArg(codexHome, platform)} ` + // fish accepts the `NAME=value cmd` prefix (3.1+), but not sh's quoting. + return `CODEX_HOME=${quoteResumeArg(codexHome, platform, shell)} ` } +/** Quotes for the live shell when one is known, else for the platform's default. */ +function quoteResumeArg( + value: string, + platform: NodeJS.Platform, + shell?: AgentStartupShell +): string { + return shell ? quoteStartupArg(value, shell) : quoteShellArg(value, platform) +} + +/** Why not the sh `'\''` idiom here: this is the same resume command the shell + * branch above builds, and fish reads that idiom differently — it would halve + * backslashes in a path and reject a trailing one. Deferring to + * `quoteStartupArg` keeps one spelling regardless of whether a caller happened + * to pass a shell. */ function quoteShellArg(value: string, platform: NodeJS.Platform): string { - if (platform === 'win32') { - return quoteWindowsCmdArg(value) - } - return `'${value.replace(/'/g, `'\\''`)}'` + return platform === 'win32' ? quoteWindowsCmdArg(value) : quoteStartupArg(value, 'posix') } function quoteWindowsCmdArg(value: string): string { diff --git a/src/shared/global-settings-types.ts b/src/shared/global-settings-types.ts index 1e41228f37e..6c331594e37 100644 --- a/src/shared/global-settings-types.ts +++ b/src/shared/global-settings-types.ts @@ -274,7 +274,8 @@ export type GlobalSettings = { claudeManagedAccounts: ClaudeManagedAccount[] activeClaudeManagedAccountId: string | null activeClaudeManagedAccountIdsByRuntime?: ClaudeManagedAccountRuntimeSelection - /** Per-worktree shell history file so ArrowUp doesn't surface other worktrees' commands. Defaults to true. */ + /** Per-worktree shell history so ArrowUp doesn't surface other worktrees' commands (a HISTFILE for + * bash/zsh, a `fish_history` session name for fish). Defaults to true. */ terminalScopeHistoryByWorktree: boolean /** Kill switch for hidden terminal view parking: unmount long-hidden panes while a pane-less watcher keeps PTY side effects alive. */ terminalHiddenViewParking?: boolean diff --git a/src/shared/hermes-startup-query.ts b/src/shared/hermes-startup-query.ts index a120305e1ee..a8536f91601 100644 --- a/src/shared/hermes-startup-query.ts +++ b/src/shared/hermes-startup-query.ts @@ -159,7 +159,11 @@ function buildQueryCommand(argv: string[], shell: AgentStartupShell): string { // Why: a fixed single-quote-safe wrapper parses in POSIX shells and pwsh; // the dynamic argv is decoded only after entering the known `sh` grammar. const script = `${POSIX_QUERY_VARIABLE}="\${${ORCA_HERMES_STARTUP_QUERY_ENV}}"; unset ${ORCA_HERMES_STARTUP_QUERY_ENV}; eval "$(printf %b "${encodedInvocation}")"` - return `sh -c ${quoteStartupArg(script, 'posix')}` + // Why `shell` and not 'posix': the body runs under `sh`, but the quoting around + // it is parsed by the shell that types the line. Today's payload survives sh + // quoting in fish only because its escapes happen to be `\0NNN` and never `\\` + // — quote for the real dialect so that stays an implementation detail. + return `sh -c ${quoteStartupArg(script, shell)}` } export function planHermesStartupQuery(args: { @@ -181,7 +185,10 @@ export function planHermesStartupQuery(args: { return null } const command = buildQueryCommand(argv, args.shell) - const env = { ...args.agentEnv, [ORCA_HERMES_STARTUP_QUERY_ENV]: args.prompt } + const env = { + ...args.agentEnv, + [ORCA_HERMES_STARTUP_QUERY_ENV]: args.prompt + } const envSize = Object.entries(env).reduce((total, [key, value]) => { if (args.platform === 'win32') { return total + key.length + value.length + 2 diff --git a/src/shared/startup-shell-portability.live-shell.test.ts b/src/shared/startup-shell-portability.live-shell.test.ts new file mode 100644 index 00000000000..b72a60ec075 --- /dev/null +++ b/src/shared/startup-shell-portability.live-shell.test.ts @@ -0,0 +1,266 @@ +/** + * Proves the central claim of the POSIX startup dialect against real shells: + * ONE emitted string is correct in sh, bash, zsh, dash and fish alike, so Orca + * never has to detect which shell will parse a queued command line. + * + * The shells are the oracle. Each case is handed to a real shell, which echoes + * the value back through `printf`, and the test compares bytes — a hand-written + * expectation would only re-state the implementation's own assumptions. + * + * Two of the quoting cases are regression pins with teeth: `\\server\share` and + * a trailing backslash are exactly what plain sh `'\''` quoting gets wrong when + * fish reads it (silently dropped backslashes, and a hard syntax error), and + * `command eval` is what zsh gets wrong — its `command` resolves external + * binaries only, so it cannot run a builtin. + */ +import { execFileSync } from 'node:child_process' +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { fishRequirementViolation, resolveFishBinary } from './fish-binary-requirement' +import { clearEnvCommand, quoteStartupArg, withoutEnvCommand } from './tui-agent-startup-shell' + +const FISH = resolveFishBinary(4) + +type LiveShell = { name: string; path: string } + +/** Every Unix shell on this machine a queued Orca command line could land in. */ +function discoverShells(): LiveShell[] { + const shells: LiveShell[] = [] + for (const path of [ + '/bin/sh', + '/bin/bash', + '/bin/zsh', + '/bin/dash', + '/usr/bin/dash', + '/bin/ksh' + ]) { + if (existsSync(path) && !shells.some((shell) => shell.name === basename(path))) { + shells.push({ name: basename(path), path }) + } + } + if (FISH.available) { + // Why absolute: runInShell scrubs PATH, so a bare `fish` would not resolve. + const path = FISH.path.includes('/') + ? FISH.path + : execFileSync('command', ['-v', FISH.path], { encoding: 'utf8', shell: true }).trim() + shells.push({ name: 'fish', path }) + } + return shells +} + +function basename(path: string): string { + return path.split('/').pop() ?? path +} + +// Why a real (empty) HOME rather than a bogus one: fish needs a writable config +// dir to hold universal variables, and warns loudly on every launch without it. +const SANDBOX_HOME = mkdtempSync(path.join(tmpdir(), 'orca-shell-portability-')) + +/** Env with no user shell config reachable, so only Orca's own text is exercised. */ +function sandboxEnv(): NodeJS.ProcessEnv { + return { + PATH: '/usr/bin:/bin:/usr/sbin:/sbin', + HOME: SANDBOX_HOME, + XDG_CONFIG_HOME: path.join(SANDBOX_HOME, 'config'), + XDG_DATA_HOME: path.join(SANDBOX_HOME, 'data') + } +} + +/** Runs one line in a real shell with no user config reachable. */ +function runInShell(shell: LiveShell, script: string): string { + return execFileSync(shell.path, ['-c', script], { + encoding: 'utf8', + timeout: 20_000, + env: sandboxEnv() + }) +} + +const SHELLS = discoverShells() + +const QUOTING_CASES: readonly string[] = [ + 'plain', + '', + 'with space', + "it's", + 'a\\b', + '\\d+', + '\\\\server\\share', + 'C:\\Users\\foo', + 'ends\\', + '\\', + '$HOME', + '`whoami`', + '$(whoami)', + '${HOME}', + '(paren)', + '{a,b}', + '*glob?', + '[bracket]', + '#hash', + 'a\nb', + 'a\tb', + 'semi;colon', + 'pipe|and&', + 'redirect>out { + rmSync(SANDBOX_HOME, { recursive: true, force: true }) +}) + +// Why skipIf: on Windows discoverShells() finds nothing, so the suite would +// fail for want of a POSIX shell rather than for any defect it tests. +describe.skipIf(process.platform === 'win32')( + 'one POSIX startup dialect is correct in every Unix shell', + () => { + // Always runs, so a CI lane cannot report green with the shell cases skipped. + it('has the fish this suite needs when CI requires one', () => { + expect(fishRequirementViolation(FISH)).toBeNull() + }) + + it('found shells to test against', () => { + expect(SHELLS.map((shell) => shell.name)).toContain('sh') + }) + + describe.each(SHELLS)('$name', (shell) => { + it.each(QUOTING_CASES)('quotes %j so the shell yields it back verbatim', (value) => { + expect(runInShell(shell, `printf '%s' ${quoteStartupArg(value, 'posix')}`)).toBe(value) + }) + + it('quotes a whole argv so every argument survives independently', () => { + const argv = QUOTING_CASES.filter((value) => value !== '') + const quoted = argv.map((value) => quoteStartupArg(value, 'posix')).join(' ') + // Why NUL: it is the one byte no case above can contain, so it cannot be + // forged by a value that was mis-split into two arguments. + const output = runInShell(shell, `printf '%s\\0' ${quoted}`) + expect(output.split('\0').slice(0, -1)).toEqual(argv) + }) + + // Why `set -u` gets its own case: the copied resume command puts the clear + // BEFORE the agent, so an aborted line takes the launch down with it. That + // is the regression that reverted #14863 (#14975). + it('runs the agent under set -u, with the vars removed', () => { + // The child must read its own environment, the way the agent binary does; + // expanding it in the caller would just echo the caller's value back. + const launch = withoutEnvCommand( + ['CODEX_HOME', 'ORCA_CODEX_HOME'], + `sh -c 'printf "LAUNCHED:%s" "\${CODEX_HOME-unset}"'`, + 'posix' + ) + const probe = + shell.name === 'fish' + ? `set -gx CODEX_HOME /bad; ${launch}` + : `set -u; CODEX_HOME=/bad; export CODEX_HOME; ${launch}` + + expect(runInShell(shell, probe)).toContain('LAUNCHED:unset') + }) + + it('clears an exported variable with no wrapper installed', () => { + const clear = clearEnvCommand('ORCA_PI_PREFILL', 'posix') + const probe = + shell.name === 'fish' + ? `set -gx ORCA_PI_PREFILL draft; ${clear}; set -q ORCA_PI_PREFILL; and echo STILL; or echo CLEARED` + : `ORCA_PI_PREFILL=draft; export ORCA_PI_PREFILL; ${clear}; echo "\${ORCA_PI_PREFILL:+STILL}\${ORCA_PI_PREFILL:-CLEARED}"` + expect(runInShell(shell, probe).trim()).toBe('CLEARED') + }) + + it('clears several variables in one statement', () => { + const clear = clearEnvCommand(['ORCA_A', 'ORCA_B'], 'posix') + const probe = + shell.name === 'fish' + ? `set -gx ORCA_A 1; set -gx ORCA_B 2; ${clear}; set -q ORCA_A; or set -q ORCA_B; and echo STILL; or echo CLEARED` + : `ORCA_A=1 ORCA_B=2; export ORCA_A ORCA_B; ${clear}; echo "\${ORCA_A:+STILL}\${ORCA_B:+STILL}\${ORCA_A:-CLEARED}"` + expect(runInShell(shell, probe).trim()).toBe('CLEARED') + }) + + // Why this case: fish's `set -e` returns non-zero for a variable that is + // already unset. An `A && B || C` spelling would fall through to the sh + // branch and print `Unknown command: unset` — the exact bug being fixed. + it.each([ + ['already set', true], + ['already unset', false] + ])('exits 0 and writes nothing to stderr when the variable is %s', (_label, preset) => { + const clear = clearEnvCommand('ORCA_PI_PREFILL', 'posix') + const setUp = preset + ? shell.name === 'fish' + ? 'set -gx ORCA_PI_PREFILL draft; ' + : 'ORCA_PI_PREFILL=draft; export ORCA_PI_PREFILL; ' + : '' + const stderr = execFileSync(shell.path, ['-c', `${setUp}${clear} 2>&1 1>/dev/null`], { + encoding: 'utf8', + timeout: 20_000, + env: sandboxEnv() + }) + expect(stderr).toBe('') + }) + + // Why INTERACTIVE (-i, script on stdin): aliases are only expanded by an + // interactive shell, which is the mode Orca types into — a `-c` run cannot + // see this class of bug at all. A user with `alias test=…` would otherwise + // silently skip both branches and keep the prefill exported. + it.runIf(['bash', 'zsh'].includes(shell.name))( + 'clears even when `test` is aliased away in an interactive shell', + () => { + const clear = clearEnvCommand('ORCA_PI_PREFILL', 'posix') + const script = [ + 'alias test=false', + 'ORCA_PI_PREFILL=draft; export ORCA_PI_PREFILL', + clear, + 'echo "RESULT=${ORCA_PI_PREFILL:+STILL}${ORCA_PI_PREFILL:-CLEARED}"' + ].join('\n') + const out = execFileSync(shell.path, ['-i'], { + input: `${script}\n`, + encoding: 'utf8', + timeout: 20_000, + env: sandboxEnv(), + stdio: ['pipe', 'pipe', 'ignore'] + }) + expect(out).toContain('RESULT=CLEARED') + } + ) + + // Why: `$fish_pid` is a heuristic — any non-empty value takes the fish + // branch. `set -e NAME` there would enable errexit, silently changing the + // semantics of every command after it for the rest of the session. + // `set --erase` cannot: `--` ends option parsing, so the worst a misfire + // costs is a usage message and the positional parameters (which an + // interactive shell does not meaningfully use). + // Scoped to bash/zsh because `set` is a POSIX special builtin: in sh, dash + // and ksh a misfire aborts a NON-interactive shell before anything can be + // observed. Interactively — the mode Orca types into — they survive without + // errexit too, but that is not scriptable here. bash and zsh report `$-` + // either way, and both DID silently enable errexit under the old `set -e`. + // Why `-g` is not optional: without it, a name that exists ONLY as a + // universal — `set -Ux CODEX_HOME …`, a normal thing for a fish user to + // have — is permanently deleted from every future session. Reachable from + // the clipboard command, which may run with no injected value at all. + it.runIf(shell.name === 'fish')('never deletes a lone universal variable', () => { + const clear = clearEnvCommand('ORCA_PI_PREFILL', 'posix') + const probe = `set -Ux ORCA_PI_PREFILL persisted; ${clear}; set -q ORCA_PI_PREFILL; and echo "RESULT=$ORCA_PI_PREFILL"; or echo RESULT=DESTROYED; set -Ue ORCA_PI_PREFILL` + expect(runInShell(shell, probe)).toContain('RESULT=persisted') + }) + + // A universal shadowed by the injected global: the global goes, theirs + // comes back. That is the wanted outcome, not a missed erase. + it.runIf(shell.name === 'fish')('reveals a shadowed universal instead of deleting it', () => { + const clear = clearEnvCommand('ORCA_PI_PREFILL', 'posix') + const probe = `set -Ux ORCA_PI_PREFILL mine; set -gx ORCA_PI_PREFILL injected; ${clear}; echo "RESULT=$ORCA_PI_PREFILL"; set -Ue ORCA_PI_PREFILL` + expect(runInShell(shell, probe)).toContain('RESULT=mine') + }) + + it.runIf(['bash', 'zsh'].includes(shell.name))( + 'never enables errexit when $fish_pid misfires', + () => { + const clear = clearEnvCommand('ORCA_PI_PREFILL', 'posix') + const probe = `fish_pid=1; export fish_pid; ${clear} 2>/dev/null; printf '%s' "$-"` + expect(runInShell(shell, probe).trim()).not.toContain('e') + } + ) + }) + } +) diff --git a/src/shared/tui-agent-startup-session-options.test.ts b/src/shared/tui-agent-startup-session-options.test.ts index cd47d81b971..45d83900b86 100644 --- a/src/shared/tui-agent-startup-session-options.test.ts +++ b/src/shared/tui-agent-startup-session-options.test.ts @@ -125,7 +125,7 @@ describe('tui agent startup session options', () => { allowEmptyPromptLaunch: true, sessionOptions: { model: "team's-model", effort: 'high' } }) - expect(plan?.launchCommand).toContain("'team'\\''s-model'") + expect(plan?.launchCommand).toContain(`'team'"'"'s-model'`) }) it('threads options through native draft launches', () => { diff --git a/src/shared/tui-agent-startup-shell.test.ts b/src/shared/tui-agent-startup-shell.test.ts index 2d155063462..1698e86b08b 100644 --- a/src/shared/tui-agent-startup-shell.test.ts +++ b/src/shared/tui-agent-startup-shell.test.ts @@ -5,7 +5,6 @@ import { commandSeparator, isPosixStartupShell, quoteStartupArg, - resolveLoginShellStartupDialect, tokenizeStartupCommand } from './tui-agent-startup-shell' import { buildAgentDraftLaunchPlan } from './tui-agent-startup' @@ -66,49 +65,86 @@ describe('tokenizeStartupCommand spans (windows shells)', () => { }) }) -describe('fish startup shell dialect', () => { - it('clears variables with fish syntax instead of the sh `unset` fish rejects', () => { - expect(clearEnvCommand('CODEX_HOME', 'fish')).toBe('set -e CODEX_HOME') - expect(clearEnvCommand('CODEX_HOME', 'posix')).toBe('unset CODEX_HOME') +describe('one Unix startup dialect', () => { + it('clears variables with a self-contained branch, not a per-shell builtin', () => { + // Why not `unset`/`set -e` alone, and why not a wrapper-defined helper: + // Orca only wraps zsh/bash/fish, so an `sh`/`dash`/`ksh` login shell — and + // any shell the user pastes copied text into — would not have the helper. + // startup-shell-portability.live-shell.test.ts proves this form works in + // real sh/bash/zsh/dash/ksh/fish. + expect(clearEnvCommand('CODEX_HOME', 'posix')).toBe( + `command test -n "$fish_pid" && set --erase -g CODEX_HOME; command test -z "$fish_pid" && unset CODEX_HOME; true` + ) + expect(clearEnvCommand(['A', 'B'], 'posix')).toBe( + `command test -n "$fish_pid" && set --erase -g A B; command test -z "$fish_pid" && unset A B; true` + ) expect(clearEnvCommand('CODEX_HOME', 'cmd')).toBe('set "CODEX_HOME="') expect(clearEnvCommand('CODEX_HOME', 'powershell')).toBe( 'Remove-Item Env:CODEX_HOME -ErrorAction SilentlyContinue' ) }) - it('shares POSIX quoting, tokenizing and chaining', () => { - expect(quoteStartupArg("it's", 'fish')).toBe(quoteStartupArg("it's", 'posix')) - expect(buildShellCommandFromArgv(['codex', 'resume', 'a b'], 'fish')).toBe( - buildShellCommandFromArgv(['codex', 'resume', 'a b'], 'posix') - ) - expect(tokenizeStartupCommand('codex --arg "a b"', 'fish')).toEqual( - tokenizeStartupCommand('codex --arg "a b"', 'posix') - ) - expect(commandSeparator('fish')).toBe('; ') - expect(isPosixStartupShell('fish')).toBe(true) + it.each([ + 'FOO; rm -rf /tmp/x', + 'FOO BAR', + '', + '1LEADING_DIGIT', + 'has-dash', + '$FOO', + 'PATH\nHOME' + ])('refuses %j as an environment variable name', (bad) => { + // Why: the name is interpolated straight into a shell line. Anything but an + // identifier is a command injection, and in fish `set --erase -g` would + // really delete whatever it names — PATH or HOME included. + expect(() => clearEnvCommand(bad, 'posix')).toThrow(/not an environment variable name/) + expect(() => clearEnvCommand(['OK', bad], 'posix')).toThrow(/not an environment variable name/) }) - it('maps login shell paths to their dialect', () => { - expect(resolveLoginShellStartupDialect('/opt/homebrew/bin/fish')).toBe('fish') - expect(resolveLoginShellStartupDialect('/usr/local/bin/FISH')).toBe('fish') - expect(resolveLoginShellStartupDialect('/bin/zsh')).toBe('posix') - expect(resolveLoginShellStartupDialect('/bin/bash')).toBe('posix') - expect(resolveLoginShellStartupDialect('')).toBe('posix') - expect(resolveLoginShellStartupDialect(undefined)).toBe('posix') + it('quotes so fish reads back the same bytes sh does', () => { + // fish single quotes are NOT literal — `\\` and `\'` are escapes inside them — + // so the sh `'\''` idiom would halve these backslashes when fish read them, + // and a trailing backslash would be a hard syntax error. + expect(quoteStartupArg("it's", 'posix')).toBe(`'it'"'"'s'`) + expect(quoteStartupArg(String.raw`\\server\share`, 'posix')).toBe( + `"\\\\""\\\\"'server'"\\\\"'share'` + ) + expect(quoteStartupArg('ends\\', 'posix')).toBe(`'ends'"\\\\"`) + expect(quoteStartupArg('', 'posix')).toBe(`''`) }) - // Contract-only: no POSIX caller threads a shell into buildAgentDraftLaunchPlan yet - // (`selectedRepoStartupShell` is Windows-only), so fish users still get `unset` here. - it('clears an agent draft prefill variable with fish syntax', () => { + it('keeps the sh-family grammar claims that do hold for fish', () => { + expect(commandSeparator('posix')).toBe('; ') + expect(isPosixStartupShell('posix')).toBe(true) + expect(isPosixStartupShell('powershell')).toBe(false) + expect(buildShellCommandFromArgv(['codex', 'resume', 'a b'], 'posix')).toBe( + `'codex' 'resume' 'a b'` + ) + }) + + it('round-trips an adversarial argument through quote then tokenize', () => { + for (const value of [ + String.raw`use \d+ and \\server\share`, + "it's mine", + 'ends\\', + 'a "b" c', + '$PATH *.ts' + ]) { + const tokenized = tokenizeStartupCommand(quoteStartupArg(value, 'posix'), 'posix') + expect(tokenized.ok && tokenized.tokens).toEqual([value]) + } + }) + + it('clears an agent draft prefill variable with the portable teardown', () => { const plan = buildAgentDraftLaunchPlan({ agent: 'pi', draft: 'hello', cmdOverrides: {}, - platform: 'darwin', - shell: 'fish' + platform: 'darwin' }) - expect(plan?.launchCommand).toBe('pi; set -e ORCA_PI_PREFILL') + expect(plan?.launchCommand).toBe( + `pi; command test -n "$fish_pid" && set --erase -g ORCA_PI_PREFILL; command test -z "$fish_pid" && unset ORCA_PI_PREFILL; true` + ) expect(plan?.env?.ORCA_PI_PREFILL).toBe('hello') }) }) diff --git a/src/shared/tui-agent-startup-shell.ts b/src/shared/tui-agent-startup-shell.ts index ef23b837e0b..94df0f80ac5 100644 --- a/src/shared/tui-agent-startup-shell.ts +++ b/src/shared/tui-agent-startup-shell.ts @@ -1,28 +1,26 @@ import { tokenizeCustomCommandTemplate, type CommandTokenSpan } from './commit-message-prompt' -// Why: fish shares POSIX word splitting, quoting and `;` chaining, so it is a -// separate dialect only where its grammar actually diverges (env clearing). -export type AgentStartupShell = 'posix' | 'fish' | 'powershell' | 'cmd' +/** + * `'posix'` covers every Unix shell Orca can type into, fish included — not + * because they agree, but because everything this module emits for it is built + * to be correct in all of them (see quoteStartupArg and clearEnvCommand). There + * is deliberately no fish member: a dialect Orca has to detect is a dialect it + * can get wrong, and it cannot detect one reliably for a remote or WSL host. + */ +export type AgentStartupShell = 'posix' | 'powershell' | 'cmd' type WindowsStartupShell = Extract -/** True for shells parsed with POSIX quoting/word rules (sh family + fish). */ +/** True for the sh-family grammar: `NAME=value cmd` prefixes, `sh -c` wrapping + * and `;` chaining. fish parses all three, so this holds for fish too. */ export function isPosixStartupShell(shell: AgentStartupShell): boolean { - return shell === 'posix' || shell === 'fish' + return shell === 'posix' } function isWindowsStartupShell(shell: AgentStartupShell): shell is WindowsStartupShell { return shell === 'powershell' || shell === 'cmd' } -/** Maps a POSIX login-shell path (`$SHELL`) to the dialect that parses queued commands. */ -export function resolveLoginShellStartupDialect( - loginShell: string | null | undefined -): AgentStartupShell { - const basename = loginShell?.trim().replaceAll('\\', '/').split('/').pop()?.toLowerCase() ?? '' - return basename === 'fish' ? 'fish' : 'posix' -} - export type StartupCommandTokens = | { ok: true; tokens: string[]; spans: CommandTokenSpan[] } | { ok: false; error: string } @@ -154,6 +152,11 @@ export function tokenizeStartupCommand( value: string, shell: AgentStartupShell ): StartupCommandTokens { + // Why one Unix parse: the input is a string the user typed into an Orca + // settings field, and the shell never parses it — every token is re-quoted by + // quoteStartupArg before the line is built. Parsing it differently per shell + // would make the same setting mean different things in different workspaces. + // (Windows is genuinely different: cmd/PowerShell re-parse the built line.) return isWindowsStartupShell(shell) ? tokenizeWindowsStartupCommand(value, shell) : tokenizeCustomCommandTemplate(value) @@ -166,6 +169,52 @@ export function resolveStartupShell( return shell ?? (platform === 'win32' ? 'powershell' : 'posix') } +/** + * Quotes one argument so the SAME text is literal in every Unix shell Orca can + * be typing into — sh, bash, zsh, dash and fish. + * + * Why not plain sh quoting: fish's single quotes are not literal. Inside `'…'` + * it treats `\\` and `\'` as escapes, so the sh `'\''` idiom collapses every + * backslash (a `\d+` regex, a `\\server\share` UNC path) and a trailing + * backslash is a hard syntax error that kills the launch outright. + * + * Both shell families agree on two things: a single-quoted run is literal apart + * from those escapes, and `"\\"` is one backslash. So emitting backslashes as + * `"\\"` and apostrophes as `"'"` between single-quoted runs round-trips in all + * of them, and Orca never has to know which shell will read the line. + * + * Verified against sh, bash, zsh, dash and fish 4.7.1 over regex escapes, UNC + * and drive paths, trailing/lone backslashes, mixed quotes, `$`/backtick/`$()` + * expansions, globs, braces, operators, comments, newlines, tabs and non-BMP + * unicode — see fish-startup-arg-quoting.live-fish.test.ts. + */ +function quotePortableUnixArg(value: string): string { + if (!value) { + return "''" + } + const parts: string[] = [] + let literal = '' + const flushLiteral = (): void => { + if (literal) { + parts.push(`'${literal}'`) + literal = '' + } + } + for (const char of value) { + if (char === "'") { + flushLiteral() + parts.push(`"'"`) + } else if (char === '\\') { + flushLiteral() + parts.push(`"\\\\"`) + } else { + literal += char + } + } + flushLiteral() + return parts.join('') +} + export function quoteStartupArg(value: string, shell: AgentStartupShell): string { if (shell === 'powershell') { return `'${value.replace(/'/g, "''")}'` @@ -173,7 +222,7 @@ export function quoteStartupArg(value: string, shell: AgentStartupShell): string if (shell === 'cmd') { return `"${value.replace(/([\^&|<>()%!"])/g, '^$1')}"` } - return `'${value.replace(/'/g, `'\\''`)}'` + return quotePortableUnixArg(value) } export function buildShellCommandFromArgv( @@ -187,19 +236,131 @@ export function buildShellCommandFromArgv( return command } -export function clearEnvCommand(name: string, shell: AgentStartupShell): string { +/** + * Clears one or more environment variables, in a form correct in every shell a + * queued command line can land in. + * + * Why it carries its own fish/sh branch rather than a single builtin: `unset` + * does not exist in fish, and `set -e` in bash enables errexit rather than + * clearing anything, so neither spelling is safe alone. + * + * Why not a helper function defined by Orca's shell wrappers: Orca only wraps + * zsh, bash and fish. A login shell of `sh`, `dash` or `ksh` launches + * UNWRAPPED, and this text is also copied to the clipboard and pasted into + * shells Orca never spawned — in all of those a helper would be `command not + * found`, which is the exact failure this exists to avoid. + * + * Why two statements rather than `A && B || C`: in fish, `set -e` on a variable + * that is already unset returns non-zero, so an `||` fallback would run the sh + * branch too and print `Unknown command: unset`. Each branch is guarded by its + * own test, so exactly one runs and the other is only parsed. + * + * The trailing `true` pins the exit status at 0: the guard that does NOT fire + * leaves a non-zero status behind, and this is the last statement of a launch + * line, so that status is what the user's prompt would render. + * + * `command test` rather than `test`, because an interactive shell expands + * aliases and `alias test=...` would otherwise skip both branches silently. + * + * `set --erase` rather than `set -e`, because `$fish_pid` is a heuristic: any + * non-empty value takes the fish branch. `set -e NAME` in the sh family enables + * errexit and replaces the positional parameters, silently changing the + * semantics of everything after it. `set --erase NAME` is the same erase in + * fish, but in sh `--` ends option parsing, so a misfire is a usage message on + * stderr and nothing else. + * + * `$fish_pid` is set by fish 3.0+ and by nothing in the sh family, and fish + * does not export it, so a fish parent cannot make a bash child misread itself. + * + * `-g` is not optional: it scopes the erase to fish's GLOBAL scope, which is + * where an inherited environment variable lands. Without it, a name that exists + * only as a UNIVERSAL — `set -Ux CODEX_HOME …`, a perfectly normal thing for a + * fish user to have — is permanently deleted from every future session. That is + * real data loss to undo one launch's injection, and it is reachable from the + * clipboard command, which a user may run with no Orca-injected value at all. + * With `-g`, a universal shadowed by an injected global is revealed again + * instead, which is the wanted outcome. + * + * KNOWN LIMIT: under `set -u` the sh side aborts on the unset `$fish_pid` and + * the variable survives. `${fish_pid-}` would be nounset-safe but is a fish + * parse error, and so is `set +u`. + * + * The one spelling that does satisfy both is a fish-builtin probe — e.g. + * `math 1 >/dev/null 2>&1 && … || …` — because it references no variable at + * all. Rejected deliberately: it EXECUTES whatever that name resolves to on + * the user's PATH, twice. `math` is a real binary (Wolfram Mathematica ships + * `/usr/local/bin/math`); measured with one on PATH, clearing a single variable + * took 10.4s and then did not clear it, because a probe that exits 0 is an + * unconditional false positive. Trading a rare uncleared prefill for an + * arbitrary program execution on the launch path is the wrong direction. + * + * Verified to clear the variables, write nothing to stderr and exit 0 — whether + * they were set or already unset — in sh, bash, zsh, dash, ksh and fish. + */ +const ENV_VAR_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/ + +export function clearEnvCommand( + name: string | readonly string[], + shell: AgentStartupShell +): string { + const names = typeof name === 'string' ? [name] : [...name] + // Why assert rather than escape: these names are interpolated straight into a + // shell line, so anything but an identifier is both a command injection and — + // in fish, where `-g` erases for real — a way to delete `PATH`/`HOME` out of + // the user's session. Every caller passes a literal from a fixed table, so + // this cannot fire today; it exists so it stays that way. + for (const each of names) { + if (!ENV_VAR_NAME.test(each)) { + throw new Error( + `clearEnvCommand: ${JSON.stringify(each)} is not an environment variable name` + ) + } + } if (shell === 'powershell') { - return `Remove-Item Env:${name} -ErrorAction SilentlyContinue` + return names.map((each) => `Remove-Item Env:${each} -ErrorAction SilentlyContinue`).join('; ') } if (shell === 'cmd') { - return `set "${name}="` + return names.map((each) => `set "${each}="`).join(' & ') } - // Why: fish has no `unset`; the sh spelling errors out and silently leaves - // the variable exported (e.g. an account-routed CODEX_HOME survives). - if (shell === 'fish') { - return `set -e ${name}` + const joined = names.join(' ') + return ( + `command test -n "$fish_pid" && set --erase -g ${joined}; ` + + `command test -z "$fish_pid" && unset ${joined}; true` + ) +} + +/** + * Prefix that runs `command` with `names` removed from its environment. + * + * Why this and not `clearEnvCommand` for a copied command: `clearEnvCommand` + * mutates the *calling* shell, so it needs a shell-specific branch, and every + * spelling of that branch touches `$fish_pid` — an unbound expansion that aborts + * the whole line under `set -u`, taking the agent launch with it. This prefix + * only has to change the CHILD's environment, which `env -u` does with no shell + * syntax and no variable expansion at all. Verified identical in sh, bash, zsh, + * dash, ksh and fish, including under `set -u`. + */ +export function withoutEnvCommand( + names: readonly string[], + command: string, + shell: AgentStartupShell +): string { + if (names.length === 0) { + return command } - return `unset ${name}` + if (isWindowsStartupShell(shell)) { + // Windows has no `env -u`; those shells clear in-place, and neither has a + // nounset mode that could abort the line. + return `${clearEnvCommand(names, shell)}${commandSeparator(shell)}${command}` + } + for (const name of names) { + if (!ENV_VAR_NAME.test(name)) { + throw new Error( + `withoutEnvCommand: ${JSON.stringify(name)} is not an environment variable name` + ) + } + } + return `env ${names.map((name) => `-u ${name}`).join(' ')} ${command}` } export function commandSeparator(shell: AgentStartupShell): string { diff --git a/src/shared/tui-agent-startup.test.ts b/src/shared/tui-agent-startup.test.ts index b3220956677..12c9f56e270 100644 --- a/src/shared/tui-agent-startup.test.ts +++ b/src/shared/tui-agent-startup.test.ts @@ -28,6 +28,26 @@ function unwrapPowerShellScript(command: string | undefined): string { return Buffer.from(encoded!, 'base64').toString('utf16le') } +describe('draft prefill teardown ordering (#14975)', () => { + // Why pinned: the teardown mutates the calling shell, so it must reference + // $fish_pid, which aborts the line under `set -u`. That is survivable ONLY + // because it runs AFTER the agent — the agent is already up. Moving it before + // the command would make an aborted line a blocked launch, which is exactly + // what reverted #14863. + it('runs the clear after the agent command, never before it', () => { + const plan = buildAgentDraftLaunchPlan({ + agent: 'pi', + draft: 'hello', + cmdOverrides: {}, + platform: 'darwin' + }) + + const command = plan?.launchCommand ?? '' + expect(command.indexOf('pi')).toBeLessThan(command.indexOf('fish_pid')) + expect(command).toMatch(/^pi;/) + }) +}) + describe('tui agent startup plans', () => { it.each(['powershell', 'cmd'] as const)( 'keeps the established invalid-quote error on %s', @@ -47,7 +67,7 @@ describe('tui agent startup plans', () => { platform: 'linux' }) - expect(plan?.launchCommand).toBe("claude 'fix Bob'\\''s branch'") + expect(plan?.launchCommand).toBe("claude 'fix Bob'\"'\"'s branch'") }) it('uses PowerShell quoting by default when the target shell is Windows', () => { @@ -184,9 +204,12 @@ describe('tui agent startup plans', () => { }) expect(plan?.launchCommand).toMatch(/^sh -c /) - expect(plan?.launchCommand).toMatch(/\\0[0-7]{3}/) expect(plan?.launchCommand).not.toContain("'sh' '-c'") - expect(plan?.launchCommand).not.toContain("'\\''") + // Why parse rather than string-match: the octal escapes must survive the + // OUTER quoting to reach the inner sh, and portable quoting emits a + // backslash as `"\\"` rather than leaving it inside a single-quoted run. + const tokens = tokenizeStartupCommand(plan?.launchCommand ?? '', 'posix') + expect(tokens.ok && tokens.tokens.at(-1)).toMatch(/\\0[0-7]{3}/) }) it('moves Hermes command override flags after the chat subcommand', () => { @@ -856,7 +879,9 @@ describe('tui agent startup plans', () => { expect(plan).not.toBeNull() expect(plan?.env).toEqual({ ORCA_OMP_PREFILL: 'fix the omp regression' }) expect(plan?.expectedProcess).toBe('omp') - expect(plan?.launchCommand).toBe('omp; unset ORCA_OMP_PREFILL') + expect(plan?.launchCommand).toBe( + `omp; command test -n "$fish_pid" && set --erase -g ORCA_OMP_PREFILL; command test -z "$fish_pid" && unset ORCA_OMP_PREFILL; true` + ) }) it('returns null for oversized Windows flag drafts so callers paste after ready', () => { From 66b599399f7fe14f18f304c25aa21e7584433844 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:01:24 -0700 Subject: [PATCH 07/10] fix(mobile): decide terminal preedit from the marked-text range, not a script table (#15007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(mobile): decide terminal preedit from the marked-text range, not a script table The live terminal capture field decided what to withhold from the PTY with a Unicode-block allowlist (Hangul jamo and syllables) and held exactly one trailing code point. Kana and kanji are not in the table, so a Japanese reading streamed to the PTY one fragment at a time and was repaired afterwards with DEL bytes (#7427). A code-point table cannot work, and the counterexample is not exotic: Chinese pinyin preedit is plain ASCII, and a Japanese romaji reading is one code point on the first keystroke and three on the fourth. Preedit is a property of the FIELD, not of the characters in it, so the only signal that identifies it is the text system's marked-text range. That is what a reference terminal implementation uses on every platform it supports - `hasMarkedText` there, the input-method context's composing state elsewhere - and neither one classifies code points anywhere in the input path. So the mirror now takes the marked-text report per change and holds the whole preedit region, whatever its length or script: - Subscribe the capture field to `onChange`, not `onChangeText`; only the raw native event carries the report at all. - A reported preedit is held entire and is never committed by the settle timer, because preedit is not text yet. Explicit boundaries still flush it. - `isTerminalLiveHangulCodePoint` and its four ranges are deleted. iOS reports the range but React Native drops it before JS, so the pinned patch forwards `markedTextRange` into the change payload. It is three hunks and it compiles because the app already sets `buildReactNativeFromSource` for iOS. The same idea was proposed in #11450, which is where the patch comes from. Android has no marked-text report in React Native at all, and a Kotlin patch would not help: Android consumes the prebuilt react-android artifact, so node_modules sources are never compiled. Until the report exists there, the fallback holds the trailing non-ASCII run. It enumerates nothing, it covers kana, kanji and Hangul, and ASCII keeps its zero-latency echo - but it cannot see an ASCII preedit, so Chinese pinyin on Android still leaks its reading. Only a report fixes that. Not-tested: no physical device or emulator was available, so no real IME drove this path. Japanese, Chinese and Korean composition are covered at the model and hook level only, and the iOS patch has not been compiled. Co-authored-by: Brennan Benson * fix(mobile): bound the fallback hold to text the pty has not received The no-report branch walked the trailing non-ASCII run over the whole field and ignored stableLength, unlike the reported branch directly above it. So after a settle-timer commit the next keystroke re-held everything already delivered and the caller erased it with DEL and retyped it — a nine-character Cyrillic word cost a DEL per already-sent character, and for the 300ms before the re-send the held text was the only copy, so a blur or reconnect destroyed characters the pty already had. Bound it the way the reported branch is bounded. Pinned by a test that drives a settle commit between every keystroke and asserts no DEL reaches the wire. --------- Co-authored-by: Brennan Benson --- .../app/h/[hostId]/session/[worktreeId].tsx | 4 +- mobile/package.json | 3 + mobile/patches/react-native@0.83.9.patch | 48 +++ mobile/pnpm-lock.yaml | 5 + .../terminal-ios-ime-keyboard.test.ts | 7 + .../terminal-live-hangul-mirror.test.ts | 178 ---------- .../terminal/terminal-live-hangul-mirror.ts | 61 ---- .../terminal-live-preedit-mirror.test.ts | 304 ++++++++++++++++++ .../terminal/terminal-live-preedit-mirror.ts | 83 +++++ .../use-terminal-live-input-commit.test.ts | 109 +++++-- .../use-terminal-live-input-commit.ts | 22 +- .../use-terminal-live-pending-input-flush.ts | 35 +- 12 files changed, 568 insertions(+), 291 deletions(-) create mode 100644 mobile/patches/react-native@0.83.9.patch delete mode 100644 mobile/src/terminal/terminal-live-hangul-mirror.test.ts delete mode 100644 mobile/src/terminal/terminal-live-hangul-mirror.ts create mode 100644 mobile/src/terminal/terminal-live-preedit-mirror.test.ts create mode 100644 mobile/src/terminal/terminal-live-preedit-mirror.ts diff --git a/mobile/app/h/[hostId]/session/[worktreeId].tsx b/mobile/app/h/[hostId]/session/[worktreeId].tsx index 6cff5a841fd..5587c204316 100644 --- a/mobile/app/h/[hostId]/session/[worktreeId].tsx +++ b/mobile/app/h/[hostId]/session/[worktreeId].tsx @@ -4944,7 +4944,9 @@ export default function SessionScreen() { ref={liveInputRef} style={styles.liveInputCapture} value={liveInputCapture} - onChangeText={handleLiveInputChange} + // Why onChange, not onChangeText: only the raw native event carries the + // marked-text report that says whether this text is still preedit. + onChange={handleLiveInputChange} onKeyPress={handleLiveInputKeyPress} onSubmitEditing={handleLiveInputSubmit} placeholder="" diff --git a/mobile/package.json b/mobile/package.json index 4b4274bb9d0..75db0998903 100644 --- a/mobile/package.json +++ b/mobile/package.json @@ -85,6 +85,9 @@ "pnpm": { "overrides": { "xcode>uuid": "11.1.1" + }, + "patchedDependencies": { + "react-native@0.83.9": "patches/react-native@0.83.9.patch" } } } diff --git a/mobile/patches/react-native@0.83.9.patch b/mobile/patches/react-native@0.83.9.patch new file mode 100644 index 00000000000..102b052746b --- /dev/null +++ b/mobile/patches/react-native@0.83.9.patch @@ -0,0 +1,48 @@ +diff --git a/React/Fabric/Mounting/ComponentViews/TextInput/RCTTextInputComponentView.mm b/React/Fabric/Mounting/ComponentViews/TextInput/RCTTextInputComponentView.mm +index d02e003a439e410ddab597b2c036256c80db0395..3caff21b0d00bfbc6dea3f8470e812b8343ac3a2 100644 +--- a/React/Fabric/Mounting/ComponentViews/TextInput/RCTTextInputComponentView.mm ++++ b/React/Fabric/Mounting/ComponentViews/TextInput/RCTTextInputComponentView.mm +@@ -703,6 +703,7 @@ static NSSet *returnKeyTypesSet; + { + return { + .text = RCTStringFromNSString(_backedTextInputView.attributedText.string), ++ .isComposing = _backedTextInputView.markedTextRange != nil, + .selectionRange = [self _selectionRange], + .eventCount = static_cast(_mostRecentEventCount), + .contentOffset = RCTPointFromCGPoint(_backedTextInputView.contentOffset), +diff --git a/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.cpp b/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.cpp +index a9bc219f8bf729111e907423ca61b991fd712b46..60e9c6f54edb295295d6e48d1d554499106b7e57 100644 +--- a/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.cpp ++++ b/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.cpp +@@ -24,6 +24,10 @@ static jsi::Value textInputMetricsPayload( + + payload.setProperty(runtime, "eventCount", textInputMetrics.eventCount); + ++ if (textInputMetrics.isComposing.has_value()) { ++ payload.setProperty(runtime, "isComposing", *textInputMetrics.isComposing); ++ } ++ + if (includeSelectionState) { + auto selection = jsi::Object(runtime); + selection.setProperty( +diff --git a/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.h b/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.h +index a3f2e01c130c1a53aee0b39e72927d21f2a6973d..c9d34f8b42644800b5c85e5be45b6ebfbe94a54d 100644 +--- a/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.h ++++ b/ReactCommon/react/renderer/components/textinput/TextInputEventEmitter.h +@@ -7,6 +7,8 @@ + + #pragma once + ++#include ++ + #include + #include + +@@ -18,6 +20,7 @@ class TextInputEventEmitter : public ViewEventEmitter { + + struct Metrics { + std::string text; ++ std::optional isComposing; + AttributedString::Range selectionRange; + // ScrollView-like metrics + Size contentSize; diff --git a/mobile/pnpm-lock.yaml b/mobile/pnpm-lock.yaml index b4b43e2308f..56e64e1d442 100644 --- a/mobile/pnpm-lock.yaml +++ b/mobile/pnpm-lock.yaml @@ -7,6 +7,11 @@ settings: overrides: xcode>uuid: 11.1.1 +patchedDependencies: + react-native@0.83.9: + hash: a6502e7d63769bf1fd639cbbaab1cc0b8aebef31d22048e8f2186fb399d0c16e + path: patches/react-native@0.83.9.patch + importers: .: diff --git a/mobile/src/terminal/terminal-ios-ime-keyboard.test.ts b/mobile/src/terminal/terminal-ios-ime-keyboard.test.ts index ac6ee0ff91f..e34aef649c4 100644 --- a/mobile/src/terminal/terminal-ios-ime-keyboard.test.ts +++ b/mobile/src/terminal/terminal-ios-ime-keyboard.test.ts @@ -12,6 +12,13 @@ describe('terminal iOS IME keyboard', () => { expect(sessionRouteSource).not.toContain('"ascii-capable"') }) + it('subscribes live capture to onChange so the marked-text report survives', () => { + // onChangeText hands over only a string, discarding the preedit report that + // decides whether the text may reach the PTY at all. + expect(sessionRouteSource).toContain('onChange={handleLiveInputChange}') + expect(sessionRouteSource).not.toContain('onChangeText={handleLiveInputChange}') + }) + it('does not put terminal keyboard capture behind iOS textContentType semantics', () => { expect(sessionRouteSource).not.toContain('textContentType="none"') expect(sessionRouteSource).toContain('autoComplete="off"') diff --git a/mobile/src/terminal/terminal-live-hangul-mirror.test.ts b/mobile/src/terminal/terminal-live-hangul-mirror.test.ts deleted file mode 100644 index b1a6669506c..00000000000 --- a/mobile/src/terminal/terminal-live-hangul-mirror.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - buildTerminalLiveMirrorPayload, - computeTerminalLiveMirrorStep, - isTerminalLiveHangulCodePoint, - type TerminalLiveMirrorStep -} from './terminal-live-hangul-mirror' - -type MirrorRun = { - readonly payloads: readonly string[] - readonly sentText: string - readonly heldText: string -} - -function runMirrorSequence( - fieldStates: readonly string[], - options: { readonly commitAtEnd: boolean } = { commitAtEnd: false } -): MirrorRun { - const payloads: string[] = [] - let sentText = '' - let heldText = '' - for (const fieldText of fieldStates) { - const step = computeTerminalLiveMirrorStep(sentText, fieldText, { commitHeld: false }) - const payload = buildTerminalLiveMirrorPayload(step) - if (payload.length > 0) { - payloads.push(payload) - } - sentText = step.nextSentText - heldText = step.heldText - } - if (options.commitAtEnd) { - const lastField = sentText + heldText - const step = computeTerminalLiveMirrorStep(sentText, lastField, { commitHeld: true }) - const payload = buildTerminalLiveMirrorPayload(step) - if (payload.length > 0) { - payloads.push(payload) - } - sentText = step.nextSentText - heldText = step.heldText - } - return { payloads, sentText, heldText } -} - -describe('terminal live hangul mirror', () => { - it('Given single-syllable composition When steps run Then leaks no jamo and commits only the final syllable', () => { - // Given / When - const run = runMirrorSequence(['ㅎ', '하', '한'], { commitAtEnd: true }) - - // Then - expect(run.payloads).toEqual(['한']) - expect(run.sentText).toBe('한') - expect(run.heldText).toBe('') - }) - - it('Given multi-syllable composition When a new syllable starts Then streams the stable prefix without erases', () => { - // Given / When - const run = runMirrorSequence(['ㅎ', '하', '한', '한ㄱ', '한그', '한글'], { commitAtEnd: true }) - - // Then - expect(run.payloads).toEqual(['한', '글']) - expect(run.sentText).toBe('한글') - }) - - it('Given dubeolsik resplit 간→가나 When steps run Then never sends the intermediate syllable', () => { - // Given / When - const run = runMirrorSequence(['ㄱ', '가', '간', '가나'], { commitAtEnd: true }) - - // Then - expect(run.payloads).toEqual(['가', '나']) - expect(run.sentText).toBe('가나') - }) - - it('Given a timer-committed syllable When composition continues Then erases and recommits via DEL correction', () => { - // Given: '하' was committed by the settle timer - const commit = computeTerminalLiveMirrorStep('', '하', { commitHeld: true }) - expect(buildTerminalLiveMirrorPayload(commit)).toBe('하') - expect(commit.nextSentText).toBe('하') - - // When: user keeps composing '하' → '한' - const correction = computeTerminalLiveMirrorStep(commit.nextSentText, '한', { - commitHeld: false - }) - - // Then: one DEL erases the stale syllable; the new one is held again - expect(buildTerminalLiveMirrorPayload(correction)).toBe('\x7f') - expect(correction.nextSentText).toBe('') - expect(correction.heldText).toBe('한') - - const recommit = computeTerminalLiveMirrorStep('', '한', { commitHeld: true }) - expect(buildTerminalLiveMirrorPayload(recommit)).toBe('한') - }) - - it('Given pure ASCII typing When steps run Then mirrors immediately with no held text', () => { - // Given / When - const run = runMirrorSequence(['a', 'ab', 'abc']) - - // Then - expect(run.payloads).toEqual(['a', 'b', 'c']) - expect(run.heldText).toBe('') - }) - - it('Given a trailing space after Hangul When the step runs Then the space commits the held syllable', () => { - // Given: '한글' typed, '한' streamed, '글' held - const beforeSpace = runMirrorSequence(['ㅎ', '하', '한', '한ㄱ', '한그', '한글']) - expect(beforeSpace.sentText).toBe('한') - expect(beforeSpace.heldText).toBe('글') - - // When - const step = computeTerminalLiveMirrorStep(beforeSpace.sentText, '한글 ', { - commitHeld: false - }) - - // Then - expect(buildTerminalLiveMirrorPayload(step)).toBe('글 ') - expect(step.heldText).toBe('') - expect(step.nextSentText).toBe('한글 ') - }) - - it('Given a trailing ASCII letter after Hangul When the step runs Then Hangul is committed with the letter', () => { - // Given - const held = computeTerminalLiveMirrorStep('', '한', { commitHeld: false }) - expect(held.heldText).toBe('한') - - // When - const step = computeTerminalLiveMirrorStep(held.nextSentText, '한a', { commitHeld: false }) - - // Then - expect(buildTerminalLiveMirrorPayload(step)).toBe('한a') - expect(step.heldText).toBe('') - }) - - it('Given sent text When the user deletes everything Then erases with one DEL per code point', () => { - // Given / When - const step = computeTerminalLiveMirrorStep('한글a', '', { commitHeld: false }) - - // Then - expect(step).toEqual({ - eraseCount: 3, - appendText: '', - nextSentText: '', - heldText: '' - }) - expect(buildTerminalLiveMirrorPayload(step)).toBe('\x7f\x7f\x7f') - }) - - it('Given empty field and empty sent text When committing Then produces a zero step', () => { - // Given / When - const step = computeTerminalLiveMirrorStep('', '', { commitHeld: true }) - - // Then - expect(buildTerminalLiveMirrorPayload(step)).toBe('') - expect(step).toEqual({ - eraseCount: 0, - appendText: '', - nextSentText: '', - heldText: '' - }) - }) - - it('Given non-Hangul IME text When the step runs Then it mirrors immediately without holding', () => { - // Given / When - const chinese = computeTerminalLiveMirrorStep('', '你好', { commitHeld: false }) - const vietnamese = computeTerminalLiveMirrorStep('', 'tiếng', { commitHeld: false }) - - // Then - expect(buildTerminalLiveMirrorPayload(chinese)).toBe('你好') - expect(chinese.heldText).toBe('') - expect(buildTerminalLiveMirrorPayload(vietnamese)).toBe('tiếng') - expect(vietnamese.heldText).toBe('') - }) - - it('Given Hangul code point ranges When checked Then jamo and syllables match and ASCII does not', () => { - expect(isTerminalLiveHangulCodePoint('ㅎ'.codePointAt(0) ?? 0)).toBe(true) - expect(isTerminalLiveHangulCodePoint('한'.codePointAt(0) ?? 0)).toBe(true) - expect(isTerminalLiveHangulCodePoint('a'.codePointAt(0) ?? 0)).toBe(false) - expect(isTerminalLiveHangulCodePoint('あ'.codePointAt(0) ?? 0)).toBe(false) - }) -}) diff --git a/mobile/src/terminal/terminal-live-hangul-mirror.ts b/mobile/src/terminal/terminal-live-hangul-mirror.ts deleted file mode 100644 index 170d1779ffd..00000000000 --- a/mobile/src/terminal/terminal-live-hangul-mirror.ts +++ /dev/null @@ -1,61 +0,0 @@ -// Why: paused composition should still reach the PTY quickly; corrections make -// a premature commit safe, so this can be short without leaking jamo forever. -export const TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS = 300 - -const TERMINAL_DEL_BYTE = '\x7f' - -export function isTerminalLiveHangulCodePoint(codePoint: number): boolean { - return ( - (codePoint >= 0x1100 && codePoint <= 0x11ff) || - (codePoint >= 0x3130 && codePoint <= 0x318f) || - (codePoint >= 0xa960 && codePoint <= 0xa97f) || - (codePoint >= 0xac00 && codePoint <= 0xd7af) - ) -} - -export type TerminalLiveMirrorStep = { - readonly eraseCount: number - readonly appendText: string - readonly nextSentText: string - readonly heldText: string -} - -// Why: React Native exposes no composition events, but Hangul composition only -// mutates the trailing syllable. Holding just that code point keeps the PTY -// echo live while preedit jamo never leak; DEL corrections repair any commit -// that later turns out to be premature. -export function computeTerminalLiveMirrorStep( - sentText: string, - fieldText: string, - options: { readonly commitHeld: boolean } -): TerminalLiveMirrorStep { - const fieldCodePoints = Array.from(fieldText) - const lastCodePoint = fieldCodePoints.at(-1) - const holdLast = - !options.commitHeld && - lastCodePoint !== undefined && - isTerminalLiveHangulCodePoint(lastCodePoint.codePointAt(0) ?? 0) - const heldText = holdLast && lastCodePoint !== undefined ? lastCodePoint : '' - const targetCodePoints = holdLast ? fieldCodePoints.slice(0, -1) : fieldCodePoints - const sentCodePoints = Array.from(sentText) - - let commonPrefixLength = 0 - while ( - commonPrefixLength < sentCodePoints.length && - commonPrefixLength < targetCodePoints.length && - sentCodePoints[commonPrefixLength] === targetCodePoints[commonPrefixLength] - ) { - commonPrefixLength += 1 - } - - return { - eraseCount: sentCodePoints.length - commonPrefixLength, - appendText: targetCodePoints.slice(commonPrefixLength).join(''), - nextSentText: targetCodePoints.join(''), - heldText - } -} - -export function buildTerminalLiveMirrorPayload(step: TerminalLiveMirrorStep): string { - return TERMINAL_DEL_BYTE.repeat(step.eraseCount) + step.appendText -} diff --git a/mobile/src/terminal/terminal-live-preedit-mirror.test.ts b/mobile/src/terminal/terminal-live-preedit-mirror.test.ts new file mode 100644 index 00000000000..4263e7335d7 --- /dev/null +++ b/mobile/src/terminal/terminal-live-preedit-mirror.test.ts @@ -0,0 +1,304 @@ +import { describe, expect, it } from 'vitest' +import { + buildTerminalLiveMirrorPayload, + computeTerminalLiveMirrorStep, + type TerminalLiveMirrorStep +} from './terminal-live-preedit-mirror' + +type MirrorRun = { + readonly payloads: readonly string[] + readonly sentText: string + readonly heldText: string +} + +type MirrorFrame = { readonly text: string; readonly composing?: boolean } + +function runMirrorSequence( + fieldStates: readonly (string | MirrorFrame)[], + options: { readonly commitAtEnd: boolean } = { commitAtEnd: false } +): MirrorRun { + const payloads: string[] = [] + let sentText = '' + let heldText = '' + for (const frame of fieldStates) { + const { text, composing } = typeof frame === 'string' ? { text: frame } : frame + const step = computeTerminalLiveMirrorStep(sentText, text, { commitHeld: false, composing }) + const payload = buildTerminalLiveMirrorPayload(step) + if (payload.length > 0) { + payloads.push(payload) + } + sentText = step.nextSentText + heldText = step.heldText + } + if (options.commitAtEnd) { + const lastField = sentText + heldText + const step = computeTerminalLiveMirrorStep(sentText, lastField, { commitHeld: true }) + const payload = buildTerminalLiveMirrorPayload(step) + if (payload.length > 0) { + payloads.push(payload) + } + sentText = step.nextSentText + heldText = step.heldText + } + return { payloads, sentText, heldText } +} + +describe('terminal live preedit mirror with a reported marked-text range', () => { + it('Given Japanese romaji composition When steps run Then no reading reaches the PTY and no DEL repairs it', () => { + // Given / When: `sa` composes to さ, then `sakura` converts to 桜 + const run = runMirrorSequence( + [ + { text: 's', composing: true }, + { text: 'さ', composing: true }, + { text: 'さく', composing: true }, + { text: 'さくら', composing: true }, + { text: '桜', composing: false } + ], + { commitAtEnd: true } + ) + + // Then + expect(run.payloads).toEqual(['桜']) + expect(run.sentText).toBe('桜') + expect(run.heldText).toBe('') + }) + + it('Given ASCII Chinese pinyin preedit When steps run Then the reading never leaks, which no code-point rule could achieve', () => { + // Given / When + const run = runMirrorSequence( + [ + { text: 'n', composing: true }, + { text: 'ni', composing: true }, + { text: 'niha', composing: true }, + { text: 'nihao', composing: true }, + { text: '你好', composing: false } + ], + { commitAtEnd: true } + ) + + // Then + expect(run.payloads).toEqual(['你好']) + expect(run.sentText).toBe('你好') + }) + + it('Given Hangul composition When the range is reported Then the syllable lands once with no DEL correction', () => { + // Given / When + const run = runMirrorSequence( + [ + { text: 'ㅎ', composing: true }, + { text: '하', composing: true }, + { text: '한', composing: true }, + { text: '한', composing: false } + ], + { commitAtEnd: true } + ) + + // Then + expect(run.payloads).toEqual(['한']) + expect(run.sentText).toBe('한') + }) + + it('Given ASCII typing When the range is reported empty Then every character mirrors immediately', () => { + // Given / When + const run = runMirrorSequence([ + { text: 'a', composing: false }, + { text: 'ab', composing: false }, + { text: 'abc', composing: false } + ]) + + // Then + expect(run.payloads).toEqual(['a', 'b', 'c']) + expect(run.heldText).toBe('') + }) + + it('Given committed text When a new composition starts Then the commit stays and only the preedit is held', () => { + // Given + const committed = computeTerminalLiveMirrorStep('', 'ls ', { + commitHeld: false, + composing: false + }) + expect(committed.nextSentText).toBe('ls ') + + // When + const step = computeTerminalLiveMirrorStep('ls ', 'ls さ', { + commitHeld: false, + composing: true + }) + + // Then + expect(buildTerminalLiveMirrorPayload(step)).toBe('') + expect(step.nextSentText).toBe('ls ') + expect(step.heldText).toBe('さ') + }) + + it('Given a composition that rewrites already-sent text When the step runs Then DEL erases only the difference', () => { + // Given / When: the input method reclaimed a code point the PTY already has + const step = computeTerminalLiveMirrorStep('abc', 'abX', { commitHeld: false, composing: true }) + + // Then + expect(buildTerminalLiveMirrorPayload(step)).toBe('\x7f') + expect(step.nextSentText).toBe('ab') + expect(step.heldText).toBe('X') + }) +}) + +describe('terminal live preedit mirror with no marked-text report', () => { + it('Given Japanese romaji When nothing is reported Then the whole multi-code-point reading is held as one unit', () => { + // Given / When: きょう is three code points, so holding one trailing code point would leak きょ + const run = runMirrorSequence(['k', 'ky', 'きょ', 'きょう'], { commitAtEnd: true }) + + // The ASCII part of the reading still leaks and is DEL-repaired — only a + // marked-text range can see an ASCII preedit — but no partial kana reaches the + // PTY, which a trailing-single-code-point hold could not achieve. + expect(run.payloads).toEqual(['k', 'y', '\x7f\x7f', 'きょう']) + expect(run.sentText).toBe('きょう') + }) + + it('Given Hangul composition When nothing is reported Then no jamo leaks and only the final syllable commits', () => { + // Given / When + const run = runMirrorSequence(['ㅎ', '하', '한'], { commitAtEnd: true }) + + // Then + expect(run.payloads).toEqual(['한']) + expect(run.sentText).toBe('한') + expect(run.heldText).toBe('') + }) + + it('Given multi-syllable Hangul When nothing is reported Then the whole run stays held until commit', () => { + // Given / When + const run = runMirrorSequence(['ㅎ', '하', '한', '한ㄱ', '한그', '한글'], { commitAtEnd: true }) + + // Then + expect(run.payloads).toEqual(['한글']) + expect(run.sentText).toBe('한글') + }) + + it('Given dubeolsik resplit 간→가나 When nothing is reported Then no intermediate syllable is ever sent', () => { + // Given / When + const run = runMirrorSequence(['ㄱ', '가', '간', '가나'], { commitAtEnd: true }) + + // Then + expect(run.payloads).toEqual(['가나']) + expect(run.sentText).toBe('가나') + }) + + it('Given a timer-committed hold When composition continues Then DEL corrections repair the premature commit', () => { + // Given: '하' was committed by the settle timer + const commit = computeTerminalLiveMirrorStep('', '하', { commitHeld: true }) + expect(buildTerminalLiveMirrorPayload(commit)).toBe('하') + + // When: user keeps composing '하' → '한' + const correction = computeTerminalLiveMirrorStep(commit.nextSentText, '한', { + commitHeld: false + }) + + // Then + expect(buildTerminalLiveMirrorPayload(correction)).toBe('\x7f') + expect(correction.nextSentText).toBe('') + expect(correction.heldText).toBe('한') + }) + + it('Given pure ASCII typing When nothing is reported Then mirroring stays immediate with no held text', () => { + // Given / When + const run = runMirrorSequence(['a', 'ab', 'abc']) + + // Then + expect(run.payloads).toEqual(['a', 'b', 'c']) + expect(run.heldText).toBe('') + }) + + it('Given a trailing space after held text When the step runs Then the space commits the hold', () => { + // Given + const beforeSpace = runMirrorSequence(['ㅎ', '하', '한']) + expect(beforeSpace.heldText).toBe('한') + + // When + const step = computeTerminalLiveMirrorStep(beforeSpace.sentText, '한 ', { commitHeld: false }) + + // Then + expect(buildTerminalLiveMirrorPayload(step)).toBe('한 ') + expect(step.heldText).toBe('') + }) + + it('Given a trailing ASCII letter after held text When the step runs Then the hold commits with the letter', () => { + // Given + const held = computeTerminalLiveMirrorStep('', '한', { commitHeld: false }) + expect(held.heldText).toBe('한') + + // When + const step = computeTerminalLiveMirrorStep(held.nextSentText, '한a', { commitHeld: false }) + + // Then + expect(buildTerminalLiveMirrorPayload(step)).toBe('한a') + expect(step.heldText).toBe('') + }) + + it('Given Vietnamese telex whose commit ends in ASCII When the step runs Then it mirrors immediately', () => { + // Given / When + const step = computeTerminalLiveMirrorStep('', 'tiếng', { commitHeld: false }) + + // Then + expect(buildTerminalLiveMirrorPayload(step)).toBe('tiếng') + expect(step.heldText).toBe('') + }) + + it('Given sent text When the user deletes everything Then erases with one DEL per code point', () => { + // Given / When + const step = computeTerminalLiveMirrorStep('한글a', '', { commitHeld: false }) + + // Then + expect(step).toEqual({ + eraseCount: 3, + appendText: '', + nextSentText: '', + heldText: '' + }) + expect(buildTerminalLiveMirrorPayload(step)).toBe('\x7f\x7f\x7f') + }) + + // A settle-timer commit promotes the held run to sent text. The next keystroke must not reach + // back over it: the fallback has no composing signal, so without a bound it re-holds everything + // already delivered and the caller erases it with DEL. Measured at 120 DELs for this word. + it('Given a non-composing script on the fallback path When the settle timer commits between keystrokes Then nothing already sent is erased', () => { + // Given + const word = Array.from('приветмир') + let sentText = '' + let field = '' + const payloads: string[] = [] + + // When — every keystroke is followed by a settle commit, the Android shape + for (const codePoint of word) { + field += codePoint + const live = computeTerminalLiveMirrorStep(sentText, field, { commitHeld: false }) + const livePayload = buildTerminalLiveMirrorPayload(live) + if (livePayload.length > 0) { + payloads.push(livePayload) + } + const settled = computeTerminalLiveMirrorStep(live.nextSentText, field, { commitHeld: true }) + const settledPayload = buildTerminalLiveMirrorPayload(settled) + if (settledPayload.length > 0) { + payloads.push(settledPayload) + } + sentText = settled.nextSentText + } + + // Then + const wire = payloads.join('') + expect(wire, 'a DEL erases a character the pty already received').not.toContain('\x7f') + expect(wire).toBe(word.join('')) + }) + + it('Given empty field and empty sent text When committing Then produces a zero step', () => { + // Given / When + const step = computeTerminalLiveMirrorStep('', '', { commitHeld: true }) + + // Then + expect(buildTerminalLiveMirrorPayload(step)).toBe('') + expect(step).toEqual({ + eraseCount: 0, + appendText: '', + nextSentText: '', + heldText: '' + }) + }) +}) diff --git a/mobile/src/terminal/terminal-live-preedit-mirror.ts b/mobile/src/terminal/terminal-live-preedit-mirror.ts new file mode 100644 index 00000000000..29c903f4366 --- /dev/null +++ b/mobile/src/terminal/terminal-live-preedit-mirror.ts @@ -0,0 +1,83 @@ +// Why: an unreported preedit that pauses should still reach the PTY quickly; +// corrections make a premature commit safe. Only armed when the platform +// reports nothing — a reported preedit is not text yet and must never time out. +export const TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS = 300 + +const TERMINAL_DEL_BYTE = '\x7f' +const LAST_ASCII_CODE_POINT = 0x7f + +export type TerminalLiveMirrorStep = { + readonly eraseCount: number + readonly appendText: string + readonly nextSentText: string + readonly heldText: string +} + +/** + * How much of the field the text system can still rewrite, in code points. + * + * Preedit is a fact about the field, never about the script — Chinese pinyin + * preedit is plain ASCII — so the marked-text range decides and no code point is + * ever classified. That is what makes this hold for input methods nobody tested. + * + * `composing` is undefined only where the platform reports no range at all — + * today React Native Android. The fallback holds the trailing non-ASCII run: a + * conversion IME always leaves one and ASCII keeps its zero-latency echo. It + * enumerates nothing, and it cannot see an ASCII preedit — only a report can. + */ +function heldPreeditLength( + fieldCodePoints: readonly string[], + stableLength: number, + composing: boolean | undefined +): number { + if (composing !== undefined) { + return composing ? fieldCodePoints.length - stableLength : 0 + } + let held = 0 + while ( + held < fieldCodePoints.length && + (fieldCodePoints[fieldCodePoints.length - 1 - held]?.codePointAt(0) ?? 0) > + LAST_ASCII_CODE_POINT + ) { + held += 1 + } + // Why the bound: the run can reach back over code points already delivered to the pty, and + // holding those makes the caller erase them with DEL and retype them. The reported branch above + // subtracts `stableLength` for the same reason; without it a settle-timer commit followed by + // another keystroke costs a DEL per already-sent character. + return Math.min(held, fieldCodePoints.length - stableLength) +} + +function commonPrefixLength(left: readonly string[], right: readonly string[]): number { + let length = 0 + while (length < left.length && length < right.length && left[length] === right[length]) { + length += 1 + } + return length +} + +export function computeTerminalLiveMirrorStep( + sentText: string, + fieldText: string, + options: { readonly commitHeld: boolean; readonly composing?: boolean } +): TerminalLiveMirrorStep { + const fieldCodePoints = Array.from(fieldText) + const sentCodePoints = Array.from(sentText) + const stableLength = commonPrefixLength(sentCodePoints, fieldCodePoints) + const heldLength = options.commitHeld + ? 0 + : heldPreeditLength(fieldCodePoints, stableLength, options.composing) + const targetCodePoints = fieldCodePoints.slice(0, fieldCodePoints.length - heldLength) + const keptLength = Math.min(stableLength, targetCodePoints.length) + + return { + eraseCount: sentCodePoints.length - keptLength, + appendText: targetCodePoints.slice(keptLength).join(''), + nextSentText: targetCodePoints.join(''), + heldText: fieldCodePoints.slice(fieldCodePoints.length - heldLength).join('') + } +} + +export function buildTerminalLiveMirrorPayload(step: TerminalLiveMirrorStep): string { + return TERMINAL_DEL_BYTE.repeat(step.eraseCount) + step.appendText +} diff --git a/mobile/src/terminal/use-terminal-live-input-commit.test.ts b/mobile/src/terminal/use-terminal-live-input-commit.test.ts index 544d57a1bf1..d4a39d2d417 100644 --- a/mobile/src/terminal/use-terminal-live-input-commit.test.ts +++ b/mobile/src/terminal/use-terminal-live-input-commit.test.ts @@ -3,12 +3,23 @@ import { act, create, type ReactTestRenderer } from 'react-test-renderer' import type { TextInput } from 'react-native' import { afterEach, describe, expect, it, vi } from 'vitest' import type { TerminalLiveInputSender } from './terminal-live-input-sender' -import { TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS } from './terminal-live-hangul-mirror' +import { TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS } from './terminal-live-preedit-mirror' import { useTerminalLiveInputCommit } from './use-terminal-live-input-commit' +type TerminalLiveInputCommitHandlers = ReturnType> + +/** `isComposing` omitted models a platform that reports no marked-text range. */ +function changeLiveInput( + handlers: TerminalLiveInputCommitHandlers, + text: string, + isComposing?: boolean +): void { + handlers.handleLiveInputChange({ nativeEvent: { text, isComposing } }) +} + type TerminalLiveInputCommitHarness = { readonly captures: readonly string[] - readonly handlers: ReturnType> + readonly handlers: TerminalLiveInputCommitHandlers readonly sent: readonly string[] readonly setActiveSessionTabType: (next: string | undefined) => void readonly setConnected: (next: boolean) => void @@ -46,7 +57,7 @@ function createTerminalLiveInputCommitHarness({ // Refs never re-render; only these variables re-run the hook's clear effects. let currentActiveSessionTabType: string | undefined = 'terminal' let currentConnected = true - let handlers: ReturnType> | null = null + let handlers: TerminalLiveInputCommitHandlers | null = null let renderer: ReactTestRenderer | null = null function Harness(): null { @@ -105,29 +116,59 @@ describe('terminal live input commit hook', () => { vi.useRealTimers() }) - it('Given Hangul composition When steps arrive Then streams the stable prefix and never leaks jamo', async () => { + it('Given Hangul composition and no marked-text report When steps arrive Then no jamo leaks', async () => { // Given vi.useFakeTimers() const { handlers, sent } = createTerminalLiveInputCommitHarness() // When: ㅎ→하→한→한ㄱ→한그→한글 (no settle pause between steps) for (const fieldText of ['ㅎ', '하', '한', '한ㄱ', '한그', '한글']) { - handlers.handleLiveInputChange(fieldText) + changeLiveInput(handlers, fieldText) await vi.advanceTimersByTimeAsync(50) } - // Then: only the stable prefix went out; the trailing syllable is held - await vi.waitFor(() => expect(sent).toEqual(['한'])) + // Then: the settle timer commits the run intact; no jamo and no DEL repair + await vi.waitFor(() => expect(sent).toEqual(['한글'])) + }) + + it('Given Japanese romaji When the marked-text range is reported Then no reading reaches the terminal', async () => { + // Given + vi.useFakeTimers() + const { handlers, sent } = createTerminalLiveInputCommitHarness() + + // When: `sa` reads as さ, converts to 桜, and only then commits + for (const fieldText of ['s', 'さ', 'さく', 'さくら']) { + changeLiveInput(handlers, fieldText, true) + await vi.advanceTimersByTimeAsync(50) + } + await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS * 4) + changeLiveInput(handlers, '桜', false) + + // Then: the reading never reached the PTY, so nothing had to be erased + await vi.waitFor(() => expect(sent).toEqual(['桜'])) + }) + + it('Given a reported preedit that goes idle When the settle delay passes Then nothing is committed on a timer', async () => { + // Given: preedit is not text yet, so no timer may promote it + vi.useFakeTimers() + const { handlers, sent } = createTerminalLiveInputCommitHarness() + + // When + changeLiveInput(handlers, 'nihao', true) + await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS * 10) + + // Then + expect(sent).toEqual([]) }) it('Given a held syllable When the settle timer elapses Then commits it to the terminal', async () => { // Given vi.useFakeTimers() const { handlers, sent } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When - await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS) + await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS) // Then await vi.waitFor(() => expect(sent).toEqual(['한'])) @@ -137,13 +178,13 @@ describe('terminal live input commit hook', () => { // Given vi.useFakeTimers() const { handlers, sent } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('하') - await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS) + changeLiveInput(handlers, '하') + await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS) await vi.waitFor(() => expect(sent).toEqual(['하'])) // When - handlers.handleLiveInputChange('한') - await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS) + changeLiveInput(handlers, '한') + await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS) // Then await vi.waitFor(() => expect(sent).toEqual(['하', '\x7f', '한'])) @@ -152,7 +193,7 @@ describe('terminal live input commit hook', () => { it('Given Hangul pending text When submit is requested Then sends composed text before carriage return', async () => { // Given const { handlers, sent } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When handlers.handleLiveInputSubmit() @@ -175,7 +216,7 @@ describe('terminal live input commit hook', () => { it('Given a rejected held-text send When submit is requested Then suppresses the carriage return', async () => { // Given const { handlers, sent } = createTerminalLiveInputCommitHarness({ sendResult: false }) - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When handlers.handleLiveInputSubmit() @@ -191,8 +232,8 @@ describe('terminal live input commit hook', () => { const { handlers, sent } = createTerminalLiveInputCommitHarness() // When - handlers.handleLiveInputChange('a') - handlers.handleLiveInputChange('ab') + changeLiveInput(handlers, 'a') + changeLiveInput(handlers, 'ab') // Then await vi.waitFor(() => expect(sent).toEqual(['a', 'b'])) @@ -203,7 +244,7 @@ describe('terminal live input commit hook', () => { const { captures, handlers, sent } = createTerminalLiveInputCommitHarness() // When: iOS smart punctuation rewrote "--" into an en dash inside the field - handlers.handleLiveInputChange('a–') + changeLiveInput(handlers, 'a–') // Then: writing "a--" back into the controlled value would kill an active // iOS dictation/IME session, so the capture must keep what iOS produced @@ -216,8 +257,8 @@ describe('terminal live input commit hook', () => { const { captures, handlers, sent } = createTerminalLiveInputCommitHarness() // When: iOS dictation replaces its hypothesis as recognition refines - handlers.handleLiveInputChange('high') - handlers.handleLiveInputChange('hi there') + changeLiveInput(handlers, 'high') + changeLiveInput(handlers, 'hi there') // Then: captures only echo the field; the mirror repairs the PTY with DELs expect(captures).toEqual(['high', 'hi there']) @@ -227,10 +268,10 @@ describe('terminal live input commit hook', () => { it('Given a trailing space after Hangul When the change arrives Then the space commits the held syllable', async () => { // Given const { handlers, sent } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When - handlers.handleLiveInputChange('한 ') + changeLiveInput(handlers, '한 ') // Then await vi.waitFor(() => expect(sent).toEqual(['한 '])) @@ -239,7 +280,7 @@ describe('terminal live input commit hook', () => { it('Given Hangul pending text When an external terminal send is requested Then flushes composed text first', async () => { // Given const { handlers, sent } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When const flushed = await handlers.flushPendingLiveInputBeforeExternalSend('terminal-a') @@ -252,7 +293,7 @@ describe('terminal live input commit hook', () => { it('Given pending text cannot be sent When an external terminal send is requested Then reports failure', async () => { // Given const { handlers, sent } = createTerminalLiveInputCommitHarness({ sendResult: false }) - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When const flushed = await handlers.flushPendingLiveInputBeforeExternalSend('terminal-a') @@ -267,7 +308,7 @@ describe('terminal live input commit hook', () => { const { handlers, sent } = createTerminalLiveInputCommitHarness() // When - handlers.handleLiveInputChange('你好') + changeLiveInput(handlers, '你好') // Then await vi.waitFor(() => expect(sent).toEqual(['你好'])) @@ -277,7 +318,7 @@ describe('terminal live input commit hook', () => { // Given vi.useFakeTimers() const { handlers, sent, unmount } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When unmount() @@ -290,7 +331,7 @@ describe('terminal live input commit hook', () => { it('Given Backspace with field text When the key arrives Then edits locally without terminal bytes', async () => { // Given const { handlers, sent } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When handlers.handleLiveInputKeyPress({ nativeEvent: { key: 'Backspace' } }) @@ -302,7 +343,7 @@ describe('terminal live input commit hook', () => { it('Given Tab with a held syllable When the key arrives Then commits the syllable before the tab bytes', async () => { // Given const { handlers, sent } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When handlers.handleLiveInputKeyPress({ nativeEvent: { key: 'Tab' } }) @@ -314,7 +355,7 @@ describe('terminal live input commit hook', () => { it('Given Hangul pending When the tab type lags to undefined Then keeps the composition state', async () => { // Given: '한' held while the active tab is still a terminal const { handlers, sent, setActiveSessionTabType } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When: the mobile tab list momentarily yields no active tab object setActiveSessionTabType(undefined) @@ -327,7 +368,7 @@ describe('terminal live input commit hook', () => { it('Given Hangul pending When the tab genuinely changes to non-terminal Then clears the composition state', async () => { // Given: '한' held while the active tab is still a terminal const { handlers, sent, setActiveSessionTabType } = createTerminalLiveInputCommitHarness() - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When: the active tab actually becomes a non-terminal (chat) tab setActiveSessionTabType('chat') @@ -341,7 +382,7 @@ describe('terminal live input commit hook', () => { // Given: a stalled link — the mirror sends but the PTY never accepts (#6713 second defect) const { captures, handlers, sent, setConnected, setSendResult } = createTerminalLiveInputCommitHarness({ sendResult: false }) - handlers.handleLiveInputChange('XYZZY') + changeLiveInput(handlers, 'XYZZY') await vi.waitFor(() => expect(sent).toEqual(['XYZZY'])) // When: the outage is finally detected, then the link recovers @@ -353,7 +394,7 @@ describe('terminal live input commit hook', () => { // 'XYZZY…' replayed and not DELs erasing PTY chars that never arrived expect(captures.at(-1)).toBe('') const sentBeforeRecovery = sent.length - handlers.handleLiveInputChange('echo CLEANLINE') + changeLiveInput(handlers, 'echo CLEANLINE') await vi.waitFor(() => expect(sent.slice(sentBeforeRecovery)).toEqual(['echo CLEANLINE'])) }) @@ -363,11 +404,11 @@ describe('terminal live input commit hook', () => { const { handlers, sent, setConnected } = createTerminalLiveInputCommitHarness({ sendResult: false }) - handlers.handleLiveInputChange('한') + changeLiveInput(handlers, '한') // When setConnected(false) - await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS) + await vi.advanceTimersByTimeAsync(TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS) // Then: the outage cleared the held text before the timer could send it expect(sent).toEqual([]) diff --git a/mobile/src/terminal/use-terminal-live-input-commit.ts b/mobile/src/terminal/use-terminal-live-input-commit.ts index d3b3e3fec09..b05241f6421 100644 --- a/mobile/src/terminal/use-terminal-live-input-commit.ts +++ b/mobile/src/terminal/use-terminal-live-input-commit.ts @@ -17,6 +17,16 @@ type TerminalLiveInputKeyPressEvent = { } } +/** `isComposing` is the text system's marked-text range, forwarded by the pinned + * react-native patch on iOS; `onChangeText` would drop the payload entirely. + * Absent means the platform reports no range — not "not composing". */ +type TerminalLiveInputChangeEvent = { + readonly nativeEvent: { + readonly text: string + readonly isComposing?: boolean + } +} + type TerminalLiveInputCommitOptions = { readonly activeHandle: string | null readonly activeHandleRef: RefObject @@ -36,7 +46,7 @@ type TerminalLiveInputCommitHandlers = { readonly handleLiveInputAccessoryBytes: ( input: TerminalLiveAccessoryInput ) => Promise - readonly handleLiveInputChange: (text: string) => void + readonly handleLiveInputChange: (event: TerminalLiveInputChangeEvent) => void readonly handleLiveInputKeyPress: (event: TerminalLiveInputKeyPressEvent) => void readonly handleLiveInputSubmit: () => void } @@ -113,7 +123,7 @@ export function useTerminalLiveInputCommit({ ) const handleLiveInputChange = useCallback( - (text: string) => { + ({ nativeEvent }: TerminalLiveInputChangeEvent) => { if (!activeHandle || !liveInputTerminalHandles.has(activeHandle)) { clearPendingLiveInputCommit() return @@ -121,8 +131,12 @@ export function useTerminalLiveInputCommit({ // Why: iOS kills an active dictation/IME session when JS writes a value // that differs from the native field text, so the controlled capture must // echo the field verbatim; only the PTY mirror sees normalized text. - setLiveInputCapture(text) - applyLiveInputMirror(activeHandle, normalizeTerminalTextInput(text)) + setLiveInputCapture(nativeEvent.text) + applyLiveInputMirror( + activeHandle, + normalizeTerminalTextInput(nativeEvent.text), + nativeEvent.isComposing + ) }, [ activeHandle, diff --git a/mobile/src/terminal/use-terminal-live-pending-input-flush.ts b/mobile/src/terminal/use-terminal-live-pending-input-flush.ts index c615a06dfaf..75dd3e49f61 100644 --- a/mobile/src/terminal/use-terminal-live-pending-input-flush.ts +++ b/mobile/src/terminal/use-terminal-live-pending-input-flush.ts @@ -4,8 +4,8 @@ import type { TerminalLiveInputSender } from './terminal-live-input-sender' import { buildTerminalLiveMirrorPayload, computeTerminalLiveMirrorStep, - TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS -} from './terminal-live-hangul-mirror' + TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS +} from './terminal-live-preedit-mirror' import { cancelTerminalLivePendingFlush, createTerminalLivePendingFlushState, @@ -22,8 +22,15 @@ type TerminalLivePendingInputFlushOptions = { readonly setLiveInputCapture: (text: string) => void } +type RunTerminalLiveMirrorStep = ( + handle: string, + fieldText: string, + commitHeld: boolean, + composing?: boolean +) => Promise + type TerminalLivePendingInputFlush = { - readonly applyLiveInputMirror: (handle: string, fieldText: string) => void + readonly applyLiveInputMirror: (handle: string, fieldText: string, composing?: boolean) => void readonly clearPendingLiveInputCommit: () => void readonly flushPendingLiveInputText: (expectedHandle: string | null) => Promise readonly heldLiveInputTextRef: RefObject @@ -45,9 +52,7 @@ export function useTerminalLivePendingInputFlush({ const heldLiveInputTextRef = useRef('') const sentLiveInputTextRef = useRef('') const pendingLiveInputHandleRef = useRef(null) - const runMirrorStepRef = useRef< - (handle: string, fieldText: string, commitHeld: boolean) => Promise - >(async () => false) + const runMirrorStepRef = useRef(async () => false) const clearHeldCommitTimer = useCallback(() => { if (heldCommitTimerRef.current) { @@ -80,8 +85,8 @@ export function useTerminalLivePendingInputFlush({ [sendLiveTerminalInputRef] ) - const runMirrorStep = useCallback( - async (handle: string, fieldText: string, commitHeld: boolean): Promise => { + const runMirrorStep = useCallback( + async (handle, fieldText, commitHeld, composing) => { if ( handle !== activeHandleRef.current || (activeSessionTabTypeRef.current != null && @@ -96,7 +101,8 @@ export function useTerminalLivePendingInputFlush({ } const step = computeTerminalLiveMirrorStep(sentLiveInputTextRef.current, fieldText, { - commitHeld + commitHeld, + composing }) sentLiveInputTextRef.current = step.nextSentText heldLiveInputTextRef.current = step.heldText @@ -104,12 +110,15 @@ export function useTerminalLivePendingInputFlush({ step.heldText.length > 0 || step.nextSentText.length > 0 ? handle : null clearHeldCommitTimer() - if (step.heldText.length > 0) { + // Why: text the platform positively marked as preedit is not text yet, so + // no idle timer may commit it. Only an unreported hold is a guess that has + // to settle on its own. + if (step.heldText.length > 0 && composing === undefined) { heldCommitTimerRef.current = setTimeout(() => { heldCommitTimerRef.current = null const heldField = sentLiveInputTextRef.current + heldLiveInputTextRef.current void runMirrorStepRef.current(handle, heldField, true) - }, TERMINAL_LIVE_HELD_SYLLABLE_COMMIT_DELAY_MS) + }, TERMINAL_LIVE_HELD_PREEDIT_COMMIT_DELAY_MS) } const payload = buildTerminalLiveMirrorPayload(step) @@ -140,8 +149,8 @@ export function useTerminalLivePendingInputFlush({ }, [runMirrorStep]) const applyLiveInputMirror = useCallback( - (handle: string, fieldText: string): void => { - void runMirrorStep(handle, fieldText, false) + (handle: string, fieldText: string, composing?: boolean): void => { + void runMirrorStep(handle, fieldText, false, composing) }, [runMirrorStep] ) From b279f66c96a7e0d4a120438b9d7a9445aeb1df44 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:02:36 -0700 Subject: [PATCH 08/10] Include descendants in Pinned section when parent is pinned (#15035) Descendants of pinned parents now appear in the Pinned section without individual pins. When an ancestor's pin state changes, reveal the active descendant to maintain focus and visibility. --- .../WorktreeList.group-headers.test.ts | 18 ++ .../src/components/sidebar/WorktreeList.tsx | 2 + .../sidebar/pinned-section-worktrees.test.ts | 173 ++++++++++++++++++ .../sidebar/pinned-section-worktrees.ts | 52 ++++++ ...rktree-list-groups-lineage-nesting.test.ts | 117 ++++++++++++ .../worktree-list/grouping/build-rows.ts | 15 +- .../worktree-list/grouping/row-builders.ts | 147 ++++++++------- .../listing/use-collapsed-groups.ts | 14 +- .../navigation/pending-reveal-inputs.ts | 12 +- .../navigation/reveal-ancestors.ts | 6 +- ...orktrees-workspace-selection-state.test.ts | 161 +++++++++++++++- .../worktrees/session/worktree-pin-reveal.ts | 59 ++++++ 12 files changed, 704 insertions(+), 72 deletions(-) create mode 100644 src/renderer/src/components/sidebar/pinned-section-worktrees.test.ts create mode 100644 src/renderer/src/components/sidebar/pinned-section-worktrees.ts diff --git a/src/renderer/src/components/sidebar/WorktreeList.group-headers.test.ts b/src/renderer/src/components/sidebar/WorktreeList.group-headers.test.ts index 091ae7a018d..092faad697a 100644 --- a/src/renderer/src/components/sidebar/WorktreeList.group-headers.test.ts +++ b/src/renderer/src/components/sidebar/WorktreeList.group-headers.test.ts @@ -152,6 +152,24 @@ describe('WorktreeList lineage child card renderer', () => { expect(markup).not.toContain('data-repo-header-collapse-affordance=""') }) + it('uncollapses pinned reveal for a descendant that only lives under a pinned parent', () => { + const child = makeWorktree({ + id: 'child-of-pinned', + instanceId: 'child-of-pinned-instance', + displayName: 'Child of pinned', + branch: 'child', + sortOrder: 2 + }) + + expect( + getPinnedWorktreeRevealCollapsedGroupKeys({ + worktree: child, + collapsedGroups: new Set(['pinned', 'all']), + inPinnedSection: true + }) + ).toEqual(['pinned']) + }) + it('uncollapses pinned reveal through the pinned section after host expansion', () => { const worktree = makeWorktree({ id: 'pinned-ssh', diff --git a/src/renderer/src/components/sidebar/WorktreeList.tsx b/src/renderer/src/components/sidebar/WorktreeList.tsx index 4733168e1fb..bcc5c65e4f0 100644 --- a/src/renderer/src/components/sidebar/WorktreeList.tsx +++ b/src/renderer/src/components/sidebar/WorktreeList.tsx @@ -117,6 +117,8 @@ const WorktreeList = React.memo(function WorktreeList({ collapsedGroups, agentSendTargetWorktreeId, groupBy, + pinnedDisplayPolicy, + visibleWorktrees, repoMap, worktreeMap, worktreeLineageById, diff --git a/src/renderer/src/components/sidebar/pinned-section-worktrees.test.ts b/src/renderer/src/components/sidebar/pinned-section-worktrees.test.ts new file mode 100644 index 00000000000..ce520b751b1 --- /dev/null +++ b/src/renderer/src/components/sidebar/pinned-section-worktrees.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from 'vitest' +import type { WorktreeLineage } from '../../../../shared/worktree/lineage-types' +import type { Worktree } from '../../../../shared/worktree/types' +import { getPinnedSectionWorktrees, isPinnedSectionWorktree } from './pinned-section-worktrees' +import { worktree, repoMap } from './worktree-list-groups-test-fixtures' +import { buildRows } from './worktree-list/grouping/build-rows' + +function makeWorktree(id: string, overrides: Partial = {}): Worktree { + return { + ...worktree, + id, + instanceId: `${id}-instance`, + displayName: id, + ...overrides + } +} + +function makeLineage(child: Worktree, parent: Worktree): WorktreeLineage { + return { + worktreeId: child.id, + worktreeInstanceId: child.instanceId!, + parentWorktreeId: parent.id, + parentWorktreeInstanceId: parent.instanceId!, + origin: 'cli', + capture: { source: 'terminal-context', confidence: 'inferred' }, + createdAt: 1 + } +} + +describe('getPinnedSectionWorktrees', () => { + const parent = makeWorktree('parent') + const child = makeWorktree('child') + const grandchild = makeWorktree('grandchild') + const sibling = makeWorktree('sibling') + const lineageById = { + [child.id]: makeLineage(child, parent), + [grandchild.id]: makeLineage(grandchild, child) + } + const worktreeMap = new Map([ + [parent.id, parent], + [child.id, child], + [grandchild.id, grandchild], + [sibling.id, sibling] + ]) + + it('includes only explicitly pinned rows when there is no lineage', () => { + const pinned = makeWorktree('pinned', { isPinned: true }) + const loose = makeWorktree('loose') + + expect(getPinnedSectionWorktrees([pinned, loose], {}, new Map([[pinned.id, pinned]]))).toEqual([ + pinned + ]) + }) + + it('includes visible descendants of a pinned parent', () => { + const pinnedParent = { ...parent, isPinned: true } + const map = new Map(worktreeMap) + map.set(pinnedParent.id, pinnedParent) + + expect( + getPinnedSectionWorktrees([pinnedParent, child, grandchild, sibling], lineageById, map).map( + (worktree) => worktree.id + ) + ).toEqual([pinnedParent.id, child.id, grandchild.id]) + }) + + it("does not pull a pinned child's unpinned parent into Pinned", () => { + const pinnedChild = { ...child, isPinned: true } + const map = new Map(worktreeMap) + map.set(pinnedChild.id, pinnedChild) + + expect( + getPinnedSectionWorktrees([parent, pinnedChild], lineageById, map).map( + (worktree) => worktree.id + ) + ).toEqual([pinnedChild.id]) + }) + + it('keeps a visible grandchild when the middle parent is absent from the visible list', () => { + const pinnedParent = { ...parent, isPinned: true } + const map = new Map(worktreeMap) + map.set(pinnedParent.id, pinnedParent) + + expect( + getPinnedSectionWorktrees([pinnedParent, grandchild], lineageById, map).map( + (worktree) => worktree.id + ) + ).toEqual([pinnedParent.id, grandchild.id]) + }) + + it('does not collect descendants of a pinned parent that is not itself visible', () => { + const pinnedParent = { ...parent, isPinned: true } + const map = new Map(worktreeMap) + map.set(pinnedParent.id, pinnedParent) + + expect( + getPinnedSectionWorktrees([child, sibling], lineageById, map).map((worktree) => worktree.id) + ).toEqual([]) + }) + + it('handles a deep pinned lineage without overflowing the renderer stack', () => { + const depth = 6_000 + const worktrees: Worktree[] = [] + const lineageById: Record = {} + for (let index = 0; index < depth; index++) { + const current = makeWorktree(`deep-${index}`, { isPinned: index === 0 }) + worktrees.push(current) + const parent = worktrees[index - 1] + if (parent) { + lineageById[current.id] = makeLineage(current, parent) + } + } + + expect( + getPinnedSectionWorktrees( + worktrees, + lineageById, + new Map(worktrees.map((worktree) => [worktree.id, worktree])) + ) + ).toHaveLength(depth) + }) + + it('builds rows for a deep expanded pinned lineage without overflowing the renderer stack', () => { + const depth = 6_000 + const worktrees: Worktree[] = [] + const lineageById: Record = {} + for (let index = 0; index < depth; index++) { + const current = makeWorktree(`rendered-deep-${index}`, { isPinned: true }) + worktrees.push(current) + const parent = worktrees[index - 1] + if (parent) { + lineageById[current.id] = makeLineage(current, parent) + } + } + + const rows = buildRows( + 'none', + worktrees, + repoMap, + null, + new Set(), + undefined, + undefined, + undefined, + lineageById, + new Map(worktrees.map((worktree) => [worktree.id, worktree])), + true + ) + + expect(rows.filter((row) => row.type === 'item')).toHaveLength(depth) + }) +}) + +describe('isPinnedSectionWorktree', () => { + const parent = makeWorktree('parent', { isPinned: true }) + const child = makeWorktree('child') + const lineageById = { [child.id]: makeLineage(child, parent) } + const worktreeMap = new Map([ + [parent.id, parent], + [child.id, child] + ]) + + it('treats an unpinned child of a visible pinned parent as pinned-section membership', () => { + expect(isPinnedSectionWorktree(child, [parent, child], lineageById, worktreeMap)).toBe(true) + }) + + it('does not treat an unrelated unpinned workspace as pinned-section membership', () => { + const other = makeWorktree('other') + expect(isPinnedSectionWorktree(other, [parent, child, other], lineageById, worktreeMap)).toBe( + false + ) + }) +}) diff --git a/src/renderer/src/components/sidebar/pinned-section-worktrees.ts b/src/renderer/src/components/sidebar/pinned-section-worktrees.ts new file mode 100644 index 00000000000..5d8e865397c --- /dev/null +++ b/src/renderer/src/components/sidebar/pinned-section-worktrees.ts @@ -0,0 +1,52 @@ +import type { WorktreeLineage } from '../../../../shared/worktree/lineage-types' +import type { Worktree } from '../../../../shared/worktree/types' +import { getProjectedWorktreeLineageChildrenByParentId } from './worktree-lineage-projection' + +// Why: pin is placement of the clicked row. Descendants keep their own isPinned +// and still follow a visible pinned ancestor into the Pinned section. +export function getPinnedSectionWorktrees( + worktrees: readonly Worktree[], + lineageById: Readonly>, + worktreeMap: ReadonlyMap +): Worktree[] { + const visibleIds = new Set(worktrees.map((worktree) => worktree.id)) + const childrenByParentId = getProjectedWorktreeLineageChildrenByParentId(lineageById, worktreeMap) + const included = new Set() + const seen = new Set() + const pendingIds = worktrees + .filter((worktree) => worktree.isPinned) + .map((worktree) => worktree.id) + + while (pendingIds.length > 0) { + const id = pendingIds.pop() + if (!id) { + continue + } + if (seen.has(id)) { + continue + } + seen.add(id) + if (visibleIds.has(id)) { + included.add(id) + } + for (const child of childrenByParentId.get(id) ?? []) { + pendingIds.push(child.id) + } + } + + return worktrees.filter((worktree) => included.has(worktree.id)) +} + +export function isPinnedSectionWorktree( + worktree: Worktree, + visibleWorktrees: readonly Worktree[], + lineageById: Readonly>, + worktreeMap: ReadonlyMap +): boolean { + if (worktree.isPinned) { + return true + } + return getPinnedSectionWorktrees(visibleWorktrees, lineageById, worktreeMap).some( + (candidate) => candidate.id === worktree.id + ) +} diff --git a/src/renderer/src/components/sidebar/worktree-list-groups-lineage-nesting.test.ts b/src/renderer/src/components/sidebar/worktree-list-groups-lineage-nesting.test.ts index 646273f6264..f5fb459e01c 100644 --- a/src/renderer/src/components/sidebar/worktree-list-groups-lineage-nesting.test.ts +++ b/src/renderer/src/components/sidebar/worktree-list-groups-lineage-nesting.test.ts @@ -464,6 +464,123 @@ describe('buildRows workspace lineage nesting', () => { expect(info).toMatchObject({ state: 'missing' }) }) + it('nests unpinned children under a pinned parent in Pinned', () => { + const pinnedParent = { ...parent, isPinned: true } + const rows = buildRows( + 'none', + [child, pinnedParent], + repoMap, + null, + new Set(), + undefined, + undefined, + undefined, + { [child.id]: lineage }, + new Map([ + [pinnedParent.id, pinnedParent], + [child.id, child] + ]), + true + ) + + const items = rows.filter((row) => row.type === 'item') + expect(rows[0]).toMatchObject({ type: 'header', key: 'pinned', count: 2 }) + expect(items.map((row) => [row.worktree.id, row.depth, row.sectionKey])).toEqual([ + [pinnedParent.id, 0, 'pinned'], + [child.id, 1, 'pinned'] + ]) + expect(rows.some((row) => row.type === 'header' && row.key === 'all')).toBe(false) + }) + + it('nests grandchildren under a pinned ancestor in Pinned', () => { + const pinnedParent = { ...parent, isPinned: true } + const rows = buildRows( + 'none', + [grandchild, child, pinnedParent], + repoMap, + null, + new Set(), + undefined, + undefined, + undefined, + { [child.id]: lineage, [grandchild.id]: grandchildLineage }, + new Map([ + [pinnedParent.id, pinnedParent], + [child.id, child], + [grandchild.id, grandchild] + ]), + true + ) + + expect( + rows.filter((row) => row.type === 'item').map((row) => [row.worktree.id, row.depth]) + ).toEqual([ + [pinnedParent.id, 0], + [child.id, 1], + [grandchild.id, 2] + ]) + }) + + it('duplicates a pinned parent tree into All when the policy allows it', () => { + const pinnedParent = { ...parent, isPinned: true } + const rows = buildRows( + 'none', + [child, pinnedParent], + repoMap, + null, + new Set(), + undefined, + undefined, + undefined, + { [child.id]: lineage }, + new Map([ + [pinnedParent.id, pinnedParent], + [child.id, child] + ]), + true, + { showPinnedWorktreesInGroups: true } as never + ) + + expect( + rows + .filter((row) => row.type === 'item') + .map((row) => [row.sectionKey, row.worktree.id, row.depth]) + ).toEqual([ + ['pinned', pinnedParent.id, 0], + ['pinned', child.id, 1], + ['all', pinnedParent.id, 0], + ['all', child.id, 1] + ]) + }) + + it('nests a pinned child under its pinned parent in Pinned', () => { + const pinnedParent = { ...parent, isPinned: true } + const pinnedChild = { ...child, isPinned: true } + const rows = buildRows( + 'none', + [pinnedChild, pinnedParent], + repoMap, + null, + new Set(), + undefined, + undefined, + undefined, + { [child.id]: lineage }, + new Map([ + [pinnedParent.id, pinnedParent], + [pinnedChild.id, pinnedChild] + ]), + true + ) + + expect( + rows.filter((row) => row.type === 'item').map((row) => [row.worktree.id, row.depth]) + ).toEqual([ + [pinnedParent.id, 0], + [pinnedChild.id, 1] + ]) + }) + it('keeps pinned children in Pinned without a parent badge', () => { const pinnedChild = { ...child, isPinned: true } const rows = buildRows( diff --git a/src/renderer/src/components/sidebar/worktree-list/grouping/build-rows.ts b/src/renderer/src/components/sidebar/worktree-list/grouping/build-rows.ts index 3abd7176929..abec111b0a5 100644 --- a/src/renderer/src/components/sidebar/worktree-list/grouping/build-rows.ts +++ b/src/renderer/src/components/sidebar/worktree-list/grouping/build-rows.ts @@ -20,6 +20,7 @@ import { import { buildProjectGroupingIndex } from './project-grouping' import type { ProjectGroupingModel } from './project-grouping' import { appendProjectGroupSections } from './project-group-sections' +import { getPinnedSectionWorktrees } from '../../pinned-section-worktrees' import { appendWorktreeRows, buildPendingCreationRow, emitPinnedGroup } from './row-builders' import { getPinnedWorktreeDisplayPolicy } from './row-types' import type { @@ -82,10 +83,14 @@ export function buildRows( } } + const pinnedSectionWorktrees = nestLineage + ? getPinnedSectionWorktrees(worktrees, lineageById, worktreeMap) + : worktrees.filter((worktree) => worktree.isPinned) + const pinnedSectionIds = new Set(pinnedSectionWorktrees.map((worktree) => worktree.id)) const naturalWorktrees = pinnedDisplayPolicy === 'duplicate-in-groups' ? worktrees - : worktrees.filter((worktree) => !worktree.isPinned) + : worktrees.filter((worktree) => !pinnedSectionIds.has(worktree.id)) const mixedWorktreeHostContextLabels = getMixedWorktreeHostContextLabels( naturalWorktrees, repoMap, @@ -103,14 +108,18 @@ export function buildRows( projectGrouping }) emitPinnedGroup( - worktrees, + pinnedSectionWorktrees, repoMap, defaultHostId, collapsedGroups, renderedNaturalAnchorRepoIds, importedWorktreesByRepo, groupBy !== 'repo', - result + result, + lineageById, + worktreeMap, + nestLineage, + cyclicLineageIds ) if (groupBy === 'none') { if (naturalWorktrees.length > 0) { diff --git a/src/renderer/src/components/sidebar/worktree-list/grouping/row-builders.ts b/src/renderer/src/components/sidebar/worktree-list/grouping/row-builders.ts index 3f9617174ce..8ae3b03d756 100644 --- a/src/renderer/src/components/sidebar/worktree-list/grouping/row-builders.ts +++ b/src/renderer/src/components/sidebar/worktree-list/grouping/row-builders.ts @@ -29,24 +29,27 @@ export function buildPendingCreationRow( } export function emitPinnedGroup( - worktrees: Worktree[], + pinnedSectionWorktrees: Worktree[], repoMap: Map, defaultHostId: ExecutionHostId, collapsedGroups: Set, renderedNaturalAnchorRepoIds: ReadonlySet, importedWorktreesByRepo: ReadonlyMap, allowImportedFallback: boolean, - result: Row[] + result: Row[], + lineageById: Record, + worktreeMap: Map, + nestLineage: boolean, + cyclicLineageIds: ReadonlySet ): void { - const pinned = worktrees.filter((w) => w.isPinned) - if (pinned.length === 0) { + if (pinnedSectionWorktrees.length === 0) { return } const hostWorktreeCounts = new Map() const hostWorktreeIds = new Map() const pinnedRepoOrder: string[] = [] const seenPinnedRepoIds = new Set() - for (const worktree of pinned) { + for (const worktree of pinnedSectionWorktrees) { const hostId = getWorktreeExecutionHostId(worktree, repoMap.get(worktree.repoId), defaultHostId) hostWorktreeCounts.set(hostId, (hostWorktreeCounts.get(hostId) ?? 0) + 1) const hostIds = hostWorktreeIds.get(hostId) ?? [] @@ -62,12 +65,12 @@ export function emitPinnedGroup( type: 'header', key: PINNED_GROUP_KEY, label: PINNED_GROUP_META.label, - count: pinned.length, + count: pinnedSectionWorktrees.length, tone: PINNED_GROUP_META.tone, icon: PINNED_GROUP_META.icon, hostWorktreeCounts, hostWorktreeIds, - worktreeIds: pinned.map((worktree) => worktree.id) + worktreeIds: pinnedSectionWorktrees.map((worktree) => worktree.id) }) if (collapsedGroups.has(PINNED_GROUP_KEY)) { for (const repoId of pinnedRepoOrder) { @@ -76,31 +79,34 @@ export function emitPinnedGroup( result.push(buildImportedWorktreesCardRow(candidate, 'pinned-fallback')) } } - } else { - const lastPinnedIndexByRepoId = new Map() - pinned.forEach((worktree, index) => lastPinnedIndexByRepoId.set(worktree.repoId, index)) - for (const [index, worktree] of pinned.entries()) { - result.push( - buildWorktreeRow(worktree, repoMap, { - rowKey: `${PINNED_GROUP_KEY}:${worktree.id}`, - sectionKey: PINNED_GROUP_KEY, - depth: 0, - groupDepth: 0, - lineageTrail: [], - isLastLineageChild: false, - lineageChildCount: 0, - lineageCollapsed: false - }) - ) - const candidate = importedWorktreesByRepo.get(worktree.repoId) - if ( - allowImportedFallback && - candidate && - !renderedNaturalAnchorRepoIds.has(worktree.repoId) && - lastPinnedIndexByRepoId.get(worktree.repoId) === index - ) { - result.push(buildImportedWorktreesCardRow(candidate, 'pinned-fallback')) - } + return + } + + const firstItemIndex = result.length + appendWorktreeRows(result, pinnedSectionWorktrees, repoMap, lineageById, worktreeMap, { + nestLineage, + collapsedGroups, + groupDepth: 0, + sectionKey: PINNED_GROUP_KEY, + cyclicLineageIds + }) + if (!allowImportedFallback) { + return + } + // Why: imported fallback sits after the last row of that repo; splice from the + // end so earlier inserts do not shift later targets. + const lastResultIndexByRepoId = new Map() + for (let index = firstItemIndex; index < result.length; index++) { + const row = result[index] + if (row?.type === 'item') { + lastResultIndexByRepoId.set(row.worktree.repoId, index) + } + } + const inserts = [...lastResultIndexByRepoId.entries()].sort((left, right) => right[1] - left[1]) + for (const [repoId, index] of inserts) { + const candidate = importedWorktreesByRepo.get(repoId) + if (candidate && !renderedNaturalAnchorRepoIds.has(repoId)) { + result.splice(index + 1, 0, buildImportedWorktreesCardRow(candidate, 'pinned-fallback')) } } } @@ -222,6 +228,51 @@ export function appendWorktreeRows( } const emitted = new Set() + const pending: { + worktree: Worktree + depth: number + lineageTrail: boolean[] + isLastChild: boolean + }[] = [] + const emitPending = (): void => { + while (pending.length > 0) { + const next = pending.pop() + if (!next || emitted.has(next.worktree.id)) { + continue + } + const { worktree, depth, lineageTrail, isLastChild } = next + const children = childrenByParentId.get(worktree.id) ?? [] + const lineageGroupKey = getLineageGroupKey(worktree.id) + const lineageCollapsed = collapsedGroups.has(lineageGroupKey) + emitted.add(worktree.id) + result.push( + buildWorktreeRow(worktree, repoMap, { + rowKey: `${sectionKey}:${worktree.id}`, + sectionKey, + depth, + groupDepth, + lineageTrail, + isLastLineageChild: isLastChild, + lineageChildCount: children.length, + lineageCollapsed, + hostContextLabel: + hostContextLabelByWorktreeId?.get(worktree.id) ?? + hostContextLabelByRepoId?.get(worktree.repoId) + }) + ) + if (lineageCollapsed) { + continue + } + for (let index = children.length - 1; index >= 0; index -= 1) { + pending.push({ + worktree: children[index], + depth: depth + 1, + lineageTrail: [...lineageTrail, index < children.length - 1], + isLastChild: index === children.length - 1 + }) + } + } + } const emit = ( worktree: Worktree, depth: number, @@ -231,36 +282,8 @@ export function appendWorktreeRows( if (emitted.has(worktree.id)) { return } - const children = childrenByParentId.get(worktree.id) ?? [] - const lineageGroupKey = getLineageGroupKey(worktree.id) - const lineageCollapsed = collapsedGroups.has(lineageGroupKey) - emitted.add(worktree.id) - result.push( - buildWorktreeRow(worktree, repoMap, { - rowKey: `${sectionKey}:${worktree.id}`, - sectionKey, - depth, - groupDepth, - lineageTrail, - isLastLineageChild: isLastChild, - lineageChildCount: children.length, - lineageCollapsed, - hostContextLabel: - hostContextLabelByWorktreeId?.get(worktree.id) ?? - hostContextLabelByRepoId?.get(worktree.repoId) - }) - ) - if (lineageCollapsed) { - return - } - children.forEach((child, index) => { - emit( - child, - depth + 1, - [...lineageTrail, index < children.length - 1], - index === children.length - 1 - ) - }) + pending.push({ worktree, depth, lineageTrail, isLastChild }) + emitPending() } const roots = worktrees.filter((worktree) => !childIds.has(worktree.id)) diff --git a/src/renderer/src/components/sidebar/worktree-list/listing/use-collapsed-groups.ts b/src/renderer/src/components/sidebar/worktree-list/listing/use-collapsed-groups.ts index f843c711e74..bc9ef82d046 100644 --- a/src/renderer/src/components/sidebar/worktree-list/listing/use-collapsed-groups.ts +++ b/src/renderer/src/components/sidebar/worktree-list/listing/use-collapsed-groups.ts @@ -5,9 +5,10 @@ import type { Repo } from '../../../../../../shared/repo-types' import type { WorkspaceStatusDefinition, Worktree } from '../../../../../../shared/worktree/types' import type { WorktreeLineage } from '../../../../../../shared/worktree/lineage-types' import { PINNED_GROUP_KEY, getLineageGroupKey } from '../grouping/group-keys' +import type { PinnedWorktreeDisplayPolicy, WorktreeGroupBy } from '../grouping/row-types' import type { ProjectGroupingModel } from '../grouping/project-grouping' -import type { WorktreeGroupBy } from '../grouping/row-types' import { getGroupKeysForWorktree } from '../grouping/worktree-group-keys' +import { isPinnedSectionWorktree } from '../../pinned-section-worktrees' import { getWorktreeLineageAncestors } from '../../worktree-lineage-projection' // While the agent send picker targets a workspace, force open every section that hides it. @@ -15,6 +16,8 @@ export function useEffectiveCollapsedGroups(args: { collapsedGroups: Set agentSendTargetWorktreeId: string | null groupBy: WorktreeGroupBy + pinnedDisplayPolicy: PinnedWorktreeDisplayPolicy + visibleWorktrees: readonly Worktree[] repoMap: Map worktreeMap: Map worktreeLineageById: Record @@ -28,6 +31,8 @@ export function useEffectiveCollapsedGroups(args: { collapsedGroups, agentSendTargetWorktreeId, groupBy, + pinnedDisplayPolicy, + visibleWorktrees, repoMap, worktreeMap, worktreeLineageById, @@ -46,7 +51,10 @@ export function useEffectiveCollapsedGroups(args: { return collapsedGroups } const next = new Set(collapsedGroups) - if (targetWorktree.isPinned) { + if ( + pinnedDisplayPolicy === 'single-location' && + isPinnedSectionWorktree(targetWorktree, visibleWorktrees, worktreeLineageById, worktreeMap) + ) { next.delete(PINNED_GROUP_KEY) } else { for (const groupKey of getGroupKeysForWorktree( @@ -75,6 +83,8 @@ export function useEffectiveCollapsedGroups(args: { agentSendTargetWorktreeId, collapsedGroups, groupBy, + pinnedDisplayPolicy, + visibleWorktrees, prCache, projectGroups, projectGrouping, diff --git a/src/renderer/src/components/sidebar/worktree-list/navigation/pending-reveal-inputs.ts b/src/renderer/src/components/sidebar/worktree-list/navigation/pending-reveal-inputs.ts index edcd6353b35..ba67a388680 100644 --- a/src/renderer/src/components/sidebar/worktree-list/navigation/pending-reveal-inputs.ts +++ b/src/renderer/src/components/sidebar/worktree-list/navigation/pending-reveal-inputs.ts @@ -14,6 +14,7 @@ import { getLineageGroupKey } from '../grouping/group-keys' import type { ProjectGroupingModel } from '../grouping/project-grouping' import type { PinnedWorktreeDisplayPolicy, WorktreeGroupBy } from '../grouping/row-types' import { getGroupKeysForWorktree } from '../grouping/worktree-group-keys' +import { isPinnedSectionWorktree } from '../../pinned-section-worktrees' import { getWorktreeLineageAncestors } from '../../worktree-lineage-projection' import { getFolderWorkspaceRevealGroupKeys } from './folder-reveal' import { getPinnedWorktreeRevealCollapsedGroupKeys } from './reveal-ancestors' @@ -90,10 +91,17 @@ export function expandGroupsForWorktreeReveal( } const groupKeys = - targetWorktree.isPinned && args.pinnedDisplayPolicy === 'single-location' + args.pinnedDisplayPolicy === 'single-location' && + isPinnedSectionWorktree( + targetWorktree, + args.worktrees, + args.worktreeLineageById, + args.worktreeMap + ) ? getPinnedWorktreeRevealCollapsedGroupKeys({ worktree: targetWorktree, - collapsedGroups: args.collapsedGroups + collapsedGroups: args.collapsedGroups, + inPinnedSection: true }) : getGroupKeysForWorktree( args.groupBy, diff --git a/src/renderer/src/components/sidebar/worktree-list/navigation/reveal-ancestors.ts b/src/renderer/src/components/sidebar/worktree-list/navigation/reveal-ancestors.ts index 95a988748eb..968118aac48 100644 --- a/src/renderer/src/components/sidebar/worktree-list/navigation/reveal-ancestors.ts +++ b/src/renderer/src/components/sidebar/worktree-list/navigation/reveal-ancestors.ts @@ -92,12 +92,14 @@ export function getSidebarRowRevealAncestorKeys(args: { export function getPinnedWorktreeRevealCollapsedGroupKeys({ worktree, - collapsedGroups + collapsedGroups, + inPinnedSection = worktree.isPinned }: { worktree: Worktree collapsedGroups: ReadonlySet + inPinnedSection?: boolean }): string[] { - if (!worktree.isPinned) { + if (!inPinnedSection) { return [] } const keys: string[] = [] diff --git a/src/renderer/src/store/slices/worktrees-workspace-selection-state.test.ts b/src/renderer/src/store/slices/worktrees-workspace-selection-state.test.ts index db7331fe797..6e9cd99f733 100644 --- a/src/renderer/src/store/slices/worktrees-workspace-selection-state.test.ts +++ b/src/renderer/src/store/slices/worktrees-workspace-selection-state.test.ts @@ -6,7 +6,12 @@ import { } from '../../components/browser-pane/webview-registry' import { folderWorkspaceKey, worktreeWorkspaceKey } from '../../../../shared/workspace-scope' import { makeDetectedResult } from './worktrees-detected-listing-fixtures' -import { createWebview, makeFolderWorkspace, makeWorktree } from './worktrees-slice-test-fixtures' +import { + createWebview, + makeFolderWorkspace, + makeLineage, + makeWorktree +} from './worktrees-slice-test-fixtures' import { createTestStore, mockApi, @@ -384,6 +389,160 @@ describe('setWorktreesPinnedAndReveal', () => { expect(reveal).not.toHaveBeenCalled() }) + it.each([ + { previousPinned: false, nextPinned: true }, + { previousPinned: true, nextPinned: false } + ])( + 'reveals the focused descendant when changing its unfocused ancestor from $previousPinned to $nextPinned', + ({ previousPinned, nextPinned }) => { + const store = createTestStore() + const parent = makeWorktree({ + id: 'repo1::/parent', + instanceId: 'parent-instance', + repoId: 'repo1', + isPinned: previousPinned + }) + const child = makeWorktree({ + id: 'repo1::/child', + instanceId: 'child-instance', + repoId: 'repo1' + }) + const reveal = vi.fn() + store.setState({ + worktreesByRepo: { repo1: [parent, child] }, + worktreeLineageById: { + [child.id]: makeLineage({ worktreeId: child.id, parentWorktreeId: parent.id }) + }, + activeWorktreeId: child.id, + revealWorktreeInSidebar: reveal + } as Partial) + + store.getState().setWorktreesPinnedAndReveal([parent.id], nextPinned) + + expect(reveal).toHaveBeenCalledWith(child.id, { behavior: 'smooth', highlight: true }) + } + ) + + it('reveals the focused descendant from embedded legacy lineage', () => { + const store = createTestStore() + const parent = makeWorktree({ + id: 'repo1::/parent', + instanceId: 'parent-instance', + repoId: 'repo1' + }) + const child = { + ...makeWorktree({ + id: 'repo1::/child', + instanceId: 'child-instance', + repoId: 'repo1' + }), + lineage: makeLineage({ worktreeId: 'repo1::/child', parentWorktreeId: parent.id }) + } + const reveal = vi.fn() + store.setState({ + worktreesByRepo: { repo1: [parent, child] }, + activeWorktreeId: child.id, + revealWorktreeInSidebar: reveal + } as Partial) + + store.getState().setWorktreesPinnedAndReveal([parent.id], true) + + expect(reveal).toHaveBeenCalledWith(child.id, { behavior: 'smooth', highlight: true }) + }) + + it('does not reveal through cyclic lineage rejected by rendering', () => { + const store = createTestStore() + const first = makeWorktree({ + id: 'repo1::/first', + instanceId: 'first-instance', + repoId: 'repo1' + }) + const second = makeWorktree({ + id: 'repo1::/second', + instanceId: 'second-instance', + repoId: 'repo1' + }) + const reveal = vi.fn() + store.setState({ + worktreesByRepo: { repo1: [first, second] }, + worktreeLineageById: { + [first.id]: makeLineage({ + worktreeId: first.id, + worktreeInstanceId: first.instanceId, + parentWorktreeId: second.id, + parentWorktreeInstanceId: second.instanceId + }), + [second.id]: makeLineage({ + worktreeId: second.id, + worktreeInstanceId: second.instanceId, + parentWorktreeId: first.id, + parentWorktreeInstanceId: first.instanceId + }) + }, + activeWorktreeId: second.id, + revealWorktreeInSidebar: reveal + } as Partial) + + store.getState().setWorktreesPinnedAndReveal([first.id], true) + + expect(reveal).not.toHaveBeenCalled() + }) + + it('does not reveal a focused descendant for duplicate pinned rows', () => { + const store = createTestStore() + const parent = makeWorktree({ + id: 'repo1::/parent', + instanceId: 'parent-instance', + repoId: 'repo1' + }) + const child = makeWorktree({ + id: 'repo1::/child', + instanceId: 'child-instance', + repoId: 'repo1' + }) + const reveal = vi.fn() + store.setState({ + worktreesByRepo: { repo1: [parent, child] }, + worktreeLineageById: { + [child.id]: makeLineage({ worktreeId: child.id, parentWorktreeId: parent.id }) + }, + activeWorktreeId: child.id, + settings: { ...store.getState().settings, showPinnedWorktreesInGroups: true }, + revealWorktreeInSidebar: reveal + } as Partial) + + store.getState().setWorktreesPinnedAndReveal([parent.id], true) + + expect(reveal).not.toHaveBeenCalled() + }) + + it('does not reveal through stale lineage', () => { + const store = createTestStore() + const parent = makeWorktree({ + id: 'repo1::/parent', + instanceId: 'replacement-parent-instance', + repoId: 'repo1' + }) + const child = makeWorktree({ + id: 'repo1::/child', + instanceId: 'child-instance', + repoId: 'repo1' + }) + const reveal = vi.fn() + store.setState({ + worktreesByRepo: { repo1: [parent, child] }, + worktreeLineageById: { + [child.id]: makeLineage({ worktreeId: child.id, parentWorktreeId: parent.id }) + }, + activeWorktreeId: child.id, + revealWorktreeInSidebar: reveal + } as Partial) + + store.getState().setWorktreesPinnedAndReveal([parent.id], true) + + expect(reveal).not.toHaveBeenCalled() + }) + it('skips a no-op toggle without requesting a reveal', () => { const store = createTestStore() const wt = makeWorktree({ id: 'repo1::/a', repoId: 'repo1', path: '/a', isPinned: true }) diff --git a/src/renderer/src/store/slices/worktrees/session/worktree-pin-reveal.ts b/src/renderer/src/store/slices/worktrees/session/worktree-pin-reveal.ts index 18a6e395bfa..e8975c76587 100644 --- a/src/renderer/src/store/slices/worktrees/session/worktree-pin-reveal.ts +++ b/src/renderer/src/store/slices/worktrees/session/worktree-pin-reveal.ts @@ -4,7 +4,56 @@ import { getActiveSidebarWorkspaceId, parseWorkspaceKey } from '../../../../../../shared/workspace-scope' +import { + getCyclicWorktreeLineageChildIds, + isValidResolvedWorktreeLineageEdge +} from '../../../../../../shared/resolved-worktree-lineage' +import type { WorktreeLineage } from '../../../../../../shared/worktree/lineage-types' import type { WorktreeMeta } from '../../../../../../shared/worktree/meta-types' +import type { Worktree } from '../../../../../../shared/worktree/types' + +type WorktreeWithEmbeddedLineage = Worktree & { lineage?: WorktreeLineage | null } + +function getProjectedLineage(get: WorktreeSliceGet, worktree: Worktree): WorktreeLineage | null { + if (Object.hasOwn(get().worktreeLineageById, worktree.id)) { + return get().worktreeLineageById[worktree.id] ?? null + } + return (worktree as WorktreeWithEmbeddedLineage).lineage ?? null +} + +function hasChangedLineageAncestor( + get: WorktreeSliceGet, + worktreeId: string, + changedWorktreeIds: ReadonlySet +): boolean { + const seen = new Set() + const validLineageByChildId = new Map() + let child = get().getKnownWorktreeById(worktreeId) + while (child && !seen.has(child.id)) { + seen.add(child.id) + const lineage = getProjectedLineage(get, child) + const parent = lineage ? get().getKnownWorktreeById(lineage.parentWorktreeId) : null + if (!lineage || !parent || !isValidResolvedWorktreeLineageEdge(child, parent, lineage)) { + break + } + validLineageByChildId.set(child.id, lineage) + child = parent + } + const cyclicIds = getCyclicWorktreeLineageChildIds(validLineageByChildId) + child = get().getKnownWorktreeById(worktreeId) + while (child && !cyclicIds.has(child.id)) { + const lineage = getProjectedLineage(get, child) + const parent = lineage ? get().getKnownWorktreeById(lineage.parentWorktreeId) : null + if (!lineage || !parent || !isValidResolvedWorktreeLineageEdge(child, parent, lineage)) { + return false + } + if (changedWorktreeIds.has(parent.id)) { + return true + } + child = parent + } + return false +} export function createSetWorktreesPinnedAndReveal( _set: WorktreeSliceSet, @@ -18,6 +67,7 @@ export function createSetWorktreesPinnedAndReveal( ) // Skip worktrees already in the target state so a no-op toggle doesn't scroll the viewport away. const updates = new Map>() + const changedWorktreeIds = new Set() let didChange = false let revealWorktreeId: string | null = null for (const worktreeId of worktreeIds) { @@ -26,6 +76,7 @@ export function createSetWorktreesPinnedAndReveal( continue } didChange = true + changedWorktreeIds.add(worktreeId) const workspaceScope = parseWorkspaceKey(worktreeId) if (workspaceScope?.type === 'folder') { void get().updateWorktreeMeta(worktreeId, { isPinned }) @@ -39,6 +90,14 @@ export function createSetWorktreesPinnedAndReveal( if (!didChange) { return } + if ( + revealWorktreeId === null && + activeSidebarWorktreeId !== null && + get().settings?.showPinnedWorktreesInGroups !== true && + hasChangedLineageAncestor(get, activeSidebarWorktreeId, changedWorktreeIds) + ) { + revealWorktreeId = activeSidebarWorktreeId + } // updateWorktreesMeta applies the store update synchronously, so the reveal below sees the row already rendered. if (updates.size > 0) { void get().updateWorktreesMeta(updates) From 7afce2ea41af48702c1642e5fb1052fa7bd9c756 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:11:19 -0700 Subject: [PATCH 09/10] fix(ssh): stop reporting a confirmed kill when the SSH provider is gone (#14977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ssh): stop reporting a confirmed kill when the SSH provider is gone A detached relay PTY is designed to outlive the provider that addressed it (it ignores SIGHUP and ships with an unlimited grace), so "the SSH provider is no longer registered" is lost contact, never evidence the remote process stopped. Both stop primitives in the PTY controller returned `true` from that branch, and every caller downstream reported the fabricated success: the CLI printed "PTY killed.", worker-stop settled the dispatch as stopped, and — because the stop "succeeded" — the unstopped-PTY gate never ran, so worktree removal walked straight past a live remote agent. `kill`/`stopAndWait` now still tombstone the local lease but report an unconfirmed stop and record why, using the three-verdict vocabulary the worktree teardown gate already spoke (`live` / `unverifiable` / `exited`), promoted out of that module into `src/shared/pty-liveness-verdict.ts`. The close receipt, the CLI wording, worker-stop and the removal gate all read that verdict instead of inferring an exit from silence. The same rule fixes the mirror-image defect: the aggregate inventory only enumerates registered providers, so a dropped relay clears `connected` for every remote PTY at once. The sweep now separates the provider answering "absent" (an exit) from no provider being able to answer (lost contact), so worker-stop stops claiming `exited` from a disconnect. The `connected` wire field is unchanged in meaning and shape. * fix(orchestration): apply the same honesty to the federation stop path The federation host runs its own copy of the worker observation and stop logic, with the same two defects: `inspectRemoteAttachment` read a dropped relay's `connected: false` as `exited`, and `federationStop` settled the dispatch as stopped from a close it never confirmed — relaying a fabricated success all the way home to the coordinator. Two guards also had to move so the honest verdict does not become a new refusal. `federationRead` gated on `status !== 'running'`, which would have rejected a connected terminal the moment a stop lost contact with it; it now gates on `status === 'exited'`, which is equivalent for every pre-existing status given the two guards beside it. Local `workerStop` likewise still attempts the close when the verdict is `unverifiable` — losing contact is a reason to report the outcome honestly, never a reason to stop trying. The show observations now carry the reason alongside the status, so a bare `unverifiable` is actionable. Both are new optional fields. * fix(ssh): preserve unconfirmed stop verdicts across consumers * fix(ssh): use canonical live verdict wording * fix(ssh): refuse wrong-host teardown verification * test(orchestration): confirm worker release teardown * fix(orchestration): negotiate honest worker stop receipts * fix(agent-teams): fence uncertain teammate respawns * fix(ssh): avoid duplicate missing-provider teardown * fix(orchestration): preserve archives across release retries * fix(ssh): preserve verdicts across synthetic kill exits * fix(ssh): preserve liveness evidence across teardown * fix(agent-teams): replace panes only after confirmed stop * fix(ssh): distinguish host exits from relay loss * fix(ssh): narrow concurrent inventory verdicts * fix(orchestration): serve archives after uncertain release * fix(orchestration): expose unverifiable read liveness * test(ssh): align liveness assertions with verdicts * fix(ssh): preserve host scope across inventory failures --- src/cli/terminal-format.test.ts | 39 ++- src/cli/terminal-format.ts | 18 +- .../process-gone-killed-one-ordering.test.ts | 7 +- .../process-gone-recorder.test.ts | 4 +- .../pty-daemon-controller-teardown.test.ts | 27 +- .../pty-daemon-ssh-lease-lifecycle.test.ts | 2 +- .../ipc/pty-runtime-kill-and-exit.test.ts | 25 +- src/main/ipc/pty-ssh-stop-verdict.test.ts | 297 ++++++++++++++++++ src/main/ipc/pty.ts | 119 +++++-- .../claude-agent-teams-service.test.ts | 57 +++- .../claude-agent-teams-tmux-dispatcher.ts | 42 ++- src/main/runtime/claude-agent-teams-types.ts | 1 + ...ntime-process-incarnation-liveness.test.ts | 16 +- ...-runtime-terminal-close-continuity.test.ts | 73 ++++- ...a-runtime-terminal-split-authority.test.ts | 6 +- src/main/runtime/orca-runtime.ts | 212 +++++++++++-- .../worker-terminal-release.ts | 5 +- .../worker-terminal-process-liveness.ts | 6 +- .../pty-inventory-liveness-verdict.test.ts | 194 ++++++++++++ .../orchestration-federation-control.ts | 72 ++++- ...ration-federation-liveness-verdict.test.ts | 183 +++++++++++ .../orchestration-federation-output.test.ts | 2 +- .../orchestration-federation-setup.test.ts | 1 + .../methods/orchestration-federation.test.ts | 5 +- .../methods/orchestration-worker-control.ts | 28 +- .../orchestration-worker-observation.ts | 21 +- .../orchestration-worker-output.test.ts | 13 + .../methods/orchestration-worker-output.ts | 13 +- ...orchestration-worker-release-completion.ts | 15 +- ...on-worker-release-liveness-verdict.test.ts | 84 +++++ ...hestration-worker-release-recovery.test.ts | 35 ++- .../orchestration-worker-release.test.ts | 15 +- .../methods/orchestration-worker-release.ts | 2 +- ...chestration-worker-stop-capability.test.ts | 57 ++++ ...ation-worker-stop-liveness-verdict.test.ts | 158 ++++++++++ .../rpc/methods/orchestration-worker-stop.ts | 37 ++- .../orchestration-workers-recovery.test.ts | 5 +- .../runtime/unstopped-pty-verification.ts | 49 ++- .../worktree-teardown-unstopped-pty.test.ts | 74 +++++ src/main/runtime/worktree-teardown.test.ts | 47 ++- src/main/runtime/worktree-teardown.ts | 21 +- .../commit-message-text-generation.ts | 4 +- src/shared/orchestration-worker-output.ts | 3 + src/shared/protocol-version.ts | 3 + src/shared/pty-liveness-verdict.ts | 34 ++ src/shared/runtime-types.ts | 8 + 46 files changed, 1978 insertions(+), 161 deletions(-) create mode 100644 src/main/ipc/pty-ssh-stop-verdict.test.ts create mode 100644 src/main/runtime/pty-inventory-liveness-verdict.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-stop-capability.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts create mode 100644 src/shared/pty-liveness-verdict.ts diff --git a/src/cli/terminal-format.test.ts b/src/cli/terminal-format.test.ts index 014f6339823..0656234e292 100644 --- a/src/cli/terminal-format.test.ts +++ b/src/cli/terminal-format.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { formatTerminalFocus } from './terminal-format' +import { formatTerminalClose, formatTerminalFocus } from './terminal-format' describe('formatTerminalFocus', () => { it('distinguishes superseded navigation from a winning focus', () => { @@ -22,3 +22,40 @@ describe('formatTerminalFocus', () => { ).toBe('Focused terminal term_winner (tab tab-winner).') }) }) + +describe('formatTerminalClose', () => { + it('prints "PTY killed." only for a confirmed kill', () => { + expect( + formatTerminalClose({ close: { handle: 'term_local', tabId: 'tab-1', ptyKilled: true } }) + ).toBe('Closed terminal term_local. PTY killed.') + }) + + it('says the remote process was not confirmed stopped instead of claiming a kill', () => { + expect( + formatTerminalClose({ + close: { + handle: 'term_remote', + tabId: 'tab-1', + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'its SSH provider is no longer registered' + } + }) + ).toBe( + 'Closed terminal term_remote. The PTY was not confirmed stopped: its SSH provider is no longer registered.' + ) + }) + + it('names a PTY known to be live', () => { + expect( + formatTerminalClose({ + close: { + handle: 'term_live', + tabId: 'tab-1', + ptyKilled: false, + ptyStopVerdict: 'live' + } + }) + ).toBe('Closed terminal term_live. The PTY is live.') + }) +}) diff --git a/src/cli/terminal-format.ts b/src/cli/terminal-format.ts index a4df4ba6951..717f1d33701 100644 --- a/src/cli/terminal-format.ts +++ b/src/cli/terminal-format.ts @@ -1,3 +1,4 @@ +import { PTY_LIVE_NOTE, describeUnconfirmedStop } from '../shared/pty-liveness-verdict' import type { RuntimeTerminalClose, RuntimeTerminalCreate, @@ -185,12 +186,25 @@ export function formatTerminalFocus(result: { focus: RuntimeTerminalFocus }): st return `Focused terminal ${result.focus.handle} (tab ${result.focus.tabId}).` } +/** "PTY killed." is a claim of observed death, so only a confirmed kill earns it. */ +function describePtyStop(close: RuntimeTerminalClose): string { + if (close.ptyKilled) { + return ' PTY killed.' + } + if (close.ptyStopVerdict === 'live') { + return ` ${PTY_LIVE_NOTE}` + } + if (close.ptyStopVerdict === 'unverifiable') { + return ` ${describeUnconfirmedStop(close.ptyStopReason ?? 'its host could not be reached')}` + } + return '' +} + export function formatTerminalClose(result: { close: RuntimeTerminalClose }): string { if (result.close.closeMode === 'tab') { return `Closed terminal tab ${result.close.tabId} (${result.close.handle}).` } - const ptyNote = result.close.ptyKilled ? ' PTY killed.' : '' - return `Closed terminal ${result.close.handle}.${ptyNote}` + return `Closed terminal ${result.close.handle}.${describePtyStop(result.close)}` } export function formatTerminalWait(result: { wait: RuntimeTerminalWait }): string { diff --git a/src/main/crash-reporting/process-gone-killed-one-ordering.test.ts b/src/main/crash-reporting/process-gone-killed-one-ordering.test.ts index 72c897088f5..2a4ded3c2a5 100644 --- a/src/main/crash-reporting/process-gone-killed-one-ordering.test.ts +++ b/src/main/crash-reporting/process-gone-killed-one-ordering.test.ts @@ -85,12 +85,7 @@ describe('recordProcessGoneCrash killed/1 ordering', () => { recordProcessGoneCrash({ record } as never, gpuKill, dedupe) recordProcessGoneCrash({ record } as never, networkServiceKill, dedupe) - recordProcessGoneCrash( - { record, attachDetails } as never, - rendererKill, - dedupe, - noMinidump - ) + recordProcessGoneCrash({ record, attachDetails } as never, rendererKill, dedupe, noMinidump) expect(record).toHaveBeenCalledOnce() // Timing proximity is evidence, not authority to discard an ambiguous report. diff --git a/src/main/crash-reporting/process-gone-recorder.test.ts b/src/main/crash-reporting/process-gone-recorder.test.ts index 70e6ca93f04..24749b1bacf 100644 --- a/src/main/crash-reporting/process-gone-recorder.test.ts +++ b/src/main/crash-reporting/process-gone-recorder.test.ts @@ -439,7 +439,9 @@ describe('recordProcessGoneCrash', () => { // Why child kills: the decode gate is source-agnostic and reads // process.platform synchronously at record time, so the platform stub is // still in force when it runs. - const nonRecoverableChildKill = (overrides: Partial): ProcessGoneCrashEvent => + const nonRecoverableChildKill = ( + overrides: Partial + ): ProcessGoneCrashEvent => event({ source: 'child', processType: 'Utility', diff --git a/src/main/ipc/pty-daemon-controller-teardown.test.ts b/src/main/ipc/pty-daemon-controller-teardown.test.ts index 25d0aa48277..73ae897b8b3 100644 --- a/src/main/ipc/pty-daemon-controller-teardown.test.ts +++ b/src/main/ipc/pty-daemon-controller-teardown.test.ts @@ -153,7 +153,7 @@ describe('registerPtyHandlers', () => { }) expect(runtime.onPtyExit).not.toHaveBeenCalled() }) - it('does not accept an incarnation-less exit as proof that the current PTY stopped', async () => { + it('uses inventory, not an incarnation-less exit, to confirm the current PTY stopped', async () => { const exitListeners = new Set< (payload: { id: string; code: number; incarnationId?: string }) => void >() @@ -203,7 +203,7 @@ describe('registerPtyHandlers', () => { await controller.spawn({ cols: 80, rows: 24 }) await expect(controller.stopAndWait('local-incarnated')).resolves.toBe(true) - expect(runtime.onPtyExit).toHaveBeenCalledWith('local-incarnated', -1, 'incarnation-live') + expect(runtime.onPtyExit).toHaveBeenCalledWith('local-incarnated', 0, 'incarnation-live') }) it('runtime controller kill routes app-scoped SSH ids through the parsed provider when ownership is absent', async () => { const localShutdown = vi.fn() @@ -319,7 +319,9 @@ describe('registerPtyHandlers', () => { kill: (ptyId: string) => boolean } - expect(controller.kill('ssh:ssh-1@@relay-pty')).toBe(true) + // The lease is tombstoned, but nothing observed the detached relay PTY + // stop, so the stop itself is reported as unconfirmed. + expect(controller.kill('ssh:ssh-1@@relay-pty')).toBe(false) expect(localShutdown).not.toHaveBeenCalled() expect(store.markSshRemotePtyLease).toHaveBeenCalledWith('ssh-1', 'relay-pty', 'terminated') @@ -347,7 +349,7 @@ describe('registerPtyHandlers', () => { kill: (ptyId: string) => boolean } - expect(controller.kill('remote-pty')).toBe(true) + expect(controller.kill('remote-pty')).toBe(false) expect(store.markSshRemotePtyLease).toHaveBeenCalledWith( 'ssh-1', @@ -356,14 +358,15 @@ describe('registerPtyHandlers', () => { ) expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', -1, undefined) }) - it('retires a rejected SSH PTY after generic kill shutdown fails transiently', async () => { + it('keeps a rejected SSH PTY unverifiable after kill shutdown fails transiently', async () => { const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) const store = { markSshRemotePtyLease: vi.fn() } const runtime = { setPtyController: vi.fn(), - onPtyExit: vi.fn() + onPtyExit: vi.fn(), + markPtyLivenessUnverifiable: vi.fn() } registerSshPtyProvider('ssh-1', { spawn: vi.fn(), @@ -399,7 +402,7 @@ describe('registerPtyHandlers', () => { ) const controller = runtime.setPtyController.mock.calls[0]?.[0] as { kill: (ptyId: string) => boolean - retireRejectedPty: (ptyId: string) => void + retireRejectedPty: (ptyId: string, stopConfirmed: boolean) => void } try { @@ -410,19 +413,23 @@ describe('registerPtyHandlers', () => { 'remote-pty', 'terminated' ) - controller.retireRejectedPty('remote-pty') + controller.retireRejectedPty('remote-pty', false) } finally { warnSpy.mockRestore() deletePtyOwnership('remote-pty') } - expect(store.markSshRemotePtyLease).toHaveBeenCalledWith( + expect(store.markSshRemotePtyLease).not.toHaveBeenCalledWith( 'ssh-1', 'remote-pty', 'terminated' ) + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + 'remote-pty', + 'a follow-up stop was issued but its outcome could not be verified' + ) expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', -1, undefined) - expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', 0, undefined) + expect(runtime.onPtyExit).not.toHaveBeenCalledWith('remote-pty', 0, undefined) }) it('strips ORCA_PANE_KEY/TAB_ID/WORKTREE_ID from SSH spawn env when remote agent hooks are disabled', async () => { const sshSpawn = vi.fn(async (_opts: { env: Record }) => ({ diff --git a/src/main/ipc/pty-daemon-ssh-lease-lifecycle.test.ts b/src/main/ipc/pty-daemon-ssh-lease-lifecycle.test.ts index 92c4b7f5d35..d2e07b8458c 100644 --- a/src/main/ipc/pty-daemon-ssh-lease-lifecycle.test.ts +++ b/src/main/ipc/pty-daemon-ssh-lease-lifecycle.test.ts @@ -459,7 +459,7 @@ describe('registerPtyHandlers', () => { 'remote-pty', 'terminated' ) - expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', -1, undefined) + expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', 0, undefined) }) it('splits the teardown budget so the liveness RPC gets only what shutdown left', async () => { // Why: sequential RPCs must share one absolute deadline; otherwise both get diff --git a/src/main/ipc/pty-runtime-kill-and-exit.test.ts b/src/main/ipc/pty-runtime-kill-and-exit.test.ts index bcef15a2492..2da8b80d4b8 100644 --- a/src/main/ipc/pty-runtime-kill-and-exit.test.ts +++ b/src/main/ipc/pty-runtime-kill-and-exit.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest' import { spawnMock } from './pty-ipc-mock-registry' import { makeDeferred } from './pty-ipc-test-constants' import { setupPtyIpcSuite } from './pty-ipc-test-harness' +import { LOCAL_EXECUTION_HOST_ID, toSshExecutionHostId } from '../../shared/execution-host' import { registerPtyHandlers, registerSshPtyProvider, @@ -102,11 +103,16 @@ describe('registerPtyHandlers', () => { }) registerSshPtyProvider('ssh-a', { listProcesses: sshAList } as never) registerSshPtyProvider('ssh-b', { listProcesses: sshBList } as never) - const runtime = { setPtyController: vi.fn() } + setPtyOwnership('ssh-b-pty', 'ssh-b') + const runtime = { + setPtyController: vi.fn(), + markPtyLivenessUnverifiable: vi.fn() + } handlers.clear() registerPtyHandlers(mainWindow as never, runtime as never) const controller = runtime.setPtyController.mock.calls[0]?.[0] as { listProcesses(connectionId?: string | null): Promise<{ id: string }[]> + listProcessesWithHostScope(): Promise<{ processes: { id: string }[]; hostIds: string[] }> } await expect(controller.listProcesses(null)).resolves.toEqual([ @@ -120,7 +126,22 @@ describe('registerPtyHandlers', () => { expect(sshAList).toHaveBeenCalledOnce() expect(sshBList).not.toHaveBeenCalled() - await expect(controller.listProcesses()).rejects.toThrow('ssh-b unavailable') + await expect(controller.listProcesses()).resolves.toEqual([ + { id: 'local-pty', title: 'Local', cwd: '/local' }, + { id: 'ssh-a-pty' } + ]) + expect(sshBList).toHaveBeenCalledOnce() + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + 'ssh-b-pty', + 'ssh-b unavailable' + ) + + await expect(controller.listProcessesWithHostScope()).resolves.toEqual({ + processes: [{ id: 'local-pty', title: 'Local', cwd: '/local' }, { id: 'ssh-a-pty' }], + hostIds: [LOCAL_EXECUTION_HOST_ID, toSshExecutionHostId('ssh-a')] + }) + expect(sshBList).toHaveBeenCalledTimes(2) + deletePtyOwnership('ssh-b-pty') }) it('returns unavailable runtime confirmation for unsupported or missing providers', async () => { registerSshPtyProvider('ssh-1', {} as never) diff --git a/src/main/ipc/pty-ssh-stop-verdict.test.ts b/src/main/ipc/pty-ssh-stop-verdict.test.ts new file mode 100644 index 00000000000..8001028ea1d --- /dev/null +++ b/src/main/ipc/pty-ssh-stop-verdict.test.ts @@ -0,0 +1,297 @@ +import { describe, expect, it, vi } from 'vitest' +import { SSH_PROVIDER_UNREGISTERED_REASON } from '../../shared/pty-liveness-verdict' +import { setupPtyIpcSuite } from './pty-ipc-test-harness' +import { + registerPtyHandlers, + deletePtyOwnership, + setPtyOwnership, + getLocalPtyProvider, + registerSshPtyProvider, + unregisterSshPtyProvider +} from './pty' + +vi.mock('electron', () => import('./pty-ipc-mock-registry').then((m) => m.electronModuleMock())) +vi.mock('fs', () => import('./pty-ipc-mock-registry').then((m) => m.fsModuleMock())) +vi.mock('node-pty', () => import('./pty-ipc-mock-registry').then((m) => m.nodePtyModuleMock())) +vi.mock('node:child_process', async (importOriginal) => + (await import('./pty-ipc-mock-registry')).childProcessModuleMock(await importOriginal()) +) +vi.mock('../opencode/hook-service', () => + import('./pty-ipc-mock-registry').then((m) => m.openCodeHookServiceModuleMock()) +) +vi.mock('../mimo/hook-service', () => + import('./pty-ipc-mock-registry').then((m) => m.mimoHookServiceModuleMock()) +) +vi.mock('../agent-hooks/server', () => + import('./pty-ipc-mock-registry').then((m) => m.agentHookServerModuleMock()) +) +vi.mock('../pi/titlebar-extension-service', () => + import('./pty-ipc-mock-registry').then((m) => m.piTitlebarExtensionModuleMock()) +) +vi.mock('../pwsh', () => import('./pty-ipc-mock-registry').then((m) => m.pwshModuleMock())) +vi.mock('../wsl', async (importOriginal) => + (await import('./pty-ipc-mock-registry')).wslModuleMock(await importOriginal()) +) +vi.mock('../telemetry/client', () => + import('./pty-ipc-mock-registry').then((m) => m.telemetryClientModuleMock()) +) +vi.mock('../telemetry/classify-error', () => + import('./pty-ipc-mock-registry').then((m) => m.classifyErrorModuleMock()) +) +vi.mock('../cli/linux-terminal-orca-cli-shim', () => + import('./pty-ipc-mock-registry').then((m) => m.linuxCliShimModuleMock()) +) +vi.mock('../memory/pty-registry', () => + import('./pty-ipc-mock-registry').then((m) => m.ptyRegistryModuleMock()) +) +vi.mock('../agent-hooks/migration-unsupported-pty-state', () => + import('./pty-ipc-mock-registry').then((m) => m.migrationUnsupportedPtyModuleMock()) +) +vi.mock('../codex/codex-pane-account-registry', () => + import('./pty-ipc-mock-registry').then((m) => m.codexPaneAccountRegistryModuleMock()) +) +vi.mock('../codex/codex-state-db-backfill-recovery', () => + import('./pty-ipc-mock-registry').then((m) => m.codexBackfillRecoveryModuleMock()) +) + +// A detached relay PTY is designed to outlive the provider that addressed it, so +// "the SSH provider is gone" is never evidence that the remote process stopped. +describe('stopping a PTY whose SSH provider is unregistered', () => { + const { handlers, mainWindow, installObservableDaemonTestProvider } = setupPtyIpcSuite() + + function installController(): { + controller: { + kill: (ptyId: string) => boolean + listProcesses: (connectionId?: string | null) => Promise<{ id: string }[]> + retireRejectedPty: (ptyId: string, stopConfirmed: boolean) => void + stopAndWait: (ptyId: string, opts?: { deadlineMs?: number }) => Promise + } + runtime: { + setPtyController: ReturnType + onPtyExit: ReturnType + markPtyLivenessUnverifiable: ReturnType + markPtyLivenessLive: ReturnType + } + } { + const runtime = { + setPtyController: vi.fn(), + onPtyExit: vi.fn(), + markPtyLivenessUnverifiable: vi.fn(), + markPtyLivenessLive: vi.fn() + } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + return { + controller: runtime.setPtyController.mock.calls[0]?.[0] as never, + runtime + } + } + + it('reports an unconfirmed stop instead of a fabricated kill', () => { + setPtyOwnership('ssh-detached-pty', 'ssh-dropped') + const { controller, runtime } = installController() + + expect(controller.kill('ssh-detached-pty')).toBe(false) + // The local lease is still tombstoned so reconnect cannot revive the pane. + expect(runtime.onPtyExit).toHaveBeenCalledWith('ssh-detached-pty', -1, undefined) + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + 'ssh-detached-pty', + expect.stringContaining('SSH') + ) + deletePtyOwnership('ssh-detached-pty') + }) + + it('reports an unconfirmed exact stop instead of a fabricated teardown', async () => { + setPtyOwnership('ssh-detached-stop', 'ssh-dropped') + const { controller, runtime } = installController() + + await expect(controller.stopAndWait('ssh-detached-stop')).resolves.toBe(false) + expect(runtime.onPtyExit).toHaveBeenCalledWith('ssh-detached-stop', -1, undefined) + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + 'ssh-detached-stop', + expect.stringContaining('SSH') + ) + deletePtyOwnership('ssh-detached-stop') + }) + + it('preserves lost-contact evidence for renderer IPC teardown', async () => { + const ptyId = 'ssh-renderer-detached' + setPtyOwnership(ptyId, 'ssh-dropped') + const { runtime } = installController() + try { + await handlers.get('pty:kill')!(null, { id: ptyId }) + + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + ptyId, + SSH_PROVIDER_UNREGISTERED_REASON + ) + expect(runtime.onPtyExit).toHaveBeenCalledWith(ptyId, -1, undefined) + } finally { + deletePtyOwnership(ptyId) + } + }) + + it('retires a rejected split without asserting an unconfirmed exit', () => { + const ptyId = 'ssh-rejected-split' + setPtyOwnership(ptyId, 'ssh-dropped') + const { controller, runtime } = installController() + try { + controller.retireRejectedPty(ptyId, false) + + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + ptyId, + 'a follow-up stop was issued but its outcome could not be verified' + ) + expect(runtime.onPtyExit).toHaveBeenCalledWith(ptyId, -1, undefined) + expect(runtime.onPtyExit).not.toHaveBeenCalledWith(ptyId, 0, expect.anything()) + } finally { + deletePtyOwnership(ptyId) + } + }) + + it('still confirms a stop the owning provider actually performed', async () => { + const daemon = installObservableDaemonTestProvider() + vi.spyOn(getLocalPtyProvider(), 'listProcesses').mockResolvedValue([]) + const { controller, runtime } = installController() + + await expect(controller.stopAndWait('wt-1@@local-session')).resolves.toBe(true) + expect(daemon.shutdown).toHaveBeenCalledWith( + 'wt-1@@local-session', + expect.objectContaining({ immediate: true }) + ) + expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalled() + }) + + it('records provider-confirmed absence when the exit event was missed', async () => { + const connectionId = 'ssh-confirmed-absent' + const ptyId = 'ssh-confirmed-absent-pty' + const provider = { + onExit: vi.fn(() => () => {}), + shutdown: vi.fn(async () => {}), + listProcesses: vi.fn(async () => []) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + + await expect(controller.stopAndWait(ptyId)).resolves.toBe(true) + expect(runtime.onPtyExit).toHaveBeenCalledWith(ptyId, 0, undefined) + expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalled() + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) + + it('reports lost contact when a registered provider drops during the stop', async () => { + const connectionId = 'ssh-mid-stop-drop' + const ptyId = 'ssh-mid-stop-pty' + const provider = { + onExit: vi.fn(() => () => {}), + shutdown: vi.fn(async () => { + throw new Error('relay disconnected during stop') + }) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + + await expect(controller.stopAndWait(ptyId)).resolves.toBe(false) + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + ptyId, + 'relay disconnected during stop' + ) + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) + + it('preserves lost-contact evidence when fire-and-forget shutdown rejects', async () => { + const connectionId = 'ssh-async-kill-drop' + const ptyId = 'ssh-async-kill-pty' + const provider = { + onExit: vi.fn(() => () => {}), + shutdown: vi.fn(async () => { + throw new Error('relay disconnected during kill') + }) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + + expect(controller.kill(ptyId)).toBe(true) + await vi.waitFor(() => + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + ptyId, + 'relay disconnected during kill' + ) + ) + expect(runtime.onPtyExit).toHaveBeenCalledWith(ptyId, -1, undefined) + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) + + it('isolates a failed SSH inventory from healthy providers', async () => { + const failedConnectionId = 'ssh-inventory-failed' + const healthyConnectionId = 'ssh-inventory-healthy' + const failedPtyId = 'ssh-inventory-failed-pty' + const healthyPtyId = 'ssh-inventory-healthy-pty' + registerSshPtyProvider(failedConnectionId, { + listProcesses: vi.fn(async () => { + throw new Error('inventory transport failed') + }) + } as never) + registerSshPtyProvider(healthyConnectionId, { + listProcesses: vi.fn(async () => [{ id: healthyPtyId }]) + } as never) + setPtyOwnership(failedPtyId, failedConnectionId) + setPtyOwnership(healthyPtyId, healthyConnectionId) + try { + const { controller, runtime } = installController() + + await expect(controller.listProcesses()).resolves.toEqual( + expect.arrayContaining([{ id: healthyPtyId }]) + ) + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + failedPtyId, + 'inventory transport failed' + ) + expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalledWith( + healthyPtyId, + expect.anything() + ) + } finally { + deletePtyOwnership(failedPtyId) + deletePtyOwnership(healthyPtyId) + unregisterSshPtyProvider(failedConnectionId) + unregisterSshPtyProvider(healthyConnectionId) + } + }) + + it('reports a provider-observed survivor as live', async () => { + const connectionId = 'ssh-still-live' + const ptyId = 'ssh-still-live-pty' + const provider = { + onExit: vi.fn(() => () => {}), + shutdown: vi.fn(async () => {}), + listProcesses: vi.fn(async () => [{ id: ptyId }]) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + + await expect(controller.stopAndWait(ptyId)).resolves.toBe(false) + expect(runtime.markPtyLivenessLive).toHaveBeenCalledWith(ptyId) + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) +}) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 376b705ef05..10ee17200db 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -15,6 +15,7 @@ export { getBashShellReadyRcfileContent } from '../providers/local-pty-shell-rea import type { OrcaRuntimeService } from '../runtime/orca-runtime' import type { PtyBindingSourceExpectation, Store } from '../persistence' import { retireTerminalSurfaceFromPersistence } from '../runtime/mobile-session-terminal-persistence-retirement' +import { SSH_PROVIDER_UNREGISTERED_REASON } from '../../shared/pty-liveness-verdict' import type { GlobalSettings } from '../../shared/global-settings-types' import type { TuiAgent } from '../../shared/tui-agent' import { @@ -264,19 +265,36 @@ function registeredPtyProviders(): RegisteredPtyProvider[] { ] } -async function listRegisteredPtyProcessesWithHostScope(): Promise<{ +async function listRegisteredPtyProcessesWithHostScope( + onSshInventoryUnavailable?: (connectionId: string, error: unknown) => void +): Promise<{ processes: PtyProcessInfo[] hostIds: ExecutionHostId[] }> { const providers = registeredPtyProviders() const providerSessions = await Promise.all( - providers.map(({ provider }) => provider.listProcesses()) + providers.map(async ({ provider, connectionId }) => { + try { + const hostId: ExecutionHostId = connectionId + ? toSshExecutionHostId(connectionId) + : LOCAL_EXECUTION_HOST_ID + return { + processes: await provider.listProcesses(), + hostId + } + } catch (error) { + if (!connectionId) { + throw error + } + onSshInventoryUnavailable?.(connectionId, error) + return null + } + }) ) + const respondingSessions = providerSessions.filter((session) => session !== null) return { - processes: providerSessions.flat(), - hostIds: providers.map(({ connectionId }) => - connectionId ? toSshExecutionHostId(connectionId) : LOCAL_EXECUTION_HOST_ID - ) + processes: respondingSessions.flatMap((session) => session.processes), + hostIds: respondingSessions.map((session) => session.hostId) } } @@ -3930,7 +3948,9 @@ export function registerPtyHandlers( return release }, finalizeExit: (event) => { - runtime?.onPtyExit(event.id, event.code, event.ptyIncarnation) + runtime?.onPtyExit(event.id, event.code, event.ptyIncarnation, { + hostExitConfirmed: true + }) finalizePtyExitForRenderer(event) }, pauseProvider: (generation, id) => { @@ -4465,6 +4485,15 @@ export function registerPtyHandlers( } } + const markSshInventoryUnverifiable = (connectionId: string, error: unknown): void => { + const reason = error instanceof Error ? error.message : String(error) + for (const [ptyId, ownerConnectionId] of ptyOwnership) { + if (ownerConnectionId === connectionId) { + runtime?.markPtyLivenessUnverifiable?.(ptyId, reason) + } + } + } + // Why: route through getProviderForPty() so CLI commands work for remote PTYs too; localProvider would silently fail for them. runtime?.setPtyController({ claimStablePaneCreate: (args) => { @@ -5539,12 +5568,15 @@ export function registerPtyHandlers( if (connectionId) { // Why: runtime/CLI close can target a detached SSH PTY after its // provider was unregistered. Tombstone the lease so reconnect does - // not revive a terminal the user explicitly closed. + // not revive a terminal the user explicitly closed — but a detached + // relay PTY outlives its provider, so report an unconfirmed stop + // rather than a kill nobody performed. const incarnationId = finishPtyShutdown(ptyId, connectionId, store) runtime?.onPtyExit(ptyId, -1, incarnationId) rememberSyntheticKillExit(ptyId) sendPtyExitToRenderer({ id: ptyId, code: -1 }) - return true + runtime?.markPtyLivenessUnverifiable?.(ptyId, SSH_PROVIDER_UNREGISTERED_REASON) + return false } return false } @@ -5576,6 +5608,12 @@ export function registerPtyHandlers( // Why: close runtime tails without clearing provider ownership, so // a retry can still target a PTY that survived the failed shutdown. if (!retired) { + if (connectionId) { + runtime?.markPtyLivenessUnverifiable?.( + ptyId, + err instanceof Error ? err.message : String(err) + ) + } runtime?.onPtyExit(ptyId, -1, ptyIncarnationById.get(ptyId)) } }) @@ -5589,6 +5627,12 @@ export function registerPtyHandlers( `[pty] Failed to stop PTY ${ptyId}: ${err instanceof Error ? err.message : String(err)}` ) if (!retiredRejectedPtyIds.has(ptyId)) { + if (connectionId) { + runtime?.markPtyLivenessUnverifiable?.( + ptyId, + err instanceof Error ? err.message : String(err) + ) + } runtime?.onPtyExit(ptyId, -1, ptyIncarnationById.get(ptyId)) } }) @@ -5596,11 +5640,23 @@ export function registerPtyHandlers( } return killWithCurrentProvider() }, - retireRejectedPty: (ptyId) => { + retireRejectedPty: (ptyId, stopConfirmed) => { rememberRetiredRejectedPty(ptyId) + if (!stopConfirmed) { + runtime?.markPtyLivenessUnverifiable?.( + ptyId, + 'a follow-up stop was issued but its outcome could not be verified' + ) + if (!ptyOwnership.has(ptyId)) { + return + } + runtime?.onPtyExit(ptyId, -1, ptyIncarnationById.get(ptyId)) + rememberSyntheticKillExit(ptyId) + sendPtyExitToRenderer({ id: ptyId, code: -1 }) + return + } // Why: a completed stop already cleared provider state, tombstoned the lease and told the - // renderer; repeating that double-fires the exit IPC. The runtime still needs code 0 so an - // SSH pane retires for good instead of staying preserved by the stop's negative exit. + // renderer; repeating that double-fires the exit IPC. if (!ptyOwnership.has(ptyId)) { runtime?.onPtyExit(ptyId, 0, ptyIncarnationById.get(ptyId)) return @@ -5669,13 +5725,14 @@ export function registerPtyHandlers( provider = connectionId ? getProvider(connectionId) : getProviderForPty(ptyId) } catch { if (connectionId) { - // Why: an absent SSH provider means there is no live target left to - // await, but the relay lease must still be tombstoned. + // Why: the relay lease must still be tombstoned, but an absent SSH + // provider is lost contact — the remote PTY is designed to survive it, + // so nothing here observed an exit to report as a confirmed stop. const incarnationId = finishPtyShutdown(ptyId, connectionId, store) runtime?.onPtyExit(ptyId, -1, incarnationId) rememberSyntheticKillExit(ptyId) sendPtyExitToRenderer({ id: ptyId, code: -1 }) - return true + runtime?.markPtyLivenessUnverifiable?.(ptyId, SSH_PROVIDER_UNREGISTERED_REASON) } return false } @@ -5688,6 +5745,12 @@ export function registerPtyHandlers( }) } catch (err) { if (!isPtyAlreadyGoneError(err)) { + if (connectionId) { + runtime?.markPtyLivenessUnverifiable?.( + ptyId, + err instanceof Error ? err.message : String(err) + ) + } console.warn( `[pty] Failed to stop PTY ${ptyId}: ${err instanceof Error ? err.message : String(err)}` ) @@ -5696,9 +5759,16 @@ export function registerPtyHandlers( } try { if (!(await verifyPtyStopped(provider, ptyId, opts))) { + runtime?.markPtyLivenessLive?.(ptyId) return false } } catch (err) { + if (connectionId) { + runtime?.markPtyLivenessUnverifiable?.( + ptyId, + err instanceof Error ? err.message : String(err) + ) + } console.warn( `[pty] Failed to verify PTY ${ptyId} stopped: ${ err instanceof Error ? err.message : String(err) @@ -5708,9 +5778,11 @@ export function registerPtyHandlers( } const incarnationId = finishPtyShutdown(ptyId, connectionId, store) if (!providerExitObserved) { - runtime?.onPtyExit(ptyId, -1, incarnationId) + // The owning provider's fresh inventory observed absence, so this is a + // death certificate even when its exit event was missed. + runtime?.onPtyExit(ptyId, 0, incarnationId) rememberSyntheticKillExit(ptyId) - sendPtyExitToRenderer({ id: ptyId, code: -1 }) + sendPtyExitToRenderer({ id: ptyId, code: 0 }) } return true }, @@ -5767,11 +5839,17 @@ export function registerPtyHandlers( return localProvider.listProcesses() } if (connectionId !== undefined) { - return getProvider(connectionId).listProcesses() + try { + return await getProvider(connectionId).listProcesses() + } catch (error) { + markSshInventoryUnverifiable(connectionId, error) + throw error + } } - return (await listRegisteredPtyProcessesWithHostScope()).processes + return (await listRegisteredPtyProcessesWithHostScope(markSshInventoryUnverifiable)).processes }, - listProcessesWithHostScope: listRegisteredPtyProcessesWithHostScope, + listProcessesWithHostScope: () => + listRegisteredPtyProcessesWithHostScope(markSshInventoryUnverifiable), serializeBuffer: (ptyId, opts) => { // Why: mobile xterm must start from the desktop's exact screen state/dimensions before live TUI chunks render correctly. return requestSerializedBuffer(ptyId, opts) @@ -7604,6 +7682,7 @@ export function registerPtyHandlers( // provider is unregistered; hydrated app-scoped ids can also arrive // before ownership is rebuilt. Tombstone instead of falling back local. const incarnationId = finishPtyShutdown(args.id, connectionId, store) + runtime?.markPtyLivenessUnverifiable?.(args.id, SSH_PROVIDER_UNREGISTERED_REASON) runtime?.onPtyExit(args.id, -1, incarnationId) rememberSyntheticKillExit(args.id) sendPtyExitToRenderer({ id: args.id, code: -1 }) diff --git a/src/main/runtime/claude-agent-teams-service.test.ts b/src/main/runtime/claude-agent-teams-service.test.ts index a41668932f0..57f867cb1c2 100644 --- a/src/main/runtime/claude-agent-teams-service.test.ts +++ b/src/main/runtime/claude-agent-teams-service.test.ts @@ -189,7 +189,7 @@ describe('ClaudeAgentTeamsService', () => { expect(api.closeTerminal).toHaveBeenLastCalledWith('teammate-2') }) - it('keeps the placeholder handle when the respawn split fails', async () => { + it('removes the pane when replacement split fails after a confirmed placeholder stop', async () => { const { service, teamId, token, leaderPane, api } = createServiceWithLeader() const request = (argv: string[], envPane = leaderPane) => service.handleTmuxCompat({ teamId, token, envPane, argv }, api) @@ -212,10 +212,59 @@ describe('ClaudeAgentTeamsService', () => { request(['respawn-pane', '-k', '-t', '%2', '--', 'claude --agent-id a']) ).resolves.toMatchObject({ ok: false, exitCode: 1 }) - // the placeholder terminal is left intact and the fake pane id still resolves. - expect(api.closeTerminal).not.toHaveBeenCalled() - await request(['kill-pane', '-t', '%2']) expect(api.closeTerminal).toHaveBeenCalledWith('teammate-1') + await expect( + request(['list-panes', '-t', 'orca:0', '-F', '#{pane_id}']) + ).resolves.toMatchObject({ stdout: '%1\n' }) + }) + + it('keeps a pane registered when its process stop is unconfirmed', async () => { + const { service, teamId, token, leaderPane, api } = createServiceWithLeader() + const request = (argv: string[], envPane = leaderPane) => + service.handleTmuxCompat({ teamId, token, envPane, argv }, api) + + await request(['split-window', '-t', leaderPane, '-h', '-P', '-F', '#{pane_id}']) + vi.mocked(api.closeTerminal).mockResolvedValueOnce({ + handle: 'teammate-1', + tabId: 'tab-1', + ptyKilled: false + }) + + await expect(request(['kill-pane', '-t', '%2'])).resolves.toMatchObject({ + ok: false, + exitCode: 1 + }) + await expect( + request(['list-panes', '-t', 'orca:0', '-F', '#{pane_id}']) + ).resolves.toMatchObject({ stdout: '%1\n%2\n' }) + }) + + it('does not launch a replacement when the placeholder stop is unconfirmed', async () => { + const { service, teamId, token, leaderPane, api, splitCalls } = createServiceWithLeader() + const request = (argv: string[], envPane = leaderPane) => + service.handleTmuxCompat({ teamId, token, envPane, argv }, api) + + await request(['split-window', '-t', leaderPane, '-h', '-P', '-F', '#{pane_id}', 'cat']) + vi.mocked(api.closeTerminal).mockResolvedValueOnce({ + handle: 'teammate-1', + tabId: 'tab-1', + ptyKilled: false, + ptyStopVerdict: 'live' + }) + + await expect( + request(['respawn-pane', '-k', '-t', '%2', '--', 'claude --agent-id a']) + ).resolves.toMatchObject({ ok: false, exitCode: 1 }) + expect(api.closeTerminal).toHaveBeenNthCalledWith(1, 'teammate-1') + expect(splitCalls).toHaveLength(1) + + await expect( + request(['respawn-pane', '-k', '-t', '%2', '--', 'claude --agent-id a']) + ).resolves.toMatchObject({ ok: false, exitCode: 1 }) + expect(splitCalls).toHaveLength(1) + + await request(['kill-pane', '-t', '%2']) + expect(api.closeTerminal).toHaveBeenLastCalledWith('teammate-1') }) it('refuses to respawn the leader pane', async () => { diff --git a/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts b/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts index 98159e8725e..a9a3b4484a8 100644 --- a/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts +++ b/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts @@ -4,6 +4,7 @@ import { tmuxSendKeysText, tmuxValue } from '../../shared/claude-agent-teams-tmux-compat' +import { describeUnconfirmedAgentStop } from '../../shared/pty-liveness-verdict' import { formatContext, paneEnv, @@ -157,27 +158,31 @@ export class ClaudeAgentTeamsTmuxDispatcher { if (!command) { return '' } + if (pane.respawnBlockedReason) { + throw new Error(pane.respawnBlockedReason) + } const origin = (pane.splitFromPane ? team.panes.get(pane.splitFromPane) : undefined) ?? team.panes.get(team.leaderPane)! - // Why: create the replacement before destroying the placeholder so a failed - // split leaves the fake pane id pointing at a still-live terminal; on cleanup - // failure, discard the new split and keep the placeholder registered. const previousHandle = pane.handle - const split = await api.splitTerminal(origin.handle, { - direction: pane.splitDirection ?? 'horizontal', - command, - env: paneEnv(team, pane.fakePaneId), - envToDelete: ['TERM_PROGRAM'], - activate: false - }) + const close = await api.closeTerminal(previousHandle) + if (!close.ptyKilled) { + pane.respawnBlockedReason = describeUnconfirmedAgentStop(close) + throw new Error(pane.respawnBlockedReason) + } try { - await api.closeTerminal(previousHandle) + const split = await api.splitTerminal(origin.handle, { + direction: pane.splitDirection ?? 'horizontal', + command, + env: paneEnv(team, pane.fakePaneId), + envToDelete: ['TERM_PROGRAM'], + activate: false + }) + pane.handle = split.handle } catch (error) { - await api.closeTerminal(split.handle).catch(() => {}) + this.removePane(team, pane) throw error } - pane.handle = split.handle return '' } @@ -266,14 +271,21 @@ export class ClaudeAgentTeamsTmuxDispatcher { if (pane.fakePaneId === team.leaderPane) { throw new Error('refusing to kill leader pane') } - await api.closeTerminal(pane.handle) + const close = await api.closeTerminal(pane.handle) + if (!close.ptyKilled) { + throw new Error(describeUnconfirmedAgentStop(close)) + } + this.removePane(team, pane) + return '' + } + + private removePane(team: AgentTeam, pane: TeamPane): void { team.panes.delete(pane.fakePaneId) team.paneOrder = team.paneOrder.filter((id) => id !== pane.fakePaneId) if (team.mainVertical?.lastColumnPane === pane.fakePaneId) { team.mainVertical.lastColumnPane = [...team.paneOrder].toReversed().find((id) => id !== team.leaderPane) ?? null } - return '' } private async lastPane( diff --git a/src/main/runtime/claude-agent-teams-types.ts b/src/main/runtime/claude-agent-teams-types.ts index 981c9eee024..10234f5226d 100644 --- a/src/main/runtime/claude-agent-teams-types.ts +++ b/src/main/runtime/claude-agent-teams-types.ts @@ -59,6 +59,7 @@ export type TeamPane = { // respawn can recreate it in the same slot while preserving its fake pane id. splitFromPane?: string splitDirection?: 'horizontal' | 'vertical' + respawnBlockedReason?: string } export type AgentTeam = { diff --git a/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts b/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts index 06a5f1cfcd0..3066367d7e8 100644 --- a/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts +++ b/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts @@ -35,12 +35,12 @@ describe('terminal process incarnation liveness', () => { ).resolves.toBe('live') await expect( runtime.inspectTerminalProcessIncarnationLiveness('remote:ssh-1:pty-1:inc-old', SSH_SCOPE) - ).resolves.toBe('dead') + ).resolves.toBe('exited') expect(listProcesses).toHaveBeenNthCalledWith(1, 'ssh-1') expect(listProcesses).toHaveBeenNthCalledWith(2, 'ssh-1') }) - it('keeps missing or malformed identity and unavailable inventory unknown', async () => { + it('keeps missing or malformed identity and unavailable inventory unverifiable', async () => { const listProcesses = vi .fn() .mockResolvedValueOnce([{ id: 'remote:ssh-1:pty-1', cwd: '', title: 'worker' }]) @@ -52,13 +52,13 @@ describe('terminal process incarnation liveness', () => { await expect( runtime.inspectTerminalProcessIncarnationLiveness(PROCESS_INCARNATION, SSH_SCOPE) - ).resolves.toBe('unknown') + ).resolves.toBe('unverifiable') await expect( runtime.inspectTerminalProcessIncarnationLiveness(PROCESS_INCARNATION, SSH_SCOPE) - ).resolves.toBe('unknown') + ).resolves.toBe('unverifiable') await expect( runtime.inspectTerminalProcessIncarnationLiveness(PROCESS_INCARNATION, SSH_SCOPE) - ).resolves.toBe('unknown') + ).resolves.toBe('unverifiable') }) it('does not inspect an unproven host scope', async () => { @@ -67,10 +67,10 @@ describe('terminal process incarnation liveness', () => { await expect( runtime.inspectTerminalProcessIncarnationLiveness(PROCESS_INCARNATION, null) - ).resolves.toBe('unknown') + ).resolves.toBe('unverifiable') await expect( runtime.inspectTerminalProcessIncarnationLiveness(PROCESS_INCARNATION, '{"kind":"ssh"}') - ).resolves.toBe('unknown') + ).resolves.toBe('unverifiable') expect(listProcesses).not.toHaveBeenCalled() }) @@ -83,7 +83,7 @@ describe('terminal process incarnation liveness', () => { await expect( runtime.inspectTerminalProcessIncarnationLiveness('local-pty:inc-1', JSON.stringify(scope)) - ).resolves.toBe('dead') + ).resolves.toBe('exited') expect(listProcesses).toHaveBeenCalledWith(connectionId) }) }) diff --git a/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts b/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts index 6d22718dcae..c3a3e747b01 100644 --- a/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts +++ b/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts @@ -311,7 +311,7 @@ describe('terminal close and handle incarnation continuity', () => { harness.retirePersistedTab() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: false }) expect(harness.kill).toHaveBeenCalledWith(PTY_ID) expect(harness.closeTerminal).not.toHaveBeenCalled() expect(harness.getSession().tabsByWorktree[WORKTREE_ID]).toEqual([]) @@ -328,7 +328,7 @@ describe('terminal close and handle incarnation continuity', () => { expect(harness.getSession().tabsByWorktree[WORKTREE_ID]).toHaveLength(1) }) - it('kills every live tab PTY after retirement when the renderer graph is stale', async () => { + it('requests a stop for every live tab PTY after retirement when the renderer graph is stale', async () => { const harness = createHarness() harness.syncSplitFixtureGraph() const terminal = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals.find( @@ -342,7 +342,7 @@ describe('terminal close and handle incarnation continuity', () => { harness.retirePersistedTab() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ ptyKilled: false }) expect(harness.kill).toHaveBeenCalledWith(PTY_ID) expect(harness.kill).toHaveBeenCalledWith(SIBLING_PTY_ID) }) @@ -365,7 +365,64 @@ describe('terminal close and handle incarnation continuity', () => { expect(harness.kill).not.toHaveBeenCalled() }) - it('falls back to kill when verified teardown rejects after retirement', async () => { + it('reports an unconfirmed stop on the close receipt rather than a bare uncertain false', async () => { + const harness = createHarness() + const [{ handle }] = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals + // Mirrors pty.ts when the SSH provider is gone: the lease is tombstoned, the + // stop reports false, and the PTY is marked as contact we lost — not a kill. + harness.kill.mockReturnValue(false) + harness.setVerifiedStopResult(false) + harness.runtime.markPtyLivenessUnverifiable(PTY_ID, 'its SSH provider is no longer registered') + + const closing = harness.runtime.closeTerminal(handle) + await vi.waitFor(() => expect(harness.closeTerminalTab).toHaveBeenCalled()) + harness.retirePersistedTab() + harness.acknowledged.resolve() + + await expect(closing).resolves.toMatchObject({ + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'its SSH provider is no longer registered' + }) + expect(harness.kill).not.toHaveBeenCalled() + }) + + it('downgrades a live verdict after issuing an unverified follow-up stop', async () => { + const harness = createHarness() + const [{ handle }] = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals + harness.setVerifiedStopResult(false) + harness.runtime.markPtyLivenessLive(PTY_ID) + + const closing = harness.runtime.closeTerminal(handle) + await vi.waitFor(() => expect(harness.closeTerminalTab).toHaveBeenCalled()) + harness.retirePersistedTab() + harness.acknowledged.resolve() + + await expect(closing).resolves.toMatchObject({ + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'a follow-up stop was issued but its outcome could not be verified' + }) + expect(harness.kill).toHaveBeenCalledWith(PTY_ID) + }) + + it('leaves a confirmed kill receipt free of any stop verdict', async () => { + const harness = createHarness() + const [{ handle }] = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals + harness.setVerifiedStopResult(true) + + const closing = harness.runtime.closeTerminal(handle) + await vi.waitFor(() => expect(harness.closeTerminalTab).toHaveBeenCalled()) + harness.retirePersistedTab() + harness.acknowledged.resolve() + + const close = await closing + expect(close.ptyKilled).toBe(true) + expect(close.ptyStopVerdict).toBeUndefined() + expect(close.ptyStopReason).toBeUndefined() + }) + + it('reports an unconfirmed stop when verified teardown rejects after retirement', async () => { const harness = createHarness() const [{ handle }] = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals harness.setVerifiedStopResult(new Error('provider_unavailable')) @@ -375,7 +432,11 @@ describe('terminal close and handle incarnation continuity', () => { harness.retirePersistedTab() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'provider_unavailable' + }) expect(harness.stopAndWait).toHaveBeenCalledWith(PTY_ID, { deadlineMs: expect.any(Number) }) @@ -398,7 +459,7 @@ describe('terminal close and handle incarnation continuity', () => { harness.makeSessionUnavailable() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: false }) expect(harness.closeTerminal).toHaveBeenCalledWith(TAB_ID) expect(harness.stopAndWait).toHaveBeenCalledWith(RUNTIME_OWNED_PTY_ID, { deadlineMs: expect.any(Number) diff --git a/src/main/runtime/orca-runtime-terminal-split-authority.test.ts b/src/main/runtime/orca-runtime-terminal-split-authority.test.ts index ca79f9e55a8..a5adbedd269 100644 --- a/src/main/runtime/orca-runtime-terminal-split-authority.test.ts +++ b/src/main/runtime/orca-runtime-terminal-split-authority.test.ts @@ -348,7 +348,7 @@ describe('remote runtime terminal split authority', () => { expect.objectContaining({ deadlineMs: expect.any(Number) }) ) expect(harness.kill).not.toHaveBeenCalled() - expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID) + expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID, true) }) it('revalidates a projected paired-runtime source after renderer adoption', async () => { @@ -374,7 +374,7 @@ describe('remote runtime terminal split authority', () => { expect.objectContaining({ deadlineMs: expect.any(Number) }) ) expect(harness.kill).toHaveBeenCalledWith(SPLIT_PTY_ID) - expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID) + expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID, false) }) it('preserves the split error when kill and retirement throw', async () => { @@ -398,6 +398,6 @@ describe('remote runtime terminal split authority', () => { await expect(split).rejects.toThrow('terminal_split_source_not_found') expect(harness.kill).toHaveBeenCalledWith(SPLIT_PTY_ID) - expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID) + expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID, false) }) }) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index fb490109950..f5af4fb0786 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -664,6 +664,12 @@ import { type RetiredTerminalSurface } from './mobile-session-terminal-retirement' import { retireTerminalSurfaceFromPersistence } from './mobile-session-terminal-persistence-retirement' +import { + NO_OBSERVING_PROVIDER_REASON, + SSH_EXIT_UNCONFIRMED_REASON, + SSH_PROVIDER_UNREGISTERED_REASON, + type PtyLivenessVerdict +} from '../../shared/pty-liveness-verdict' import { advanceTerminalTopologyRevision, hasHostAuthoritativeTerminalMembership @@ -1568,6 +1574,14 @@ function isAgentSessionOperationOutcomeUnknown(error: unknown): boolean { ) } +// Orphaned verdicts are bounded; active PTYs retain theirs until new evidence resolves them. +const MAX_TRACKED_PTY_LIVENESS_VERDICTS = 256 + +type TrackedPtyLivenessVerdict = { + verdict: PtyLivenessVerdict + observedAt: number +} + const AGENT_SESSION_OPERATION_PER_CLIENT_LIMIT = 512 const AGENT_SESSION_OPERATION_GLOBAL_LIMIT = 4_096 @@ -1865,7 +1879,7 @@ type RuntimePtyController = { * False on doubt (absent session, SSH-scoped id, non-daemon provider). */ attach?(ptyId: string): Promise kill(ptyId: string): boolean - retireRejectedPty?(ptyId: string): void + retireRejectedPty?(ptyId: string, stopConfirmed: boolean): void stopAndWait?( ptyId: string, opts?: { keepHistory?: boolean; deadlineMs?: number } @@ -3206,6 +3220,12 @@ export class OrcaRuntimeService { // iterates them all. Listeners are cleaned up via subscriptionCleanups. private notificationListeners = new Set<(event: MobileNotificationEvent) => void>() private ptysById = new Map() + // Why a separate map: `connected` is a wire field that any inventory gap + // clears, so it cannot distinguish an observed exit from lost contact. This + // records the last liveness verdict we actually earned, and outlives the pty + // record so a close/stop receipt can still say the stop was unconfirmed. + private ptyLivenessVerdictByPtyId = new Map() + private ptyLivenessObservationSequence = 0 private readonly pairedRendererSessionOwnedPtyIds = new Set() private wslDistroByPtyId = new Map() private titleObservationSequence = 0 @@ -10579,6 +10599,7 @@ export class OrcaRuntimeService { incarnationId?: PtyIncarnationId, options: { awaitsRegistration?: boolean } = {} ): void { + this.forgetPtyLivenessVerdict(ptyId) if (options.awaitsRegistration !== false) { // Why: surface absence cannot distinguish an in-flight admission from a completed headless lifecycle. this.pendingPtyRegistrationIncarnations.set(ptyId, incarnationId ?? null) @@ -10612,6 +10633,7 @@ export class OrcaRuntimeService { isWsl?: boolean ): void { this.assertPtyDidNotExitBeforeRegistration(ptyId, binding?.incarnationId) + this.forgetPtyLivenessVerdict(ptyId) this.spawnPublishedPtys.add(ptyId) // Why: record the renderer pane identity at spawn time so a stalled graph // sync can't hide that a live PTY already backs a pending mobile create. @@ -14906,14 +14928,25 @@ export class OrcaRuntimeService { } } - onPtyExit(ptyId: string, exitCode: number, exitIncarnationId?: PtyIncarnationId): void { + onPtyExit( + ptyId: string, + exitCode: number, + exitIncarnationId?: PtyIncarnationId, + options?: { hostExitConfirmed?: boolean } + ): void { const pty = this.ptysById.get(ptyId) if (exitIncarnationId && pty?.incarnationId && exitIncarnationId !== pty.incarnationId) { return } const preservesAbnormalSshSurface = - this.isSshOwnedPtyId(ptyId) && pty?.connectionId != null && exitCode < 0 + this.isSshOwnedPtyId(ptyId) && + pty?.connectionId != null && + exitCode < 0 && + options?.hostExitConfirmed !== true if (preservesAbnormalSshSurface) { + if (this.getPtyLivenessVerdict(ptyId)?.status !== 'unverifiable') { + this.markPtyLivenessUnverifiable(ptyId, SSH_EXIT_UNCONFIRMED_REASON) + } this.restoredOrchestrationAuthorityByPtyId.delete(ptyId) } else { this.retirePtyAgentLaunchAuthority(ptyId) @@ -15047,6 +15080,11 @@ export class OrcaRuntimeService { this.setPairedRendererSessionOwnership(pty.ptyId, false) pty.disconnectedAt = Date.now() pty.lastExitCode = exitCode + if (exitCode >= 0 || options?.hostExitConfirmed === true) { + // A real wait status from the owning host is the death certificate; the + // synthetic -1 we emit on a failed/unroutable stop is not. + this.forgetPtyLivenessVerdict(ptyId) + } // Why: the exited process's live frames say nothing about a replacement. // A same-id respawn makes the leaf writable again before any new title, // so leaving this true would let push delivery type into the new process @@ -16929,17 +16967,17 @@ export class OrcaRuntimeService { async inspectTerminalProcessIncarnationLiveness( processIncarnation: string, serializedHostScope: string | null - ): Promise<'live' | 'dead' | 'unknown'> { + ): Promise<'live' | 'exited' | 'unverifiable'> { const hostScope = parseWorkerTerminalHostScope(serializedHostScope) if (!hostScope || !this.ptyController?.listProcesses) { - return 'unknown' + return 'unverifiable' } const listed = await withTimeoutResult( this.ptyController.listProcesses(hostScope.kind === 'ssh' ? hostScope.targetId : null), PTY_CONTROLLER_LIST_TIMEOUT_MS ) if (!listed.ok) { - return 'unknown' + return 'unverifiable' } return classifyWorkerTerminalProcessIncarnation(processIncarnation, listed.value) } @@ -17787,6 +17825,68 @@ export class OrcaRuntimeService { return `${this.runtimeId}:${record.ptyId}:${record.ptyGeneration}` } + /** + * Records that we lost contact with a PTY's owning host. Callers must never + * read this as an exit: a detached relay PTY is designed to outlive the + * provider that addressed it. + */ + markPtyLivenessUnverifiable(ptyId: string, reason: string): void { + this.rememberPtyLivenessVerdict(ptyId, { status: 'unverifiable', reason }) + } + + markPtyLivenessLive(ptyId: string): void { + this.rememberPtyLivenessVerdict(ptyId, { status: 'live', ptyIds: [ptyId] }) + } + + /** Null when nothing has been observed either way, so callers keep their own default. */ + getPtyLivenessVerdict(ptyId: string): PtyLivenessVerdict | null { + return this.ptyLivenessVerdictByPtyId.get(ptyId)?.verdict ?? null + } + + getTerminalLivenessVerdict(handle: string): PtyLivenessVerdict | null { + const record = this.getLivePtyForHandle(handle)?.record ?? this.handles.get(handle) + return record?.ptyId ? this.getPtyLivenessVerdict(record.ptyId) : null + } + + private rememberPtyLivenessVerdict(ptyId: string, verdict: PtyLivenessVerdict): void { + if (verdict.status === 'exited') { + // An earned death certificate ends the question; nothing left to remember. + this.ptyLivenessVerdictByPtyId.delete(ptyId) + return + } + this.ptyLivenessVerdictByPtyId.delete(ptyId) + this.ptyLivenessObservationSequence += 1 + this.ptyLivenessVerdictByPtyId.set(ptyId, { + verdict, + observedAt: this.ptyLivenessObservationSequence + }) + while (this.ptyLivenessVerdictByPtyId.size > MAX_TRACKED_PTY_LIVENESS_VERDICTS) { + let oldestOrphaned: string | null = null + for (const candidate of this.ptyLivenessVerdictByPtyId.keys()) { + if ( + !this.ptysById.has(candidate) && + !this.handleByPtyId.has(candidate) && + !this.leafExistsForPty(candidate) + ) { + oldestOrphaned = candidate + break + } + } + if (!oldestOrphaned) { + return + } + this.ptyLivenessVerdictByPtyId.delete(oldestOrphaned) + } + } + + private forgetPtyLivenessVerdict(ptyId: string, observedNoLaterThan?: number): void { + const tracked = this.ptyLivenessVerdictByPtyId.get(ptyId) + if (observedNoLaterThan !== undefined && tracked && tracked.observedAt > observedNoLaterThan) { + return + } + this.ptyLivenessVerdictByPtyId.delete(ptyId) + } + getExactWorkerProviderSession( handle: string, observedAfter: number @@ -28845,13 +28945,34 @@ export class OrcaRuntimeService { let addressedPtyStopped = false const deadlineMs = Date.now() + EXPLICIT_TERMINAL_CLOSE_STOP_TIMEOUT_MS for (const ptyId of ptyIds) { - let verifiedStopped = false - try { - verifiedStopped = (await this.ptyController?.stopAndWait?.(ptyId, { deadlineMs })) ?? false - } catch { - // Why: verified teardown is preferred, but its transport failure must not suppress the legacy provider kill. + let stopped = false + if (this.ptyController?.stopAndWait) { + try { + stopped = await this.ptyController.stopAndWait(ptyId, { deadlineMs }) + } catch (error) { + this.markPtyLivenessUnverifiable( + ptyId, + error instanceof Error ? error.message : String(error) + ) + } + if (!stopped) { + const verdict = this.getPtyLivenessVerdict(ptyId) + const providerAlreadyRetiredPty = + verdict?.status === 'unverifiable' && + verdict.reason === SSH_PROVIDER_UNREGISTERED_REASON + if (!providerAlreadyRetiredPty) { + this.ptyController.kill(ptyId) + if (!verdict || verdict.status === 'live') { + this.markPtyLivenessUnverifiable( + ptyId, + 'a follow-up stop was issued but its outcome could not be verified' + ) + } + } + } + } else { + stopped = this.ptyController?.kill(ptyId) ?? false } - const stopped = verifiedStopped || (this.ptyController?.kill(ptyId) ?? false) if (ptyId === addressedPtyId) { addressedPtyStopped = stopped } @@ -29053,15 +29174,15 @@ export class OrcaRuntimeService { this.notifier.closeTerminal?.(tabId) } const ptyKilled = await this.stopExplicitlyClosedTabPtys(ptyIdsToKill, pty.pty.ptyId) - return { handle, tabId, ptyKilled } + return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, ptyKilled) } if (siblingCount <= 1 && !surface && pty.pty.tabId && this.notifier?.closeTerminalTab) { const ptyIdsToKill = this.getPtyIdsForExplicitTabClose(pty.pty.worktreeId, tabId) await this.notifier.closeTerminalTab(tabId, { localPtyTeardownOwnedExternally: true }) const ptyKilled = await this.stopExplicitlyClosedTabPtys(ptyIdsToKill, pty.pty.ptyId) - return { handle, tabId, ptyKilled } + return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, ptyKilled) } - const ptyKilled = this.ptyController?.kill(pty.pty.ptyId) ?? false + const ptyKilled = await this.stopExplicitlyClosedTabPtys([pty.pty.ptyId], pty.pty.ptyId) if (!ptyKilled || siblingCount <= 1) { if (surface) { // Why: paired viewers keep ended streams mounted until the HUB publishes removal, so explicit close uses the durable host-tab transaction instead of viewer-local exit handling. @@ -29077,7 +29198,7 @@ export class OrcaRuntimeService { this.notifier?.closeTerminal(tabId) } } - return { handle, tabId, ptyKilled } + return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, ptyKilled) } this.assertGraphReady() const { leaf } = this.getLiveLeafForHandle(handle) @@ -29100,7 +29221,36 @@ export class OrcaRuntimeService { if (siblingCount > 1 ? !ptyKilled : !this.notifier?.closeTerminalTab) { this.notifier?.closeTerminal(leaf.tabId, leaf.paneRuntimeId) } - return { handle, tabId: leaf.tabId, ptyKilled } + return this.describeTerminalClose(handle, leaf.tabId, leaf.ptyId ?? null, ptyKilled) + } + + /** + * A close receipt must not read as a kill nobody performed: when the stop was + * not confirmed, the receipt carries why so the CLI and callers can say so. + */ + private describeTerminalClose( + handle: string, + tabId: string, + ptyId: string | null, + ptyKilled: boolean + ): RuntimeTerminalClose { + if (ptyKilled || !ptyId) { + return { handle, tabId, ptyKilled } + } + const verdict = this.getPtyLivenessVerdict(ptyId) + if (verdict?.status === 'unverifiable') { + return { + handle, + tabId, + ptyKilled, + ptyStopVerdict: 'unverifiable', + ptyStopReason: verdict.reason + } + } + if (verdict?.status === 'live') { + return { handle, tabId, ptyKilled, ptyStopVerdict: 'live' } + } + return { handle, tabId, ptyKilled } } async closeTerminalTab(handle: string): Promise { @@ -29331,7 +29481,7 @@ export class OrcaRuntimeService { } } try { - this.ptyController.retireRejectedPty?.(result.id) + this.ptyController.retireRejectedPty?.(result.id, stopped) } catch { // Best-effort cleanup; preserve the original split authority error. } @@ -31651,6 +31801,7 @@ export class OrcaRuntimeService { } const inventoryGeneration = this.ptyControllerInventorySequence + 1 this.ptyControllerInventorySequence = inventoryGeneration + const livenessObservationAtStart = this.ptyLivenessObservationSequence const providerKey = typeof connectionId === 'string' ? `ssh:${connectionId}` : 'local' if (connectionId === undefined) { this.ptyControllerAggregateInventoryGeneration = inventoryGeneration @@ -31760,6 +31911,8 @@ export class OrcaRuntimeService { const allLivePtyIds = new Set(sessions.map((session) => session.id)) const selectedLivePtyIds = new Set() for (const session of sessions) { + // The owning inventory positively observed this PTY again; prior lost-contact doubt is stale. + this.forgetPtyLivenessVerdict(session.id, livenessObservationAtStart) const sessionConnectionId = parseAppSshPtyId(session.id)?.connectionId ?? (typeof connectionId === 'string' ? connectionId : null) @@ -31862,7 +32015,18 @@ export class OrcaRuntimeService { continue } if (!allLivePtyIds.has(pty.ptyId) && !this.leafExistsForPty(pty.ptyId)) { - if (this.ptyController.hasPty?.(pty.ptyId) === true) { + const currentVerdict = this.ptyLivenessVerdictByPtyId.get(pty.ptyId) + if ( + currentVerdict && + currentVerdict.observedAt > livenessObservationAtStart && + currentVerdict.verdict.status === 'unverifiable' + ) { + pty.connected = false + pty.disconnectedAt ??= Date.now() + continue + } + const observed = this.ptyController.hasPty?.(pty.ptyId) + if (observed === true) { // Why: an SSH spawn can become addressable before an overlapping relay list includes it. allLivePtyIds.add(pty.ptyId) if ( @@ -31873,10 +32037,19 @@ export class OrcaRuntimeService { } pty.connected = true pty.disconnectedAt = null + this.forgetPtyLivenessVerdict(pty.ptyId) continue } pty.connected = false pty.disconnectedAt ??= Date.now() + // Why: this list only enumerates registered providers, so a dropped relay + // clears `connected` for every one of its PTYs at once. Only `false` here + // is an observed absence; `null` means no provider could be asked. + if (observed === false) { + this.forgetPtyLivenessVerdict(pty.ptyId) + } else if (observed === null) { + this.markPtyLivenessUnverifiable(pty.ptyId, NO_OBSERVING_PROVIDER_REASON) + } } } this.pruneDisconnectedPtyRecords() @@ -31958,6 +32131,7 @@ export class OrcaRuntimeService { if (pty) { pty.connected = true pty.disconnectedAt = null + this.forgetPtyLivenessVerdict(ptyId) this.refreshPtyForegroundAgent(ptyId) } } else if (pty && !this.leafExistsForPty(ptyId)) { diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts index 01a20948a8d..03035aa2eec 100644 --- a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-release.ts @@ -64,10 +64,7 @@ export function requestWorkerTerminalRelease( this.db.exec('COMMIT') return { disposition: 'retained', resource, reason: 'ownership_transferred' } } - if ( - resource.release_state === 'unknown' || - (resource.release_state === 'retained' && resource.retained_reason === 'user_requested') - ) { + if (resource.release_state === 'retained' && resource.retained_reason === 'user_requested') { this.db.prepare('DELETE FROM worker_terminal_archives WHERE dispatch_id = ?').run(dispatchId) } this.db diff --git a/src/main/runtime/orchestration/worker-terminal-process-liveness.ts b/src/main/runtime/orchestration/worker-terminal-process-liveness.ts index 5a6e92c7878..67277644bfc 100644 --- a/src/main/runtime/orchestration/worker-terminal-process-liveness.ts +++ b/src/main/runtime/orchestration/worker-terminal-process-liveness.ts @@ -39,7 +39,7 @@ export function parseWorkerTerminalHostScope(value: string | null): WorkerTermin export function classifyWorkerTerminalProcessIncarnation( processIncarnation: string, sessions: readonly PtyProcessInfo[] -): 'live' | 'dead' | 'unknown' { +): 'live' | 'exited' | 'unverifiable' { const possibleMatches = sessions.filter((session) => processIncarnation.startsWith(`${session.id}:`) ) @@ -57,6 +57,6 @@ export function classifyWorkerTerminalProcessIncarnation( return possibleMatches.some( (session) => !session.incarnationId || session.incarnationId !== session.incarnationId.trim() ) - ? 'unknown' - : 'dead' + ? 'unverifiable' + : 'exited' } diff --git a/src/main/runtime/pty-inventory-liveness-verdict.test.ts b/src/main/runtime/pty-inventory-liveness-verdict.test.ts new file mode 100644 index 00000000000..38e3dda20eb --- /dev/null +++ b/src/main/runtime/pty-inventory-liveness-verdict.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' +import { getDefaultWorkspaceSession } from '../../shared/constants' +import { SSH_EXIT_UNCONFIRMED_REASON } from '../../shared/pty-liveness-verdict' + +// The aggregate inventory only enumerates registered providers, so a dropped +// relay clears `connected` for every one of its PTYs at once. Only the +// provider's own answer separates an observed exit from lost contact. + +const WORKTREE_ID = 'repo-1::/tmp/inventory-verdict' +const REMOTE_PTY_ID = 'ssh:conn-1@@relay-9' + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +function makeStore() { + const session = getDefaultWorkspaceSession() + return { + getWorkspaceSession: vi.fn(() => session), + setWorkspaceSession: vi.fn(), + getRepos: vi.fn(() => [ + { + id: 'repo-1', + path: '/tmp/inventory-verdict', + displayName: 'inventory-verdict', + badgeColor: '#000000', + addedAt: 0 + } + ]), + getAllWorktreeMeta: vi.fn(() => ({})), + getWorktreeMeta: vi.fn(() => undefined), + setWorktreeMeta: vi.fn(), + removeWorktreeMeta: vi.fn(), + getSettings: vi.fn(() => ({ workspaceDir: '/tmp/workspaces' })), + getProjects: vi.fn(() => []) + } +} + +function makeRuntimeMissingFromInventory( + hasPty: () => boolean | null, + listProcesses: () => Promise<{ id: string; worktreeId: string }[]> = vi.fn(async () => []) +): OrcaRuntimeService { + const runtime = new OrcaRuntimeService(makeStore() as never) + runtime.setPtyController({ + write: () => true, + kill: () => true, + hasPty, + listProcesses, + getForegroundProcess: async () => null + } as never) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + runtime.registerPty(REMOTE_PTY_ID, WORKTREE_ID, 'conn-1') + return runtime +} + +describe('inventory sweep liveness verdicts', () => { + it('records an abnormal SSH exit as unverifiable at the runtime boundary', () => { + const runtime = makeRuntimeMissingFromInventory(() => null) + + runtime.onPtyExit(REMOTE_PTY_ID, -1) + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toEqual({ + status: 'unverifiable', + reason: SSH_EXIT_UNCONFIRMED_REASON + }) + }) + + it('preserves a more specific lost-contact reason across an abnormal SSH exit', () => { + const runtime = makeRuntimeMissingFromInventory(() => null) + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'inventory transport failed') + + runtime.onPtyExit(REMOTE_PTY_ID, -1) + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toEqual({ + status: 'unverifiable', + reason: 'inventory transport failed' + }) + }) + + it('accepts a current owning-host exit even when its numeric code is negative', () => { + const runtime = makeRuntimeMissingFromInventory(() => null) + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'inventory transport failed') + + runtime.onPtyExit(REMOTE_PTY_ID, -1, undefined, { hostExitConfirmed: true }) + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + }) + + it('records lost contact when no provider can answer for the PTY', async () => { + const runtime = makeRuntimeMissingFromInventory(() => null) + + await runtime.listTerminals(`id:${WORKTREE_ID}`) + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toEqual({ + status: 'unverifiable', + reason: 'no registered provider can observe its host' + }) + }) + + it('records no doubt when the owning provider reports the PTY absent', async () => { + const runtime = makeRuntimeMissingFromInventory(() => false) + + await runtime.listTerminals(`id:${WORKTREE_ID}`) + + // An observed absence is the death certificate callers already act on. + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + }) + + it('clears lost-contact doubt when reconnect inventory observes the PTY live', async () => { + let reconnected = false + const runtime = makeRuntimeMissingFromInventory( + () => null, + vi.fn(async () => (reconnected ? [{ id: REMOTE_PTY_ID, worktreeId: WORKTREE_ID }] : [])) + ) + + await runtime.listTerminals(`id:${WORKTREE_ID}`) + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)?.status).toBe('unverifiable') + + reconnected = true + await runtime.listTerminals(`id:${WORKTREE_ID}`) + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + }) + + it('does not let a pre-drop inventory clear a newer lost-contact verdict', async () => { + const inventory = deferred<{ id: string; worktreeId: string }[]>() + const listProcesses = vi.fn(() => inventory.promise) + const runtime = makeRuntimeMissingFromInventory(() => null, listProcesses) + + const listing = runtime.listTerminals(`id:${WORKTREE_ID}`) + await vi.waitFor(() => expect(listProcesses).toHaveBeenCalled()) + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'relay disconnected during stop') + inventory.resolve([{ id: REMOTE_PTY_ID, worktreeId: WORKTREE_ID }]) + await listing + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toEqual({ + status: 'unverifiable', + reason: 'relay disconnected during stop' + }) + }) + + it('does not let a partial inventory overwrite a concurrent provider failure', async () => { + const inventory = deferred<{ id: string; worktreeId: string }[]>() + const listProcesses = vi.fn(() => inventory.promise) + const runtime = makeRuntimeMissingFromInventory(() => false, listProcesses) + + const listing = runtime.listTerminals(`id:${WORKTREE_ID}`) + await vi.waitFor(() => expect(listProcesses).toHaveBeenCalled()) + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'inventory transport failed') + inventory.resolve([]) + await listing + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toEqual({ + status: 'unverifiable', + reason: 'inventory transport failed' + }) + }) + + it('clears stale doubt when a new PTY lifecycle is positively registered', () => { + const runtime = makeRuntimeMissingFromInventory(() => null) + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'old incarnation lost contact') + + runtime.onPtySpawned(REMOTE_PTY_ID, 'incarnation-2') + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'registration raced reconnect') + runtime.registerPty(REMOTE_PTY_ID, WORKTREE_ID, 'conn-1', { + tabId: 'tab-new', + leafId: '00000000-0000-4000-8000-000000000001', + incarnationId: 'incarnation-2' + }) + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + }) + + it('retains unresolved verdicts for every still-addressable PTY', () => { + const runtime = new OrcaRuntimeService(makeStore() as never) + for (let index = 0; index < 257; index += 1) { + const ptyId = `ssh:conn-1@@relay-${index}` + runtime.registerPty(ptyId, WORKTREE_ID, 'conn-1') + runtime.markPtyLivenessUnverifiable(ptyId, 'provider disconnected') + } + + expect(runtime.getPtyLivenessVerdict('ssh:conn-1@@relay-0')).toEqual({ + status: 'unverifiable', + reason: 'provider disconnected' + }) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-control.ts b/src/main/runtime/rpc/methods/orchestration-federation-control.ts index 215a45c5676..eb8a29bce74 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-control.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-control.ts @@ -6,6 +6,7 @@ import type { RemoteDispatchAttachmentRow } from '../../orchestration/types' import { defineMethod, type RpcMethod } from '../core' import { OptionalFiniteNumber, requiredString } from '../schemas' import { readExactWorkerOutput } from './orchestration-worker-output' +import { describeUnconfirmedAgentStop } from '../../../../shared/pty-liveness-verdict' const FederationDispatchParams = z.object({ dispatchId: requiredString('Missing Dispatch ID') @@ -36,7 +37,11 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS: RpcMethod[] = [ runtimeEpoch: runtime.getRuntimeId(), attachment: exposeRemoteAttachment(attachment), terminal: observation.exact ? observation.terminal : null, - observation: { status: observation.status, exactWorker: observation.exact } + observation: { + status: observation.status, + exactWorker: observation.exact, + ...(observation.reason ? { reason: observation.reason } : {}) + } } } }), @@ -46,7 +51,10 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS: RpcMethod[] = [ handler: async (params, { runtime, authenticatedCallerFingerprint }) => { requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) const observation = await inspectRemoteAttachment(runtime, params.dispatchId) - if (!observation.exact || !observation.terminal || observation.status !== 'running') { + // Why `=== 'exited'` rather than `!== 'live'`: the other non-live + // statuses are already covered by the two guards, and an unverifiable + // terminal is still readable — losing stop-contact is not an exit. + if (!observation.exact || !observation.terminal || observation.status === 'exited') { throw new OrchestrationError( 'worker_identity_changed', `Remote Dispatch ${params.dispatchId} no longer resolves to its exact process.` @@ -83,7 +91,18 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS: RpcMethod[] = [ dispatchId: params.dispatchId, terminalHandle: observation.terminal.handle, workerState: attachment.state, - terminalStatus: observation.status === 'exited' ? 'exited' : 'running', + terminalStatus: + observation.status === 'exited' + ? 'exited' + : observation.status === 'unverifiable' + ? 'unknown' + : 'running', + terminalLiveness: + observation.status === 'unverifiable' + ? 'unverifiable' + : observation.status === 'exited' + ? 'exited' + : 'live', attachedAt: attachment.created_at, source: params.source, cursor: params.cursor, @@ -134,6 +153,22 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS: RpcMethod[] = [ } try { const close = await runtime.closeTerminal(observation.terminal.handle) + if (!close.ptyKilled) { + // The tab is retired but the process was never confirmed stopped, so + // the coordinator must not be told this dispatch reached 'stopped'. + const attachment = db.markRemoteAttachmentStopUnknown( + params.dispatchId, + describeUnconfirmedAgentStop(close) + ) + return { + dispatchId: params.dispatchId, + state: attachment.state, + alreadySettled: false, + processAction: 'closed_agent_terminal', + lastError: attachment.last_error, + close + } + } const attachment = db.settleRemoteAttachmentStop(params.dispatchId) return { dispatchId: params.dispatchId, @@ -172,29 +207,44 @@ function requireHomeAttachment( return attachment } -async function inspectRemoteAttachment(runtime: OrcaRuntimeService, dispatchId: string) { +async function inspectRemoteAttachment( + runtime: OrcaRuntimeService, + dispatchId: string +): Promise<{ + terminal: Awaited> | null + exact: boolean + status: 'unattached' | 'missing' | 'identity_changed' | 'live' | 'exited' | 'unverifiable' + /** Set with `unverifiable`; names what we lost contact with. */ + reason?: string +}> { const db = runtime.getOrchestrationDb() const attachment = db.getRemoteDispatchAttachment(dispatchId) if (!attachment?.terminal_handle) { - return { terminal: null, exact: false, status: 'unattached' as const } + return { terminal: null, exact: false, status: 'unattached' } } const terminal = await runtime.showTerminal(attachment.terminal_handle).catch(() => null) if (!terminal) { - return { terminal: null, exact: false, status: 'missing' as const } + return { terminal: null, exact: false, status: 'missing' } } const exact = db.isRemoteAttachmentProcessCurrent({ dispatchId, paneKey: runtime.getTerminalPaneKey(attachment.terminal_handle), processIncarnation: runtime.getTerminalProcessIncarnation(attachment.terminal_handle) }) + if (!exact) { + return { terminal, exact, status: 'identity_changed' } + } + // Why: the same rule as the local worker observation — the inventory only + // iterates registered providers, so a dropped relay clears `connected` for + // every remote PTY at once. Lost contact is not a death certificate. + const verdict = runtime.getTerminalLivenessVerdict?.(attachment.terminal_handle) ?? null + if (verdict?.status === 'unverifiable') { + return { terminal, exact, status: 'unverifiable', reason: verdict.reason } + } return { terminal, exact, - status: exact - ? terminal.connected === false - ? ('exited' as const) - : ('running' as const) - : ('identity_changed' as const) + status: verdict?.status !== 'live' && terminal.connected === false ? 'exited' : 'live' } } diff --git a/src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts new file mode 100644 index 00000000000..9f8436f96ef --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts @@ -0,0 +1,183 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import { ORCHESTRATION_METHODS } from './orchestration' + +// The federation host runs its own copy of the observation and stop logic, so +// it needs the same rule: lost contact with a worker's host is not an exit, and +// a close it could not confirm must not be relayed home as a settled stop. + +const HOME_FINGERPRINT = 'home-peer-fingerprint' +const DISPATCH_ID = 'ctx_federation_verdict' +const HANDLE = 'term_remote_worker' +const PANE_KEY = 'tab_remote:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' +const INCARNATION = 'runtime:pty:7' +const SSH_PROVIDER_GONE = 'its SSH provider is no longer registered' + +describe('federation host liveness verdicts', () => { + let db: OrchestrationDb + let runtime: OrcaRuntimeService + + beforeEach(() => { + db = new OrchestrationDb(':memory:') + runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(PANE_KEY) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue(INCARNATION) + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: HANDLE, + worktreeId: 'repo::remote-worktree', + connected: false, + status: 'exited' + } as never) + db.createRemoteDispatchAttachment({ + dispatchId: DISPATCH_ID, + taskId: 'task_remote', + homePeerFingerprint: HOME_FINGERPRINT, + protocolVersion: ORCHESTRATION_CONTRACT_VERSION, + runtimeEpoch: runtime.getRuntimeId(), + mutationReceipt: { + callerFingerprint: HOME_FINGERPRINT, + requestId: 'rpc_attach', + method: 'orchestration.federationStart', + payloadHash: 'hash' + } + }) + db.prepareRemoteAttachmentAuthority({ + dispatchId: DISPATCH_ID, + paneKey: PANE_KEY, + processIncarnation: INCARNATION, + worktreeId: 'repo::remote-worktree', + terminalHandle: HANDLE, + setupState: 'not_applicable', + effects: [{ kind: 'terminal', action: 'created', id: HANDLE }] + }) + db.markRemoteAttachmentReady(DISPATCH_ID) + }) + + afterEach(() => db.close()) + + async function call(name: string, params: Record) { + const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name) + if (!method) { + throw new Error(`Method not found: ${name}`) + } + return method.handler(method.params!.parse(params), { + runtime, + authenticatedCallerFingerprint: HOME_FINGERPRINT + } as never) + } + + it('reports lost contact as unverifiable rather than an observed exit', async () => { + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'unverifiable', + reason: SSH_PROVIDER_GONE + }) + + await expect( + call('orchestration.federationShow', { dispatchId: DISPATCH_ID }) + ).resolves.toMatchObject({ + observation: { status: 'unverifiable', exactWorker: true, reason: SSH_PROVIDER_GONE } + }) + }) + + it('uses the canonical live verdict for an observed process', async () => { + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: HANDLE, + worktreeId: 'repo::remote-worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'live', + ptyIds: [HANDLE] + }) + + await expect( + call('orchestration.federationShow', { dispatchId: DISPATCH_ID }) + ).resolves.toMatchObject({ observation: { status: 'live', exactWorker: true } }) + }) + + it('still reports a locally observed exit as exited', async () => { + await expect( + call('orchestration.federationShow', { dispatchId: DISPATCH_ID }) + ).resolves.toMatchObject({ observation: { status: 'exited', exactWorker: true } }) + }) + + it('still serves output for a terminal we merely lost stop-contact with', async () => { + // Why this matters: the read gate used to reject every status except live, which + // would refuse a connected terminal the moment a stop lost contact with it. + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: HANDLE, + worktreeId: 'repo::remote-worktree', + connected: true + } as never) + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'unverifiable', + reason: SSH_PROVIDER_GONE + }) + + const outcome = await call('orchestration.federationRead', { + dispatchId: DISPATCH_ID + }).catch((error: unknown) => error) + + expect(outcome).not.toMatchObject({ code: 'worker_identity_changed' }) + }) + + it('does not relay an unconfirmed close home as a settled stop', async () => { + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'unverifiable', + reason: SSH_PROVIDER_GONE + }) + const closeTerminal = vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: HANDLE, + tabId: 'tab_remote', + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: SSH_PROVIDER_GONE + }) + + const stopped = (await call('orchestration.federationStop', { dispatchId: DISPATCH_ID })) as { + state: string + lastError?: string + } + + // Losing contact is a reason to report honestly, never to stop trying. + expect(closeTerminal).toHaveBeenCalledWith(HANDLE) + expect(stopped.state).not.toBe('stopped') + expect(stopped.lastError).toContain('could not be confirmed stopped') + }) + + it('does not settle a bare false close as a stop', async () => { + vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: HANDLE, + tabId: 'tab_remote', + ptyKilled: false + }) + + const stopped = (await call('orchestration.federationStop', { dispatchId: DISPATCH_ID })) as { + state: string + lastError?: string + } + + expect(stopped.state).not.toBe('stopped') + expect(stopped.lastError).toContain('could not be confirmed stopped') + }) + + it('still settles a confirmed close as a stop', async () => { + vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: HANDLE, + tabId: 'tab_remote', + ptyKilled: true + }) + + const stopped = (await call('orchestration.federationStop', { dispatchId: DISPATCH_ID })) as { + state: string + processAction: string + } + + expect(stopped.state).toBe('stopped') + expect(stopped.processAction).toBe('closed_agent_terminal') + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts index 9617c05c125..adb2cc5459a 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts @@ -192,7 +192,7 @@ describe('orchestration federated worker output', () => { ok: true, result: { server: { environmentId: 'environment_windows', name: 'windows' }, - observation: { status: 'running', exactWorker: true }, + observation: { status: 'live', exactWorker: true }, terminal: { handle: 'term_windows_worker' } } }) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts index 70c08e2ce95..6eec52cd93f 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts @@ -181,6 +181,7 @@ describe('orchestration federated setup evidence', () => { await expect( workerShow.handler(workerShow.params!.parse({ dispatch: started.dispatch.id }), { runtime }) ).resolves.toMatchObject({ + observation: { status: 'live', exactWorker: true }, worker: { state: 'ready', stage: 'input_accepted', diff --git a/src/main/runtime/rpc/methods/orchestration-federation.test.ts b/src/main/runtime/rpc/methods/orchestration-federation.test.ts index 18db8750ff0..59dd8a7a5f8 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation.test.ts @@ -152,7 +152,8 @@ describe('orchestration federation', () => { } as never) vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: 'term_windows_worker', - closed: true + tabId: 'tab-windows-worker', + ptyKilled: true } as never) } @@ -664,7 +665,7 @@ describe('orchestration federation', () => { expect(shown).toMatchObject({ ok: true, - result: { observation: { status: 'running', exactWorker: true } } + result: { observation: { status: 'live', exactWorker: true } } }) expect(homeDb.getFederatedDispatch(dispatch.id)?.remote_runtime_epoch).not.toBe(oldEpoch) expect(homeDb.getFederatedDispatch(dispatch.id)?.peer_fingerprint).toBe( diff --git a/src/main/runtime/rpc/methods/orchestration-worker-control.ts b/src/main/runtime/rpc/methods/orchestration-worker-control.ts index 87133ede7b2..480e7d3db88 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-control.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-control.ts @@ -99,7 +99,11 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [ server: { environmentId: server.environmentId, name: server.name }, remoteRuntimeEpoch: remote.runtimeEpoch, terminal: remote.terminal, - observation: remote.observation + observation: { + ...remote.observation, + // Legacy servers published `running`; normalize at the compatibility boundary. + status: remote.observation.status === 'running' ? 'live' : remote.observation.status + } } } if (worker.runtime_epoch && worker.runtime_epoch !== runtime.getRuntimeId()) { @@ -122,7 +126,12 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [ dispatch, worker: exposeWorker(worker), terminal: observation.exact ? observation.terminal : null, - observation: { status: observation.status, exactWorker: observation.exact }, + observation: { + status: observation.status, + exactWorker: observation.exact, + // Why: a bare `unverifiable` is not actionable without naming what we lost. + ...(observation.reason ? { reason: observation.reason } : {}) + }, terminalResource: resource ? exposeWorkerTerminalResource(resource) : null } } @@ -176,7 +185,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [ ) } const resource = db.getWorkerTerminalResourceByOwner(params.dispatch) - if (resource && ['releasing', 'released'].includes(resource.release_state)) { + if (resource && ['releasing', 'unknown', 'released'].includes(resource.release_state)) { return readArchivedWorkerOutput({ db, dispatchId: params.dispatch, @@ -199,7 +208,18 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [ dispatchId: params.dispatch, terminalHandle: worker.agent_terminal_handle, workerState: worker.state, - terminalStatus: observation.status === 'exited' ? 'exited' : 'running', + terminalStatus: + observation.status === 'exited' + ? 'exited' + : observation.status === 'unverifiable' + ? 'unknown' + : 'running', + terminalLiveness: + observation.status === 'unverifiable' + ? 'unverifiable' + : observation.status === 'exited' + ? 'exited' + : 'live', attachedAt: worker.created_at, source: params.source, cursor: params.cursor, diff --git a/src/main/runtime/rpc/methods/orchestration-worker-observation.ts b/src/main/runtime/rpc/methods/orchestration-worker-observation.ts index 7148fe32dcc..fe6ce087b7f 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-observation.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-observation.ts @@ -10,7 +10,9 @@ export async function inspectWorkerTerminal( ): Promise<{ terminal: Awaited> | null exact: boolean - status: 'unattached' | 'missing' | 'identity_changed' | 'running' | 'exited' + status: 'unattached' | 'missing' | 'identity_changed' | 'live' | 'exited' | 'unverifiable' + /** Set with `unverifiable`; names what we lost contact with. */ + reason?: string }> { const worker = db.getWorkerDispatch(dispatchId) if (!worker?.agent_terminal_handle) { @@ -25,10 +27,23 @@ export async function inspectWorkerTerminal( paneKey: runtime.getTerminalPaneKey(worker.agent_terminal_handle), processIncarnation: runtime.getTerminalProcessIncarnation(worker.agent_terminal_handle) }) + if (!exact) { + return { terminal, exact, status: 'identity_changed' } + } + // Why: the aggregate inventory only iterates registered providers, so a dropped + // relay clears `connected` for every remote PTY at once. Lost contact is not a + // death certificate, and the verdict is the only field that can tell them apart. + const verdict = runtime.getTerminalLivenessVerdict?.(worker.agent_terminal_handle) ?? null + if (verdict?.status === 'unverifiable') { + return { terminal, exact, status: 'unverifiable', reason: verdict.reason } + } + if (verdict?.status === 'live') { + return { terminal, exact, status: 'live' } + } return { terminal, exact, - status: exact ? (terminal.connected === false ? 'exited' : 'running') : 'identity_changed' + status: terminal.connected === false ? 'exited' : 'live' } } @@ -71,7 +86,7 @@ export async function callFederatedWorkerShow( residualResources: unknown[] } terminal: unknown - observation: { status: string; exactWorker: boolean } + observation: { status: string; exactWorker: boolean; reason?: string } }> { return (await runtime.callOrchestrationWorkerServer( federated.environment_id, diff --git a/src/main/runtime/rpc/methods/orchestration-worker-output.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-output.test.ts index 6db96efe33d..0e24a6bced8 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-output.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-output.test.ts @@ -83,6 +83,19 @@ describe('exact orchestration worker output', () => { expect(readTerminal).not.toHaveBeenCalled() }) + it('reports unverifiable liveness without claiming the terminal is running', async () => { + const result = await read({ + terminalStatus: 'unknown', + terminalLiveness: 'unverifiable' + }) + + expect(result.status).toEqual({ + worker: 'ready', + terminal: 'unknown', + liveness: 'unverifiable' + }) + }) + it('reads Grok through the shared Native Chat transcript decoder', async () => { await writeFile( transcriptA, diff --git a/src/main/runtime/rpc/methods/orchestration-worker-output.ts b/src/main/runtime/rpc/methods/orchestration-worker-output.ts index 2b222f01586..bd63da46546 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-output.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-output.ts @@ -20,6 +20,7 @@ export async function readExactWorkerOutput(args: { terminalHandle: string workerState: string terminalStatus: RuntimeTerminalState + terminalLiveness?: 'live' | 'unverifiable' | 'exited' attachedAt: string source?: OrchestrationWorkerReadSource cursor?: string | number @@ -100,7 +101,11 @@ export async function readExactWorkerOutput(args: { returnedMessageCount: transcript.messages.length }, cursor: nextCursor, - status: { worker: args.workerState, terminal: args.terminalStatus }, + status: { + worker: args.workerState, + terminal: args.terminalStatus, + ...(args.terminalLiveness ? { liveness: args.terminalLiveness } : {}) + }, fallbackReason: null, warnings: transcript.warnings } @@ -145,7 +150,11 @@ async function readTerminalOutput( sourceIdentity, terminal: { ...terminal, tail: redactedTerminal.lines }, cursor: nextCursor, - status: { worker: args.workerState, terminal: terminal.status }, + status: { + worker: args.workerState, + terminal: args.terminalLiveness === 'unverifiable' ? args.terminalStatus : terminal.status, + ...(args.terminalLiveness ? { liveness: args.terminalLiveness } : {}) + }, fallbackReason: null, warnings: redactedTerminal.warnings } diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts b/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts index 4beb0323fd1..ecac17b8cbb 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts @@ -10,6 +10,7 @@ import { type WorkerTerminalTailArchive } from '../../orchestration/worker-output-archive' import type { OrcaRuntimeService } from '../../orca-runtime' +import { describeUnconfirmedAgentStop } from '../../../../shared/pty-liveness-verdict' import { inspectWorkerTerminal } from './orchestration-worker-observation' import { orchestrationTimestampToMs } from './orchestration-worker-output' @@ -212,7 +213,19 @@ async function completeWorkerTerminalReleaseOnce( } try { - await runtime.closeTerminal(resource.terminal_handle) + const close = await runtime.closeTerminal(resource.terminal_handle) + if (!close.ptyKilled) { + const reason = describeUnconfirmedAgentStop(close) + const unknown = db.markWorkerTerminalReleaseUnknown(resource.id, reason) + return { + dispatchId, + state: 'release_unknown', + processAction: 'closed_agent_terminal', + archive: { source: archiveSource, status: archiveStatus }, + lastError: unknown.release_error ?? reason, + recovery: `Inspect with: orca orchestration worker-show --dispatch ${dispatchId} --json — then repeat worker-release with the same --retry-request. Never substitute a broad terminal close.` + } + } } catch (error) { const reason = error instanceof Error ? error.message : String(error) if (/disposed|not connected|unavailable/i.test(reason)) { diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts new file mode 100644 index 00000000000..6124b8b6cae --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from 'vitest' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationDb } from '../../orchestration/db' +import type { WorkerTerminalResourceRow } from '../../orchestration/worker-terminal-ownership' +import { completeWorkerTerminalRelease } from './orchestration-worker-release-completion' + +describe('orchestration worker release liveness verdict', () => { + it.each([ + { + name: 'an explicit unverifiable verdict', + close: { + handle: 'term_worker', + tabId: 'tab-worker', + ptyKilled: false, + ptyStopVerdict: 'unverifiable' as const, + ptyStopReason: 'its SSH provider is no longer registered' + }, + detail: 'its SSH provider is no longer registered' + }, + { + name: 'a bare unconfirmed close', + close: { handle: 'term_worker', tabId: 'tab-worker', ptyKilled: false }, + detail: 'the stop outcome could not be verified' + } + ])('does not release a worker after $name', async ({ close, detail }) => { + const reason = 'its SSH provider is no longer registered' + const resource = { + id: 'resource-1', + terminal_handle: 'term_worker', + host_scope: JSON.stringify({ kind: 'ssh', targetId: 'target-1' }), + archive_source: 'terminal', + archive_status: 'captured', + ownership_state: 'owned', + release_state: 'requested' + } as WorkerTerminalResourceRow + const runtime = { + showTerminal: vi.fn(async () => ({ handle: 'term_worker', connected: false })), + getTerminalPaneKey: vi.fn(() => 'tab-worker:leaf-worker'), + getTerminalProcessIncarnation: vi.fn(() => 'pty-worker:incarnation-1'), + getTerminalLivenessVerdict: vi.fn(() => ({ status: 'unverifiable', reason })), + getOrchestrationDispatchAuthority: vi.fn(() => ({ + hostScope: { kind: 'ssh', targetId: 'target-1' } + })), + closeTerminal: vi.fn(async () => close), + notifyMessageArrived: vi.fn() + } as unknown as OrcaRuntimeService + const markWorkerTerminalReleaseUnknown = vi.fn((_resourceId: string, releaseError: string) => ({ + ...resource, + release_state: 'unknown', + release_error: releaseError + })) + const db = { + getWorkerDispatch: vi.fn(() => ({ + agent_terminal_handle: 'term_worker', + created_at: '2026-08-16T00:00:00.000Z' + })), + isDispatchProcessCurrent: vi.fn(() => true), + workerTerminalResourceHasIdentityConflict: vi.fn(() => false), + getWorkerTerminalArchive: vi.fn(() => ({ kind: 'transcript_pin' })), + commitWorkerTerminalArchiveForRelease: vi.fn(() => ({ + ...resource, + release_state: 'releasing' + })), + markWorkerTerminalReleaseUnknown + } as unknown as OrchestrationDb + + await expect( + completeWorkerTerminalRelease({ + runtime, + db, + dispatchId: 'ctx-worker', + resource + }) + ).resolves.toMatchObject({ + state: 'release_unknown', + processAction: 'closed_agent_terminal', + lastError: `The agent terminal was closed but its process could not be confirmed stopped: ${detail}.` + }) + expect(markWorkerTerminalReleaseUnknown).toHaveBeenCalledWith( + 'resource-1', + `The agent terminal was closed but its process could not be confirmed stopped: ${detail}.` + ) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release-recovery.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-release-recovery.test.ts index d090e15103a..b29d765dcad 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-release-recovery.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-release-recovery.test.ts @@ -80,8 +80,9 @@ describe('orchestration worker release recovery', () => { }) vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: 'term_worker', - closed: true - } as never) + tabId: 'tab-worker', + ptyKilled: true + }) vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) activeRunId = db.createRun({ objective: 'Release recovery test Run', @@ -162,6 +163,36 @@ describe('orchestration worker release recovery', () => { expect(db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('releasing') }) + it('preserves archived output when an unconfirmed release retry cannot find the terminal', async () => { + setup() + const { dispatchId } = await startSettledWorker() + vi.mocked(runtime.closeTerminal).mockResolvedValueOnce({ + handle: 'term_worker', + tabId: 'tab-worker', + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'its SSH provider is no longer registered' + }) + + await expect( + call('orchestration.workerRelease', { dispatch: dispatchId }) + ).resolves.toMatchObject({ state: 'release_unknown' }) + expect(db.getWorkerTerminalArchive(dispatchId)).toBeDefined() + + vi.mocked(runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale')) + await expect( + call('orchestration.workerRelease', { dispatch: dispatchId }) + ).resolves.toMatchObject({ state: 'release_unknown' }) + const read = (await call('orchestration.workerRead', { dispatch: dispatchId })) as { + archived?: boolean + terminal: { tail: string[] } + } + expect(read).toMatchObject({ + archived: true, + terminal: { tail: ['worker output line 1', 'worker output line 2'] } + }) + }) + it('never touches resources without requested releases', async () => { setup() await startSettledWorker() diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts index b5ba5b0000a..5bdac829942 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts @@ -93,7 +93,8 @@ describe('orchestration worker release', () => { }) vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: 'term_worker', - closed: true + tabId: 'tab-worker', + ptyKilled: true } as never) vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) activeRunId = db.createRun({ @@ -242,7 +243,7 @@ describe('orchestration worker release', () => { it('reconciles a dead external terminal without closing a process', async () => { setup() const { dispatchId } = await startSettledWorker('succeeded', { terminal: 'term_worker' }) - inspectProcessLiveness.mockResolvedValue('dead') + inspectProcessLiveness.mockResolvedValue('exited') await expect( call('orchestration.workerRelease', { dispatch: dispatchId }) @@ -268,7 +269,7 @@ describe('orchestration worker release', () => { raw .prepare('UPDATE worker_terminal_resources SET prior_owner_dispatch_ids = ? WHERE id = ?') .run('{invalid', resource?.id) - inspectProcessLiveness.mockResolvedValue('dead') + inspectProcessLiveness.mockResolvedValue('exited') await expect( call('orchestration.workerRelease', { dispatch: dispatchId }) @@ -297,7 +298,7 @@ describe('orchestration worker release', () => { setup() const { dispatchId } = await startSettledWorker() await call('orchestration.workerTerminalUserInput', { paneKey: workerPaneKey }) - inspectProcessLiveness.mockResolvedValue('dead') + inspectProcessLiveness.mockResolvedValue('exited') await expect( call('orchestration.workerRelease', { dispatch: dispatchId }) @@ -324,7 +325,7 @@ describe('orchestration worker release', () => { } else { db.abandonWorkerDispatch(dispatchId) } - inspectProcessLiveness.mockResolvedValue('dead') + inspectProcessLiveness.mockResolvedValue('exited') await expect( call('orchestration.workerRelease', { dispatch: dispatchId }) @@ -706,7 +707,7 @@ describe('orchestration worker release', () => { expect(transferred?.terminal_handle).toBe('term_reminted') expect(db.getWorkerTerminalResourceByOwner(first.dispatchId)).toBeUndefined() - inspectProcessLiveness.mockResolvedValueOnce('dead') + inspectProcessLiveness.mockResolvedValueOnce('exited') const oldRelease = (await call('orchestration.workerRelease', { dispatch: first.dispatchId })) as { state: string; reason?: string } @@ -727,7 +728,7 @@ describe('orchestration worker release', () => { const first = await startSettledWorker('succeeded') const second = await startWorker({ terminal: 'term_reminted' }) settle(second.taskId, second.dispatchId, 'succeeded') - inspectProcessLiveness.mockResolvedValue('dead') + inspectProcessLiveness.mockResolvedValue('exited') await expect( call('orchestration.workerRelease', { dispatch: first.dispatchId }) diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release.ts b/src/main/runtime/rpc/methods/orchestration-worker-release.ts index 446887f581a..dd5b5ed4aa3 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-release.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-release.ts @@ -61,7 +61,7 @@ export const ORCHESTRATION_WORKER_RELEASE_METHODS: RpcMethod[] = [ (await runtime.inspectTerminalProcessIncarnationLiveness( processIncarnation, resource.host_scope - )) === 'dead' + )) === 'exited' ) { const reconciled = db.settleDeadWorkerTerminalRelease({ requestingDispatchId: params.dispatch, diff --git a/src/main/runtime/rpc/methods/orchestration-worker-stop-capability.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-stop-capability.test.ts new file mode 100644 index 00000000000..15d9ef62612 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-stop-capability.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationDb } from '../../orchestration/db' +import { ORCHESTRATION_WORKER_STOP_METHODS } from './orchestration-worker-stop' + +describe('federated worker stop capability', () => { + it('does not trust a legacy server stop receipt', async () => { + const markWorkerStopUnknown = vi.fn((_dispatchId: string, reason: string) => ({ + state: 'stop_unknown', + last_error: reason + })) + const db = { + getFederatedDispatch: vi.fn(() => ({ + dispatch_id: 'ctx_remote', + environment_id: 'environment_linux', + environment_name: 'linux', + peer_fingerprint: 'peer-linux' + })), + beginWorkerStop: vi.fn(() => ({ disposition: 'started' })), + markWorkerStopUnknown + } as unknown as OrchestrationDb + const callOrchestrationWorkerServer = vi.fn(async (_environmentId, method) => { + if (method === 'status.get') { + return { capabilities: [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY] } + } + return { state: 'stopped', processAction: 'closed_agent_terminal' } + }) + const runtime = { + getOrchestrationDb: () => db, + resolveOrchestrationWorkerServer: () => ({ + environmentId: 'environment_linux', + name: 'linux', + peerFingerprint: 'peer-linux' + }), + callOrchestrationWorkerServer + } as unknown as OrcaRuntimeService + const method = ORCHESTRATION_WORKER_STOP_METHODS[0]! + + await expect( + method.handler(method.params!.parse({ dispatch: 'ctx_remote' }), { + runtime, + orchestrationMutation: { + callerFingerprint: 'coordinator', + requestId: 'request_stop', + method: 'orchestration.workerStop', + payloadHash: 'hash' + } + }) + ).resolves.toMatchObject({ state: 'stop_unknown', processAction: 'none' }) + expect(markWorkerStopUnknown).toHaveBeenCalledWith( + 'ctx_remote', + 'Connected server linux cannot prove the worker stop outcome.' + ) + expect(callOrchestrationWorkerServer).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts new file mode 100644 index 00000000000..60ff3f66575 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts @@ -0,0 +1,158 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import { ORCHESTRATION_METHODS } from './orchestration' + +// The aggregate terminal inventory only iterates registered providers, so a +// dropped relay clears `connected` for every remote PTY at once. That is lost +// contact, not a death certificate. +describe('worker-stop against a terminal we lost contact with', () => { + let db: OrchestrationDb + let runtime: OrcaRuntimeService + + beforeEach(() => { + db = new OrchestrationDb(':memory:') + runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue( + 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1') + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + connected: false, + status: 'exited' + } as never) + }) + + afterEach(() => db.close()) + + async function call(name: string, params: Record) { + const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name) + if (!method) { + throw new Error(`Method not found: ${name}`) + } + return method.handler(method.params!.parse(params), { runtime }) + } + + function createWorker() { + const run = db.createRun({ + objective: 'Verdict', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + const task = db.createTask({ spec: 'stop worker', runId: run.id }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + runtimeEpoch: runtime.getRuntimeId() + }) + db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_worker', + paneKey: 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb', + processIncarnation: 'runtime:pty:1', + worktreeId: 'repo::worktree', + setupState: 'not_applicable', + effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }] + }) + db.markWorkerDispatchReady(started.dispatch.id) + return started.dispatch + } + + it('reports the process as unverifiable, not exited, when the link dropped', async () => { + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'unverifiable', + reason: 'its SSH provider is no longer registered' + }) + const closeTerminal = vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: 'term_worker', + tabId: 'tab_worker', + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'its SSH provider is no longer registered' + }) + const dispatch = createWorker() + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + observation: { + status: 'unverifiable', + exactWorker: true, + reason: 'its SSH provider is no longer registered' + } + }) + + const stopped = (await call('orchestration.workerStop', { dispatch: dispatch.id })) as { + state: string + processAction: string + lastError: string + } + // Losing contact is a reason to report honestly, never to stop trying. + expect(closeTerminal).toHaveBeenCalledWith('term_worker') + expect(stopped.processAction).toBe('closed_agent_terminal') + expect(stopped.state).toBe('stop_unknown') + expect(stopped.lastError).toContain('could not be confirmed stopped') + expect(stopped.lastError).not.toContain('exited') + }) + + it('does not settle a bare false close as stopped', async () => { + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: 'term_worker', + tabId: 'tab_worker', + ptyKilled: false + }) + const dispatch = createWorker() + + const stopped = (await call('orchestration.workerStop', { dispatch: dispatch.id })) as { + state: string + lastError: string + } + + expect(stopped.state).toBe('stop_unknown') + expect(stopped.lastError).toContain('could not be confirmed stopped') + }) + + it('uses the canonical live verdict for an observed process', async () => { + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'live', + ptyIds: ['runtime:pty:1'] + }) + const dispatch = createWorker() + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + observation: { status: 'live', exactWorker: true } + }) + }) + + it('still reports a locally observed exit as exited', async () => { + const dispatch = createWorker() + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + observation: { status: 'exited', exactWorker: true } + }) + + const stopped = (await call('orchestration.workerStop', { dispatch: dispatch.id })) as { + lastError: string + } + expect(stopped.lastError).toBe('The recorded worker process is exited; no terminal was closed.') + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-worker-stop.ts b/src/main/runtime/rpc/methods/orchestration-worker-stop.ts index 0802fac3954..c02b43d5dc2 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-stop.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-stop.ts @@ -2,6 +2,9 @@ import { z } from 'zod' import { OrchestrationError } from '../../orchestration/orchestration-error' import { defineMethod, type RpcMethod } from '../core' import { requiredString } from '../schemas' +import { describeUnconfirmedAgentStop } from '../../../../shared/pty-liveness-verdict' +import { ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import type { RuntimeStatus } from '../../../../shared/runtime-types' import { inspectWorkerTerminal, resolvePinnedFederatedServer @@ -29,6 +32,24 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [ return settledReceipt(params.dispatch, begun.worker.state) } try { + const status = (await runtime.callOrchestrationWorkerServer( + server.environmentId, + 'status.get', + undefined, + 30_000 + )) as RuntimeStatus + if ( + !status.capabilities?.includes(ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY) + ) { + return unknownReceipt( + params.dispatch, + db.markWorkerStopUnknown( + params.dispatch, + `Connected server ${server.name} cannot prove the worker stop outcome.` + ), + 'none' + ) + } const remote = (await runtime.callOrchestrationWorkerServer( server.environmentId, 'orchestration.federationStop', @@ -101,7 +122,12 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [ ) } const observation = await inspectWorkerTerminal(runtime, db, params.dispatch) - if (!observation.exact || observation.status !== 'running') { + // Why `unverifiable` still proceeds: losing contact is a reason to report + // the outcome honestly, never a reason to stop trying to stop the worker. + if ( + !observation.exact || + (observation.status !== 'live' && observation.status !== 'unverifiable') + ) { return unknownReceipt( params.dispatch, db.markWorkerStopUnknown( @@ -113,6 +139,15 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [ } try { const close = await runtime.closeTerminal(handle) + if (!close.ptyKilled) { + // The tab is retired, but the agent process was never confirmed stopped — + // settling here is the false success this receipt exists to prevent. + return unknownReceipt( + params.dispatch, + db.markWorkerStopUnknown(params.dispatch, describeUnconfirmedAgentStop(close)), + 'closed_agent_terminal' + ) + } const worker = db.settleWorkerStop(params.dispatch) runtime.notifyMessageArrived(`dispatch:${params.dispatch}`, 'status') return { diff --git a/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts b/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts index 880620eee43..3f4679353bf 100644 --- a/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts @@ -30,7 +30,8 @@ describe('orchestration worker recovery', () => { }) vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: 'term_worker', - closed: true + tabId: 'tab-worker', + ptyKilled: true } as never) }) @@ -80,7 +81,7 @@ describe('orchestration worker recovery', () => { call('orchestration.workerShow', { dispatch: dispatch.id }) ).resolves.toMatchObject({ worker: { state: 'ready' }, - observation: { status: 'running', exactWorker: true }, + observation: { status: 'live', exactWorker: true }, terminal: { handle: 'term_worker' } }) await expect( diff --git a/src/main/runtime/unstopped-pty-verification.ts b/src/main/runtime/unstopped-pty-verification.ts index 2dc9aac5960..a2cd83b8eec 100644 --- a/src/main/runtime/unstopped-pty-verification.ts +++ b/src/main/runtime/unstopped-pty-verification.ts @@ -1,18 +1,20 @@ import type { IPtyProvider } from '../providers/types' +import type { OrcaRuntimeService } from './orca-runtime' import { UNSTOPPED_PTY_DETAIL_SEPARATOR, UNSTOPPED_PTY_LIVE_DETAIL_PREFIX, UNSTOPPED_PTY_REMOVAL_PREFIX } from '../../shared/worktree/removal' +import { + NO_OBSERVING_PROVIDER_REASON, + type PtyLivenessVerdict +} from '../../shared/pty-liveness-verdict' import { settleBeforeDeadline } from './settle-before-deadline' // Floor for the verification window when the sweep ran on a very short budget. export const WORKTREE_TEARDOWN_VERIFY_GRACE_MS = 2_000 -export type UnstoppedPtyVerdict = - | { status: 'exited' } - | { status: 'live'; ptyIds: string[] } - | { status: 'unverifiable'; reason: string } +export type UnstoppedPtyVerdict = PtyLivenessVerdict /** * Re-lists the provider's processes to decide what a failed stop RPC actually @@ -56,6 +58,45 @@ export async function verifyUnstoppedPtys( return stillLive.length > 0 ? { status: 'live', ptyIds: stillLive } : { status: 'exited' } } +/** + * A stop that lost contact with the PTY's own host stays unverifiable: the + * surviving provider's inventory is silent about a host it cannot reach, and + * silence is not evidence of an exit. Force Delete is still the escape hatch. + */ +export function unverifiableStopVerdict( + failedPtyIds: readonly string[], + runtime: OrcaRuntimeService | undefined +): UnstoppedPtyVerdict | null { + for (const ptyId of failedPtyIds) { + const verdict = runtime?.getPtyLivenessVerdict?.(ptyId) + if (verdict?.status === 'unverifiable') { + return verdict + } + } + return null +} + +export async function resolveUnstoppedPtyVerdict( + failedPtyIds: readonly string[], + provider: IPtyProvider, + sweepBudgetMs: number, + providerObservesOwningHost: boolean, + runtime?: OrcaRuntimeService +): Promise { + if (failedPtyIds.length === 0) { + return { status: 'exited' } + } + if (!providerObservesOwningHost) { + return ( + unverifiableStopVerdict(failedPtyIds, runtime) ?? { + status: 'unverifiable', + reason: NO_OBSERVING_PROVIDER_REASON + } + ) + } + return verifyUnstoppedPtys(failedPtyIds, provider, sweepBudgetMs) +} + /** Names the blocking PTYs so a wedged removal is diagnosable, not just refused. */ export function describeUnstoppedPtys( worktreeId: string, diff --git a/src/main/runtime/worktree-teardown-unstopped-pty.test.ts b/src/main/runtime/worktree-teardown-unstopped-pty.test.ts index 31ba390c3ec..c89c5e93747 100644 --- a/src/main/runtime/worktree-teardown-unstopped-pty.test.ts +++ b/src/main/runtime/worktree-teardown-unstopped-pty.test.ts @@ -175,6 +175,80 @@ describe('destructive teardown when a PTY stop cannot be proven', () => { expect(error.message).not.toContain('w1@@gone-2') }) + // Why: the client's own provider list is silent about an SSH host, so a stop + // it could not confirm must stay unverifiable instead of reading as exited — + // otherwise removal walks straight past a live remote agent. + it("blocks removal when the stop lost contact with the PTY's own host", async () => { + // The surviving provider lists nothing for a host it cannot reach, so without + // the recorded verdict this empty list reads as "exited" and removal walks + // straight past a live remote agent. + const localProvider = createProviderStub(async () => []) + listRegisteredPtysMock.mockReturnValue([]) + const runtime = { + stopTerminalsForWorktree: async ( + _worktreeId: string, + opts: { + stopPty?: ( + ptyId: string, + stop: () => boolean | Promise + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => { + await opts.stopPty?.('ssh:conn-1@@relay-9', () => false) + return { stopped: 0 } + }, + getPtyLivenessVerdict: (ptyId: string) => + ptyId === 'ssh:conn-1@@relay-9' + ? { status: 'unverifiable', reason: 'its SSH provider is no longer registered' } + : null + } + + await expect( + killAllProcessesForWorktree('w1', { + runtime: runtime as never, + localProvider, + resolvedConnectionId: 'conn-1', + includeProviderInventory: false, + includeLocalRegistry: false, + requirePhysicalStop: true + }) + ).rejects.toThrow( + /could not verify[\s\S]*ssh:conn-1@@relay-9[\s\S]*SSH provider is no longer registered[\s\S]*--force/ + ) + }) + + it('does not infer a remote exit from fallback local inventory without a cached verdict', async () => { + const localProvider = createProviderStub(async () => []) + listRegisteredPtysMock.mockReturnValue([]) + const runtime = { + stopTerminalsForWorktree: async ( + _worktreeId: string, + opts: { + stopPty?: ( + ptyId: string, + stop: () => boolean | Promise + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => { + await opts.stopPty?.('ssh:conn-1@@relay-10', () => false) + return { stopped: 0 } + }, + getPtyLivenessVerdict: () => null + } + + await expect( + killAllProcessesForWorktree('w1', { + runtime: runtime as never, + localProvider, + resolvedConnectionId: 'conn-1', + includeProviderInventory: false, + includeLocalRegistry: false, + requirePhysicalStop: true + }) + ).rejects.toThrow(/could not verify[\s\S]*no registered provider can observe its host/) + expect(localProvider.listProcesses).not.toHaveBeenCalled() + }) + it('reports unverifiable separately from live when the process list fails', async () => { const localProvider = createProviderStub(async () => { throw new Error('daemon socket closed') diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index b51f292c29c..ed29bc26e2f 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -521,7 +521,7 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.shutdown).toHaveBeenCalledTimes(1) }) - it('accepts a failed Windows stop when a fresh inventory proves the PTY exited', async () => { + it('accepts a failed stop when exact-owner inventory supersedes cached uncertainty', async () => { const worktreeId = 'repo-1::C:/Users/User/orca/workspaces/repo/feature' const ptyId = `${worktreeId}@@windows-pty` const stopTerminalsForWorktree = vi.fn( @@ -538,7 +538,11 @@ describe('killAllProcessesForWorktree', () => { }) ) const runtime = { - stopTerminalsForWorktree + stopTerminalsForWorktree, + getPtyLivenessVerdict: vi.fn(() => ({ + status: 'unverifiable', + reason: 'the provider disconnected during stop' + })) } as unknown as Parameters[1]['runtime'] let inventoryCount = 0 const localProvider = createProviderStub(async () => { @@ -568,6 +572,45 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.listProcesses).toHaveBeenCalledTimes(2) }) + it('keeps cached uncertainty when no provider can observe the owning host', async () => { + const worktreeId = 'repo-1::/remote/worktree' + const ptyId = `${worktreeId}@@remote-pty` + const stopTerminalsForWorktree = vi.fn( + async ( + _worktreeId: string, + options: { + stopPty: ( + ptyId: string, + stop: () => boolean + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => ({ + stopped: (await options.stopPty(ptyId, () => false)).owner ? 1 : 0 + }) + ) + const runtime = { + stopTerminalsForWorktree, + getPtyLivenessVerdict: vi.fn(() => ({ + status: 'unverifiable', + reason: 'no registered provider can observe its host' + })) + } as unknown as Parameters[1]['runtime'] + const fallbackProvider = createProviderStub(async () => []) + listRegisteredPtysMock.mockReturnValue([]) + + await expect( + killAllProcessesForWorktree(worktreeId, { + runtime, + resolvedConnectionId: 'missing-connection', + localProvider: fallbackProvider, + includeProviderInventory: false, + includeLocalRegistry: false, + requirePhysicalStop: true + }) + ).rejects.toThrow('no registered provider can observe its host') + expect(fallbackProvider.listProcesses).not.toHaveBeenCalled() + }) + it('keeps duplicate sweeps behind the runtime physical-stop promise', async () => { let releasePhysicalStop: () => void = () => undefined const physicalStop = new Promise((resolve) => { diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index ed0df69e8f7..a28875dff24 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -15,8 +15,7 @@ import { describeError, describeFailedPtySweep, describeUnstoppedPtys, - verifyUnstoppedPtys, - type UnstoppedPtyVerdict + resolveUnstoppedPtyVerdict } from './unstopped-pty-verification' // Why: normal inventories still coalesce into one process scan, while a stale @@ -231,10 +230,15 @@ export async function killAllProcessesForWorktree( [...stopAttempts].map(async ([ptyId, stopped]) => [ptyId, await stopped] as const) ) const failedPtyIds = stopResults.filter(([, stopped]) => !stopped).map(([ptyId]) => ptyId) - const verdict: UnstoppedPtyVerdict = - failedPtyIds.length === 0 - ? { status: 'exited' } - : await verifyUnstoppedPtys(failedPtyIds, deps.localProvider, sweepBudgetMs) + const verdict = await resolveUnstoppedPtyVerdict( + failedPtyIds, + deps.localProvider, + sweepBudgetMs, + deps.includeProviderInventory !== false || + (deps.resolvedConnectionId === undefined && + deps.resolvedRuntimeEnvironmentId === undefined), + deps.runtime + ) if (verdict.status === 'exited') { for (const ptyId of failedPtyIds) { clearStoppedPtyState(ptyId, deps.onPtyStopped) @@ -363,13 +367,10 @@ async function sweepRegistryForWorktree( } function clearStoppedPtyState(ptyId: string, onPtyStopped?: (ptyId: string) => void): void { - if (!onPtyStopped) { - return - } try { // Why: daemon shutdown does not always fan a local pty:exit event back // through pty.ts, but removed worktrees must immediately drop memory rows. - onPtyStopped(ptyId) + onPtyStopped?.(ptyId) } catch { /* cleanup is best-effort and must not block git-level removal */ } diff --git a/src/main/text-generation/commit-message-text-generation.ts b/src/main/text-generation/commit-message-text-generation.ts index 6370117066c..e4077b3bddf 100644 --- a/src/main/text-generation/commit-message-text-generation.ts +++ b/src/main/text-generation/commit-message-text-generation.ts @@ -845,9 +845,7 @@ export function commandBackslashMode( target: CommitMessageGenerationTarget, platform: NodeJS.Platform = process.platform ): CommandTemplateBackslash { - return platform === 'win32' && target.kind === 'local' && !target.wslDistro - ? 'literal' - : 'escape' + return platform === 'win32' && target.kind === 'local' && !target.wslDistro ? 'literal' : 'escape' } type LocalGenerationTarget = Extract diff --git a/src/shared/orchestration-worker-output.ts b/src/shared/orchestration-worker-output.ts index 8e61d1d3b97..767623f96f8 100644 --- a/src/shared/orchestration-worker-output.ts +++ b/src/shared/orchestration-worker-output.ts @@ -1,6 +1,7 @@ import type { AgentProviderSessionMetadata } from './agent-session-resume' import type { AgentType, NativeChatMessage } from './native-chat-types' import type { RuntimeTerminalRead, RuntimeTerminalState } from './runtime-types' +import type { PtyLivenessVerdict } from './pty-liveness-verdict' export const ORCHESTRATION_WORKER_READ_SOURCES = ['auto', 'transcript', 'terminal'] as const export type OrchestrationWorkerReadSource = (typeof ORCHESTRATION_WORKER_READ_SOURCES)[number] @@ -41,6 +42,7 @@ export type OrchestrationWorkerReadTranscriptResult = { status: { worker: string terminal: RuntimeTerminalState + liveness?: PtyLivenessVerdict['status'] } fallbackReason: null warnings: string[] @@ -57,6 +59,7 @@ export type OrchestrationWorkerReadTerminalResult = { status: { worker: string terminal: RuntimeTerminalState + liveness?: PtyLivenessVerdict['status'] } fallbackReason: OrchestrationWorkerReadFallbackReason | null warnings: string[] diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index ddaa9a6fc53..15d376192e1 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -45,6 +45,8 @@ export const ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY = 'orchestration.federation-control-mail.v1' as const export const ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_RUNTIME_CAPABILITY = 'orchestration.federation-lifecycle-settlement.v1' as const +export const ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY = + 'orchestration.worker-stop-verdict.v1' as const export const ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY = 'orchestration.worker-launch-preferences.v1' as const export const ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION = 2 as const @@ -115,6 +117,7 @@ export const RUNTIME_CAPABILITIES = [ ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY, ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY, ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_RUNTIME_CAPABILITY, + ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY, ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY, ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, BROWSER_SCREENCAST_RUNTIME_CAPABILITY, diff --git a/src/shared/pty-liveness-verdict.ts b/src/shared/pty-liveness-verdict.ts new file mode 100644 index 00000000000..f16e756ca9a --- /dev/null +++ b/src/shared/pty-liveness-verdict.ts @@ -0,0 +1,34 @@ +/** + * The one vocabulary Orca uses to talk about whether a PTY is live. + * + * `exited` requires positive evidence of absence from the owning host. Losing + * contact with that host — an unregistered SSH provider, a dropped relay, an + * inventory that only enumerates registered providers — is `unverifiable`, never + * a death certificate and never a successful stop. + */ +export type PtyLivenessVerdict = + | { status: 'exited' } + | { status: 'live'; ptyIds: string[] } + | { status: 'unverifiable'; reason: string } + +export const SSH_PROVIDER_UNREGISTERED_REASON = 'its SSH provider is no longer registered' +export const NO_OBSERVING_PROVIDER_REASON = 'no registered provider can observe its host' +export const SSH_EXIT_UNCONFIRMED_REASON = 'the owning SSH host did not confirm the PTY exit' +export const PTY_LIVE_NOTE = 'The PTY is live.' + +/** The one sentence every surface uses to admit a stop was not confirmed. */ +export function describeUnconfirmedStop(reason: string): string { + return `The PTY was not confirmed stopped: ${reason}.` +} + +/** Words a close whose PTY teardown was never confirmed, for a stop receipt. */ +export function describeUnconfirmedAgentStop(close: { + ptyStopVerdict?: 'live' | 'unverifiable' + ptyStopReason?: string +}): string { + const detail = + close.ptyStopVerdict === 'live' + ? 'it is live' + : (close.ptyStopReason ?? 'the stop outcome could not be verified') + return `The agent terminal was closed but its process could not be confirmed stopped: ${detail}.` +} diff --git a/src/shared/runtime-types.ts b/src/shared/runtime-types.ts index fa2e28bc5bc..a2fa36ecadb 100644 --- a/src/shared/runtime-types.ts +++ b/src/shared/runtime-types.ts @@ -734,6 +734,14 @@ export type RuntimeTerminalClose = { /** Present for the durable whole-tab lifecycle without changing legacy receipts. */ closeMode?: 'tab' ptyKilled: boolean + /** + * Why the PTY was not killed, when we know. Absent means today's answer — + * nothing observed either way — so older clients reading only `ptyKilled` are + * unaffected. `exited` never appears here: that is what `ptyKilled` reports. + */ + ptyStopVerdict?: 'live' | 'unverifiable' + /** Set with `ptyStopVerdict: 'unverifiable'`; names what we lost contact with. */ + ptyStopReason?: string } export type RuntimeTerminalWaitCondition = 'exit' | 'tui-idle' From 8e9b5c908c8dc61d27b292e3c28550200011cbd4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:12:14 -0700 Subject: [PATCH 10/10] fix(github): fail closed instead of running client git against a remote repoPath when the SSH provider is unregistered (#14945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(github): fail closed when the SSH git provider is gone getCurrentHeadOid and probeTrackedUpstreamBranches only routed through the SSH provider when one was registered. With connectionId set but the provider unregistered (dropped connection, not yet reattached) they fell through to client-side git with cwd pointing at the remote repoPath — on a machine with a same-named local path that silently answers for the wrong repository. getCurrentHeadOid feeds shouldHideMergedImplicitPR, so a wrong OID changes which PR the UI attributes to a worktree. Both now take their existing unknown path (null / probeFailed) instead, matching repo-default-branch.ts. Local and WSL routing is unchanged. * fix(github): preserve PR state when SSH probes fail * fix(github): keep failed SSH discovery unverifiable * fix(github): propagate SSH identity failures * fix(github): scope verified SSH identity probes * test(github): preserve tolerant resolver calls * fix(github): preserve indeterminate auth discovery * fix(github): isolate SSH repository probe generations * test(github): expose SSH probe generation in mocks --- src/main/git/remote-url-probe.test.ts | 9 + src/main/git/remote-url-probe.ts | 13 +- .../github/client-pr-local-runtime.test.ts | 1 + ...nt-ssh-provider-execution-boundary.test.ts | 266 ++++++++++++++++++ src/main/github/client-test-mocks.ts | 12 +- src/main/github/client.ts | 46 ++- src/main/github/gh-utils.ts | 1 + .../github/github-api-repository-probe.ts | 25 ++ .../github-api-repository-validation.ts | 20 ++ src/main/github/github-api-repository.test.ts | 66 +++++ src/main/github/github-api-repository.ts | 115 ++++---- .../github/github-enterprise-repository.ts | 33 ++- src/main/github/github-repository-host.ts | 18 ++ ...repository-identity.ssh-host-alias.test.ts | 67 ++++- src/main/github/github-repository-identity.ts | 47 +++- 15 files changed, 641 insertions(+), 98 deletions(-) create mode 100644 src/main/github/client-ssh-provider-execution-boundary.test.ts create mode 100644 src/main/github/github-api-repository-probe.ts create mode 100644 src/main/github/github-api-repository-validation.ts create mode 100644 src/main/github/github-repository-host.ts diff --git a/src/main/git/remote-url-probe.test.ts b/src/main/git/remote-url-probe.test.ts index 8a7672cb229..f9e0348c3ac 100644 --- a/src/main/git/remote-url-probe.test.ts +++ b/src/main/git/remote-url-probe.test.ts @@ -97,6 +97,15 @@ describe('remote URL probe', () => { await expect(assertRemoteUrlReadable({ repoPath: '/repo' })).resolves.toBeUndefined() }) + it('keeps a failed SSH command unverifiable even without a transport error string', async () => { + const failure = new Error('relay request failed') + getSshGitProviderMock.mockReturnValue({ exec: vi.fn().mockRejectedValue(failure) }) + + await expect( + assertRemoteUrlReadable({ repoPath: '/repo', connectionId: 'ssh-1' }) + ).rejects.toBe(failure) + }) + it('does not turn a missing SSH provider into a false readable result', async () => { getSshGitProviderMock.mockReturnValue(null) diff --git a/src/main/git/remote-url-probe.ts b/src/main/git/remote-url-probe.ts index 7b18c1deac9..d237bdd98fa 100644 --- a/src/main/git/remote-url-probe.ts +++ b/src/main/git/remote-url-probe.ts @@ -3,6 +3,7 @@ import { SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE } from '../providers/ssh-git-dispatch' import { gitExecFileAsync } from './runner' +import { isStableMissingGitRemoteError } from './stable-missing-git-remote-error' /** * The `git remote get-url` probe every forge integration runs to decide whether @@ -91,10 +92,9 @@ export function isTransientGitProbeError(error: unknown): boolean { } /** - * Throws when the remote could not be read because the probe was wedged or its - * transport was gone, and returns normally for every answer — including a repo - * with no remote at all. Lets a caller that treats "nothing found" as a cacheable - * answer tell that apart from a lookup that never got to ask. + * Throws when the remote could not be read, and returns normally for every + * answer — including a repo with no remote at all. Lets a caller that treats + * "nothing found" as cacheable tell that apart from a lookup that never got to ask. */ export async function assertRemoteUrlReadable( context: RemoteUrlProbeContext, @@ -106,8 +106,9 @@ export async function assertRemoteUrlReadable( try { await readRemoteUrl(context, remoteName) } catch (error) { - if (isTransientGitProbeError(error)) { - throw error + if (isStableMissingGitRemoteError(error)) { + return } + throw error } } diff --git a/src/main/github/client-pr-local-runtime.test.ts b/src/main/github/client-pr-local-runtime.test.ts index 543d6f239f2..0633b137b1c 100644 --- a/src/main/github/client-pr-local-runtime.test.ts +++ b/src/main/github/client-pr-local-runtime.test.ts @@ -85,6 +85,7 @@ vi.mock('./github-enterprise-repository', () => ({ })) vi.mock('../providers/ssh-git-dispatch', () => ({ + getSshGitProviderGeneration: vi.fn(() => 0), getSshGitProvider: vi.fn() })) diff --git a/src/main/github/client-ssh-provider-execution-boundary.test.ts b/src/main/github/client-ssh-provider-execution-boundary.test.ts new file mode 100644 index 00000000000..1f9bb1b1704 --- /dev/null +++ b/src/main/github/client-ssh-provider-execution-boundary.test.ts @@ -0,0 +1,266 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as GithubApiRepositoryModule from './github-api-repository' +import type * as GitHubEnterpriseRepositoryModule from './github-enterprise-repository' + +const { clientMocks, moduleMocks } = await vi.hoisted(async () => { + const moduleMocks = await import('./client-test-mocks') + return { clientMocks: moduleMocks.createGitHubClientMocks(), moduleMocks } +}) + +vi.mock('./gh-utils', () => moduleMocks.ghUtilsModuleMock(clientMocks)) +vi.mock('../git/runner', () => moduleMocks.gitRunnerModuleMock(clientMocks)) +vi.mock('../providers/ssh-git-dispatch', () => moduleMocks.sshGitDispatchModuleMock(clientMocks)) +vi.mock('./local-git-config-signature', () => + moduleMocks.localGitConfigSignatureModuleMock(clientMocks) +) +vi.mock('./github-enterprise-repository', async (importOriginal) => + moduleMocks.githubEnterpriseRepositoryModuleMock( + await importOriginal() + ) +) +vi.mock('./rate-limit', () => moduleMocks.rateLimitModuleMock(clientMocks)) +vi.mock('./github-api-repository', async (importOriginal) => + moduleMocks.githubApiRepositoryModuleMock( + clientMocks, + await importOriginal() + ) +) + +import { getPRForBranch, getPRForBranchOutcome } from './client' +import { resetPRForBranchMocks } from './client-test-harness' + +const { + ghExecFileAsyncMock, + getOwnerRepoMock, + gitExecFileAsyncMock, + getSshGitProviderMock, + resolvePRRepositoryCandidatesMock +} = clientMocks + +const MERGED_BRANCH = 'fix-tab-strip-layout-test' +const MERGED_HEAD_OID = 'current-head-oid' + +function mockMergedBranchPRLookup(): void { + ghExecFileAsyncMock + .mockResolvedValueOnce({ + stdout: JSON.stringify([ + { + number: 5875, + title: 'Merged current branch PR', + state: 'closed', + merged_at: '2026-06-20T04:53:05Z', + html_url: 'https://github.com/acme/widgets/pull/5875', + updated_at: '2026-06-20T04:53:05Z', + draft: false, + mergeable_state: 'clean', + head: { ref: MERGED_BRANCH, sha: MERGED_HEAD_OID }, + base: { ref: 'main', sha: 'base-oid' } + } + ]) + }) + .mockResolvedValueOnce({ + stdout: JSON.stringify({ + number: 5875, + title: 'Merged current branch PR', + state: 'MERGED', + url: 'https://github.com/acme/widgets/pull/5875', + statusCheckRollup: [], + updatedAt: '2026-06-20T04:53:05Z', + isDraft: false, + mergeable: 'MERGEABLE', + baseRefName: 'main', + headRefName: MERGED_BRANCH, + baseRefOid: 'base-oid', + headRefOid: MERGED_HEAD_OID + }) + }) +} + +function mockUpstreamOnlyPRLookup(): void { + ghExecFileAsyncMock + .mockResolvedValueOnce({ stdout: JSON.stringify([]) }) + .mockResolvedValueOnce({ + stdout: JSON.stringify([ + { + number: 78, + title: 'Upstream branch PR', + state: 'open', + html_url: 'https://github.com/acme/widgets/pull/78', + updated_at: '2026-03-28T00:00:00Z', + draft: false, + mergeable: true, + base: { ref: 'main', sha: 'base-oid' }, + head: { ref: 'contributor/original', sha: 'upstream-head-oid' } + } + ]) + }) + .mockResolvedValueOnce({ + stdout: JSON.stringify({ + number: 78, + title: 'Hydrated upstream branch PR', + state: 'OPEN', + url: 'https://github.com/acme/widgets/pull/78', + statusCheckRollup: [], + updatedAt: '2026-03-28T00:00:00Z', + isDraft: false, + mergeable: 'MERGEABLE', + baseRefName: 'main', + headRefName: 'contributor/original', + baseRefOid: 'base-oid', + headRefOid: 'upstream-head-oid' + }) + }) +} + +// The strict execution boundary: for an SSH-hosted repoPath, an unregistered +// provider means "we could not ask" — it must never degrade into a client-side +// git run, which on a same-named local path answers for the wrong repository. +describe('getPRForBranch SSH execution boundary', () => { + beforeEach(() => { + resetPRForBranchMocks(clientMocks) + getOwnerRepoMock.mockResolvedValue({ owner: 'acme', repo: 'widgets' }) + }) + + it('does not rev-parse HEAD locally when the SSH provider is unregistered', async () => { + getSshGitProviderMock.mockReturnValue(undefined) + mockMergedBranchPRLookup() + gitExecFileAsyncMock.mockResolvedValue({ stdout: `${MERGED_HEAD_OID}\n`, stderr: '' }) + + const outcome = await getPRForBranchOutcome('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(outcome).toMatchObject({ kind: 'upstream-error', errorType: 'unknown' }) + }) + + it('reports an upstream error when SSH candidate discovery is unverifiable', async () => { + resolvePRRepositoryCandidatesMock.mockRejectedValue( + new Error('Remote repository identity is unverifiable.') + ) + + const outcome = await getPRForBranchOutcome('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(ghExecFileAsyncMock).not.toHaveBeenCalled() + expect(outcome).toMatchObject({ kind: 'upstream-error', errorType: 'unknown' }) + }) + + it('keeps a verified empty SSH candidate set as no PR', async () => { + resolvePRRepositoryCandidatesMock.mockResolvedValue({ candidates: [], headRepo: null }) + + const outcome = await getPRForBranchOutcome('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(ghExecFileAsyncMock).not.toHaveBeenCalled() + expect(outcome).toMatchObject({ kind: 'no-pr' }) + }) + + it('rev-parses HEAD through the SSH provider when it is registered', async () => { + const sshGitProvider = { + exec: vi.fn(async (args: string[]) => + args[0] === 'rev-parse' && args[1] === 'HEAD' + ? { stdout: `${MERGED_HEAD_OID}\n`, stderr: '' } + : { stdout: '', stderr: '' } + ) + } + getSshGitProviderMock.mockReturnValue(sshGitProvider) + mockMergedBranchPRLookup() + + const pr = await getPRForBranch('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(sshGitProvider.exec).toHaveBeenCalledWith(['rev-parse', 'HEAD'], '/remote/repo') + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(pr).toMatchObject({ number: 5875, state: 'merged', headSha: MERGED_HEAD_OID }) + }) + + it('rev-parses HEAD through the local runtime for a WSL repository', async () => { + mockMergedBranchPRLookup() + gitExecFileAsyncMock.mockResolvedValue({ stdout: `${MERGED_HEAD_OID}\n`, stderr: '' }) + + const pr = await getPRForBranch('/repo-root', MERGED_BRANCH, null, null, null, { + localGitExecOptions: { wslDistro: 'Ubuntu' } + }) + + expect(gitExecFileAsyncMock).toHaveBeenCalledWith(['rev-parse', 'HEAD'], { + cwd: '/repo-root', + wslDistro: 'Ubuntu' + }) + expect(pr).toMatchObject({ number: 5875, state: 'merged', headSha: MERGED_HEAD_OID }) + }) + + it('does not read tracked upstreams locally when the SSH provider is unregistered', async () => { + getSshGitProviderMock.mockReturnValue(undefined) + mockUpstreamOnlyPRLookup() + gitExecFileAsyncMock.mockResolvedValue({ + stdout: `${MERGED_BRANCH}\0origin/contributor/original\n`, + stderr: '' + }) + + const outcome = await getPRForBranchOutcome('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(outcome).toMatchObject({ kind: 'upstream-error', errorType: 'unknown' }) + expect(ghExecFileAsyncMock).toHaveBeenCalledTimes(1) + }) + + it('reports an upstream error when the SSH HEAD probe loses its provider mid-flight', async () => { + getSshGitProviderMock.mockReturnValue({ + exec: vi.fn().mockRejectedValue(new Error('SSH transport closed')) + }) + mockMergedBranchPRLookup() + + const outcome = await getPRForBranchOutcome('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(outcome).toMatchObject({ kind: 'upstream-error', errorType: 'unknown' }) + }) + + it('reports an upstream error when the SSH upstream probe loses its provider mid-flight', async () => { + getSshGitProviderMock.mockReturnValue({ + exec: vi.fn().mockRejectedValue(new Error('SSH transport closed')) + }) + mockUpstreamOnlyPRLookup() + + const outcome = await getPRForBranchOutcome('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(outcome).toMatchObject({ kind: 'upstream-error', errorType: 'unknown' }) + }) + + it('reads tracked upstreams through the SSH provider when it is registered', async () => { + const forEachRefArgs = ['for-each-ref', '--format=%(refname)%00%(upstream)', 'refs/heads'] + const sshGitProvider = { + exec: vi.fn(async (args: string[]) => + args[0] === 'for-each-ref' + ? { stdout: `${MERGED_BRANCH}\0origin/contributor/original\n`, stderr: '' } + : { stdout: '', stderr: '' } + ) + } + getSshGitProviderMock.mockReturnValue(sshGitProvider) + mockUpstreamOnlyPRLookup() + + const pr = await getPRForBranch('/remote/repo', MERGED_BRANCH, null, 'ssh-1') + + expect(sshGitProvider.exec).toHaveBeenCalledWith(forEachRefArgs, '/remote/repo') + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + expect(pr).toMatchObject({ number: 78, title: 'Hydrated upstream branch PR' }) + }) + + it('reads tracked upstreams through the local runtime for a WSL repository', async () => { + const forEachRefArgs = ['for-each-ref', '--format=%(refname)%00%(upstream)', 'refs/heads'] + mockUpstreamOnlyPRLookup() + gitExecFileAsyncMock.mockResolvedValue({ + stdout: `${MERGED_BRANCH}\0origin/contributor/original\n`, + stderr: '' + }) + + const pr = await getPRForBranch('/repo-root', MERGED_BRANCH, null, null, null, { + localGitExecOptions: { wslDistro: 'Ubuntu' } + }) + + expect(gitExecFileAsyncMock).toHaveBeenCalledWith(forEachRefArgs, { + cwd: '/repo-root', + wslDistro: 'Ubuntu' + }) + expect(pr).toMatchObject({ number: 78, title: 'Hydrated upstream branch PR' }) + }) +}) diff --git a/src/main/github/client-test-mocks.ts b/src/main/github/client-test-mocks.ts index 7852134552b..15fe0f36214 100644 --- a/src/main/github/client-test-mocks.ts +++ b/src/main/github/client-test-mocks.ts @@ -131,10 +131,18 @@ export function gitRunnerModuleMock(mocks: GitHubClientMocks): GitRunnerModuleMo } } -export type SshGitDispatchModuleMock = { getSshGitProvider: Mock } +export type SshGitDispatchModuleMock = { + getSshGitProviderGeneration: Mock + getSshGitProvider: Mock + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE: string +} export function sshGitDispatchModuleMock(mocks: GitHubClientMocks): SshGitDispatchModuleMock { - return { getSshGitProvider: mocks.getSshGitProviderMock } + return { + getSshGitProviderGeneration: vi.fn(() => 0), + getSshGitProvider: mocks.getSshGitProviderMock, + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE: 'Remote connection dropped.' + } } export type LocalGitConfigSignatureModuleMock = { readLocalGitConfigSignature: Mock } diff --git a/src/main/github/client.ts b/src/main/github/client.ts index c3cae80520f..b9e8d94226f 100644 --- a/src/main/github/client.ts +++ b/src/main/github/client.ts @@ -73,7 +73,10 @@ import { isCommitPartOfMergedPR, type MergedPRCommitMembership } from './merged-pr-commit-membership' -import { getSshGitProvider } from '../providers/ssh-git-dispatch' +import { + getSshGitProvider, + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE +} from '../providers/ssh-git-dispatch' import { hasHostedReviewLocalGitOptions, getHostedReviewLocalGitOptions, @@ -2315,14 +2318,19 @@ async function getCurrentHeadOid( connectionId?: string | null, localGitOptions: { wslDistro?: string } = {} ): Promise { + const provider = connectionId ? getSshGitProvider(connectionId) : null + if (connectionId && !provider) { + throw new Error(SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE) + } + if (provider) { + const result = await provider.exec(['rev-parse', 'HEAD'], repoPath) + return result.stdout.trim() || null + } try { - const provider = connectionId ? getSshGitProvider(connectionId) : null - const result = provider - ? await provider.exec(['rev-parse', 'HEAD'], repoPath) - : await gitExecFileAsync(['rev-parse', 'HEAD'], { - cwd: repoPath, - ...(localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) - }) + const result = await gitExecFileAsync(['rev-parse', 'HEAD'], { + cwd: repoPath, + ...(localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) + }) return result.stdout.trim() || null } catch { return null @@ -2814,14 +2822,22 @@ async function probeTrackedUpstreamBranches( upstreamsByBranchName: Map }> { const args = ['for-each-ref', '--format=%(refname)%00%(upstream)', 'refs/heads'] + const provider = connectionId ? getSshGitProvider(connectionId) : null + if (connectionId && !provider) { + throw new Error(SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE) + } + if (provider) { + const result = await provider.exec(args, repoPath) + return { + probeFailed: false, + upstreamsByBranchName: parseTrackedUpstreamBranches(result.stdout) + } + } try { - const provider = connectionId ? getSshGitProvider(connectionId) : null - const result = provider - ? await provider.exec(args, repoPath) - : await gitExecFileAsync(args, { - cwd: repoPath, - ...(localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) - }) + const result = await gitExecFileAsync(args, { + cwd: repoPath, + ...(localGitOptions.wslDistro ? { wslDistro: localGitOptions.wslDistro } : {}) + }) return { probeFailed: false, upstreamsByBranchName: parseTrackedUpstreamBranches(result.stdout) diff --git a/src/main/github/gh-utils.ts b/src/main/github/gh-utils.ts index ffeb2ca22c7..7f4c4e05830 100644 --- a/src/main/github/gh-utils.ts +++ b/src/main/github/gh-utils.ts @@ -26,6 +26,7 @@ export { } from './github-repository-identity' export type { GitHubRemoteIdentity, + GitHubRemoteIdentityProbeOptions, GitHubRepoContext, LocalGitExecOptions, OwnerRepo diff --git a/src/main/github/github-api-repository-probe.ts b/src/main/github/github-api-repository-probe.ts new file mode 100644 index 00000000000..08d0c5fd6e3 --- /dev/null +++ b/src/main/github/github-api-repository-probe.ts @@ -0,0 +1,25 @@ +import type { LocalGitExecOptions } from './github-repository-identity' +import { getSshGitProviderGeneration } from '../providers/ssh-git-dispatch' + +export function githubApiRepositoryProbeCacheKey( + repoPath: string, + remoteName: string, + connectionId: string | null | undefined, + localGitOptions: LocalGitExecOptions, + requireVerifiedSshProbe: boolean +): string { + const runtimeKey = connectionId + ? `ssh:${connectionId}:${getSshGitProviderGeneration(connectionId)}` + : `local:${localGitOptions.wslDistro ?? 'host'}` + return `${runtimeKey}\0${repoPath}\0${remoteName}\0${requireVerifiedSshProbe ? 'verified' : 'tolerant'}` +} + +export function resolveGitHubApiRepositoryProbe( + value: T | undefined, + requireVerifiedSshProbe: boolean +): T | null { + if (value === undefined && requireVerifiedSshProbe) { + throw new Error('GitHub repository identity is unverifiable.') + } + return value ?? null +} diff --git a/src/main/github/github-api-repository-validation.ts b/src/main/github/github-api-repository-validation.ts new file mode 100644 index 00000000000..e70197288cd --- /dev/null +++ b/src/main/github/github-api-repository-validation.ts @@ -0,0 +1,20 @@ +import type { GitHubOwnerRepo } from '../../shared/github/pull-request-types' + +export type GitHubApiRepositoryResolution = + | GitHubOwnerRepo + | null + | undefined + | (() => Promise) + +// Why: renderer/RPC overrides reach authenticated REST paths. +const OWNER_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9-]*$/ +const REPOSITORY_SLUG_RE = /^[A-Za-z0-9._-]+$/ + +export function isValidGitHubApiRepository(repository: GitHubOwnerRepo): boolean { + return ( + OWNER_SLUG_RE.test(repository.owner) && + REPOSITORY_SLUG_RE.test(repository.repo) && + repository.repo !== '.' && + repository.repo !== '..' + ) +} diff --git a/src/main/github/github-api-repository.test.ts b/src/main/github/github-api-repository.test.ts index 2ddfc507630..bf1948588ed 100644 --- a/src/main/github/github-api-repository.test.ts +++ b/src/main/github/github-api-repository.test.ts @@ -1,19 +1,27 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type * as GitHubEnterpriseRepository from './github-enterprise-repository' import type * as GhUtils from './gh-utils' +import type * as SshGitDispatch from '../providers/ssh-git-dispatch' const { getEnterpriseGitHubRepoSlugMock, getOwnerRepoMock, getOwnerRepoForRemoteMock, + getSshGitProviderGenerationMock, isGitHubHostAuthenticatedMock } = vi.hoisted(() => ({ getEnterpriseGitHubRepoSlugMock: vi.fn(), getOwnerRepoMock: vi.fn(), getOwnerRepoForRemoteMock: vi.fn(), + getSshGitProviderGenerationMock: vi.fn(() => 0), isGitHubHostAuthenticatedMock: vi.fn() })) +vi.mock('../providers/ssh-git-dispatch', async (importOriginal) => ({ + ...(await importOriginal()), + getSshGitProviderGeneration: getSshGitProviderGenerationMock +})) + vi.mock('./gh-utils', async (importOriginal) => ({ ...(await importOriginal()), getOwnerRepo: getOwnerRepoMock, @@ -29,6 +37,7 @@ vi.mock('./github-enterprise-repository', async (importOriginal) => ({ import { _resetOriginGitHubApiRepositoryCache, + getGitHubApiRepositoryForRemote, getOriginGitHubApiRepository, githubHostExecOptions, resolveGitHubApiRepository, @@ -40,6 +49,7 @@ beforeEach(() => { getEnterpriseGitHubRepoSlugMock.mockReset().mockResolvedValue(null) getOwnerRepoMock.mockReset().mockResolvedValue(null) getOwnerRepoForRemoteMock.mockReset().mockResolvedValue(null) + getSshGitProviderGenerationMock.mockReset().mockReturnValue(0) isGitHubHostAuthenticatedMock.mockReset().mockResolvedValue(false) }) @@ -182,6 +192,62 @@ describe('resolveGitHubRepoExecution', () => { }) describe('origin repository cache', () => { + it('isolates Enterprise identity across SSH provider generations', async () => { + const beforeReconnect = { owner: 'acme', repo: 'widgets', host: 'github.acme-corp.com' } + const afterReconnect = { owner: 'acme', repo: 'other', host: 'github.acme-corp.com' } + getEnterpriseGitHubRepoSlugMock + .mockResolvedValueOnce(beforeReconnect) + .mockResolvedValueOnce(afterReconnect) + + await expect(getOriginGitHubApiRepository('/remote/repo', 'ssh-1')).resolves.toEqual( + beforeReconnect + ) + getSshGitProviderGenerationMock.mockReturnValue(1) + await expect(getOriginGitHubApiRepository('/remote/repo', 'ssh-1')).resolves.toEqual( + afterReconnect + ) + expect(getEnterpriseGitHubRepoSlugMock).toHaveBeenCalledTimes(2) + }) + + it('keeps an indeterminate auth inventory unverifiable during candidate discovery', async () => { + getEnterpriseGitHubRepoSlugMock.mockResolvedValue(undefined) + + await expect( + getGitHubApiRepositoryForRemote( + '/remote/repo', + 'origin', + 'ssh-1', + {}, + { + requireVerifiedSshProbe: true + } + ) + ).rejects.toThrow('GitHub repository identity is unverifiable.') + }) + + it('does not reuse a tolerant SSH miss for verified candidate discovery', async () => { + const enterprise = { + owner: 'acme', + repo: 'widgets', + host: 'github.acme-corp.com' + } + getEnterpriseGitHubRepoSlugMock.mockResolvedValueOnce(null).mockResolvedValueOnce(enterprise) + + await expect(getOriginGitHubApiRepository('/remote/repo', 'ssh-1')).resolves.toBeNull() + await expect( + getGitHubApiRepositoryForRemote( + '/remote/repo', + 'origin', + 'ssh-1', + {}, + { + requireVerifiedSshProbe: true + } + ) + ).resolves.toEqual(enterprise) + expect(getEnterpriseGitHubRepoSlugMock).toHaveBeenCalledTimes(2) + }) + it('does not cache an indeterminate Enterprise auth probe', async () => { const enterprise = { owner: 'acme', diff --git a/src/main/github/github-api-repository.ts b/src/main/github/github-api-repository.ts index d7fabcd4f5b..7e394789714 100644 --- a/src/main/github/github-api-repository.ts +++ b/src/main/github/github-api-repository.ts @@ -8,6 +8,7 @@ import { getOwnerRepoForRemote, ghRepoExecOptions, githubRepoContext, + type GitHubRemoteIdentityProbeOptions, type LocalGitExecOptions } from './gh-utils' import { @@ -15,7 +16,21 @@ import { getEnterpriseGitHubRepoSlugForRemote, isGitHubHostAuthenticated } from './github-enterprise-repository' +import { githubHostExecOptions } from './github-repository-host' +import { + isValidGitHubApiRepository, + type GitHubApiRepositoryResolution +} from './github-api-repository-validation' +import { + githubApiRepositoryProbeCacheKey, + resolveGitHubApiRepositoryProbe +} from './github-api-repository-probe' +export { + githubHostExecOptions, + githubRepositorySlugArg, + githubRepositoryWebHost +} from './github-repository-host' export type GitHubApiRepository = GitHubOwnerRepo export type GitHubRepoExecOptions = ReturnType & { host?: string } export type GitHubRepoExecution = { @@ -23,43 +38,12 @@ export type GitHubRepoExecution = { ghOptions: GitHubRepoExecOptions } -type GitHubApiRepositoryResolution = - | GitHubApiRepository - | null - | undefined - | (() => Promise) - -// Why: renderer/RPC repository overrides are interpolated into REST paths. -// Reject path syntax before an authenticated gh process can target it. -const GITHUB_OWNER_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9-]*$/ -const GITHUB_REPO_SLUG_RE = /^[A-Za-z0-9._-]+$/ - -function isValidGitHubApiRepository(repository: GitHubApiRepository): boolean { - return ( - GITHUB_OWNER_SLUG_RE.test(repository.owner) && - GITHUB_REPO_SLUG_RE.test(repository.repo) && - repository.repo !== '.' && - repository.repo !== '..' - ) -} - -// Why: the enterprise branch spawns an uncached `git remote get-url` (an SSH -// round trip on connection-backed repos) — hot paths like per-file contents -// and viewed-state toggles resolve per call, so cache like ownerRepoCache does. +// Why: cache the uncached Enterprise remote probe used by hot paths. const ORIGIN_REPO_CACHE_TTL_MS = 30_000 const ORIGIN_REPO_CACHE_MAX_ENTRIES = 512 const originRepoCache = new Map() const originRepoInFlight = new Map>() -function originRepoCacheKey( - repoPath: string, - remoteName: string, - connectionId?: string | null, - localGitOptions: LocalGitExecOptions = {} -): string { - return `${connectionId ?? 'local'}\0${localGitOptions.wslDistro ?? ''}\0${repoPath}\0${remoteName}` -} - /** @internal - exposed for tests only */ export function _resetOriginGitHubApiRepositoryCache(): void { originRepoCache.clear() @@ -90,15 +74,30 @@ export async function getGitHubApiRepositoryForRemote( repoPath: string, remoteName: string, connectionId?: string | null, - localGitOptions: LocalGitExecOptions = {} + localGitOptions: LocalGitExecOptions = {}, + probeOptions: GitHubRemoteIdentityProbeOptions = {} ): Promise { // Why: generic PR resolution prefers upstream, but this API represents the // caller-selected remote exactly (#7331). - const ownerRepo = await getOwnerRepoForRemote(repoPath, remoteName, connectionId, localGitOptions) + const requireVerifiedSshProbe = probeOptions.requireVerifiedSshProbe === true + const verifiedIdentityArgs = requireVerifiedSshProbe ? ([probeOptions] as const) : [] + const ownerRepo = await getOwnerRepoForRemote( + repoPath, + remoteName, + connectionId, + localGitOptions, + ...verifiedIdentityArgs + ) if (ownerRepo) { return { ...ownerRepo, host: 'github.com' } } - const cacheKey = originRepoCacheKey(repoPath, remoteName, connectionId, localGitOptions) + const cacheKey = githubApiRepositoryProbeCacheKey( + repoPath, + remoteName, + connectionId, + localGitOptions, + requireVerifiedSshProbe + ) const now = Date.now() pruneOriginRepoCache(now) const cached = originRepoCache.get(cacheKey) @@ -112,14 +111,21 @@ export async function getGitHubApiRepositoryForRemote( const probe = (async () => { const enterpriseOptions = Object.keys(localGitOptions).length > 0 ? { localGitExecOptions: localGitOptions } : {} + const verifiedEnterpriseArgs = requireVerifiedSshProbe ? ([true] as const) : [] const slug = remoteName === 'origin' - ? await getEnterpriseGitHubRepoSlug(repoPath, connectionId, enterpriseOptions) + ? await getEnterpriseGitHubRepoSlug( + repoPath, + connectionId, + enterpriseOptions, + ...verifiedEnterpriseArgs + ) : await getEnterpriseGitHubRepoSlugForRemote( repoPath, remoteName, connectionId, - enterpriseOptions + enterpriseOptions, + ...verifiedEnterpriseArgs ) // Why: undefined means the gh auth inventory could not be read. Caching it // as a negative would turn a transient spawn failure into a 30-second miss. @@ -130,7 +136,7 @@ export async function getGitHubApiRepositoryForRemote( }) pruneOriginRepoCache(Date.now()) } - return slug ?? null + return resolveGitHubApiRepositoryProbe(slug, requireVerifiedSshProbe) })() originRepoInFlight.set(cacheKey, probe) try { @@ -180,8 +186,12 @@ export async function resolveGitHubApiRepositoryCandidates( localGitOptions: LocalGitExecOptions = {} ): Promise { const [upstream, origin] = await Promise.all([ - getGitHubApiRepositoryForRemote(repoPath, 'upstream', connectionId, localGitOptions), - getGitHubApiRepositoryForRemote(repoPath, 'origin', connectionId, localGitOptions) + getGitHubApiRepositoryForRemote(repoPath, 'upstream', connectionId, localGitOptions, { + requireVerifiedSshProbe: true + }), + getGitHubApiRepositoryForRemote(repoPath, 'origin', connectionId, localGitOptions, { + requireVerifiedSshProbe: true + }) ]) const seen = new Set() const candidates: GitHubApiRepository[] = [] @@ -292,15 +302,6 @@ export async function resolveGitHubApiRepository( return null } -// Why: the gh runner host-qualifies argv from `options.host`, so every known -// host must be carried through. Pinning github.com prevents a process-level -// GH_HOST from silently redirecting an otherwise unambiguous API request. -export function githubHostExecOptions(repository: GitHubApiRepository | null | undefined): { - host?: string -} { - return repository?.host ? { host: repository.host } : {} -} - export async function resolveGitHubRepoExecution( repoPath: string, repository?: GitHubApiRepositoryResolution, @@ -329,19 +330,3 @@ export async function resolveGitHubRepoExecution( } } } - -export function githubRepositoryWebHost(repository: GitHubApiRepository): string { - return repository.host ?? 'github.com' -} - -/** - * Positional `HOST/OWNER/REPO` argv value (e.g. `gh repo view `). - * Positional slugs bypass the runner's `--repo` qualifier, so they must be - * qualified here whenever the host is known. - */ -export function githubRepositorySlugArg(repository: GitHubApiRepository): string { - const slug = `${repository.owner}/${repository.repo}` - // Why: github.com must be explicit too; otherwise process-level GH_HOST can - // redirect positional `gh repo view OWNER/REPO` calls to an Enterprise host. - return repository.host ? `${repository.host}/${slug}` : slug -} diff --git a/src/main/github/github-enterprise-repository.ts b/src/main/github/github-enterprise-repository.ts index f91975097fc..0d244039145 100644 --- a/src/main/github/github-enterprise-repository.ts +++ b/src/main/github/github-enterprise-repository.ts @@ -18,6 +18,11 @@ import { } from './github-remote-identity-parsing' import { resolveSshConfigHostname } from './github-ssh-host-alias-resolution' import { parseWslPath } from '../wsl' +import { + getSshGitProvider, + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE +} from '../providers/ssh-git-dispatch' +import { isStableMissingGitRemoteError } from '../git/stable-missing-git-remote-error' export type GitHubEnterpriseRepoSlug = GitHubOwnerRepo & { host: string } @@ -214,16 +219,26 @@ export async function getEnterpriseGitHubRepoSlugForRemote( repoPath: string, remoteName: string, connectionId?: string | null, - options: HostedReviewExecutionOptions = {} + options: HostedReviewExecutionOptions = {}, + requireVerifiedSshProbe = false ): Promise { const localGitOptions = getHostedReviewLocalGitOptions(options) const context = githubRepoContext(repoPath, connectionId, localGitOptions) + if (requireVerifiedSshProbe && connectionId && !getSshGitProvider(connectionId)) { + throw new Error(SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE) + } let remoteUrl: string | null try { remoteUrl = await getRemoteUrlForRepo(context, remoteName) - } catch { + } catch (error) { + if (requireVerifiedSshProbe && connectionId && !isStableMissingGitRemoteError(error)) { + throw error + } return null } + if (requireVerifiedSshProbe && connectionId && !remoteUrl && !getSshGitProvider(connectionId)) { + throw new Error(SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE) + } const identity = remoteUrl ? parseGitHubRemoteIdentity(remoteUrl) : null if (!identity) { return null @@ -234,6 +249,9 @@ export async function getEnterpriseGitHubRepoSlugForRemote( if (aliasHost) { const { hostname, resolved } = await resolveSshConfigHostname(aliasHost, context) if (!resolved || !hostname) { + if (requireVerifiedSshProbe && connectionId) { + throw new Error('Remote repository identity is unverifiable.') + } const authenticatedLiteralHost = await resolveAuthenticatedGitHubHost( identity.host, repoPath, @@ -266,7 +284,14 @@ export async function getEnterpriseGitHubRepoSlugForRemote( export async function getEnterpriseGitHubRepoSlug( repoPath: string, connectionId?: string | null, - options: HostedReviewExecutionOptions = {} + options: HostedReviewExecutionOptions = {}, + requireVerifiedSshProbe = false ): Promise { - return getEnterpriseGitHubRepoSlugForRemote(repoPath, 'origin', connectionId, options) + return getEnterpriseGitHubRepoSlugForRemote( + repoPath, + 'origin', + connectionId, + options, + requireVerifiedSshProbe + ) } diff --git a/src/main/github/github-repository-host.ts b/src/main/github/github-repository-host.ts new file mode 100644 index 00000000000..250f71d38c3 --- /dev/null +++ b/src/main/github/github-repository-host.ts @@ -0,0 +1,18 @@ +import type { GitHubOwnerRepo } from '../../shared/github/pull-request-types' + +export function githubHostExecOptions(repository: GitHubOwnerRepo | null | undefined): { + host?: string +} { + return repository?.host ? { host: repository.host } : {} +} + +export function githubRepositoryWebHost(repository: GitHubOwnerRepo): string { + return repository.host ?? 'github.com' +} + +/** Host-qualified positional slug for commands that bypass the runner's `--repo`. */ +export function githubRepositorySlugArg(repository: GitHubOwnerRepo): string { + const slug = `${repository.owner}/${repository.repo}` + // Why: pin dotcom too so process-level GH_HOST cannot redirect the request. + return repository.host ? `${repository.host}/${slug}` : slug +} diff --git a/src/main/github/github-repository-identity.ssh-host-alias.test.ts b/src/main/github/github-repository-identity.ssh-host-alias.test.ts index 26ad3504bfa..d08d5335f6e 100644 --- a/src/main/github/github-repository-identity.ssh-host-alias.test.ts +++ b/src/main/github/github-repository-identity.ssh-host-alias.test.ts @@ -25,7 +25,8 @@ vi.mock('../git/runner', async (importOriginal) => ({ vi.mock('../providers/ssh-git-dispatch', () => ({ getSshGitProvider: getSshGitProviderMock, - getSshGitProviderGeneration: getSshGitProviderGenerationMock + getSshGitProviderGeneration: getSshGitProviderGenerationMock, + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE: 'Remote connection dropped.' })) vi.mock('./local-git-config-signature', () => ({ @@ -174,6 +175,70 @@ describe('#10284 SSH Host alias → github.com owner/repo', () => { expect(resolveWithSshGMock).not.toHaveBeenCalled() }) + it('does not read an SSH repoPath locally when its provider is missing', async () => { + getSshGitProviderMock.mockReturnValue(undefined) + + await expect( + getOwnerRepoForRemote( + '/remote/repo', + 'origin', + 'ssh-1', + {}, + { + requireVerifiedSshProbe: true + } + ) + ).rejects.toThrow('Remote connection dropped.') + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + }) + + it('preserves a one-shot SSH remote failure through candidate discovery', async () => { + const failure = new Error('relay request failed') + const provider = { + exec: vi + .fn() + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ stdout: 'git@github.com:team/orca.git\n', stderr: '' }) + } + getSshGitProviderMock.mockReturnValue(provider) + + await expect( + getOwnerRepoForRemote( + '/remote/repo', + 'origin', + 'ssh-1', + {}, + { + requireVerifiedSshProbe: true + } + ) + ).rejects.toBe(failure) + await expect(getOwnerRepoForRemote('/remote/repo', 'origin', 'ssh-1')).resolves.toEqual({ + owner: 'team', + repo: 'orca' + }) + expect(provider.exec).toHaveBeenCalledTimes(2) + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + }) + + it('keeps failed SSH alias classification unverifiable', async () => { + const provider = sshProvider('') + getSshGitProviderMock.mockReturnValue(provider) + + await expect( + getOwnerRepoForRemote( + '/remote/repo', + 'origin', + 'ssh-1', + {}, + { + requireVerifiedSshProbe: true + } + ) + ).rejects.toThrow('Remote repository identity is unverifiable.') + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + }) + it('isolates the same alias across native and WSL runtimes', async () => { resolveWithSshGMock.mockResolvedValueOnce(sshConfig('github.com')) commandExecFileAsyncMock.mockResolvedValueOnce({ diff --git a/src/main/github/github-repository-identity.ts b/src/main/github/github-repository-identity.ts index b860c7b7d31..c912f424417 100644 --- a/src/main/github/github-repository-identity.ts +++ b/src/main/github/github-repository-identity.ts @@ -1,7 +1,11 @@ import { runCoalescedProbe, type CoalescedProbes } from '../git/coalesced-probe' import { readRemoteUrl } from '../git/remote-url-probe' import type { GitHubOwnerRepo } from '../../shared/github/pull-request-types' -import { getSshGitProviderGeneration } from '../providers/ssh-git-dispatch' +import { + getSshGitProvider, + getSshGitProviderGeneration, + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE +} from '../providers/ssh-git-dispatch' import { readLocalGitConfigSignature } from './local-git-config-signature' import { parseGitHubOwnerRepo, @@ -26,6 +30,10 @@ export type LocalGitExecOptions = { wslDistro?: string } +export type GitHubRemoteIdentityProbeOptions = { + requireVerifiedSshProbe?: boolean +} + export function githubRepoContext( repoPath: string, connectionId?: string | null, @@ -108,9 +116,17 @@ export async function getOwnerRepoForRemote( repoPath: string, remoteName: string, connectionId?: string | null, - localGitOptions: LocalGitExecOptions = {} + localGitOptions: LocalGitExecOptions = {}, + probeOptions: GitHubRemoteIdentityProbeOptions = {} ): Promise { const context = githubRepoContext(repoPath, connectionId, localGitOptions) + if ( + probeOptions.requireVerifiedSshProbe && + context.connectionId && + !getSshGitProvider(context.connectionId) + ) { + throw new Error(SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE) + } const runtimeKey = context.connectionId ? `ssh:${context.connectionId}:${getSshGitProviderGeneration(context.connectionId)}` : `local:${context.wslDistro ?? 'host'}` @@ -145,8 +161,15 @@ export async function getOwnerRepoForRemote( // for the same repo concurrently. Coalesce missing-remote probes — but only // onto one young enough to still answer, so a wedged probe cannot pin the // repo's identity for the life of the process (P1-D). - return runCoalescedProbe(ownerRepoInFlight, cacheKey, () => - resolveOwnerRepoForRemote(context, remoteName, cacheKey, nextConfigSignature) + const inFlightKey = `${cacheKey}\0${probeOptions.requireVerifiedSshProbe ? 'verified' : 'tolerant'}` + return runCoalescedProbe(ownerRepoInFlight, inFlightKey, () => + resolveOwnerRepoForRemote( + context, + remoteName, + cacheKey, + nextConfigSignature, + probeOptions.requireVerifiedSshProbe === true + ) ) } @@ -154,12 +177,20 @@ async function resolveOwnerRepoForRemote( context: GitHubRepoContext, remoteName: string, cacheKey: string, - configSignature?: string + configSignature: string | undefined, + requireVerifiedSshProbe: boolean ): Promise { const now = Date.now() try { const remoteUrl = await getRemoteUrlForRepo(context, remoteName) if (!remoteUrl) { + if ( + requireVerifiedSshProbe && + context.connectionId && + !getSshGitProvider(context.connectionId) + ) { + throw new Error(SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE) + } // Empty remote URL is stable until git config changes. ownerRepoCache.set(cacheKey, { value: null, @@ -181,6 +212,9 @@ async function resolveOwnerRepoForRemote( } if (classification.kind === 'indeterminate') { // Why: a failed ssh -G probe is not a stable "not GitHub" result. + if (requireVerifiedSshProbe && context.connectionId) { + throw new Error('Remote repository identity is unverifiable.') + } return null } const stableConfigSignature = classification.cacheWithGitConfigSignature @@ -197,6 +231,9 @@ async function resolveOwnerRepoForRemote( // Why: only stable "no such remote" misses are safe to hold for minutes. // Transient git lock/IO failures must retry on the next lookup. if (!isStableMissingGitRemoteError(error)) { + if (requireVerifiedSshProbe && context.connectionId) { + throw error + } return null } }