From 3a8daa2418d0ab929e509a1d154364fd6b9c432f Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:50:17 +0000 Subject: [PATCH 01/38] fix(terminal): let a paste made while the terminal starts wait for it like typed keys (#40840) Co-authored-by: Wooseong Kim <2222333+innocarpe@users.noreply.github.com> --- .../terminal-pane-paste-dispatch.test.ts | 73 +++++++++++++++++++ .../paste/terminal-pane-paste-dispatch.ts | 12 ++- .../paste/terminal-paste-target-state.test.ts | 51 +++++++++++++ .../paste/terminal-paste-target-state.ts | 19 +++-- .../pty-transport/ipc/pty-transport.ts | 2 + .../legacy-relay-bridge-transport.ts | 2 + .../pty-transport/pty-transport-types.ts | 2 + .../remote-runtime-pty-transport.ts | 2 + 8 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.test.ts create mode 100644 src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.test.ts diff --git a/src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.test.ts b/src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.test.ts new file mode 100644 index 00000000000..192a23bc4c0 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.test.ts @@ -0,0 +1,73 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + createDeferred, + flushAsyncTicks +} from '../../pty-connection/test-fixtures/pty-connection-test-async' +import { + installIpcPtyWindow, + restorePtySpecWindow +} from '../../pty-transport/ipc/pty-transport-test-harness' +import { pasteTextIntoTerminalPane } from './terminal-pane-paste-dispatch' + +describe('pasteTextIntoTerminalPane into a starting terminal', () => { + const originalWindow = globalThis.window + + beforeEach(() => { + vi.resetModules() + installIpcPtyWindow(originalWindow, {}) + }) + + afterEach(() => { + restorePtySpecWindow(originalWindow) + }) + + async function pasteBeforeConnect(text: string): Promise<{ status: string; written: string }> { + const spawn = createDeferred<{ id: string }>() + vi.mocked(window.api.pty.spawn).mockReturnValue(spawn.promise) + const written: string[] = [] + vi.mocked(window.api.pty.write).mockImplementation((_id, data) => { + written.push(data) + }) + vi.mocked(window.api.pty.writeAccepted).mockImplementation(async (_id, data) => { + written.push(data) + return true + }) + const { createIpcPtyTransport } = await import('../../pty-transport/ipc/pty-transport') + const transport = createIpcPtyTransport({ bufferInputUntilConnect: true }) + const connecting = transport.connect({ url: '', callbacks: {} }) + const terminal = { + modes: { bracketedPasteMode: false }, + options: {}, + paste: (data: string) => { + transport.sendInput(data, 'driving') + }, + input: (data: string) => { + transport.sendInput(data, 'driving') + } + } + + const result = await pasteTextIntoTerminalPane({ + pane: { id: 1, leafId: 'leaf-1', terminal }, + text, + source: 'keyboard', + ptyId: null, + runtime: { kind: 'local', platform: 'darwin', runtimeKey: 'local:darwin' }, + transport, + inputKind: 'driving', + isTargetCurrent: () => true, + canContinue: () => true + }) + transport.sendInput('\r', 'driving') + spawn.resolve({ id: 'pty-1' }) + await connecting + await flushAsyncTicks() + return { status: result.status, written: written.join('') } + } + + it('refuses a paste too large for one held entry before writing any of it', async () => { + await expect(pasteBeforeConnect('x'.repeat(70_000))).resolves.toEqual({ + status: 'rejected', + written: '\r' + }) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.ts b/src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.ts index 3b2115f2ff5..bca71e75a57 100644 --- a/src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.ts +++ b/src/renderer/src/components/terminal-pane/input/paste/terminal-pane-paste-dispatch.ts @@ -1,3 +1,4 @@ +import type { PtyTransport } from '../../pty-transport/pty-transport-types' import { pasteTerminalText } from './terminal-bracketed-paste' import { executeTerminalPastePlan } from './terminal-paste-executor' import { planTerminalPasteWithYield } from './terminal-paste-coordinator' @@ -15,7 +16,7 @@ type PasteTextIntoTerminalPaneArgs = { source: TerminalPasteSource ptyId: string | null runtime: TerminalPasteRuntime - transport: Parameters[0] + transport: Pick | undefined inputKind: 'driving' | 'launch' isTargetCurrent: () => boolean canContinue: () => boolean @@ -46,11 +47,16 @@ export async function pasteTextIntoTerminalPane({ windowsInputRecordNewline: planOptions?.windowsInputRecordNewline, terminalBracketedPasteMode: pane.terminal.modes?.bracketedPasteMode === true }) + // Why: a held terminal takes a paste only as one entry, like a typed key; chunks need a bound, live PTY. + const isHolding = (): boolean => transport?.isHoldingInput?.() === true return executeTerminalPastePlan(plan, { pasteText: (pasteText, pasteOptions) => pasteTerminalText(pane.terminal, pasteText, pasteOptions), - writePty: (data, signal) => writeTerminalPastePtyInput(transport, data, inputKind, signal), + writePty: + ptyId !== null && !isHolding() + ? (data, signal) => writeTerminalPastePtyInput(transport, data, inputKind, signal) + : undefined, isTargetCurrent, - canContinue + canContinue: () => canContinue() && !isHolding() }) } diff --git a/src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.test.ts b/src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.test.ts new file mode 100644 index 00000000000..fa36f764e0a --- /dev/null +++ b/src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.test.ts @@ -0,0 +1,51 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createDeferred } from '../../pty-connection/test-fixtures/pty-connection-test-async' +import { + installIpcPtyWindow, + restorePtySpecWindow +} from '../../pty-transport/ipc/pty-transport-test-harness' +import type { PtyTransport } from '../../pty-transport/pty-transport-types' +import { isTerminalPanePasteTargetCurrent } from './terminal-paste-target-state' + +describe('isTerminalPanePasteTargetCurrent', () => { + const originalWindow = globalThis.window + const manager = { getPanes: () => [{ id: 1, leafId: 'leaf-1' }] } + + beforeEach(() => { + vi.resetModules() + installIpcPtyWindow(originalWindow, {}) + }) + + afterEach(() => { + restorePtySpecWindow(originalWindow) + }) + + const isCurrent = (transport: PtyTransport, ptyId: string | null): boolean => + isTerminalPanePasteTargetCurrent({ + manager, + paneTransports: new Map([[1, transport]]), + paneId: 1, + leafId: 'leaf-1', + transport, + ptyId + }) + + it('accepts a paste into a starting terminal that holds typed input until it connects', async () => { + const spawn = createDeferred<{ id: string }>() + vi.mocked(window.api.pty.spawn).mockReturnValue(spawn.promise) + const { createIpcPtyTransport } = await import('../../pty-transport/ipc/pty-transport') + const transport = createIpcPtyTransport({ bufferInputUntilConnect: true }) + + const connecting = transport.connect({ url: '', callbacks: {} }) + expect(isCurrent(transport, null)).toBe(true) + + spawn.resolve({ id: 'pty-1' }) + await connecting + expect(isCurrent(transport, null)).toBe(true) + expect(isCurrent(transport, 'pty-1')).toBe(true) + expect(isCurrent(transport, 'pty-0')).toBe(false) + + transport.disconnect() + expect(isCurrent(transport, null)).toBe(false) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.ts b/src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.ts index e1f7ff426f2..de8fffee77a 100644 --- a/src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.ts +++ b/src/renderer/src/components/terminal-pane/input/paste/terminal-paste-target-state.ts @@ -12,6 +12,7 @@ type TerminalPastePaneManager = { type TerminalPasteTransport = { getPtyId: () => string | null isConnected: () => boolean + isHoldingInput?: () => boolean } export type TerminalPasteTargetState = { @@ -31,12 +32,18 @@ export function isTerminalPanePasteTargetCurrent({ transport, ptyId }: TerminalPasteTargetState): boolean { - return Boolean( - manager?.getPanes().some((pane) => pane.id === paneId && pane.leafId === leafId) && - transport && - paneTransports.get(paneId) === transport && - transport.isConnected() && - transport.getPtyId() === ptyId + if ( + !manager?.getPanes().some((pane) => pane.id === paneId && pane.leafId === leafId) || + !transport || + paneTransports.get(paneId) !== transport + ) { + return false + } + // A paste made before the first bind (ptyId null) may follow this transport onto the PTY it binds. + const livePtyId = transport.getPtyId() + return ( + (livePtyId === ptyId || ptyId === null) && + (transport.isConnected() || transport.isHoldingInput?.() === true) ) } diff --git a/src/renderer/src/components/terminal-pane/pty-transport/ipc/pty-transport.ts b/src/renderer/src/components/terminal-pane/pty-transport/ipc/pty-transport.ts index e9912c7aad3..0da077fb4a4 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport/ipc/pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport/ipc/pty-transport.ts @@ -181,6 +181,8 @@ export function createIpcPtyTransport(opts: IpcPtyTransportOptions = {}): PtyTra } }, + isHoldingInput: () => !destroyed && preconnectInputBuffer?.isBuffering() === true, + abandonPreconnectInput() { dropPreconnectInput('its pending connection was abandoned') }, diff --git a/src/renderer/src/components/terminal-pane/pty-transport/legacy-relay/legacy-relay-bridge-transport.ts b/src/renderer/src/components/terminal-pane/pty-transport/legacy-relay/legacy-relay-bridge-transport.ts index 0848e801f56..3e5ee0d728b 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport/legacy-relay/legacy-relay-bridge-transport.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport/legacy-relay/legacy-relay-bridge-transport.ts @@ -292,6 +292,8 @@ export function createLegacyRelayBridgeTransport(ptyId: string, inner: PtyTransp isConnected: () => (bridged() ? viewer !== null : inner.isConnected()), getPtyId: () => (bridged() ? ptyId : inner.getPtyId()), isConnectPending: () => (bridged() ? false : (inner.isConnectPending?.() ?? false)), + // Why not pending: selecting the bridge never drains the inner hold. + isHoldingInput: () => mode === 'inner' && (inner.isHoldingInput?.() ?? false), getRuntimeEnvironmentId: () => (bridged() ? null : (inner.getRuntimeEnvironmentId?.() ?? null)), getExecutionHostId: () => (bridged() ? null : (inner.getExecutionHostId?.() ?? null)), onExecutionHostChange: (listener) => inner.onExecutionHostChange?.(listener) ?? (() => {}), diff --git a/src/renderer/src/components/terminal-pane/pty-transport/pty-transport-types.ts b/src/renderer/src/components/terminal-pane/pty-transport/pty-transport-types.ts index 324787a93ad..ad8594d0ee7 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport/pty-transport-types.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport/pty-transport-types.ts @@ -252,6 +252,8 @@ export type PtyTransport = { ownsRecovery?: () => boolean /** Typed input the host has not applied yet, which a remount would discard. */ hasUndeliveredInput?: () => boolean + /** Input sent now is held and delivered to this pane's terminal once it connects. */ + isHoldingInput?: () => boolean /** The user dismissed the error surface; the next occurrence of the same message must surface again. */ notifyErrorSurfaceDismissed?: () => void getPtyId: () => string | null diff --git a/src/renderer/src/components/terminal-pane/pty-transport/remote-runtime/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/pty-transport/remote-runtime/remote-runtime-pty-transport.ts index faefc6be092..43f4e9f1fc4 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport/remote-runtime/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport/remote-runtime/remote-runtime-pty-transport.ts @@ -4151,6 +4151,8 @@ export function createRemoteRuntimePtyTransport( return !destroyed && !terminalEnded && recoveryBlocksIo() }, + isHoldingInput: shouldHoldInput, + hasUndeliveredInput() { return ( inputJournal.hasUnacknowledgedInput() || From 6843d8fd08a5b3dd5798e2a8d2be4a5cf60abaa9 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:54:49 +0000 Subject: [PATCH 02/38] fix(browser): stop a stalled tab capture keeping the Mac's display awake for hours (#40847) * fix(browser): hold a page drawable until its capture settles A stalled capture stays pending until the page draws and keeps macOS display sleep off ("Capturing"); dropping the paint hold at our timeout left it there. Fixes #24946 * fix(browser): keep the debugger attached while a CDP capture is pending * refactor(browser): share one settle-scoped hold helper for paint and debugger leases --- .../browser/cdp/commands/cdp-screenshot.ts | 39 +++++++++------- .../browser/cdp/debugger/guest-paint-hold.ts | 39 ++++++++++++++++ ...ser-screencast-first-frame-timeout.test.ts | 44 +++++++++++++++++++ ...rowser-screencast-guest-view-limit.test.ts | 1 + .../browser-screencast-lifecycle.test.ts | 2 + ...r-screencast-live-frame-allocation.test.ts | 1 + .../browser-screencast-snapshot-capture.ts | 20 +++++++-- .../browser-screencast-stream-types.ts | 2 + .../browser-screencast-stream.test.ts | 3 ++ .../browser/screencast/browser-screencasts.ts | 2 + ...browser-screencast-frame-admission.test.ts | 1 + ...r-publication-throttle-electron-fixture.ts | 1 + 12 files changed, 136 insertions(+), 19 deletions(-) diff --git a/src/main/browser/cdp/commands/cdp-screenshot.ts b/src/main/browser/cdp/commands/cdp-screenshot.ts index f43f5c1e194..76cc4726365 100644 --- a/src/main/browser/cdp/commands/cdp-screenshot.ts +++ b/src/main/browser/cdp/commands/cdp-screenshot.ts @@ -1,4 +1,8 @@ -import type { CapturePaintHold } from '../debugger/guest-paint-hold' +import { + holdPaintUntilCdpCaptureSettles, + holdPaintUntilSettled, + type CapturePaintHold +} from '../debugger/guest-paint-hold' import type { WebContents } from 'electron' import { sendGuestCdpCommand } from '../debugger/guest-cdp-command' import { @@ -137,7 +141,8 @@ export function withTimeout( // Undrawn pages can leave captures pending; cheap probes trigger a frame for the original request. function captureUntilDrawn( webContents: WebContents, - params: Record + params: Record, + holdPaint: CapturePaintHold ): Promise<{ data: string } | null> { return new Promise((resolve, reject) => { let settled = false @@ -173,12 +178,12 @@ function captureUntilDrawn( } catch { // Guest teardown can reject repaint requests; still try CDP. } - return sendGuestCdpCommand(webContents, 'Page.captureScreenshot', requestParams).then( - (result) => { - const data = isRecord(result) ? result.data : undefined - return typeof data === 'string' ? { data } : undefined - } - ) + return holdPaintUntilCdpCaptureSettles(webContents, holdPaint, () => + sendGuestCdpCommand(webContents, 'Page.captureScreenshot', requestParams) + ).then((result) => { + const data = isRecord(result) ? result.data : undefined + return typeof data === 'string' ? { data } : undefined + }) } const deadline = setTimeout(() => finish(() => resolve(null)), SCREENSHOT_TIMEOUT_MS) let probes: NodeJS.Timeout[] = [] @@ -223,11 +228,15 @@ export async function captureFullPageScreenshot( if (!clip) { throw new Error('Unable to determine full-page screenshot bounds') } - const frame = await captureUntilDrawn(webContents, { - format, - captureBeyondViewport: true, - clip - }) + const frame = await captureUntilDrawn( + webContents, + { + format, + captureBeyondViewport: true, + clip + }, + holdPaint + ) if (!frame) { throw new Error(SCREENSHOT_TIMEOUT_MESSAGE) } @@ -269,13 +278,13 @@ export async function captureScreenshot( const release = holdPaint() try { - const frame = await captureUntilDrawn(webContents, screenshotParams) + const frame = await captureUntilDrawn(webContents, screenshotParams, holdPaint) if (frame) { return frame } const fallback = await readRemoteCliAdmittedPage(webContents.id, () => withTimeout( - Promise.resolve().then(() => webContents.capturePage()), + holdPaintUntilSettled(holdPaint, () => webContents.capturePage()), FALLBACK_CAPTURE_TIMEOUT_MS, SCREENSHOT_TIMEOUT_MESSAGE ) diff --git a/src/main/browser/cdp/debugger/guest-paint-hold.ts b/src/main/browser/cdp/debugger/guest-paint-hold.ts index 885a3e98a84..c235cf72541 100644 --- a/src/main/browser/cdp/debugger/guest-paint-hold.ts +++ b/src/main/browser/cdp/debugger/guest-paint-hold.ts @@ -1,2 +1,41 @@ +import type { WebContents } from 'electron' +import { acquireElectronDebugger } from './electron-debugger-lease' + /** Draws a hidden guest while a capture or agent input needs its frames; returns its release. */ export type CapturePaintHold = () => () => void + +function retainUntilSettled(acquire: () => () => void, run: () => Promise): Promise { + let release: () => void + try { + release = acquire() + } catch (error) { + return Promise.reject(error) + } + const pending = new Promise((resolve) => resolve(run())) + void pending.then(release, release) + return pending +} + +/** + * Holds paint until Chromium answers the capture, not just while the caller waits. A capture stays + * pending until the guest draws, and while pending it can hold macOS display sleep off ("Capturing"); + * timeouts do not cancel it (#24946). + */ +export function holdPaintUntilSettled( + holdPaint: CapturePaintHold, + capture: () => Promise +): Promise { + return retainUntilSettled(holdPaint, capture) +} + +/** Also keeps the debugger attached: a detach rejects the command while Chromium keeps the capture. */ +export function holdPaintUntilCdpCaptureSettles( + webContents: WebContents, + holdPaint: CapturePaintHold, + capture: () => Promise +): Promise { + return retainUntilSettled( + () => acquireElectronDebugger(webContents).release, + () => retainUntilSettled(holdPaint, capture) + ) +} diff --git a/src/main/browser/cdp/screencast/browser-screencast-first-frame-timeout.test.ts b/src/main/browser/cdp/screencast/browser-screencast-first-frame-timeout.test.ts index 797f2ff07a5..8129fb06766 100644 --- a/src/main/browser/cdp/screencast/browser-screencast-first-frame-timeout.test.ts +++ b/src/main/browser/cdp/screencast/browser-screencast-first-frame-timeout.test.ts @@ -24,6 +24,7 @@ function startOptions(viewport: boolean) { maxHeight: 1200, everyNthFrame: 2, minFrameIntervalMs: 0, + holdPaint: () => () => {}, ...(viewport ? { viewportWidth: 390, viewportHeight: 844 } : {}), onFrame: vi.fn(() => true), onError: vi.fn() @@ -59,6 +60,49 @@ describe('browser screencast first-frame deadline', () => { await session.done }) + it('keeps the page drawable until Chromium answers the captures the stream gave up on', async () => { + let settleCapture!: (image: unknown) => void + const webContents = Object.assign(createMockScreencastWebContents(), { + capturePage: vi.fn( + () => + new Promise((resolve) => { + settleCapture = resolve + }) + ) + }) + let settleScreenshot!: (result: unknown) => void + let rejectScreenshot!: (error: Error) => void + webContents.debugger.sendCommand.mockImplementation(async (method: string) => + method === 'Page.captureScreenshot' + ? new Promise((resolve, reject) => { + settleScreenshot = resolve + rejectScreenshot = reject + }) + : {} + ) + // Electron rejects pending commands on detach, but Chromium keeps the capture. + webContents.debugger.detach.mockImplementation(() => + rejectScreenshot(new Error('target closed while handling command')) + ) + const releasePaint = vi.fn() + const options = { ...startOptions(true), holdPaint: vi.fn(() => releasePaint) } + const session = await start(webContents, options) + + // capturePage's 10 s bound, then the CDP fallback's 8 s. + await vi.advanceTimersByTimeAsync(18_000) + session.stop() + await session.done + expect(options.holdPaint).toHaveBeenCalledTimes(2) + expect(webContents.debugger.detach).not.toHaveBeenCalled() + expect(releasePaint).not.toHaveBeenCalled() + + settleCapture({ getSize: () => ({ width: 0, height: 0 }) }) + settleScreenshot({}) + await vi.advanceTimersByTimeAsync(0) + expect(releasePaint).toHaveBeenCalledTimes(2) + expect(webContents.debugger.detach).toHaveBeenCalledOnce() + }) + it('drops a capture that settles after the limit', async () => { let settleCapture!: (image: unknown) => void const webContents = Object.assign(createMockScreencastWebContents(), { diff --git a/src/main/browser/cdp/screencast/browser-screencast-guest-view-limit.test.ts b/src/main/browser/cdp/screencast/browser-screencast-guest-view-limit.test.ts index 5968bf42069..a07487bfd5b 100644 --- a/src/main/browser/cdp/screencast/browser-screencast-guest-view-limit.test.ts +++ b/src/main/browser/cdp/screencast/browser-screencast-guest-view-limit.test.ts @@ -70,6 +70,7 @@ async function startOversizedStream(guest: ReturnType, onFra deviceScaleFactor: 2, everyNthFrame: 1, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame }) } diff --git a/src/main/browser/cdp/screencast/browser-screencast-lifecycle.test.ts b/src/main/browser/cdp/screencast/browser-screencast-lifecycle.test.ts index 09889e723c7..4443f28f7e3 100644 --- a/src/main/browser/cdp/screencast/browser-screencast-lifecycle.test.ts +++ b/src/main/browser/cdp/screencast/browser-screencast-lifecycle.test.ts @@ -35,6 +35,7 @@ describe('browser screencast lifecycle', () => { maxHeight: 2160, everyNthFrame: 2, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame }) @@ -80,6 +81,7 @@ describe('browser screencast lifecycle', () => { maxHeight: 1200, everyNthFrame: 2, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame: vi.fn() }, isClosed: () => false, diff --git a/src/main/browser/cdp/screencast/browser-screencast-live-frame-allocation.test.ts b/src/main/browser/cdp/screencast/browser-screencast-live-frame-allocation.test.ts index 0d61e942a80..86185d11b0b 100644 --- a/src/main/browser/cdp/screencast/browser-screencast-live-frame-allocation.test.ts +++ b/src/main/browser/cdp/screencast/browser-screencast-live-frame-allocation.test.ts @@ -40,6 +40,7 @@ function createLiveFrameRoute( maxHeight: 1200, everyNthFrame: 2, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame } const isClosed = () => false diff --git a/src/main/browser/cdp/screencast/browser-screencast-snapshot-capture.ts b/src/main/browser/cdp/screencast/browser-screencast-snapshot-capture.ts index 733a4cb542a..b9b3c1f037d 100644 --- a/src/main/browser/cdp/screencast/browser-screencast-snapshot-capture.ts +++ b/src/main/browser/cdp/screencast/browser-screencast-snapshot-capture.ts @@ -1,7 +1,11 @@ import { Buffer } from 'node:buffer' import type { Debugger, WebContents } from 'electron' import { readBrowserScreencastImageSize } from './browser-screencast-image-size' -import { sendDebuggerCommand } from './browser-screencast-debugger-command' +import { runDebuggerCommandWithTimeout } from './browser-screencast-debugger-command' +import { + holdPaintUntilCdpCaptureSettles, + holdPaintUntilSettled +} from '../debugger/guest-paint-hold' import type { BrowserScreencastOptions, PendingScreencastFrame @@ -35,6 +39,7 @@ export function createBrowserScreencastSnapshotCapture( ): BrowserScreencastSnapshotCapture { const { webContents, dbg, options, isClosed, isStopping, getSeq, queueFrame } = deps const { applyDeviceMetricsOverride } = deps + const { holdPaint } = options let snapshotGeneration = 0 let navigationCaptureTimer: ReturnType | null = null @@ -81,7 +86,9 @@ export function createBrowserScreencastSnapshotCapture( try { // Prefer capturePage to avoid CDP tiling; stopped compositing can leave it pending. const nativeImage = await withTimeout( - webContents.capturePage({ x: 0, y: 0, width: viewportWidth, height: viewportHeight }), + holdPaintUntilSettled(holdPaint, () => + webContents.capturePage({ x: 0, y: 0, width: viewportWidth, height: viewportHeight }) + ), SNAPSHOT_CAPTURE_TIMEOUT_MS, null ) @@ -101,7 +108,7 @@ export function createBrowserScreencastSnapshotCapture( } } if (!image) { - const result = await sendDebuggerCommand(dbg, 'Page.captureScreenshot', { + const screenshotParams = { format: options.format, ...(options.format === 'jpeg' ? { quality: options.quality } : {}), ...(viewportWidth && viewportHeight @@ -117,7 +124,12 @@ export function createBrowserScreencastSnapshotCapture( } : {}), captureBeyondViewport: false - }) + } + const result = await runDebuggerCommandWithTimeout('Page.captureScreenshot', () => + holdPaintUntilCdpCaptureSettles(webContents, holdPaint, () => + dbg.sendCommand('Page.captureScreenshot', screenshotParams) + ) + ) if (isSnapshotStale(initialOnly, generation)) { return } diff --git a/src/main/browser/cdp/screencast/browser-screencast-stream-types.ts b/src/main/browser/cdp/screencast/browser-screencast-stream-types.ts index 0676e970411..666bba266b2 100644 --- a/src/main/browser/cdp/screencast/browser-screencast-stream-types.ts +++ b/src/main/browser/cdp/screencast/browser-screencast-stream-types.ts @@ -2,6 +2,7 @@ import type { BrowserScreencastFormat, BrowserScreencastFrameMetadata } from '../../../../shared/browser/screencast/browser-screencast-protocol' +import type { CapturePaintHold } from '../debugger/guest-paint-hold' export type BrowserScreencastOptions = { format: BrowserScreencastFormat @@ -17,6 +18,7 @@ export type BrowserScreencastOptions = { onFrame: (bytes: Uint8Array) => boolean | void onEvent?: (event: BrowserScreencastEvent) => void onError?: (message: string) => void + holdPaint: CapturePaintHold } export type BrowserScreencastViewport = Pick< diff --git a/src/main/browser/cdp/screencast/browser-screencast-stream.test.ts b/src/main/browser/cdp/screencast/browser-screencast-stream.test.ts index 6870a27f172..456ff1f213c 100644 --- a/src/main/browser/cdp/screencast/browser-screencast-stream.test.ts +++ b/src/main/browser/cdp/screencast/browser-screencast-stream.test.ts @@ -27,6 +27,7 @@ describe('startBrowserScreencast', () => { maxHeight: 1200, everyNthFrame: 1, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame }) @@ -84,6 +85,7 @@ describe('startBrowserScreencast', () => { maxHeight: 1200, everyNthFrame: 1, minFrameIntervalMs: 100, + holdPaint: () => () => {}, onFrame }) @@ -136,6 +138,7 @@ describe('startBrowserScreencast', () => { maxHeight: 1200, everyNthFrame: 2, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame: vi.fn(), onEvent }) diff --git a/src/main/runtime/browser/screencast/browser-screencasts.ts b/src/main/runtime/browser/screencast/browser-screencasts.ts index 192aa7412ed..d8a97faa304 100644 --- a/src/main/runtime/browser/screencast/browser-screencasts.ts +++ b/src/main/runtime/browser/screencast/browser-screencasts.ts @@ -2,6 +2,7 @@ import type { BrowserScreencastResult } from '../../../../shared/browser/runtime import { BrowserError } from '../../../browser/browser-error' import { randomUUID } from 'node:crypto' import { startBrowserScreencast } from '../../../browser/cdp/screencast/browser-screencast-stream' +import { browserManager } from '../../../browser/guest-lifecycle/manager/browser-manager' import { rendererPublicationThrottle } from '../../../window/renderer-publication-throttle' import { mintBrowserScreencastSubscriptionId } from './browser-screencast-subscription-id' import { @@ -102,6 +103,7 @@ export class RuntimeBrowserScreencasts { format: params.format, ...budget, ...viewport, + holdPaint: () => browserManager.holdPaintForCapture(guest.id), onFrame: (bytes) => { const ghosts: string[] = [] for (const [subscriptionId, subscriber] of pendingRecord.subscribers) { diff --git a/src/main/runtime/browser/screencast/remote-browser-screencast-frame-admission.test.ts b/src/main/runtime/browser/screencast/remote-browser-screencast-frame-admission.test.ts index 6e5d0b3db0c..c87652a7309 100644 --- a/src/main/runtime/browser/screencast/remote-browser-screencast-frame-admission.test.ts +++ b/src/main/runtime/browser/screencast/remote-browser-screencast-frame-admission.test.ts @@ -92,6 +92,7 @@ describe('sendRemoteBrowserScreencastFrame', () => { maxHeight: 2160, everyNthFrame: 1, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame: (bytes) => sendRemoteBrowserScreencastFrame(transport.sendBinary, bytes) !== 'refused' }) diff --git a/src/main/window/renderer-publication-throttle-electron-fixture.ts b/src/main/window/renderer-publication-throttle-electron-fixture.ts index 9adaac8227b..3a417f1dc9f 100644 --- a/src/main/window/renderer-publication-throttle-electron-fixture.ts +++ b/src/main/window/renderer-publication-throttle-electron-fixture.ts @@ -47,6 +47,7 @@ async function run(): Promise { maxHeight: 480, everyNthFrame: 1, minFrameIntervalMs: 0, + holdPaint: () => () => {}, onFrame: () => { frames += 1 firstFrame() From 88880fca7a49a39600008bfbea9bde5e45b84992 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:56:02 -0500 Subject: [PATCH 03/38] feat(orchestration): a paired Orca as a Run's coordinator on the server it placed work on (inert) (#40717) * feat(orchestration): a paired Orca as a Run's coordinator on the server it placed work on (inert) The server half of remote-owned workers. A paired Orca's call names one of its Runs; the server admits it as a peer caller kind, writes that Run at frame receipt, and holds it to the one access check. Its workers' mail and state queue home under a sequence no reset reuses. Nothing sends the envelope until the coordinator side ships, so this changes nothing today. * fix(orchestration): reserve the peer address, keep conversation kinds apart, queue a correction once with its result * fix(orchestration): hold back a corrected Task's change whatever its latest attempt; rotate pulls in a stable order per database * chore(orchestration): typed field read, cast rationales, a wake test on fake timers * fix(orchestration): a peer's receipts per conversation, its sends only into its Run, a page that fits every conversation, pending adopted Tasks keep their gate * fix(orchestration): a peer's follow-up places where its worker ran * fix(orchestration): a peer's current/new-child start gets the --on placement refusal --- .../orchestration/worker-agent-views.ts | 3 + .../orchestration/worker-terminal-handlers.ts | 5 +- .../orchestration-caller-identity.ts | 46 ++ .../db/attach-orchestration-db-methods.ts | 2 + .../worker-report-settlement.ts | 5 + .../db/federation/peer-mail-store.ts | 10 +- .../orchestration/db/lifecycle-transition.ts | 4 +- .../db/messages/message-inbox.ts | 32 +- .../db/messages/message-insert.ts | 15 + .../db/orchestration-db-methods.ts | 2 + .../orchestration/db/orchestration-db.ts | 2 + .../db/peer-runs/peer-run-queue-sql.ts | 175 +++++++ .../db/peer-runs/peer-run-queue.test.ts | 265 +++++++++++ .../db/peer-runs/peer-run-store.ts | 146 ++++++ .../db/questions/question-threads.ts | 7 +- .../db/reset/orchestration-reset.ts | 32 +- .../orchestration/db/runs/run-binding.ts | 9 + .../orchestration/db/runs/run-create.ts | 78 ++-- .../orchestration/db/schema/create-tables.ts | 5 +- .../orchestration/db/tasks/task-store.ts | 4 +- .../worker-dispatch/worker-dispatch-start.ts | 71 ++- .../worker-terminal-listing.ts | 5 + .../federation/peer-mail/peer-mail-waiters.ts | 13 +- .../federated-worker-child-parent.ts | 4 +- .../federated-worker-start-request.ts | 4 +- .../launch/worker-dispatch-preamble.ts | 10 +- .../workers/launch/worker-start-card.ts | 9 +- src/main/runtime/rpc/core.ts | 8 +- .../runtime/rpc/dispatcher-stream-options.ts | 4 +- src/main/runtime/rpc/dispatcher.ts | 12 +- .../orchestration-access-outcomes.test.ts | 2 + .../access/orchestration-method-access.ts | 59 ++- .../orchestration/access/peer-run-access.ts | 208 +++++++++ .../rpc/methods/orchestration/caller-show.ts | 10 +- .../federation/federation-methods.ts | 4 +- .../federation/peer-coordinator.test.ts | 437 ++++++++++++++++++ .../federation/peer-mail-methods.ts | 34 +- .../federation/peer-mail.test.ts | 2 +- .../federation/peer-run-methods.ts | 54 +++ .../orchestration/messaging/check-methods.ts | 1 + .../messaging/check-run-pending-mail.ts | 4 +- .../orchestration/messaging/check-run.ts | 4 +- .../messaging/helper-question-answerer.ts | 4 +- .../messaging/message-methods.ts | 8 +- .../messaging/peer-mail-reply.ts | 6 +- .../orchestration/messaging/peer-mail-send.ts | 6 +- .../orchestration/messaging/reply-method.ts | 1 + .../orchestration/messaging/send-methods.ts | 13 + .../messaging/send-point-to-point.ts | 3 + .../settled-worker-follow-through.ts | 19 +- .../orchestration-caller-workspace.ts | 10 +- .../orchestration/runs/dispatch-creator.ts | 4 +- .../methods/orchestration/runs/run-scope.ts | 8 +- .../deliver-worker-dispatch-preamble.ts | 1 + .../explicit-worker-terminal-validation.ts | 4 +- .../worker/local-worker-start-dispatch.ts | 1 + .../worker/local-worker-start.ts | 4 +- .../worker/worker-list-method.ts | 1 + .../worker/worker-start-authority.ts | 6 +- .../methods/orchestration/worker/workers.ts | 24 +- .../orchestration-mutation-executor.test.ts | 25 + .../rpc/orchestration-mutation-executor.ts | 25 +- .../rpc/orchestration-mutation-receipt.ts | 15 +- .../runtime/rpc/orchestration-peer-caller.ts | 72 +++ .../rpc/orchestration-session-caller.ts | 7 +- .../rpc/rpc-request-admission-prologue.ts | 11 + .../orchestration-rpc-contract.ts | 3 +- src/shared/orchestration/remote-owned-wire.ts | 79 ++++ .../remote-runtime-memory-limits.ts | 1 + .../rpc-contract/orchestration-params.ts | 4 +- .../orchestration-worker-start-params.ts | 4 +- .../rpc-params-catalog.generated.ts | 2 + src/shared/runtime-rpc-envelope.ts | 3 + 73 files changed, 1997 insertions(+), 183 deletions(-) create mode 100644 src/main/runtime/orchestration/db/peer-runs/peer-run-queue-sql.ts create mode 100644 src/main/runtime/orchestration/db/peer-runs/peer-run-queue.test.ts create mode 100644 src/main/runtime/orchestration/db/peer-runs/peer-run-store.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/access/peer-run-access.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/federation/peer-coordinator.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/federation/peer-run-methods.ts create mode 100644 src/main/runtime/rpc/orchestration-peer-caller.ts create mode 100644 src/shared/orchestration/remote-owned-wire.ts diff --git a/src/cli/handlers/orchestration/worker-agent-views.ts b/src/cli/handlers/orchestration/worker-agent-views.ts index ac76e9c214a..ca993bc66c1 100644 --- a/src/cli/handlers/orchestration/worker-agent-views.ts +++ b/src/cli/handlers/orchestration/worker-agent-views.ts @@ -108,6 +108,7 @@ export type WorkerListResult = { dispatchStatus: string agentTerminalHandle: string | null terminalState: string | null + coordinatedFrom?: string | null resource?: { worktreeId?: string | null; autoReleaseAt?: string | null } | null projection?: WorkerProjection }[] @@ -133,6 +134,7 @@ type WorkerListRow = { terminalState?: string workspace?: string host?: string + coordinatedFrom?: string attention?: string[] autoReleaseAt?: string next?: string @@ -168,6 +170,7 @@ export const WORKER_LIST_VIEW: OrchestrationAgentView +/** A paired Orca acting as the coordinator of its Run here, as its authenticated frame names it. + * Its handle and binding key are its address, which no terminal or session here can have. */ +export type OrchestrationPeerCaller = OrchestrationCallerIdentity & + Readonly<{ + kind: 'peer' + terminalHandle: string + paneKey: string + orcaSessionId: null + peerRun: PeerRunRow + }> + +/** The caller the dispatch entry resolved, which wins over any declared handle: a session from + * its environment, or a paired Orca from its pairing. */ +export type OrchestrationEntryCaller = OrchestrationSessionCaller | OrchestrationPeerCaller + +/** The address, and Run binding key, of the paired Orca coordinating a conversation's Run here. */ +export function peerCoordinatorAddress(conversationId: string): string { + return `${PEER_ADDRESS_PREFIX}${conversationId}` +} + +const PEER_ADDRESS_PREFIX = 'peer:' + +export function isPeerCoordinatorAddress(value: string | null | undefined): boolean { + return !!value?.startsWith(PEER_ADDRESS_PREFIX) +} + +export function peerCoordinatorCaller(peerRun: PeerRunRow): OrchestrationPeerCaller { + const address = peerCoordinatorAddress(peerRun.conversation_id) + return Object.freeze({ + kind: 'peer', + address, + terminalHandle: address, + paneKey: address, + orcaSessionId: null, + peerRun + }) +} + +export function isPeerCaller( + caller: OrchestrationCallerIdentity | null | undefined +): caller is OrchestrationPeerCaller { + return !!caller && 'kind' in caller && caller.kind === 'peer' +} + /** A caller with neither a pane nor an Orca session id can never be bound to a Run. */ export function hasRunBindingKey(caller: OrchestrationCoordinatorKey): boolean { return caller.paneKey !== null || caller.orcaSessionId !== null diff --git a/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts b/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts index 7d5bc369062..68624a6ade3 100644 --- a/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts +++ b/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts @@ -11,6 +11,7 @@ import { attachFederationRelayAck } from './federation/federation-relay-ack' import { attachLegacyRemoteDbMethods } from '../legacy-remote/legacy-remote-db-methods' import { attachLegacyRemoteHomeDbMethods } from '../legacy-remote-home/legacy-remote-home-db-methods' import { attachPeerMailStore } from './federation/peer-mail-store' +import { attachPeerRunStore } from './peer-runs/peer-run-store' import { attachLegacyAskOperation } from './legacy/legacy-ask-operation' import { attachLegacyCompatibilityCandidates } from './legacy/legacy-compatibility-candidates' import { attachLegacyCompatibilityPrincipals } from './legacy/legacy-compatibility-principals' @@ -107,6 +108,7 @@ export function attachOrchestrationDbMethods(ctor: { prototype: object }): void attachLegacyRemoteDbMethods(ctor) attachFederationRelayAck(ctor) attachPeerMailStore(ctor) + attachPeerRunStore(ctor) attachWorkerTerminalResourceStore(ctor) attachWorkerTerminalTransfer(ctor) attachWorkerTerminalRelease(ctor) diff --git a/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts b/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts index 9152202ada9..4f812539964 100644 --- a/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts +++ b/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts @@ -168,6 +168,10 @@ export function settleWorkerReportInTransaction( let dispatchUpdate: { changes: number } let taskUpdate: { changes: number } if (settledByUnobservedPrompt) { + // A paired coordinator gets the correction once, after all its writes; see peer_run_corrections. + this.db + .prepare('INSERT INTO peer_run_corrections (dispatch_id) VALUES (?)') + .run(params.dispatchId) const now = new Date().toISOString() const dispatchTransition = transitionLifecycleWithDb(this.db, { entity: 'dispatch', @@ -239,6 +243,7 @@ export function settleWorkerReportInTransaction( projection: { stage: 'settled', updated_at: new Date().toISOString() }, correction: 'unobserved_prompt_report' }) + this.db.prepare('DELETE FROM peer_run_corrections WHERE dispatch_id = ?').run(params.dispatchId) } else if ((unprovenWorker || reportingStart) && params.outcome === 'succeeded') { transitionLifecycleWithDb(this.db, { entity: 'worker', diff --git a/src/main/runtime/orchestration/db/federation/peer-mail-store.ts b/src/main/runtime/orchestration/db/federation/peer-mail-store.ts index da829b40e5e..ffe1e3b2cb9 100644 --- a/src/main/runtime/orchestration/db/federation/peer-mail-store.ts +++ b/src/main/runtime/orchestration/db/federation/peer-mail-store.ts @@ -78,7 +78,7 @@ export function listHomePeerMailEnvironments(this: OrchestrationDb): string[] { export function listPeerConversationIds(this: OrchestrationDb, peerFingerprint: string): string[] { const statement = this.db.prepare( `SELECT conversation_id FROM peer_mail_conversations - WHERE role = 'peer' AND peer_fingerprint = ?` + WHERE role = 'peer' AND peer_fingerprint = ? ORDER BY conversation_id` ) // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT over this table returns exactly these columns. const rows = statement.all(peerFingerprint) as { conversation_id: string }[] @@ -118,6 +118,14 @@ export function insertPeerMailMessage( | { message: MessageRow | undefined; duplicate: true } { this.db.exec('SAVEPOINT peer_mail_insert') try { + // A peer Run's conversation keeps its own queue sequence, so it never carries peer mail. + if (this.getPeerRunForConversation(params.conversationId)) { + throw new OrchestrationError( + 'request_mismatch', + `Conversation ${params.conversationId} belongs to a coordinated Run. Nothing was delivered.`, + { effectsApplied: false } + ) + } if (params.peerFingerprint !== undefined) { const conversation = this.getPeerMailConversation(params.conversationId) if ( diff --git a/src/main/runtime/orchestration/db/lifecycle-transition.ts b/src/main/runtime/orchestration/db/lifecycle-transition.ts index 5eefd879ecb..f0b44ee85ae 100644 --- a/src/main/runtime/orchestration/db/lifecycle-transition.ts +++ b/src/main/runtime/orchestration/db/lifecycle-transition.ts @@ -1,6 +1,7 @@ import type Database from '../../../sqlite/sync-database' import { OrchestrationError } from '../orchestration-error' import type { OrchestrationDb } from './orchestration-db' +import { notifyPeerMailQueued } from '../federation/peer-mail/peer-mail-waiters' /** Callers own transactions so lifecycle projections and mailbox effects commit atomically together. */ export type LifecycleEntity = 'task' | 'dispatch' | 'worker' @@ -189,7 +190,8 @@ export function transitionLifecycleWithDb( `${params.entity} ${params.id} changed while transitioning.` ) } - + // A peer Run's change is queued home by trigger in this statement. + notifyPeerMailQueued(db) return { changed: true } } diff --git a/src/main/runtime/orchestration/db/messages/message-inbox.ts b/src/main/runtime/orchestration/db/messages/message-inbox.ts index e0ec891fd63..f201d19e808 100644 --- a/src/main/runtime/orchestration/db/messages/message-inbox.ts +++ b/src/main/runtime/orchestration/db/messages/message-inbox.ts @@ -226,11 +226,14 @@ export function markAsReadAndDelivered(this: OrchestrationDb, ids: string[]): vo ) } -export function getInbox(this: OrchestrationDb, limit = 20): MessageRow[] { +export function getInbox(this: OrchestrationDb, limit = 20, runId?: string): MessageRow[] { return exposeMessageListTimestamps( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over messages returns the row shape its schema and MessageRow define, like every row cast in db/. this.db - .prepare('SELECT * FROM messages ORDER BY sequence DESC LIMIT ?') - .all(limit) as MessageRow[] + .prepare( + 'SELECT * FROM messages WHERE (?1 IS NULL OR run_id = ?1) ORDER BY sequence DESC LIMIT ?2' + ) + .all(runId ?? null, limit) as MessageRow[] ) } @@ -239,22 +242,19 @@ export function getAllMessagesForHandle( this: OrchestrationDb, toHandle: string, limit = 100, - types?: MessageType[] + types?: MessageType[], + runId?: string ): MessageRow[] { - if (types && types.length > 0) { - const placeholders = types.map(() => '?').join(',') - return exposeMessageListTimestamps( - this.db - .prepare( - `SELECT * FROM messages WHERE to_handle = ? AND type IN (${placeholders}) ORDER BY sequence DESC LIMIT ?` - ) - .all(toHandle, ...types, limit) as MessageRow[] - ) - } + const typed = types && types.length > 0 + const typeFilter = typed ? `AND type IN (${types.map(() => '?').join(',')})` : '' return exposeMessageListTimestamps( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over messages returns the row shape its schema and MessageRow define, like every row cast in db/. this.db - .prepare('SELECT * FROM messages WHERE to_handle = ? ORDER BY sequence DESC LIMIT ?') - .all(toHandle, limit) as MessageRow[] + .prepare( + `SELECT * FROM messages WHERE to_handle = ? AND (? IS NULL OR run_id = ?) ${typeFilter} + ORDER BY sequence DESC LIMIT ?` + ) + .all(toHandle, runId ?? null, runId ?? null, ...(typed ? types : []), limit) as MessageRow[] ) } diff --git a/src/main/runtime/orchestration/db/messages/message-insert.ts b/src/main/runtime/orchestration/db/messages/message-insert.ts index cc590b76b1a..24c45a19e7d 100644 --- a/src/main/runtime/orchestration/db/messages/message-insert.ts +++ b/src/main/runtime/orchestration/db/messages/message-insert.ts @@ -8,6 +8,7 @@ import { generateId } from '../generated-id' import { exposeMessageTimestamps } from '../utc-timestamp' import type { OrchestrationDb } from '../orchestration-db' import { runLifecycleWriteTransaction } from '../lifecycle-write-transaction-runner' +import { notifyPeerMailQueued } from '../../federation/peer-mail/peer-mail-waiters' import { ORCHESTRATION_UNBOUND_RUN_ID as UNBOUND_RUN_ID } from '../../../../../shared/orchestration/orchestration-rpc-contract' const MESSAGE_INSERT_SAVEPOINT = 'message_insert_batch' @@ -27,6 +28,8 @@ export type MessageInsert = { runId?: string answerSenderSnapshot?: string deliveryContract?: MessageDeliveryContract + /** The Dispatch whose worker sent this, as the sender's own check found it; kept for a peer Run's queue. */ + senderDispatchId?: string } export function insertMessage(this: OrchestrationDb, msg: MessageInsert): MessageRow { @@ -44,6 +47,16 @@ export function insertMessage(this: OrchestrationDb, msg: MessageInsert): Messag this.requireRun(runId) const id = msg.id ?? generateId('msg') const type = msg.type ?? 'status' + if (msg.senderDispatchId) { + // Before the insert, whose trigger queues a peer Run's mail home with it; immutable after. + this.db + .prepare( + `INSERT OR IGNORE INTO message_dispatch_provenance (message_id, dispatch_id, task_id) + SELECT ?, d.id, d.task_id FROM dispatch_contexts d + WHERE d.id = ? AND d.run_id IN (SELECT run_id FROM peer_runs)` + ) + .run(id, msg.senderDispatchId) + } // Heartbeats only record liveness: store them handled so they never reach a mailbox or wake it. const handled = type === 'heartbeat' ? 1 : 0 const stmt = this.db.prepare(` @@ -70,6 +83,8 @@ export function insertMessage(this: OrchestrationDb, msg: MessageInsert): Messag handled, handled ) + // Mail filed in a peer Run is queued home by trigger in this statement. + notifyPeerMailQueued(this.db) return exposeMessageTimestamps( this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow ) diff --git a/src/main/runtime/orchestration/db/orchestration-db-methods.ts b/src/main/runtime/orchestration/db/orchestration-db-methods.ts index 0e06a05191d..9072fe5c5fe 100644 --- a/src/main/runtime/orchestration/db/orchestration-db-methods.ts +++ b/src/main/runtime/orchestration/db/orchestration-db-methods.ts @@ -10,6 +10,7 @@ import type { FederationRelayAckMethods } from './federation/federation-relay-ac import type { LegacyRemoteDbMethods } from '../legacy-remote/legacy-remote-db-methods' import type { LegacyRemoteHomeDbMethods } from '../legacy-remote-home/legacy-remote-home-db-methods' import type { PeerMailStoreMethods } from './federation/peer-mail-store' +import type { PeerRunStoreMethods } from './peer-runs/peer-run-store' import type { LegacyAskOperationMethods } from './legacy/legacy-ask-operation' import type { LegacyCompatibilityCandidatesMethods } from './legacy/legacy-compatibility-candidates' import type { LegacyCompatibilityPrincipalsMethods } from './legacy/legacy-compatibility-principals' @@ -104,6 +105,7 @@ export type OrchestrationDbMethods = AttemptObservationStoreMethods & LegacyRemoteDbMethods & FederationRelayAckMethods & PeerMailStoreMethods & + PeerRunStoreMethods & StructuredMailSendStoreMethods & WorkerTerminalResourceStoreMethods & WorkerTerminalTransferMethods & diff --git a/src/main/runtime/orchestration/db/orchestration-db.ts b/src/main/runtime/orchestration/db/orchestration-db.ts index 25bec3ae31c..3fcd88f18b1 100644 --- a/src/main/runtime/orchestration/db/orchestration-db.ts +++ b/src/main/runtime/orchestration/db/orchestration-db.ts @@ -15,6 +15,7 @@ import { rememberCurrentRunCoordinatorHandles } from './runs/run-coordinator-mail-routing' import { createTables } from './schema/create-tables' +import { createPeerRunQueueTriggers } from './peer-runs/peer-run-queue-sql' import { migrate } from './schema/migrate' import { backfillStructuredWorkerOrcaSessionIds } from './schema/structured-worker-orca-session-backfill' import { reconcileSettledWorkerDispatches } from './worker-dispatch/worker-dispatch-settlement' @@ -63,6 +64,7 @@ class OrchestrationDbCore { } } createRunCoordinatorAddressTriggers(this.db) + createPeerRunQueueTriggers(this.db) backfillFederatedStubHomeRuns(this.db) // After every open step that creates Runs, before any step that deletes work. installRemovedWorkCapture(this.db, removedWorkCaptureWasIntact) diff --git a/src/main/runtime/orchestration/db/peer-runs/peer-run-queue-sql.ts b/src/main/runtime/orchestration/db/peer-runs/peer-run-queue-sql.ts new file mode 100644 index 00000000000..c2862a64508 --- /dev/null +++ b/src/main/runtime/orchestration/db/peer-runs/peer-run-queue-sql.ts @@ -0,0 +1,175 @@ +import type Database from '../../../../sqlite/sync-database' +import { REMOTE_OWNED_WIRE_VERSION } from '../../../../../shared/orchestration/remote-owned-wire' + +/** + * A Run here whose coordinator is a paired Orca, keyed by the conversation its queue goes home + * under. It outlives this Orca's resets, so the conversation keeps its Run ID and its sequence. + * Each conversation's next `to_home` sequence is persistent too, so a reset never reuses one. + * A worker message's Dispatch is recorded when it is filed, before any reroute into the Run. + */ +export const PEER_RUN_SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS peer_runs ( + run_id TEXT PRIMARY KEY, + conversation_id TEXT NOT NULL UNIQUE, + peer_fingerprint TEXT NOT NULL, + home_run_id TEXT NOT NULL, + home_label TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'open' CHECK(state IN ('open', 'closed')), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS federation_relay_sequences ( + dispatch_id TEXT NOT NULL, + direction TEXT NOT NULL, + last_sequence INTEGER NOT NULL, + PRIMARY KEY (dispatch_id, direction) +); + +-- A Dispatch whose report owner's correction is being written: its edges queue nothing until the +-- row is removed, which queues the corrected state once, flagged. +CREATE TABLE IF NOT EXISTS peer_run_corrections ( + dispatch_id TEXT PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS message_dispatch_provenance ( + message_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL, + task_id TEXT NOT NULL +); +` + +/** A peer-Run Dispatch as a `worker_state` item carries it, with its Task and Run. */ +const PEER_RUN_DISPATCH_STATE_VIEW_SQL = ` +DROP VIEW IF EXISTS peer_run_dispatch_state; +CREATE VIEW peer_run_dispatch_state AS +SELECT d.id AS dispatch_id, pr.conversation_id, pr.home_run_id, pr.run_id AS peer_run_id, + json_object( + 'id', d.id, 'taskId', d.task_id, 'status', d.status, 'workerState', w.state, + 'lastFailure', d.last_failure, 'agent', json_extract(w.start_options, '$.agent') + ) AS dispatch_json, + json_object('status', t.status, 'result', t.result) AS task_json +FROM dispatch_contexts d +JOIN peer_runs pr ON pr.run_id = d.run_id +JOIN tasks t ON t.id = d.task_id +LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id; +` + +const SEQUENCE = `(SELECT last_sequence FROM federation_relay_sequences + WHERE dispatch_id = conversation_id AND direction = 'to_home')` + +/** + * Takes the conversation's next sequence, then queues the item under it. `rows` selects at most one + * row with `conversation_id` and the item's `message_id` and `payload`; both statements read it. + */ +function queueItemStatements(rows: string, kind: string): [string, string] { + return [ + `INSERT INTO federation_relay_sequences (dispatch_id, direction, last_sequence) + SELECT conversation_id, 'to_home', 1 FROM (${rows}) WHERE true + ON CONFLICT(dispatch_id, direction) DO UPDATE SET last_sequence = last_sequence + 1`, + `INSERT INTO federation_relay_items + (dispatch_id, direction, sequence, message_id, kind, payload, byte_count) + SELECT conversation_id, 'to_home', ${SEQUENCE}, message_id, '${kind}', payload, length(payload) + FROM (${rows})` + ] +} + +/** One `worker_state` item for a peer-Run Dispatch; its ID is the sequence it is queued under. */ +export function peerRunDispatchStateStatements( + dispatchId: string, + corrected: boolean, + removed: string +): [string, string] { + return queueItemStatements( + `SELECT conversation_id, 'state_' || ${SEQUENCE} AS message_id, + json_object( + 'v', ${REMOTE_OWNED_WIRE_VERSION}, 'id', 'state_' || ${SEQUENCE}, + 'homeRunId', home_run_id, 'peerRunId', peer_run_id, + 'dispatch', json(dispatch_json), 'task', json(task_json), + 'correctedUnobservedPrompt', json('${corrected}'), + 'removed', ${removed} + ) AS payload + FROM peer_run_dispatch_state WHERE dispatch_id = ${dispatchId}`, + 'worker_state' + ) +} + +/** A message filed in a peer Run's mailbox, once, with the Dispatch its sender worked for. */ +function peerRunMailStatements(row: string): [string, string] { + return queueItemStatements( + `SELECT pr.conversation_id, ${row}.id AS message_id, + json_object( + 'v', ${REMOTE_OWNED_WIRE_VERSION}, 'id', ${row}.id, + 'homeRunId', pr.home_run_id, 'peerRunId', pr.run_id, + 'message', json_object( + 'from', ${row}.from_handle, 'type', ${row}.type, 'subject', ${row}.subject, + 'body', ${row}.body, 'payload', ${row}.payload, 'priority', ${row}.priority, + 'threadId', ${row}.thread_id + ), + 'dispatchId', p.dispatch_id, 'taskId', p.task_id + ) AS payload + FROM peer_runs pr + LEFT JOIN message_dispatch_provenance p ON p.message_id = ${row}.id + WHERE ${row}.to_handle = 'run:' || pr.run_id + AND NOT EXISTS ( + SELECT 1 FROM federation_relay_items + WHERE dispatch_id = pr.conversation_id AND direction = 'to_home' + AND message_id = ${row}.id + )`, + 'worker_mail' + ) +} + +const LATEST_TASK_DISPATCH = (taskId: string) => + `(SELECT id FROM dispatch_contexts WHERE task_id = ${taskId} ORDER BY rowid DESC LIMIT 1)` + +const NOT_CORRECTING = (dispatchId: string) => + `NOT EXISTS (SELECT 1 FROM peer_run_corrections WHERE dispatch_id = ${dispatchId})` + +const trigger = (statements: [string, string]) => `BEGIN ${statements.join(';\n')}; END;` + +/** + * Every final message of a peer Run's mailbox and every lifecycle change of its Dispatches, their + * workers and their Tasks queue home in the transaction that made them. Rebuilt on every open. + * The report owner's unobserved-prompt correction queues once, flagged, after all its writes. + */ +export function createPeerRunQueueTriggers(db: Database.Database): void { + db.exec('BEGIN IMMEDIATE') + try { + db.exec(` + ${PEER_RUN_DISPATCH_STATE_VIEW_SQL} + DROP TRIGGER IF EXISTS trg_peer_run_mail_insert; + DROP TRIGGER IF EXISTS trg_peer_run_mail_reroute; + DROP TRIGGER IF EXISTS trg_peer_run_dispatch_insert; + DROP TRIGGER IF EXISTS trg_peer_run_dispatch_status; + DROP TRIGGER IF EXISTS trg_peer_run_worker_state; + DROP TRIGGER IF EXISTS trg_peer_run_task_status; + DROP TRIGGER IF EXISTS trg_peer_run_correction; + CREATE TRIGGER trg_peer_run_mail_insert AFTER INSERT ON messages + WHEN NEW.to_handle LIKE 'run:%' AND NEW.type <> 'heartbeat' + ${trigger(peerRunMailStatements('NEW'))} + CREATE TRIGGER trg_peer_run_mail_reroute AFTER UPDATE OF to_handle ON messages + WHEN NEW.to_handle LIKE 'run:%' AND NEW.to_handle <> OLD.to_handle AND NEW.type <> 'heartbeat' + ${trigger(peerRunMailStatements('NEW'))} + CREATE TRIGGER trg_peer_run_dispatch_insert AFTER INSERT ON dispatch_contexts + ${trigger(peerRunDispatchStateStatements('NEW.id', false, 'NULL'))} + CREATE TRIGGER trg_peer_run_dispatch_status AFTER UPDATE OF status ON dispatch_contexts + WHEN NEW.status <> OLD.status AND ${NOT_CORRECTING('NEW.id')} + ${trigger(peerRunDispatchStateStatements('NEW.id', false, 'NULL'))} + CREATE TRIGGER trg_peer_run_worker_state AFTER UPDATE OF state ON worker_dispatches + WHEN NEW.state <> OLD.state AND ${NOT_CORRECTING('NEW.dispatch_id')} + ${trigger(peerRunDispatchStateStatements('NEW.dispatch_id', false, 'NULL'))} + CREATE TRIGGER trg_peer_run_task_status AFTER UPDATE OF status ON tasks + WHEN NEW.status <> OLD.status AND NOT EXISTS ( + SELECT 1 FROM peer_run_corrections c JOIN dispatch_contexts d ON d.id = c.dispatch_id + WHERE d.task_id = NEW.id + ) + ${trigger(peerRunDispatchStateStatements(LATEST_TASK_DISPATCH('NEW.id'), false, 'NULL'))} + CREATE TRIGGER trg_peer_run_correction AFTER DELETE ON peer_run_corrections + ${trigger(peerRunDispatchStateStatements('OLD.dispatch_id', true, 'NULL'))} + `) + db.exec('COMMIT') + } catch (error) { + db.exec('ROLLBACK') + throw error + } +} diff --git a/src/main/runtime/orchestration/db/peer-runs/peer-run-queue.test.ts b/src/main/runtime/orchestration/db/peer-runs/peer-run-queue.test.ts new file mode 100644 index 00000000000..a8b9bd4ce3b --- /dev/null +++ b/src/main/runtime/orchestration/db/peer-runs/peer-run-queue.test.ts @@ -0,0 +1,265 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { OrchestrationDb } from '../orchestration-db' +import { generateId } from '../generated-id' +import { + RemoteOwnedWorkerMail, + RemoteOwnedWorkerState +} from '../../../../../shared/orchestration/remote-owned-wire' +import { AGENT_PROMPT_STALLED_ERROR } from '../../../terminal/input/agent-prompt-submission-verification' +import type { PeerRunRow } from './peer-run-store' + +describe("a paired Orca's Run: its queue home", () => { + let db: OrchestrationDb + let peerRun: PeerRunRow + + beforeEach(() => { + db = new OrchestrationDb(':memory:') + peerRun = db.admitPeerRun({ + peerFingerprint: 'fingerprint_a', + peerRun: { runId: 'run_home', conversationId: generateId('peerconv'), label: 'devA' } + }) + }) + + afterEach(() => db.close()) + + function start(params: { adoptTaskId?: string; retryOf?: string } = {}) { + return db.createStartingWorkerDispatch({ + taskSpec: 'Build it', + taskRunId: peerRun.run_id, + startOptions: { agent: 'claude' }, + creator: { kind: 'system' }, + maxDepth: 5, + ...params + }) + } + + function pending() { + return db.listPendingFederationRelay(peerRun.conversation_id, 'to_home', 500) + } + + function states() { + return pending() + .filter((item) => item.kind === 'worker_state') + .map((item) => RemoteOwnedWorkerState.parse(JSON.parse(item.payload))) + } + + function fail(dispatchId: string, taskId: string) { + db.transitionLifecycle({ entity: 'worker', id: dispatchId, from: 'starting', to: 'failed' }) + db.transitionLifecycle({ entity: 'dispatch', id: dispatchId, from: 'pending', to: 'failed' }) + db.transitionLifecycle({ entity: 'task', id: taskId, from: 'dispatched', to: 'failed' }) + } + + it('queues every lifecycle change of a Dispatch and its Task, in order, as the wire schema', () => { + const { dispatch, task } = start({ adoptTaskId: 'task_home1' }) + fail(dispatch.id, task.id) + + expect(task.id).toBe('task_home1') + const items = pending() + expect(items.map((item) => item.sequence)).toEqual(items.map((_, index) => index + 1)) + expect( + states().map((state) => [ + state.dispatch.status, + state.dispatch.workerState, + state.task.status + ]) + ).toEqual([ + ['pending', null, 'ready'], + ['pending', 'starting', 'dispatched'], + ['pending', 'failed', 'dispatched'], + ['failed', 'failed', 'dispatched'], + ['failed', 'failed', 'failed'] + ]) + expect(states().every((state) => state.homeRunId === 'run_home')).toBe(true) + for (const item of items) { + expect(JSON.parse(item.payload).id).toBe(item.message_id) + } + }) + + it("queues the report owner's unobserved-prompt correction once, flagged, with its result", () => { + const { dispatch, task } = start() + fail(dispatch.id, task.id) + db.transitionLifecycle({ + entity: 'dispatch', + id: dispatch.id, + from: 'failed', + to: 'failed', + projection: { last_failure: AGENT_PROMPT_STALLED_ERROR } + }) + const before = states().length + + db.settleWorkerReport({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded', + result: 'actually done' + }) + + expect(states().slice(before)).toEqual([ + expect.objectContaining({ + correctedUnobservedPrompt: true, + dispatch: expect.objectContaining({ status: 'completed', workerState: 'succeeded' }), + task: { status: 'completed', result: 'actually done' } + }) + ]) + }) + + it('holds back the Task change of a correction even when a newer attempt is the latest', () => { + const first = start() + fail(first.dispatch.id, first.task.id) + db.transitionLifecycle({ + entity: 'dispatch', + id: first.dispatch.id, + from: 'failed', + to: 'failed', + projection: { last_failure: AGENT_PROMPT_STALLED_ERROR } + }) + const retry = start({ adoptTaskId: first.task.id }) + fail(retry.dispatch.id, first.task.id) + const before = states().length + + db.settleWorkerReport({ + taskId: first.task.id, + dispatchId: first.dispatch.id, + outcome: 'succeeded', + result: 'actually done' + }) + + expect(states().slice(before)).toEqual([ + expect.objectContaining({ + correctedUnobservedPrompt: true, + dispatch: expect.objectContaining({ id: first.dispatch.id }) + }) + ]) + }) + + it("queues a worker's question with its thread, so the coordinator can answer in it", () => { + const { dispatch } = start() + const { message } = db.createQuestion({ + runId: peerRun.run_id, + dispatchId: dispatch.id, + askerHandle: 'term_w', + question: 'Which branch?' + }) + const queued = pending().find((item) => item.message_id === message.id)! + expect(RemoteOwnedWorkerMail.parse(JSON.parse(queued.payload)).message.threadId).toBe( + message.id + ) + }) + + it('never reuses a sequence after a reset, and a reset tells the coordinator what it removed', () => { + const first = start() + const throughOld = Math.max(...pending().map((item) => item.sequence)) + + db.resetTasks() + const removed = states().filter((state) => state.removed) + expect(removed.map((state) => [state.dispatch.id, state.removed?.reason])).toEqual([ + [first.dispatch.id, 'reset on the server running it'] + ]) + start() + db.acknowledgeFederationRelay({ + dispatchId: peerRun.conversation_id, + direction: 'to_home', + throughSequence: throughOld + }) + + const after = pending() + expect(after.length).toBeGreaterThan(0) + expect(after.every((item) => item.sequence > throughOld)).toBe(true) + }) + + it('keeps the queue and the Run ID through a full reset, with a removed item per Dispatch', () => { + const { dispatch } = start() + db.resetAll() + + expect(states().find((state) => state.removed)?.dispatch.id).toBe(dispatch.id) + const again = db.admitPeerRun({ + peerFingerprint: 'fingerprint_a', + peerRun: { runId: 'run_home', conversationId: peerRun.conversation_id, label: 'devA' } + }) + expect(again.run_id).toBe(peerRun.run_id) + expect(db.getRun(peerRun.run_id)).toBeDefined() + }) + + it('attributes mail rerouted into the Run to the Dispatch its sender worked for when it sent it', () => { + const coordinator = `peer:${peerRun.conversation_id}` + const first = start() + const note = db.insertMessage({ + from: 'term_w', + to: 'term_unbound', + subject: 'late note', + runId: peerRun.run_id, + senderDispatchId: first.dispatch.id + }) + fail(first.dispatch.id, first.task.id) + start() + db.db.prepare('UPDATE messages SET to_handle = ? WHERE id = ?').run(coordinator, note.id) + db.db + .prepare('UPDATE messages SET to_handle = ? WHERE id = ?') + .run(`run:${peerRun.run_id}`, note.id) + + const mail = pending() + .filter((item) => item.kind === 'worker_mail') + .map((item) => RemoteOwnedWorkerMail.parse(JSON.parse(item.payload))) + expect(mail.map((item) => [item.id, item.dispatchId, item.taskId])).toEqual([ + [note.id, first.dispatch.id, first.task.id] + ]) + }) + + it('reopens an adopted Task whose last attempt settled, so the coordinator can run it again', () => { + const first = start({ adoptTaskId: 'task_home1' }) + fail(first.dispatch.id, first.task.id) + + const again = start({ adoptTaskId: 'task_home1' }) + expect(again.task.id).toBe('task_home1') + expect(again.dispatch.id).not.toBe(first.dispatch.id) + expect(() => start({ adoptTaskId: 'task_home1' })).toThrow(/only a ready Task can start/) + }) + + it('keeps a peer Run conversation and a peer-mail conversation apart, so their sequences never collide', () => { + const mailConversation = generateId('peerconv') + db.insertPeerMailMessage({ + conversationId: mailConversation, + peerFingerprint: 'fingerprint_a', + remoteParty: null, + message: { id: generateId('msg'), from: 'devA:term_x', to: 'term_w', subject: 'hi' } + }) + expect(() => + db.admitPeerRun({ + peerFingerprint: 'fingerprint_a', + peerRun: { runId: 'run_home2', conversationId: mailConversation, label: 'devA' } + }) + ).toThrow(expect.objectContaining({ code: 'request_mismatch' })) + expect(() => + db.insertPeerMailMessage({ + conversationId: peerRun.conversation_id, + peerFingerprint: 'fingerprint_a', + remoteParty: null, + message: { id: generateId('msg'), from: 'devA:term_x', to: 'term_w', subject: 'hi' } + }) + ).toThrow(expect.objectContaining({ code: 'request_mismatch' })) + }) + + it("follows a rename of the coordinating Orca's machine", () => { + const renamed = db.admitPeerRun({ + peerFingerprint: 'fingerprint_a', + peerRun: { runId: 'run_home', conversationId: peerRun.conversation_id, label: 'devA2' } + }) + expect(renamed.home_label).toBe('devA2') + }) + + it('leaves an adopted Task that still waits on its dependencies waiting', () => { + const dependency = db.createTask({ spec: 'first', runId: peerRun.run_id }) + const task = db.createTask({ + spec: 'second', + runId: peerRun.run_id, + deps: [dependency.id] + }) + expect(() => start({ adoptTaskId: task.id })).toThrow(/only a ready Task can start/) + expect(db.getTask(task.id)?.status).toBe('pending') + }) + + it('admits no start into a Run its coordinator cancelled', () => { + db.closePeerRun(peerRun.run_id) + expect(() => start()).toThrow(/cancelled by devA/) + }) +}) diff --git a/src/main/runtime/orchestration/db/peer-runs/peer-run-store.ts b/src/main/runtime/orchestration/db/peer-runs/peer-run-store.ts new file mode 100644 index 00000000000..20f70500176 --- /dev/null +++ b/src/main/runtime/orchestration/db/peer-runs/peer-run-store.ts @@ -0,0 +1,146 @@ +import type { RemoteOwnedPeerRun } from '../../../../../shared/orchestration/remote-owned-wire' +import { OrchestrationError } from '../../orchestration-error' +import { generateId } from '../generated-id' +import type { OrchestrationDb } from '../orchestration-db' +import { peerRunDispatchStateStatements } from './peer-run-queue-sql' +import { peerCoordinatorAddress } from '../../addressing/orchestration-caller-identity' + +export type PeerRunRow = { + run_id: string + conversation_id: string + peer_fingerprint: string + home_run_id: string + home_label: string + state: 'open' | 'closed' + created_at: string +} + +const NO_EFFECTS = { effectsApplied: false } as const + +/** + * The Run a paired Orca coordinates here for one conversation, written when its frame arrives and + * before any await, so a later frame of the same conversation (a cancel) always finds it. The Run + * keeps its ID across this Orca's resets: a reset that removed its row gets it back on the next call. + */ +export function admitPeerRun( + this: OrchestrationDb, + params: { peerFingerprint: string; peerRun: RemoteOwnedPeerRun } +): PeerRunRow { + const { peerFingerprint, peerRun } = params + this.db.exec('BEGIN IMMEDIATE') + try { + const conversation = this.getPeerMailConversation(peerRun.conversationId) + const existing = this.getPeerRunForConversation(peerRun.conversationId) + // A peer-mail conversation keeps its own queue sequence, so it never becomes a peer Run's. + if ( + existing + ? existing.peer_fingerprint !== peerFingerprint || existing.home_run_id !== peerRun.runId + : conversation + ) { + throw new OrchestrationError( + 'request_mismatch', + `Conversation ${peerRun.conversationId} belongs to another Run or pairing. No effects were applied.`, + NO_EFFECTS + ) + } + const runId = existing?.run_id ?? generateId('run') + if (existing && existing.home_label !== peerRun.label) { + this.db + .prepare('UPDATE peer_runs SET home_label = ? WHERE run_id = ?') + .run(peerRun.label, runId) + } + if (!existing) { + this.db + .prepare( + `INSERT OR IGNORE INTO peer_mail_conversations (conversation_id, role, peer_fingerprint) + VALUES (?, 'peer', ?)` + ) + .run(peerRun.conversationId, peerFingerprint) + this.db + .prepare( + `INSERT INTO peer_runs (run_id, conversation_id, peer_fingerprint, home_run_id, home_label) + VALUES (?, ?, ?, ?, ?)` + ) + .run(runId, peerRun.conversationId, peerFingerprint, peerRun.runId, peerRun.label) + } + if (!this.getRunRaw(runId)) { + const coordinator = peerCoordinatorAddress(peerRun.conversationId) + this.insertRunRow(runId, { + objective: `Coordinated from ${peerRun.label}`, + coordinatorHandle: coordinator, + coordinatorPaneKey: coordinator + }) + } + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the committed transaction above wrote or found this conversation's row. + return this.getPeerRunForConversation(peerRun.conversationId) as PeerRunRow +} + +export function getPeerRun(this: OrchestrationDb, runId: string): PeerRunRow | undefined { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table returns exactly its row shape. + return this.db.prepare('SELECT * FROM peer_runs WHERE run_id = ?').get(runId) as + | PeerRunRow + | undefined +} + +export function hasPeerRuns(this: OrchestrationDb): boolean { + return this.db.prepare('SELECT 1 FROM peer_runs LIMIT 1').get() !== undefined +} + +export function getPeerRunForConversation( + this: OrchestrationDb, + conversationId: string +): PeerRunRow | undefined { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table returns exactly its row shape. + return this.db + .prepare('SELECT * FROM peer_runs WHERE conversation_id = ?') + .get(conversationId) as PeerRunRow | undefined +} + +/** Closed for good: the claim refuses every start, follow-up and helper into it from now on. */ +export function closePeerRun(this: OrchestrationDb, runId: string): void { + this.db.prepare(`UPDATE peer_runs SET state = 'closed' WHERE run_id = ?`).run(runId) +} + +/** Inside a reset's transaction, before it deletes them: one final item per peer-Run Dispatch. */ +export function queueRemovedPeerRunDispatches(this: OrchestrationDb, reason: string): void { + const [allocate, queue] = peerRunDispatchStateStatements( + '?1', + false, + `json_object('reason', ?2)` + ).map((sql) => this.db.prepare(sql)) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column. + const dispatches = this.db + .prepare( + `SELECT d.id FROM dispatch_contexts d JOIN peer_runs pr ON pr.run_id = d.run_id ORDER BY d.rowid` + ) + .all() as { id: string }[] + for (const { id } of dispatches) { + allocate.run(id, reason) + queue.run(id, reason) + } +} + +export type PeerRunStoreMethods = { + admitPeerRun: typeof admitPeerRun + getPeerRun: typeof getPeerRun + getPeerRunForConversation: typeof getPeerRunForConversation + hasPeerRuns: typeof hasPeerRuns + closePeerRun: typeof closePeerRun + queueRemovedPeerRunDispatches: typeof queueRemovedPeerRunDispatches +} + +export function attachPeerRunStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + admitPeerRun, + getPeerRun, + getPeerRunForConversation, + hasPeerRuns, + closePeerRun, + queueRemovedPeerRunDispatches + }) +} diff --git a/src/main/runtime/orchestration/db/questions/question-threads.ts b/src/main/runtime/orchestration/db/questions/question-threads.ts index 4f0be287206..5741dc5750a 100644 --- a/src/main/runtime/orchestration/db/questions/question-threads.ts +++ b/src/main/runtime/orchestration/db/questions/question-threads.ts @@ -2,6 +2,7 @@ import type { MessageRow, QuestionRow } from '../orchestration-records' import { OrchestrationError } from '../../orchestration-error' import { exposeQuestionTimestamps } from '../utc-timestamp' import type { OrchestrationDb } from '../orchestration-db' +import { generateId } from '../generated-id' import { handHelperMailToRun } from '../messages/helper-mailbox-hand-back' export function createQuestion( @@ -27,7 +28,11 @@ export function createQuestion( `Dispatch ${params.dispatchId} is not in Run ${params.runId}.` ) } + // A question is its own thread, set at insert so mail queued for a paired coordinator carries it. + const id = generateId('msg') const message = this.insertMessage({ + id, + threadId: id, from: `dispatch:${params.dispatchId}`, to: params.to ?? `run:${params.runId}`, subject: 'Question', @@ -39,9 +44,9 @@ export function createQuestion( question: params.question, options: params.options ?? [] }), + senderDispatchId: dispatch.id, runId: params.runId }) - this.db.prepare('UPDATE messages SET thread_id = ? WHERE id = ?').run(message.id, message.id) this.db .prepare( `INSERT INTO question_threads ( diff --git a/src/main/runtime/orchestration/db/reset/orchestration-reset.ts b/src/main/runtime/orchestration/db/reset/orchestration-reset.ts index ac68ba65767..b8af86962be 100644 --- a/src/main/runtime/orchestration/db/reset/orchestration-reset.ts +++ b/src/main/runtime/orchestration/db/reset/orchestration-reset.ts @@ -1,5 +1,6 @@ import { ORCHESTRATION_LEGACY_RUN_ID as LEGACY_RUN_ID } from '../../../../../shared/orchestration/orchestration-rpc-contract' import type { OrchestrationDb } from '../orchestration-db' +import { notifyPeerMailQueued } from '../../federation/peer-mail/peer-mail-waiters' import type { ResetRemovedWork } from '../../../../../shared/orchestration/orchestration-removed-work' import { readRemovedWorkWatermark, @@ -7,13 +8,22 @@ import { } from './removed-orchestration-work-read' /** Returns the removed-work watermark, read inside the transaction so the reply counts only this reset. */ -export function runResetTransaction(this: OrchestrationDb, statements: string): number { +export function runResetTransaction( + this: OrchestrationDb, + statements: string, + { removesDispatches = false }: { removesDispatches?: boolean } = {} +): number { this.db.exec('BEGIN IMMEDIATE') try { // Why unguarded: an error that ends the transaction must roll the whole reset back, not let the deletes autocommit. const watermark = readRemovedWorkWatermark(this.db) + if (removesDispatches) { + // Their paired coordinator learns of it from its queue, which the reset keeps. + this.queueRemovedPeerRunDispatches('reset on the server running it') + } this.db.exec(statements) this.db.exec('COMMIT') + notifyPeerMailQueued(this.db) return watermark } catch (error) { try { @@ -27,7 +37,8 @@ export function runResetTransaction(this: OrchestrationDb, statements: string): export function resetAll(this: OrchestrationDb): void { // Keep receipts so a lost reset reply cannot replay the mutation. - this.runResetTransaction(` + this.runResetTransaction( + ` DELETE FROM coordinator_runs; DELETE FROM decision_gates; DELETE FROM remote_questions; @@ -37,7 +48,7 @@ export function resetAll(this: OrchestrationDb): void { DELETE FROM legacy_operation_receipts; DELETE FROM legacy_compatibility_principals; DELETE FROM legacy_adoptions; - DELETE FROM federation_relay_items; + DELETE FROM federation_relay_items WHERE NOT (direction = 'to_home' AND dispatch_id IN (SELECT conversation_id FROM peer_runs)); DELETE FROM remote_dispatch_attachments; DELETE FROM federated_dispatches; DELETE FROM worker_terminal_archives; @@ -48,18 +59,22 @@ export function resetAll(this: OrchestrationDb): void { DELETE FROM dispatch_contexts; DELETE FROM tasks; DELETE FROM messages; + DELETE FROM message_dispatch_provenance; DELETE FROM run_coordinator_handles; DELETE FROM runs; DELETE FROM removed_orchestration_work; INSERT INTO runs (id, objective, home_database, consumer_generation, legacy) VALUES ('${LEGACY_RUN_ID}', 'Legacy orchestration state (inspect only)', 'this_database', 0, 1); - `) + `, + { removesDispatches: true } + ) this.hasAnyDispatchContextsCache = undefined } export function resetTasks(this: OrchestrationDb): ResetRemovedWork | undefined { // Messages survive, and so do replies waiting for a paired Orca to pull them; close threads so orphaned questions cannot masquerade as generic replies. - const watermark = this.runResetTransaction(` + const watermark = this.runResetTransaction( + ` DELETE FROM coordinator_runs; DELETE FROM decision_gates; DELETE FROM remote_questions; @@ -70,7 +85,7 @@ export function resetTasks(this: OrchestrationDb): ResetRemovedWork | undefined DELETE FROM legacy_operation_receipts; DELETE FROM legacy_compatibility_principals; DELETE FROM legacy_adoptions; - DELETE FROM federation_relay_items WHERE kind <> 'peer_mail'; + DELETE FROM federation_relay_items WHERE kind <> 'peer_mail' AND NOT (direction = 'to_home' AND dispatch_id IN (SELECT conversation_id FROM peer_runs)); DELETE FROM remote_dispatch_attachments; DELETE FROM federated_dispatches; DELETE FROM worker_terminal_archives; @@ -80,7 +95,9 @@ export function resetTasks(this: OrchestrationDb): ResetRemovedWork | undefined DELETE FROM worker_dispatches; DELETE FROM dispatch_contexts; DELETE FROM tasks; - `) + `, + { removesDispatches: true } + ) this.hasAnyDispatchContextsCache = undefined return summarizeRemovedWorkSince(this.db, watermark) } @@ -92,6 +109,7 @@ export function resetMessages(this: OrchestrationDb): void { DELETE FROM question_threads; DELETE FROM deliveries; DELETE FROM messages; + DELETE FROM message_dispatch_provenance; DELETE FROM structured_pointer_operations; `) } diff --git a/src/main/runtime/orchestration/db/runs/run-binding.ts b/src/main/runtime/orchestration/db/runs/run-binding.ts index 0d47fd2e34c..6950c8cba8b 100644 --- a/src/main/runtime/orchestration/db/runs/run-binding.ts +++ b/src/main/runtime/orchestration/db/runs/run-binding.ts @@ -42,6 +42,15 @@ export function bindRun( return undefined } const sameBinding = runBoundToCoordinator(run, coordinator) + const peerRun = this.getPeerRun(params.runId) + if (peerRun && !sameBinding) { + // A paired Orca's Run has that Orca as its only consumer; it never binds anything here. + throw new OrchestrationError( + 'consumer_fenced', + `Run ${params.runId} is coordinated from ${peerRun.home_label}, which stays its only coordinator. No effects were applied.`, + { effectsApplied: false } + ) + } const explicitTakeover = params.takeoverLegacy === true // Why: before the legacy gate, so a worker never sees its --takeover-legacy recovery command. const workerRefusal = sameBinding diff --git a/src/main/runtime/orchestration/db/runs/run-create.ts b/src/main/runtime/orchestration/db/runs/run-create.ts index 53b17414ea6..6ce7877082c 100644 --- a/src/main/runtime/orchestration/db/runs/run-create.ts +++ b/src/main/runtime/orchestration/db/runs/run-create.ts @@ -4,42 +4,19 @@ import type { OrchestrationDb } from '../orchestration-db' import type { OrcaSessionId } from '../../../../../shared/orca-session-address' import { mailboxAddressOf } from '../../addressing/orchestration-caller-identity' -export function createRun( - this: OrchestrationDb, - params: { - objective: string - coordinatorHandle: string | null - coordinatorPaneKey: string | null - /** The coordinator's bare Orca session id when it is a structured session; see orca-session-address. */ - coordinatorOrcaSessionId?: OrcaSessionId | null - } -): RunRow { - const coordinator = { - terminalHandle: params.coordinatorHandle, - paneKey: params.coordinatorPaneKey, - orcaSessionId: params.coordinatorOrcaSessionId ?? null - } +type RunCoordinatorParams = { + objective: string + coordinatorHandle: string | null + coordinatorPaneKey: string | null + /** The coordinator's bare Orca session id when it is a structured session; see orca-session-address. */ + coordinatorOrcaSessionId?: OrcaSessionId | null +} + +export function createRun(this: OrchestrationDb, params: RunCoordinatorParams): RunRow { const id = generateId('run') this.db.exec('BEGIN IMMEDIATE') try { - this.db - .prepare( - `INSERT INTO runs ( - id, objective, coordinator_handle, coordinator_pane_key, coordinator_orca_session_id, - coordinator_orca_session_id_generation, consumer_generation, legacy - ) VALUES (?, ?, ?, ?, ?, 1, 1, 0)` - ) - .run( - id, - params.objective, - coordinator.terminalHandle, - coordinator.paneKey, - coordinator.orcaSessionId - ) - const address = mailboxAddressOf(coordinator) - if (address !== null) { - this.rememberRunCoordinatorHandle(id, address) - } + this.insertRunRow(id, params) this.db.exec('COMMIT') } catch (error) { this.db.exec('ROLLBACK') @@ -48,12 +25,45 @@ export function createRun( return this.getRun(id) as RunRow } +/** Writes a Run bound to its coordinator inside the caller's transaction. */ +export function insertRunRow( + this: OrchestrationDb, + id: string, + params: RunCoordinatorParams +): void { + const coordinator = { + terminalHandle: params.coordinatorHandle, + paneKey: params.coordinatorPaneKey, + orcaSessionId: params.coordinatorOrcaSessionId ?? null + } + this.db + .prepare( + `INSERT INTO runs ( + id, objective, coordinator_handle, coordinator_pane_key, coordinator_orca_session_id, + coordinator_orca_session_id_generation, consumer_generation, legacy + ) VALUES (?, ?, ?, ?, ?, 1, 1, 0)` + ) + .run( + id, + params.objective, + coordinator.terminalHandle, + coordinator.paneKey, + coordinator.orcaSessionId + ) + const address = mailboxAddressOf(coordinator) + if (address !== null) { + this.rememberRunCoordinatorHandle(id, address) + } +} + export type RunCreateMethods = { createRun: typeof createRun + insertRunRow: typeof insertRunRow } export function attachRunCreate(ctor: { prototype: object }): void { Object.assign(ctor.prototype, { - createRun + createRun, + insertRunRow }) } diff --git a/src/main/runtime/orchestration/db/schema/create-tables.ts b/src/main/runtime/orchestration/db/schema/create-tables.ts index 1895230e847..5378ee71629 100644 --- a/src/main/runtime/orchestration/db/schema/create-tables.ts +++ b/src/main/runtime/orchestration/db/schema/create-tables.ts @@ -3,9 +3,12 @@ import { createCoreTablesSql } from './create-core-tables-sql' import { createGraphTablesSql } from './create-graph-tables-sql' import { DERIVED_DELIVERY_SCHEMA_SQL } from './migrate-v41' import { REMOVED_WORK_SCHEMA_SQL } from '../reset/removed-orchestration-work-store' +import { PEER_RUN_SCHEMA_SQL } from '../peer-runs/peer-run-queue-sql' export function createTables(this: OrchestrationDb): void { - this.db.exec(`${createCoreTablesSql()}\n${createGraphTablesSql()}\n${REMOVED_WORK_SCHEMA_SQL}`) + this.db.exec( + `${createCoreTablesSql()}\n${createGraphTablesSql()}\n${REMOVED_WORK_SCHEMA_SQL}\n${PEER_RUN_SCHEMA_SQL}` + ) // Unreleased builds saved each send's fence as NOT NULL; nothing reads it, and new rows carry none. if (this.hasColumn('structured_mail_sends', 'fence')) { this.db.exec('ALTER TABLE structured_mail_sends DROP COLUMN fence') diff --git a/src/main/runtime/orchestration/db/tasks/task-store.ts b/src/main/runtime/orchestration/db/tasks/task-store.ts index 401d3d68544..aebef0d890f 100644 --- a/src/main/runtime/orchestration/db/tasks/task-store.ts +++ b/src/main/runtime/orchestration/db/tasks/task-store.ts @@ -10,6 +10,8 @@ import { selectColumns, TASK_COLUMNS } from '../row-column-lists' export function createTask( this: OrchestrationDb, task: { + /** Set only for a Task placed on a paired server: one ID for it here and there, from the start's retry key. */ + id?: string spec: string taskTitle?: string displayName?: string @@ -39,7 +41,7 @@ export function createTask( throw new Error(`Dependency task ${depId} must belong to run ${runId}`) } } - const id = generateId('task') + const id = task.id ?? generateId('task') const depsJson = JSON.stringify(task.deps ?? []) const display = buildOrchestrationTaskDisplayMetadata({ spec: task.spec, diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts index b65da6e2f61..f45cf6677e0 100644 --- a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts @@ -18,6 +18,8 @@ export function createStartingWorkerDispatch( params: { taskId?: string taskSpec?: string + /** The paired coordinator's ID for this Task: an existing one is reopened, else the new one takes it. */ + adoptTaskId?: string taskRunId?: string taskCreatedByTerminalHandle?: string taskCreatedByPaneKey?: string @@ -89,26 +91,42 @@ export function createStartingWorkerDispatch( { followUpDispatchId: claimedFollowUp.id } ) } + const adopted = params.adoptTaskId ? this.getTask(params.adoptTaskId) : undefined + if (adopted && !params.retryOf) { + reopenSettledAdoptedTask(this, adopted, params.taskRunId) + } const task = params.taskId ? this.getTask(params.taskId) - : params.taskSpec - ? this.createTask({ - spec: params.taskSpec, - taskTitle: params.taskTitle, - deps: params.taskDeps, - parentId: params.taskParentId, - createdByTerminalHandle: params.taskCreatedByTerminalHandle, - createdByPaneKey: params.taskCreatedByPaneKey, - createdByProcessIncarnation: params.taskCreatedByProcessIncarnation, - createdByRunGeneration: params.taskCreatedByRunGeneration, - runId: params.taskRunId - }) - : undefined + : adopted + ? this.getTask(adopted.id) + : params.taskSpec + ? this.createTask({ + id: params.adoptTaskId, + spec: params.taskSpec, + taskTitle: params.taskTitle, + deps: params.taskDeps, + parentId: params.taskParentId, + createdByTerminalHandle: params.taskCreatedByTerminalHandle, + createdByPaneKey: params.taskCreatedByPaneKey, + createdByProcessIncarnation: params.taskCreatedByProcessIncarnation, + createdByRunGeneration: params.taskCreatedByRunGeneration, + runId: params.taskRunId + }) + : undefined if (!task) { // An inline spec already creates its Task; missing rows here imply an explicit id. const taskId = params.taskId ?? '' throw taskNotFoundError(`Task ${taskId} was not found.`, { taskId }) } + const peerRun = this.getPeerRun(task.run_id) + if (peerRun?.state === 'closed') { + // Starts, follow-ups and helpers alike: its coordinator cancelled it. + throw new OrchestrationError( + 'run_not_found', + `Run ${task.run_id} was cancelled by ${peerRun.home_label}. No effects were applied.`, + { effectsApplied: false } + ) + } let priorFailures = 0 if (params.retryOf) { const prior = this.getDispatchContextById(params.retryOf) @@ -251,3 +269,30 @@ export function attachWorkerDispatchStart(ctor: { prototype: object }): void { createStartingWorkerDispatch }) } + +/** A Task its paired coordinator reopened gets a new attempt here once its latest attempt settled. */ +function reopenSettledAdoptedTask( + db: OrchestrationDb, + task: TaskRow, + runId: string | undefined +): void { + if (task.run_id !== runId) { + throw taskNotFoundError(`Task ${task.id} was not found in Run ${runId ?? ''}.`, { + taskId: task.id + }) + } + const latest = db.getDispatchContext(task.id) + const worker = latest ? db.getWorkerDispatch(latest.id) : undefined + const settled = worker + ? ['succeeded', 'failed', 'stopped', 'abandoned'].includes(worker.state) + : !latest || ['completed', 'failed', 'circuit_broken'].includes(latest.status) + // A pending Task still waits on its dependencies; the ordinary promotion readies it. + if (settled && task.status !== 'ready' && task.status !== 'pending') { + transitionLifecycleWithDb(db.db, { + entity: 'task', + id: task.id, + from: task.status, + to: 'ready' + }) + } +} diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts index 3e625c2126c..c2ce51ded5e 100644 --- a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts @@ -103,6 +103,8 @@ export function listWorkerTerminalResources( pendingApproval: boolean terminationReason: TerminalExitCause['kind'] | null resource: WorkerTerminalResourceRow | null + /** The paired Orca's name when its Run here coordinates this worker. */ + coordinatedFrom: string | null createdAt: string databaseId: number }[] { @@ -170,6 +172,7 @@ export function listWorkerTerminalResources( t.parent_id AS parent_task_id, d.task_id, d.run_id, d.status AS dispatch_status, d.termination_reason, + (SELECT home_label FROM peer_runs WHERE run_id = d.run_id) AS coordinated_from, EXISTS ( SELECT 1 FROM question_threads q WHERE q.dispatch_id = d.id AND q.status = 'pending' @@ -199,6 +202,7 @@ export function listWorkerTerminalResources( termination_reason: TerminalExitCause['kind'] | null pending_input: number pending_approval: number + coordinated_from: string | null created_at: string database_id: number }[] @@ -236,6 +240,7 @@ export function listWorkerTerminalResources( pendingApproval: row.pending_approval === 1, terminationReason: row.termination_reason, resource, + coordinatedFrom: row.coordinated_from, createdAt: row.created_at, databaseId: row.database_id } diff --git a/src/main/runtime/orchestration/federation/peer-mail/peer-mail-waiters.ts b/src/main/runtime/orchestration/federation/peer-mail/peer-mail-waiters.ts index 0e2b13c329d..5b88ac6206d 100644 --- a/src/main/runtime/orchestration/federation/peer-mail/peer-mail-waiters.ts +++ b/src/main/runtime/orchestration/federation/peer-mail/peer-mail-waiters.ts @@ -1,17 +1,18 @@ -import type { OrchestrationDb } from '../../db/orchestration-db' +import type Database from '../../../../sqlite/sync-database' -const waiters = new WeakMap void>>() +const waiters = new WeakMap void>>() -/** Wakes every pull waiting on this store; each re-reads its own conversations. */ -export function notifyPeerMailQueued(db: OrchestrationDb): void { +/** Wakes every pull waiting on this database; each re-reads its own conversations. Called by every + * write that can queue an item home: a reply, a message filed in a Run, a lifecycle change, a reset. */ +export function notifyPeerMailQueued(db: Database.Database): void { for (const wake of waiters.get(db) ?? []) { wake() } } -/** Resolves when a reply is queued, the wait ends, or the caller goes away. */ +/** Resolves when an item may have been queued, the wait ends, or the caller goes away. */ export function waitForPeerMail( - db: OrchestrationDb, + db: Database.Database, timeoutMs: number, signal?: AbortSignal ): Promise { diff --git a/src/main/runtime/orchestration/legacy-remote-home/federated-worker-child-parent.ts b/src/main/runtime/orchestration/legacy-remote-home/federated-worker-child-parent.ts index e0249544e10..be0580a0127 100644 --- a/src/main/runtime/orchestration/legacy-remote-home/federated-worker-child-parent.ts +++ b/src/main/runtime/orchestration/legacy-remote-home/federated-worker-child-parent.ts @@ -1,5 +1,5 @@ import type { OrcaRuntimeService } from '../../orca-runtime' -import type { OrchestrationSessionCaller } from '../addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../addressing/orchestration-caller-identity' import { OrchestrationError } from '../orchestration-error' import { readCallerStoreInstallationId } from '../../hosts/platform-probe/host-descriptor' import { getCanonicalUserDataPath } from '../../../persistence/loading-store/user-data-path' @@ -22,7 +22,7 @@ export async function resolveFederatedChildParent(args: { | 'getRuntimeId' > from: string - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined serverCapabilities: readonly string[] | undefined serverName: string }): Promise< diff --git a/src/main/runtime/orchestration/legacy-remote-home/federated-worker-start-request.ts b/src/main/runtime/orchestration/legacy-remote-home/federated-worker-start-request.ts index 21dd2d80613..d4a6950dfc6 100644 --- a/src/main/runtime/orchestration/legacy-remote-home/federated-worker-start-request.ts +++ b/src/main/runtime/orchestration/legacy-remote-home/federated-worker-start-request.ts @@ -1,4 +1,4 @@ -import type { OrchestrationSessionCaller } from '../addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../addressing/orchestration-caller-identity' import type { OrcaRuntimeService } from '../../orca-runtime' import type { OrchestrationDb } from '../db/orchestration-db' import type { WorkerStartInput } from '../../rpc/methods/orchestration/worker/worker-start-schema' @@ -14,5 +14,5 @@ export type FederatedWorkerStartRequest = { taskDeps: string[] | undefined orchestrationMutation?: WorkerStartMutation /** The coordinator's resolved session, when it is one; recorded as the Dispatch creator. */ - callerSession?: OrchestrationSessionCaller + callerSession?: OrchestrationEntryCaller } diff --git a/src/main/runtime/orchestration/workers/launch/worker-dispatch-preamble.ts b/src/main/runtime/orchestration/workers/launch/worker-dispatch-preamble.ts index bc1fa776441..a68c56d5f2e 100644 --- a/src/main/runtime/orchestration/workers/launch/worker-dispatch-preamble.ts +++ b/src/main/runtime/orchestration/workers/launch/worker-dispatch-preamble.ts @@ -10,6 +10,8 @@ export type PreambleParams = { taskSpec: string /** A terminal handle, or a session's `orca_session_id:`; each is labelled by its kind. */ coordinatorHandle: string + /** The paired Orca's name when it coordinates this worker's Run from another computer. */ + coordinatedFrom?: string workerHandle: string devMode?: boolean // Packaged WSL uses orca-ide; other hosts retain their existing launcher. @@ -38,9 +40,11 @@ function isSessionHandle(handle: string): boolean { /** Terminal agents keep their handle's wording; only a session is named by its Orca session ID. */ function dispatchIdentityLines(params: PreambleParams): string { return [ - isSessionHandle(params.coordinatorHandle) - ? `Your coordinator's Orca session ID is: ${params.coordinatorHandle}` - : `Your coordinator's terminal handle is: ${params.coordinatorHandle}`, + params.coordinatedFrom + ? `You are coordinated from ${params.coordinatedFrom}. Your coordinator's address here is: ${params.coordinatorHandle}` + : isSessionHandle(params.coordinatorHandle) + ? `Your coordinator's Orca session ID is: ${params.coordinatorHandle}` + : `Your coordinator's terminal handle is: ${params.coordinatorHandle}`, `Your task ID is: ${params.taskId}`, ...(isSessionHandle(params.workerHandle) ? [`Your Orca session ID is: ${params.workerHandle}`] diff --git a/src/main/runtime/orchestration/workers/launch/worker-start-card.ts b/src/main/runtime/orchestration/workers/launch/worker-start-card.ts index 7f114fd9544..c68dca11f1a 100644 --- a/src/main/runtime/orchestration/workers/launch/worker-start-card.ts +++ b/src/main/runtime/orchestration/workers/launch/worker-start-card.ts @@ -6,7 +6,7 @@ import type { import type { AgentMessageSender } from '../../../../../shared/agent-session/agent-session-message-source' import type { AgentType } from '../../../../../shared/agent-status/contracts/agent-status-types' import type { OrcaRuntimeService } from '../../../orca-runtime' -import type { OrchestrationSessionCaller } from '../../addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../addressing/orchestration-caller-identity' export type AgentCreationCard = AgentJournalAgentStartItem | AgentJournalAgentHandoffItem @@ -48,7 +48,7 @@ export function startedAgentSender( export function prepareWorkerStartCard( runtime: OrcaRuntimeService, - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined ): (args: { dispatchId: string workerAddress: string @@ -56,7 +56,10 @@ export function prepareWorkerStartCard( /** What this start launched; the only kind known for a worker on another host. */ agent: AgentType | null }) => void { - const publish = prepareAgentCreationCard(callerSession?.sessionId) + // A paired Orca has no chat here to show the card in. + const publish = prepareAgentCreationCard( + callerSession?.kind === 'peer' ? undefined : callerSession?.sessionId + ) if (!publish) { return () => {} } diff --git a/src/main/runtime/rpc/core.ts b/src/main/runtime/rpc/core.ts index 729f5cf8e54..f135d2ae3f8 100644 --- a/src/main/runtime/rpc/core.ts +++ b/src/main/runtime/rpc/core.ts @@ -20,7 +20,7 @@ import type { DelegatedMobileDeviceSyncResult } from '../../../shared/delegated-mobile-device-contract' import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration/orchestration-compatibility-evidence' -import type { OrchestrationSessionCaller } from '../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../orchestration/addressing/orchestration-caller-identity' import type { RpcCallerIdentity } from './rpc-caller-identity' import type { OrcadPeerDesktopsResult } from '../../../shared/orcad-peer-desktops-contract' import type { RpcBinarySender } from './rpc-binary-sender' @@ -91,6 +91,8 @@ export type RpcRequest = { params?: unknown orchestrationContractVersion?: number orchestrationRequestId?: string + // Untrusted wire input; only a paired Orca's runtime pairing admits it (orchestration-peer-caller). + orchestrationPeerRun?: unknown compatibilityInvocationId?: string orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence // Untrusted wire evidence; runtime-source-admission validates it. @@ -151,8 +153,8 @@ export type RpcContext = { replayedMutationReceipt?: unknown // Compare declared Run handles against request attestation. orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence - // Resolve the injected Orca session id once at dispatch; session identity wins. - orchestrationCaller?: OrchestrationSessionCaller + // Resolved once at dispatch from a session's environment or a paired Orca's pairing; it wins. + orchestrationCaller?: OrchestrationEntryCaller // Only the trusted compatibility router sets this; params cannot bypass Run consumer binding. legacyCoordinatorRunId?: string legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof diff --git a/src/main/runtime/rpc/dispatcher-stream-options.ts b/src/main/runtime/rpc/dispatcher-stream-options.ts index 551b7db074d..21b77e43870 100644 --- a/src/main/runtime/rpc/dispatcher-stream-options.ts +++ b/src/main/runtime/rpc/dispatcher-stream-options.ts @@ -1,6 +1,6 @@ import type { OrcaRuntimeService } from '../orca-runtime' import { captureStructuredAgentCliSource } from '../../native-chat/structured-agent-cli-source' -import type { OrchestrationSessionCaller } from '../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../orchestration/addressing/orchestration-caller-identity' import { isRegistrationFencedUnsubscribe, type RpcContext, type RpcRequest } from './core' import { resolveRpcCallerIdentity } from './rpc-caller-identity' import type { RpcCallerScope } from './rpc-caller-scope' @@ -41,7 +41,7 @@ export function rpcContextFromTransport( runtime: OrcaRuntimeService, request: RpcRequest, options: RpcDispatchStreamingOptions | undefined, - orchestrationCaller: OrchestrationSessionCaller | undefined, + orchestrationCaller: OrchestrationEntryCaller | undefined, params?: unknown ): RpcContext { return { diff --git a/src/main/runtime/rpc/dispatcher.ts b/src/main/runtime/rpc/dispatcher.ts index a4daf612975..8ad691b1fda 100644 --- a/src/main/runtime/rpc/dispatcher.ts +++ b/src/main/runtime/rpc/dispatcher.ts @@ -15,7 +15,7 @@ import { withoutMethodsThisHostDoesNotServe } from './methods/terminal/session-t import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe' import type { OrcaRuntimeService } from '../orca-runtime' import { isDurableMutation } from '../../../shared/orchestration/orchestration-rpc-contract' -import type { OrchestrationSessionCaller } from '../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../orchestration/addressing/orchestration-caller-identity' import { getOrchestrationMutationExecutor, type DurableMutationInvocation, @@ -45,7 +45,7 @@ type AdmittedUnaryCall = { request: RpcRequest method: RpcMethod params: unknown - caller: OrchestrationSessionCaller | undefined + caller: OrchestrationEntryCaller | undefined } export class RpcDispatcher { @@ -231,7 +231,11 @@ export class RpcDispatcher { return desktopBrowser.result } - const compatibility = await legacyOrchestration.tryHandle(request, params, context.signal) + // A paired Orca is never a legacy caller, so its calls reach only the declared access check. + const compatibility = + caller?.kind === 'peer' + ? { handled: false as const } + : await legacyOrchestration.tryHandle(request, params, context.signal) if (compatibility.handled) { return compatibility.result } @@ -275,7 +279,7 @@ export class RpcDispatcher { effectiveParams, invoke, callerFingerprint, - context.orchestrationCaller?.orcaSessionId + context.orchestrationCaller ) if (request.method.startsWith('orchestration.')) { queueMicrotask(() => { diff --git a/src/main/runtime/rpc/methods/orchestration/access/orchestration-access-outcomes.test.ts b/src/main/runtime/rpc/methods/orchestration/access/orchestration-access-outcomes.test.ts index 0165250f962..694ff102ae1 100644 --- a/src/main/runtime/rpc/methods/orchestration/access/orchestration-access-outcomes.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/access/orchestration-access-outcomes.test.ts @@ -223,6 +223,7 @@ const CASES = { }) }, peerMailPull: { method: 'orchestration.peerMailPull', params: () => ({ waitMs: 0 }) }, + peerRunCancel: { method: 'orchestration.peerRunCancel', params: () => ({}) }, replyHelperQuestion: { method: 'orchestration.reply', params: (from, ids) => ({ id: ids.helperQuestion, body: 'a', from }) @@ -376,6 +377,7 @@ const EXPECTED: Record & Record = { 'mail_requires_server_pairing mail_requires_server_pairing mail_requires_server_pairing mail_requires_server_pairing', peerMailPull: 'mail_requires_server_pairing mail_requires_server_pairing mail_requires_server_pairing mail_requires_server_pairing', + peerRunCancel: 'forbidden forbidden forbidden forbidden', replyHelperQuestion: 'ok consumer_fenced consumer_fenced ok', replyMail: 'ok ok ok ok', replyQuestion: 'ok consumer_fenced consumer_fenced worker_cannot_coordinate_own_run', diff --git a/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts b/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts index 01c58ffda86..c36f9dd7336 100644 --- a/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts +++ b/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts @@ -3,6 +3,11 @@ import { types } from 'node:util' import type { ZodType } from 'zod' import type { RpcContext, RpcTypedMethod, RpcTypedStreamingMethod } from '../../../core' +import { + checkPeerAccess, + refusePeerAddressClaim, + refuseServerCallFromPeerRun +} from './peer-run-access' type ParsedParams = TSchema extends ZodType ? TSchema['_output'] @@ -28,6 +33,9 @@ export type OrchestrationSubjectKind = /** A field naming who is calling, as opposed to what the call acts on. */ export type OrchestrationCallerField = 'from' | 'terminal' | 'callerTerminalHandle' +/** The pane key a caller sends with its caller field, which only its own host can fill. */ +export type OrchestrationCallerPaneField = 'terminalPaneKey' | 'senderPaneKey' + type FieldOf

= P extends object ? Extract : never export type OrchestrationAccessRuleInput

= { @@ -43,6 +51,9 @@ export type OrchestrationAccessRule = (input: OrchestrationAccessRuleInput export type OrchestrationMethodAccess = { readonly kind: OrchestrationAccessKind readonly caller?: Extract, OrchestrationCallerField> + readonly callerPane?: Extract, OrchestrationCallerPaneField> + /** The field that sends the call on to a paired server (`--on`). */ + readonly server?: Extract, 'on'> /** Every field naming something the call acts on, kept apart from the caller field. */ readonly targets: Readonly, OrchestrationSubjectKind>>> /** A read returning a collection; a narrower caller restricts its query, never its page. */ @@ -54,32 +65,55 @@ export type OrchestrationMethodAccess = { export type OrchestrationMethodAccessDeclaration = { readonly kind: OrchestrationAccessKind readonly caller?: OrchestrationCallerField + readonly callerPane?: OrchestrationCallerPaneField + readonly server?: 'on' readonly targets: Readonly>> readonly collection?: true readonly rule?: OrchestrationAccessRule } -/** The one access check: every caller on this Orca is held to the method's own rule. */ +/** The one access check: every caller on this Orca is held to the method's own rule, and a paired + * Orca also to the Run it coordinates here. Returns the params the method runs with and its grant. */ export function checkOrchestrationAccess( access: OrchestrationMethodAccess, params: P, ctx: RpcContext -): G { +): { params: P; grant: G } { + const caller = ctx.orchestrationCaller + if (caller?.kind !== 'peer') { + refusePeerAddressClaim(access.caller && fieldValue(access.caller, params)) + refusePeerAddressClaim(access.callerPane && fieldValue(access.callerPane, params)) + } + const scoped = + caller?.kind === 'peer' + ? checkPeerAccess(access, params, ctx.runtime.getOrchestrationDb(), caller) + : params + if (access.server && fieldValue(access.server, scoped) !== undefined) { + refuseServerCallFromPeerRun(ctx, declaredCaller(access.caller, scoped)) + } if (!access.rule) { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: G defaults to undefined exactly when a method declares no rule. - return undefined as G + return { params: scoped, grant: undefined as G } + } + return { + params: scoped, + grant: access.rule({ params: scoped, ctx, caller: declaredCaller(access.caller, scoped) }) } - return access.rule({ params, ctx, caller: declaredCaller(access.caller, params) }) } function declaredCaller

( field: Extract, OrchestrationCallerField> | undefined, params: P ): string | undefined { - if (field === undefined) { - return undefined - } - const value: unknown = params[field] + return field === undefined ? undefined : fieldValue(field, params) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function fieldValue(field: string, params: unknown): string | undefined { + const value: unknown = isRecord(params) ? params[field] : undefined return typeof value === 'string' ? value : undefined } @@ -154,7 +188,8 @@ export function defineOrchestrationMethod< // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a replay returns the receipt this handler produced when the call was accepted; a promise-returning handler's wrapper is async. return replayed.receipt as TResult } - return handler(params, ctx, checkOrchestrationAccess(access, params, ctx)) + const checked = checkOrchestrationAccess(access, params, ctx) + return handler(checked.params, ctx, checked.grant) }) } } @@ -191,8 +226,10 @@ export function defineOrchestrationStreamingMethod< handler: checked( handler, declaration, - (params: ParsedParams, ctx: RpcContext, emit: (result: unknown) => void) => - handler(params, ctx, emit, checkOrchestrationAccess(access, params, ctx)) + (params: ParsedParams, ctx: RpcContext, emit: (result: unknown) => void) => { + const checked = checkOrchestrationAccess(access, params, ctx) + return handler(checked.params, ctx, emit, checked.grant) + } ) } } diff --git a/src/main/runtime/rpc/methods/orchestration/access/peer-run-access.ts b/src/main/runtime/rpc/methods/orchestration/access/peer-run-access.ts new file mode 100644 index 00000000000..6755ba30dbb --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/access/peer-run-access.ts @@ -0,0 +1,208 @@ +import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' +import { + isPeerCoordinatorAddress, + type OrchestrationPeerCaller +} from '../../../../orchestration/addressing/orchestration-caller-identity' +import { OrchestrationError } from '../../../../orchestration/orchestration-error' +import { parseOrcaSessionAddress } from '../../../../../../shared/orca-session-address' +import type { RpcContext } from '../../../core' +import { resolveDispatchCreator } from '../runs/dispatch-creator' +import type { + OrchestrationMethodAccessDeclaration, + OrchestrationSubjectKind +} from './orchestration-method-access' + +const NO_EFFECTS = { effectsApplied: false } as const + +/** Kinds named by an ID: one this Orca does not have is in no Run, and the method answers it as + * not found. Addresses also name terminals and sessions that work in no Run, so they must resolve. */ +const ID_KINDS: ReadonlySet = new Set([ + 'dispatch', + 'task', + 'message', + 'gate' +]) + +/** + * A paired Orca acts only inside the Run it coordinates here: every subject a call names must be in + * that Run, and a collection read is narrowed to it before its query runs, so counts and cursors + * stay right. Calls that act on this Orca as a whole are this Orca's own. Returns the params the + * method runs with. + */ +export function checkPeerAccess

( + access: OrchestrationMethodAccessDeclaration, + params: P, + db: OrchestrationDb, + peer: OrchestrationPeerCaller +): P { + const { run_id: runId, home_label: label } = peer.peerRun + if (access.kind === 'localOnly') { + throw new OrchestrationError( + 'forbidden', + `This command acts on this server as a whole, so only this server can run it, not ${label}. No effects were applied.`, + NO_EFFECTS + ) + } + const values: Record = + params && typeof params === 'object' ? { ...(params as Record) } : {} + for (const [field, kind] of Object.entries(access.targets)) { + for (const subject of subjectsOf(values[field])) { + const subjectRunId = kind && resolveSubjectRunId(db, kind, subject, peer) + if (kind && subjectRunId !== runId && !(subjectRunId === undefined && ID_KINDS.has(kind))) { + throw new OrchestrationError( + 'forbidden', + `${subject} is not in the Run ${label} coordinates here. No effects were applied.`, + NO_EFFECTS + ) + } + } + } + if (access.collection) { + const runField = Object.entries(access.targets).find(([, kind]) => kind === 'run')?.[0] + if (!runField) { + throw new OrchestrationError( + 'forbidden', + `This list spans every Run on this server, so ${label} cannot read it. No effects were applied.`, + NO_EFFECTS + ) + } + values[runField] = runId + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the same params with at most the declared Run field set to a string, which every Run field accepts. + return (params && typeof params === 'object' ? values : params) as P +} + +function subjectsOf(value: unknown): string[] { + if (Array.isArray(value)) { + return value.filter((item): item is string => typeof item === 'string') + } + if (typeof value !== 'string' || value === '') { + return [] + } + if (!value.startsWith('[') && !value.startsWith('{')) { + return [value] + } + // A JSON list of IDs (`--deps`), or send's JSON payload, which names its Dispatch by `dispatchId`. + let parsed: unknown + try { + parsed = JSON.parse(value) + } catch { + return [value] + } + if (Array.isArray(parsed)) { + return subjectsOf(parsed) + } + const dispatchId = + parsed && typeof parsed === 'object' && 'dispatchId' in parsed ? parsed.dispatchId : undefined + return typeof dispatchId === 'string' ? [dispatchId] : [] +} + +/** The Run a subject belongs to on this Orca, or undefined when it names nothing here. */ +export function resolveSubjectRunId( + db: OrchestrationDb, + kind: OrchestrationSubjectKind, + subject: string, + peer: OrchestrationPeerCaller +): string | undefined { + switch (kind) { + case 'run': + return subject + case 'dispatch': + return db.getDispatchContextById(subject)?.run_id + case 'task': + return db.getTask(subject)?.run_id + case 'message': + return db.getMessageById(subject)?.run_id + case 'gate': + return db.getGate(subject)?.run_id + case 'request': + // Receipts are keyed by the pairing, so they are its own; the same paired Orca may read another of its Runs' receipts. + return peer.peerRun.run_id + case 'worker': + case 'session': + return assigneeRunId(db, subject) + case 'party': + return partyRunId(db, subject, peer) + case 'conversation': + return undefined + } +} + +function partyRunId( + db: OrchestrationDb, + address: string, + peer: OrchestrationPeerCaller +): string | undefined { + if (address === peer.address) { + return peer.peerRun.run_id + } + if (address.startsWith('run:')) { + return address.slice('run:'.length) + } + if (address.startsWith('dispatch:')) { + return db.getDispatchContextById(address.slice('dispatch:'.length))?.run_id + } + // A group spans Runs, so it never resolves into one. + return address.startsWith('@') ? undefined : assigneeRunId(db, address) +} + +/** The Run of the newest Dispatch a terminal, pane or session was given. */ +function assigneeRunId(db: OrchestrationDb, subject: string): string | undefined { + const session = parseOrcaSessionAddress(subject) ?? subject + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column. + const row = db.db + .prepare( + `SELECT run_id FROM dispatch_contexts + WHERE assignee_handle = ?1 OR assignee_pane_key = ?1 OR assignee_orca_session_id = ?2 + ORDER BY rowid DESC LIMIT 1` + ) + .get(subject, session) as { run_id: string } | undefined + return row?.run_id +} + +/** Only a paired Orca's own frames speak as it: its address is predictable, so no other caller may + * declare it as its handle or pane and read or bind its Run's mail. */ +export function refusePeerAddressClaim(declared: string | undefined): void { + if (isPeerCoordinatorAddress(declared)) { + throw new OrchestrationError( + 'forbidden', + `${declared} is a paired Orca's coordinator address, and only that Orca can act as it. No effects were applied.`, + NO_EFFECTS + ) + } +} + +/** Pairing is one-way: work running in a paired Orca's Run, its helpers' included, never calls on + * to another server, and neither does that Orca through this one. */ +export function refuseServerCallFromPeerRun( + ctx: RpcContext, + callerHandle: string | undefined +): void { + const db = ctx.runtime.getOrchestrationDb() + const caller = ctx.orchestrationCaller + // Most servers coordinate no paired Orca's Run, and then the creator lookup (which can refuse) is skipped. + const peerRun = + caller?.kind === 'peer' + ? caller.peerRun + : db.hasPeerRuns() + ? peerRunOfCallerWork(ctx, db, callerHandle) + : undefined + if (peerRun) { + throw new OrchestrationError( + 'forbidden', + `This work is coordinated from ${peerRun.home_label}, so it cannot use --on. Ask that coordinator instead. No effects were applied.`, + NO_EFFECTS + ) + } +} + +function peerRunOfCallerWork( + ctx: RpcContext, + db: OrchestrationDb, + callerHandle: string | undefined +): ReturnType { + const creator = resolveDispatchCreator(ctx.runtime, callerHandle, ctx.orchestrationCaller) + const dispatchId = db.resolveCreatorDispatchId(creator) + const runId = dispatchId ? db.getDispatchContextById(dispatchId)?.run_id : undefined + return runId ? db.getPeerRun(runId) : undefined +} diff --git a/src/main/runtime/rpc/methods/orchestration/caller-show.ts b/src/main/runtime/rpc/methods/orchestration/caller-show.ts index 9ee58a20e4d..5d0acbef31b 100644 --- a/src/main/runtime/rpc/methods/orchestration/caller-show.ts +++ b/src/main/runtime/rpc/methods/orchestration/caller-show.ts @@ -75,9 +75,13 @@ export const ORCHESTRATION_CALLER_METHODS = [ _params, { orchestrationCaller, orchestrationCompatibilityEvidence, runtime } ): OrchestrationCallerShowResult => ({ - caller: orchestrationCaller - ? { orcaSessionId: formatOrcaSessionAddress(orchestrationCaller.orcaSessionId), live: true } - : null, + caller: + orchestrationCaller && orchestrationCaller.kind !== 'peer' + ? { + orcaSessionId: formatOrcaSessionAddress(orchestrationCaller.orcaSessionId), + live: true + } + : null, // An Orca-launched agent's own pane and launch secret prove it is one; no hook needs to have fired. terminal: !orchestrationCaller && diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-methods.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-methods.ts index c0e0e2f50a8..5d4c9640030 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-methods.ts @@ -4,10 +4,12 @@ import { ORCHESTRATION_FEDERATION_RELAY_METHODS } from '../../../../orchestration/legacy-remote/legacy-remote' import { ORCHESTRATION_PEER_MAIL_METHODS } from './peer-mail-methods' +import { ORCHESTRATION_PEER_RUN_METHODS } from './peer-run-methods' export const ORCHESTRATION_FEDERATION_METHODS = [ ...ORCHESTRATION_FEDERATION_ATTACH_METHODS, ...ORCHESTRATION_FEDERATION_RELAY_METHODS, ...ORCHESTRATION_FEDERATION_CONTROL_METHODS, - ...ORCHESTRATION_PEER_MAIL_METHODS + ...ORCHESTRATION_PEER_MAIL_METHODS, + ...ORCHESTRATION_PEER_RUN_METHODS ] diff --git a/src/main/runtime/rpc/methods/orchestration/federation/peer-coordinator.test.ts b/src/main/runtime/rpc/methods/orchestration/federation/peer-coordinator.test.ts new file mode 100644 index 00000000000..8d06f4d4123 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/federation/peer-coordinator.test.ts @@ -0,0 +1,437 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../../../shared/orchestration/orchestration-runtime-capabilities' +import { OrcaRuntimeService } from '../../../../orca-runtime' +import { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' +import { generateId } from '../../../../orchestration/db/generated-id' +import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture' +import { RpcDispatcher } from '../../../dispatcher' +import type { RpcResponse } from '../../../core' +import { ORCHESTRATION_METHODS } from '../orchestration' +import { startWorker } from '../worker/workers' +import { peerCoordinatorCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' + +const WORKER_PANE = 'tab_w:11111111-1111-4111-8111-111111111111' +const OTHER_PANE = 'tab_o:22222222-2222-4222-8222-222222222222' +const LOCAL_PANE = 'tab_l:33333333-3333-4333-8333-333333333333' +const PAIRED = { + authenticatedCallerFingerprint: 'fingerprint_a', + pairedDeviceId: 'device_a', + clientKind: 'runtime' as const +} + +describe('a paired Orca coordinating its Run on this server', () => { + let dir: string + let db: OrchestrationDb + let runtime: OrcaRuntimeService + let dispatcher: RpcDispatcher + let conversationId: string + let sequence = 0 + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'orca-peer-coordinator-')) + db = new OrchestrationDb(join(dir, 'b.db')) + runtime = new OrcaRuntimeService(null) + const panes: Record = { + term_w: WORKER_PANE, + term_o: OTHER_PANE, + term_l: LOCAL_PANE + } + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => panes[handle] ?? null) + vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockImplementation( + (handle) => panes[handle] ?? null + ) + runtime.setOrchestrationDb(db) + dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + conversationId = generateId('peerconv') + }) + + afterEach(() => { + db.close() + rmSync(dir, { recursive: true, force: true }) + vi.restoreAllMocks() + vi.useRealTimers() + }) + + function peerCall( + method: string, + params: Record, + options: { requestId?: string; conversation?: string } = {} + ): Promise { + sequence += 1 + return dispatcher.dispatch( + { + id: `peer_${sequence}`, + authToken: 'token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationPeerRun: { + runId: 'run_home', + conversationId: options.conversation ?? conversationId, + label: 'devA' + }, + ...(options.requestId ? { orchestrationRequestId: options.requestId } : {}) + }, + PAIRED + ) + } + + function localCall(method: string, params: Record): Promise { + sequence += 1 + return dispatcher.dispatch({ + id: `local_${sequence}`, + authToken: 'token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION + }) + } + + function peerRunId(): string { + return db.getPeerRunForConversation(conversationId)!.run_id + } + + /** The peer's Run with one worker in it, and a local Run with another. */ + async function twoRuns() { + await peerCall('orchestration.runCurrent', {}) + const peerTask = db.createTask({ spec: 'peer work', runId: peerRunId() }) + const peerWork = createRootDispatch(db, peerTask.id, 'term_w', WORKER_PANE) + const localRun = db.createRun({ + objective: 'local', + coordinatorHandle: 'term_l', + coordinatorPaneKey: LOCAL_PANE + }) + const localTask = db.createTask({ spec: 'local work', runId: localRun.id }) + const localWork = createRootDispatch(db, localTask.id, 'term_o', OTHER_PANE) + return { peerWork, localRun, localWork } + } + + async function pull(id: string, params: Record = {}) { + const response = await dispatcher.dispatch( + { + id, + authToken: 'token', + method: 'orchestration.peerMailPull', + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION + }, + PAIRED + ) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pull's result schema; a refusal reads as an empty page. + return (response.ok ? response.result : { items: [] }) as { + items: { conversationId: string; kind: string }[] + } + } + + function refused(response: RpcResponse): string | undefined { + return response.ok ? undefined : response.error.code + } + + it("writes the peer's Run when its frame arrives, before the call awaits anything", () => { + const pending = peerCall('orchestration.runCurrent', {}) + const run = db.getPeerRunForConversation(conversationId) + expect(run).toMatchObject({ home_run_id: 'run_home', home_label: 'devA', state: 'open' }) + expect(db.getRun(run!.run_id)).toBeDefined() + return pending + }) + + it('admits a peer Run only from a runtime pairing', async () => { + sequence += 1 + const response = await dispatcher.dispatch( + { + id: 'mobile', + authToken: 'token', + method: 'orchestration.runCurrent', + params: {}, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationPeerRun: { runId: 'run_home', conversationId, label: 'devA' } + }, + { ...PAIRED, clientKind: 'mobile' } + ) + expect(refused(response)).toBe('mail_requires_server_pairing') + expect(db.getPeerRunForConversation(conversationId)).toBeUndefined() + }) + + it('refuses commands that act on this server as a whole', async () => { + const { localRun } = await twoRuns() + expect(refused(await peerCall('orchestration.reset', { all: true }))).toBe('forbidden') + expect(refused(await peerCall('orchestration.runUse', { id: localRun.id }))).toBe('forbidden') + expect(db.getRun(localRun.id)).toBeDefined() + }) + + it("keeps another Run's worker out of reach by terminal, pane or session, but not its own", async () => { + const { peerWork, localWork } = await twoRuns() + db.db + .prepare('UPDATE dispatch_contexts SET assignee_orca_session_id = ? WHERE id = ?') + .run('33333333-3333-4333-8333-333333333334', localWork.id) + + for (const target of [ + { terminal: 'term_o' }, + { paneKey: OTHER_PANE }, + { sessionId: '33333333-3333-4333-8333-333333333334' } + ]) { + expect( + refused(await peerCall('orchestration.workerKeepOpen', { ...target, keep: true })) + ).toBe('forbidden') + } + expect(peerWork.assignee_handle).toBe('term_w') + expect( + (await peerCall('orchestration.workerKeepOpen', { terminal: 'term_w', keep: true })).ok + ).toBe(true) + }) + + it('reads only its own Run from an unqualified worker list and inbox', async () => { + const { peerWork, localRun } = await twoRuns() + db.insertMessage({ + from: 'term_w', + to: `run:${peerRunId()}`, + subject: 'mine', + runId: peerRunId() + }) + db.insertMessage({ + from: 'term_o', + to: `run:${localRun.id}`, + subject: 'theirs', + runId: localRun.id + }) + + const list = await peerCall('orchestration.workerList', { paginate: true }) + const inbox = await peerCall('orchestration.inbox', {}) + + expect(list.ok && list.result).toMatchObject({ + workers: [{ dispatchId: peerWork.id, coordinatedFrom: 'devA' }], + page: { total: 1 } + }) + expect(inbox.ok && inbox.result).toMatchObject({ messages: [{ subject: 'mine' }], count: 1 }) + }) + + it('sends and replies only inside its own Run, by any address form', async () => { + const { localRun, localWork } = await twoRuns() + const question = db.insertMessage({ + from: 'term_o', + to: `run:${localRun.id}`, + subject: 'Question', + type: 'question', + runId: localRun.id + }) + for (const to of ['term_o', `dispatch:${localWork.id}`, `run:${localRun.id}`, '@all']) { + expect(refused(await peerCall('orchestration.send', { to, subject: 'hi' }))).toBe('forbidden') + } + expect(refused(await peerCall('orchestration.reply', { id: question.id, body: 'yes' }))).toBe( + 'forbidden' + ) + }) + + it('cannot reach its former worker once that terminal coordinates local work', async () => { + await twoRuns() + db.createRun({ + objective: 'later local', + coordinatorHandle: 'term_w', + coordinatorPaneKey: WORKER_PANE + }) + expect(refused(await peerCall('orchestration.send', { to: 'term_w', subject: 'hi' }))).toBe( + 'forbidden' + ) + }) + + it('refuses a start that would place in its workspace, which it has none of here', async () => { + for (const worktree of ['current', 'new-child']) { + const response = await peerCall('orchestration.workerStart', { spec: 'work', worktree }) + expect(response.ok ? undefined : response.error).toMatchObject({ + code: 'invalid_argument', + message: expect.stringContaining('--on requires an exact remote worktree selector') + }) + } + }) + + it('follows up with a finished worker where that worker ran, though it has no workspace here', async () => { + await peerCall('orchestration.runCurrent', {}) + const peer = peerCoordinatorCaller(db.getPeerRunForConversation(conversationId)!) + const placed = startWorker( + { from: peer.address, run: peerRunId(), spec: 'One more thing' }, + { + runtime, + orchestrationMutation: undefined, + orchestrationCompatibilityEvidence: undefined, + orchestrationCaller: peer + }, + { + of: 'ctx_finished', + addressed: 'ctx_finished', + place: async () => { + throw new Error('placed where the worker ran') + } + } + ) + await expect(placed).rejects.toThrow('placed where the worker ran') + }) + + it("never answers one Run's cancel with another Run's receipt", async () => { + const other = generateId('peerconv') + await peerCall('orchestration.peerRunCancel', {}, { requestId: 'cancel:1.1' }) + const second = await peerCall( + 'orchestration.peerRunCancel', + {}, + { requestId: 'cancel:1.1', conversation: other } + ) + expect(second.ok && second.result).not.toMatchObject({ mutation: { replayed: true } }) + }) + + it('stays the only coordinator of its Run: a local terminal cannot bind it', async () => { + await twoRuns() + const response = await localCall('orchestration.runUse', { id: peerRunId(), from: 'term_l' }) + expect(refused(response)).toBe('consumer_fenced') + expect(db.getRun(peerRunId())?.coordinator_handle).toBe(`peer:${conversationId}`) + }) + + it('cannot be impersonated by a local caller declaring its address', async () => { + await twoRuns() + const address = `peer:${conversationId}` + const mail = db.insertMessage({ + from: 'term_w', + to: address, + subject: 'done', + runId: peerRunId() + }) + const check = await localCall('orchestration.check', { + terminal: address, + terminalPaneKey: address + }) + expect(refused(check)).toBe('forbidden') + expect(db.getMessageById(mail.id)?.read).toBe(0) + const send = await localCall('orchestration.send', { + from: 'term_l', + senderPaneKey: address, + to: 'term_w', + subject: 'hi' + }) + expect(refused(send)).toBe('forbidden') + }) + + it('keeps work in its Run, helpers included, from calling on to another server', async () => { + const { localRun } = await twoRuns() + const helperTask = db.createTask({ spec: 'helper', runId: peerRunId() }) + createRootDispatch(db, helperTask.id, 'term_l', LOCAL_PANE) + for (const [method, params] of [ + ['orchestration.send', { from: 'term_w', to: 'term_x', subject: 'hi', on: 'devC' }], + ['orchestration.workerStart', { from: 'term_l', spec: 'more', on: 'devC' }] + ] as const) { + const response = await localCall(method, params) + expect(response.ok ? undefined : response.error.message).toMatch(/coordinated from devA/) + } + expect(localRun.id).toBeDefined() + }) + + it('cancels over the ordinary stop and answers what it could not yet prove stopped', async () => { + await peerCall('orchestration.runCurrent', {}) + const task = db.createTask({ spec: 'peer work', runId: peerRunId() }) + const { dispatch } = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 5 + }) + const remaining = async (requestId: string) => { + const response = await peerCall('orchestration.peerRunCancel', {}, { requestId }) + return response.ok ? response.result : response.error + } + + // A worker still starting cannot be stopped yet, so the cancel is owed again. + expect(await remaining('cancel:1.1')).toMatchObject({ remaining: 1 }) + expect(db.getPeerRunForConversation(conversationId)?.state).toBe('closed') + db.transitionLifecycle({ entity: 'worker', id: dispatch.id, from: 'starting', to: 'failed' }) + // A lost answer's retry replays it; a new attempt runs the stops again. + expect(await remaining('cancel:1.1')).toMatchObject({ + remaining: 1, + mutation: { replayed: true } + }) + expect(await remaining('cancel:1.2')).toMatchObject({ remaining: 0 }) + }) + + it('cancels a Run it never had, and closes it', async () => { + const response = await peerCall('orchestration.peerRunCancel', {}, { requestId: 'cancel:1.1' }) + expect(response.ok && response.result).toMatchObject({ remaining: 0 }) + expect(db.getPeerRunForConversation(conversationId)?.state).toBe('closed') + // A closed Run still admits its coordinator's cancel. + const again = await peerCall('orchestration.peerRunCancel', {}, { requestId: 'cancel:1.2' }) + expect(again.ok && again.result).toMatchObject({ remaining: 0 }) + }) + + it('gives every conversation a share of each pull, so an unimportable head starves no other', async () => { + const stuck = conversationId + const other = generateId('peerconv') + await peerCall('orchestration.runCurrent', {}) + await peerCall('orchestration.runCurrent', {}, { conversation: other }) + const stuckRun = db.getPeerRunForConversation(stuck)!.run_id + const otherRun = db.getPeerRunForConversation(other)!.run_id + for (let index = 0; index < 60; index += 1) { + db.insertMessage({ + from: 'term_w', + to: `run:${stuckRun}`, + subject: `s${index}`, + runId: stuckRun + }) + } + db.insertMessage({ from: 'term_w', to: `run:${otherRun}`, subject: 'done', runId: otherRun }) + + const items = await pull('pull') + expect(items.items.filter((item) => item.conversationId === other)).toEqual([ + expect.objectContaining({ kind: 'worker_mail' }) + ]) + }) + + it('fits every pending conversation in one page when they number fewer than its size', async () => { + const conversations = Array.from({ length: 30 }, () => generateId('peerconv')) + for (const conversation of conversations) { + await peerCall('orchestration.runCurrent', {}, { conversation }) + const runId = db.getPeerRunForConversation(conversation)!.run_id + for (const subject of ['one', 'two']) { + db.insertMessage({ from: 'term_w', to: `run:${runId}`, subject, runId }) + } + } + const page = await pull('pull') + expect(new Set(page.items.map((item) => item.conversationId)).size).toBe(30) + }) + + it('rotates the page when more conversations are pending than it holds, so none starves', async () => { + const conversations = Array.from({ length: 51 }, () => generateId('peerconv')) + for (const conversation of conversations) { + await peerCall('orchestration.runCurrent', {}, { conversation }) + const runId = db.getPeerRunForConversation(conversation)!.run_id + db.insertMessage({ from: 'term_w', to: `run:${runId}`, subject: 'stuck', runId }) + } + const seen = new Set() + for (const id of ['pull_1', 'pull_2']) { + for (const item of (await pull(id)).items) { + seen.add(item.conversationId) + } + } + expect(seen.size).toBe(51) + }) + + it('wakes a waiting pull when a worker in its Run changes state', async () => { + await peerCall('orchestration.runCurrent', {}) + const task = db.createTask({ spec: 'peer work', runId: peerRunId() }) + const { dispatch } = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 5 + }) + db.acknowledgeFederationRelay({ + dispatchId: conversationId, + direction: 'to_home', + throughSequence: 1_000 + }) + // The pull's own deadline never passes, so only the state change can answer it. + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + const waiting = pull('pull', { waitMs: 20_000 }) + await new Promise((resolve) => setImmediate(resolve)) + db.transitionLifecycle({ entity: 'worker', id: dispatch.id, from: 'starting', to: 'ready' }) + + expect((await waiting).items).toEqual([expect.objectContaining({ kind: 'worker_state' })]) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/peer-mail-methods.ts b/src/main/runtime/rpc/methods/orchestration/federation/peer-mail-methods.ts index 6d79a0577dd..7c7b5df0a94 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/peer-mail-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/peer-mail-methods.ts @@ -35,17 +35,24 @@ function requireRuntimePairing( return ctx.authenticatedCallerFingerprint } +// Where each pairing's next page starts, per database, when more conversations are pending than a page holds. +const pullRotations = new WeakMap>() + +/** Every conversation with pending items gets an equal share of the page, so an item the paired Orca + * cannot import yet, which stays at its conversation's head, never starves another conversation. */ function pendingReplies(db: OrchestrationDb, peerFingerprint: string): FederationRelayItemRow[] { - const items: FederationRelayItemRow[] = [] - for (const conversationId of db.listPeerConversationIds(peerFingerprint)) { - if (items.length >= PULL_PAGE_SIZE) { - break - } - items.push( - ...db.listPendingFederationRelay(conversationId, 'to_home', PULL_PAGE_SIZE - items.length) - ) - } - return items + const pending = db + .listPeerConversationIds(peerFingerprint) + .filter((id) => db.listPendingFederationRelay(id, 'to_home', 1).length > 0) + const share = Math.max(1, Math.floor(PULL_PAGE_SIZE / pending.length)) + const pullRotation = pullRotations.get(db.db) ?? new Map() + pullRotations.set(db.db, pullRotation) + const start = pending.length > PULL_PAGE_SIZE ? (pullRotation.get(peerFingerprint) ?? 0) : 0 + pullRotation.set(peerFingerprint, (start + PULL_PAGE_SIZE) % Math.max(1, pending.length)) + return [...pending.slice(start), ...pending.slice(0, start)] + .slice(0, PULL_PAGE_SIZE) + .flatMap((id) => db.listPendingFederationRelay(id, 'to_home', share)) + .slice(0, PULL_PAGE_SIZE) } export const ORCHESTRATION_PEER_MAIL_METHODS = [ @@ -154,8 +161,10 @@ export const ORCHESTRATION_PEER_MAIL_METHODS = [ const db = ctx.runtime.getOrchestrationDb() let items = pendingReplies(db, peerFingerprint) const waitMs = Math.min(Math.max(params.waitMs ?? 0, 0), MAX_PULL_WAIT_MS) - if (items.length === 0 && waitMs > 0) { - await waitForPeerMail(db, waitMs, ctx.signal) + const deadline = Date.now() + waitMs + // A wake is any queued change on this Orca, so one that queued nothing for this peer waits on. + while (items.length === 0 && Date.now() < deadline && !ctx.signal?.aborted) { + await waitForPeerMail(db.db, deadline - Date.now(), ctx.signal) items = pendingReplies(ctx.runtime.getOrchestrationDb(), peerFingerprint) } return { @@ -163,6 +172,7 @@ export const ORCHESTRATION_PEER_MAIL_METHODS = [ conversationId: item.dispatch_id, sequence: item.sequence, messageId: item.message_id, + kind: item.kind, payload: item.payload })) } diff --git a/src/main/runtime/rpc/methods/orchestration/federation/peer-mail.test.ts b/src/main/runtime/rpc/methods/orchestration/federation/peer-mail.test.ts index c624b2989fa..3ee7865d728 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/peer-mail.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/peer-mail.test.ts @@ -385,7 +385,7 @@ describe('agent mail between paired Orca servers', () => { a.db.resetMessages() // The ack was lost: the server serves the stored reply again. b.db['db'].prepare('UPDATE federation_relay_items SET acked_at = NULL').run() - notifyPeerMailQueued(b.db) + notifyPeerMailQueued(b.db.db) await vi.waitFor(() => expect(b.db.listPendingFederationRelay(conversationId!, 'to_home')).toEqual([]) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/peer-run-methods.ts b/src/main/runtime/rpc/methods/orchestration/federation/peer-run-methods.ts new file mode 100644 index 00000000000..a64ce31c976 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/federation/peer-run-methods.ts @@ -0,0 +1,54 @@ +import { defineOrchestrationMethod } from '../access/orchestration-method-access' +import { OrchestrationError } from '../../../../orchestration/orchestration-error' +import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' +import { WORKER_SETTLED_STATES } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership' +import { + PeerRunCancelParams, + type PeerRunCancelResult +} from '../../../../../../shared/orchestration/remote-owned-wire' +import type { z } from 'zod' +import { stopWorkerAndHelpers } from '../worker/worker-stop' + +export const ORCHESTRATION_PEER_RUN_METHODS = [ + defineOrchestrationMethod({ + name: 'orchestration.peerRunCancel', + access: { kind: 'writes', targets: {} }, + permission: 'workspace', + params: PeerRunCancelParams, + /** The paired Orca reset its Run: close it here, then run the ordinary stop on each live worker. */ + handler: async (_params, { runtime, orchestrationCaller }) => { + if (orchestrationCaller?.kind !== 'peer') { + throw new OrchestrationError( + 'forbidden', + 'Only the paired Orca that coordinates a Run cancels it. No effects were applied.', + { effectsApplied: false } + ) + } + const db = runtime.getOrchestrationDb() + const runId = orchestrationCaller.peerRun.run_id + // Committed before any stop, so nothing more starts in it while the stops run. + db.closePeerRun(runId) + for (const dispatchId of liveWorkerDispatchIds(db, runId)) { + // Each stop's own outcome stays on its Dispatch; the count below is the answer. + await stopWorkerAndHelpers(runtime, dispatchId, undefined).catch(() => undefined) + } + const result: z.infer = { + remaining: liveWorkerDispatchIds(db, runId).length + } + return result + } + }) +] + +/** Workers in the Run not yet proven settled; a Run this Orca never had work in has none. */ +function liveWorkerDispatchIds(db: OrchestrationDb, runId: string): string[] { + const settled = WORKER_SETTLED_STATES.map(() => '?').join(', ') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column. + const rows = db.db + .prepare( + `SELECT d.id FROM dispatch_contexts d JOIN worker_dispatches w ON w.dispatch_id = d.id + WHERE d.run_id = ? AND w.state NOT IN (${settled}) ORDER BY d.rowid` + ) + .all(runId, ...WORKER_SETTLED_STATES) as { id: string }[] + return rows.map((row) => row.id) +} diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts index cc9251aeff5..11af8dd3c25 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts @@ -22,6 +22,7 @@ export const ORCHESTRATION_CHECK_METHODS = [ access: { kind: 'writes', caller: 'terminal', + callerPane: 'terminalPaneKey', targets: { run: 'run', ack: 'message' }, collection: true }, diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-run-pending-mail.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-run-pending-mail.ts index 2b90558c392..31ec54fb285 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-run-pending-mail.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-run-pending-mail.ts @@ -7,7 +7,7 @@ import { acknowledgedOnlyReceipt, handsOverBatch } from './mailbox-message-recei import { currentDispatchAssigneeRun } from './recipient-routing' import { callerHoldsDispatchPane, dispatchFenced } from './dispatch-mailbox-fence' import { orchestrationCallerIdentity } from '../runs/run-scope' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { CheckParams } from '../../../../../../shared/rpc-contract/orchestration-params' import type { z } from 'zod' @@ -18,7 +18,7 @@ export async function checkRunPendingMail(args: { run: RunRow handle: string paneKey: string | undefined - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined typeFilter: MessageType[] | undefined signal: AbortSignal | undefined revalidateConsumer: () => void diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts index 49ef6a58dbe..b52ac709c7e 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts @@ -11,7 +11,7 @@ import { routeAllMailboxPages } from './mailbox-page-routing' import { resolveRunScope } from '../runs/run-scope' import type { CheckParams } from '../../../../../../shared/rpc-contract/orchestration-params' import type { z } from 'zod' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import { checkRunPendingMail } from './check-run-pending-mail' import { FEDERATION_RELAY_WAIT_REARM_MS } from '../../../../orchestration/federation/relay/federation-relay-scheduler' @@ -23,7 +23,7 @@ export async function checkRunMailbox(args: { db: OrchestrationDb handle: string paneKey: string | undefined - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined typeFilter: MessageType[] | undefined signal: AbortSignal | undefined legacyCoordinatorRunId: string | undefined diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts b/src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts index d38486eb25d..944a3dfcb14 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts @@ -1,5 +1,5 @@ import type { OrchestrationCompatibilityEvidence } from '../../../../../../shared/orchestration/orchestration-compatibility-evidence' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { MessageRow } from '../../../../orchestration/db/orchestration-records' import type { OrcaRuntimeService } from '../../../../orca-runtime' import { assertCallerHandleMatchesEvidence, orchestrationCallerIdentity } from '../runs/run-scope' @@ -10,7 +10,7 @@ export function helperQuestionAnswerer( args: { question: MessageRow from: string | undefined - session: OrchestrationSessionCaller | undefined + session: OrchestrationEntryCaller | undefined evidence: OrchestrationCompatibilityEvidence | undefined } ): { dispatchId: string; consumerGeneration: number } | undefined { diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts index 027c7f424e3..fff196ea676 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts @@ -18,16 +18,18 @@ export const ORCHESTRATION_MESSAGE_METHODS = [ name: 'orchestration.inbox', permission: 'workspace', params: InboxParams, - access: { kind: 'reads', targets: { terminal: 'party' }, collection: true }, + access: { kind: 'reads', targets: { terminal: 'party', run: 'run' }, collection: true }, handler: (params, { runtime }) => { const db = runtime.getOrchestrationDb() // Return empty for stale/unknown handles because historical mail outlives handle deletion (design doc §3.3). const messages = params.terminal ? db.getAllMessagesForHandle( resolveOrchestrationParty(params.terminal, db).address, - params.limit + params.limit, + undefined, + params.run ) - : db.getInbox(params.limit) + : db.getInbox(params.limit, params.run) return { messages: messages.map(({ answer_sender_snapshot: _answer, ...message }) => message), count: messages.length diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-reply.ts b/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-reply.ts index e78a9abda6a..f23c85fdc82 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-reply.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-reply.ts @@ -1,5 +1,5 @@ import type { OrcaRuntimeService } from '../../../../orca-runtime' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { PeerMailMessageRow } from '../../../../orchestration/db/federation/peer-mail-store' import type { MessageRow } from '../../../../orchestration/db/orchestration-records' import { OrchestrationError } from '../../../../orchestration/orchestration-error' @@ -21,7 +21,7 @@ export async function replyOverPeerMail(args: { original: MessageRow route: PeerMailMessageRow params: { body: string; from?: string; on?: string } - orchestrationCaller: OrchestrationSessionCaller | undefined + orchestrationCaller: OrchestrationEntryCaller | undefined orchestrationMutation: { callerFingerprint: string; requestId: string } | undefined recordMutationReceipt: ((receipt: unknown) => void) | undefined }): Promise { @@ -67,7 +67,7 @@ export async function replyOverPeerMail(args: { return recordReceiptBeforeNudge( args.recordMutationReceipt, { message: { id, subject }, to: original.from_handle }, - () => notifyPeerMailQueued(db), + () => notifyPeerMailQueued(db.db), { kind: 'messages', targets: [] } ) } diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-send.ts b/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-send.ts index a9fa69344b3..a18969d81c4 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-send.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/peer-mail-send.ts @@ -1,6 +1,6 @@ import type { z } from 'zod' import type { OrcaRuntimeService } from '../../../../orca-runtime' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { OrchestrationCompatibilityCallerAuthority } from '../../../../orchestration/terminal-authority/orchestration-compatibility-authority' import { OrchestrationError } from '../../../../orchestration/orchestration-error' import { @@ -20,7 +20,7 @@ const NOTHING_SENT = { effectsApplied: false } as const export function peerMailLocalCaller(args: { runtime: OrcaRuntimeService from: string | undefined - orchestrationCaller: OrchestrationSessionCaller | undefined + orchestrationCaller: OrchestrationEntryCaller | undefined attestedCaller: OrchestrationCompatibilityCallerAuthority | undefined }): { address: string; paneKey: string | null } { if (args.orchestrationCaller) { @@ -51,7 +51,7 @@ export function peerMailLocalCaller(args: { export async function sendPeerMail(args: { params: z.infer & { on: string } runtime: OrcaRuntimeService - orchestrationCaller: OrchestrationSessionCaller | undefined + orchestrationCaller: OrchestrationEntryCaller | undefined attestedCaller: OrchestrationCompatibilityCallerAuthority | undefined orchestrationMutation: { callerFingerprint: string; requestId: string } | undefined recordMutationReceipt: ((receipt: unknown) => void) | undefined diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/reply-method.ts b/src/main/runtime/rpc/methods/orchestration/messaging/reply-method.ts index 94e0996ac2d..19d52a02e68 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/reply-method.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/reply-method.ts @@ -29,6 +29,7 @@ export const ORCHESTRATION_REPLY_METHOD = defineOrchestrationMethod({ access: { kind: 'writes', caller: 'from', + server: 'on', targets: { id: 'message', run: 'run' }, rule: replyAccess }, diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts index 1d9c485e919..a0ad726c0cd 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts @@ -39,6 +39,8 @@ export const ORCHESTRATION_SEND_METHODS = [ access: { kind: 'writes', caller: 'from', + callerPane: 'senderPaneKey', + server: 'on', targets: { to: 'party', threadId: 'message', run: 'run', payload: 'dispatch' } }, permission: 'workspace', @@ -172,6 +174,17 @@ export const ORCHESTRATION_SEND_METHODS = [ if (!recipient.ok) { throw new OrchestrationError(recipient.code, recipient.message) } + // A paired Orca's former worker may since have taken up local work; it is no longer its to reach. + if ( + orchestrationCaller?.kind === 'peer' && + recipient.runId !== orchestrationCaller.peerRun.run_id + ) { + throw new OrchestrationError( + 'forbidden', + `${params.to} now works outside the Run ${orchestrationCaller.peerRun.home_label} coordinates here. No effects were applied.`, + { effectsApplied: false } + ) + } to = recipient.to messageRunId = recipient.runId if (recipient.warning) { diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-point-to-point.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-point-to-point.ts index 56273cfec5a..85752b5aa3d 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-point-to-point.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-point-to-point.ts @@ -8,6 +8,7 @@ import { reconcileLifecycleMessage } from '../../../../orchestration/workers/lif import { bindCoordinatorMutationPayload } from '../../../../orchestration/workers/reports/dispatch-message-binding' import { isDispatchMutationMessageType } from '../../../../../../shared/rpc-contract/orchestration-params' import { parseMessageTaskId } from './message-payload' +import { isDispatchAssignee } from '../../../../orchestration/db/dispatch-context/dispatch-lookup' import type { SendParams } from './send-params' import { legacyWorkerDeliveryContract } from '../routing' import { exposeMessage } from './mailbox-message-receipt' @@ -74,6 +75,8 @@ export function sendPointToPointMessage(args: { ? bindCoordinatorMutationPayload(messageType, params.payload, dispatch.id) : params.payload, senderPaneKey, + senderDispatchId: + dispatch && isDispatchAssignee(dispatch, from, senderPaneKey) ? dispatch.id : undefined, runId: messageRunId, deliveryContract: legacyWorkerDeliveryContract( runtime, diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/settled-worker-follow-through.ts b/src/main/runtime/rpc/methods/orchestration/messaging/settled-worker-follow-through.ts index fc223637a6a..3aec8ace463 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/settled-worker-follow-through.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/settled-worker-follow-through.ts @@ -10,7 +10,11 @@ import { receivesMailWhole } from '../../../../orchestration/db/dispatch-context/worker-follow-up-lineage' import { FOLLOW_UP_MESSAGE_TYPES } from '../../../../orchestration/db/dispatch-context/worker-follow-up-plan' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import { + peerCoordinatorAddress, + peerCoordinatorCaller, + type OrchestrationEntryCaller +} from '../../../../orchestration/addressing/orchestration-caller-identity' import { resolveOrcaSessionParty } from '../../../../orchestration/addressing/orchestration-party' import { readStructuredAgentSessionRecord } from '../../../../orchestration/terminal-authority/structured-sessions/structured-worker-authority' import { createHash } from 'node:crypto' @@ -111,7 +115,7 @@ async function continueUnreadMail( outcome = plan?.kind === 'start' ? await startWorkerFollowUp({ - ctx: { runtime, orchestrationCaller: senderSession(db, first.from_handle) }, + ctx: { runtime, orchestrationCaller: senderCaller(db, first) }, db, plan, from: first.from_handle, @@ -304,7 +308,16 @@ export function continueAllUnreadWorkerMail( } } -function senderSession(db: OrchestrationDb, from: string): OrchestrationSessionCaller | undefined { +/** The sender continues as itself: a session as its session, a paired Orca as the coordinator of its Run. */ +function senderCaller( + db: OrchestrationDb, + message: { from_handle: string; run_id: string } +): OrchestrationEntryCaller | undefined { + const peerRun = db.getPeerRun(message.run_id) + if (peerRun && message.from_handle === peerCoordinatorAddress(peerRun.conversation_id)) { + return peerCoordinatorCaller(peerRun) + } + const from = message.from_handle const sessionId = parseOrcaSessionAddress(from) const record = sessionId ? readStructuredAgentSessionRecord(sessionId) : null return sessionId && record diff --git a/src/main/runtime/rpc/methods/orchestration/orchestration-caller-workspace.ts b/src/main/runtime/rpc/methods/orchestration/orchestration-caller-workspace.ts index d22964215f7..ce78dca6226 100644 --- a/src/main/runtime/rpc/methods/orchestration/orchestration-caller-workspace.ts +++ b/src/main/runtime/rpc/methods/orchestration/orchestration-caller-workspace.ts @@ -2,14 +2,18 @@ * showTerminal requires a real PTY; fabricating its writable fields would break terminal verbs for chat workers. */ import type { OrcaRuntimeService } from '../../../orca-runtime' -import type { OrchestrationSessionCaller } from '../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../orchestration/addressing/orchestration-caller-identity' import { isStructuredWorkerHandle } from '../../../../../shared/orchestration/workers/structured-worker-handle' export async function resolveDispatchCallerWorktreeId( runtime: Pick, callerHandle: string, - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined ): Promise { + // A paired Orca has no workspace here, so `current` and `new-child` refuse as for an unmanaged one. + if (callerSession?.kind === 'peer') { + return PEER_COORDINATOR_HAS_NO_WORKSPACE + } // A session caller's workspace is on its record, whichever owner (chat or terminal) holds it. if (callerSession) { return callerSession.workspaceId @@ -22,3 +26,5 @@ export async function resolveDispatchCallerWorktreeId( } return (await runtime.showTerminal(callerHandle)).worktreeId } + +const PEER_COORDINATOR_HAS_NO_WORKSPACE = 'peer-coordinator-has-no-workspace' diff --git a/src/main/runtime/rpc/methods/orchestration/runs/dispatch-creator.ts b/src/main/runtime/rpc/methods/orchestration/runs/dispatch-creator.ts index c43da58338e..fb415f5b858 100644 --- a/src/main/runtime/rpc/methods/orchestration/runs/dispatch-creator.ts +++ b/src/main/runtime/rpc/methods/orchestration/runs/dispatch-creator.ts @@ -1,5 +1,5 @@ import type { DispatchCreator } from '../../../../orchestration/db/dispatch-depth' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { OrcaRuntimeService } from '../../../../orca-runtime' import { orchestrationCallerIdentity } from './run-scope' @@ -7,7 +7,7 @@ import { orchestrationCallerIdentity } from './run-scope' export function resolveDispatchCreator( runtime: OrcaRuntimeService, callerHandle: string | undefined, - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined ): DispatchCreator { if (!callerHandle) { // No declared caller resolves no parent; retain legacy Run-binding depth zero. diff --git a/src/main/runtime/rpc/methods/orchestration/runs/run-scope.ts b/src/main/runtime/rpc/methods/orchestration/runs/run-scope.ts index a5993600973..5d59eb85261 100644 --- a/src/main/runtime/rpc/methods/orchestration/runs/run-scope.ts +++ b/src/main/runtime/rpc/methods/orchestration/runs/run-scope.ts @@ -8,7 +8,7 @@ import { hasRunBindingKey, mailboxAddressOf, runCoordinatorKey, - type OrchestrationSessionCaller + type OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { OrchestrationPartyIdentity as OrchestrationCallerIdentity } from '../../../../../../shared/orchestration/mailbox/orchestration-party-identity' import { resolveDeclaredCallerParty } from '../../../../orchestration/addressing/orchestration-party' @@ -24,7 +24,7 @@ export type RunScopeParams = { callerTerminalHandle?: string callerPaneKey?: string /** Resolved at the dispatch entry; when set it is the caller, whatever the declared handle. */ - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined requireCurrentConsumer: boolean legacyCoordinatorRunId?: string // Attest user-supplied handles against this trusted handle. @@ -56,7 +56,7 @@ export function orchestrationCallerIdentity( caller: { handle: string paneKey: string | null | undefined - session: OrchestrationSessionCaller | undefined + session: OrchestrationEntryCaller | undefined } ): OrchestrationCallerIdentity { if (caller.session) { @@ -70,7 +70,7 @@ export type OrchestrationCallerParams = { callerTerminalHandle: string callerEvidence?: OrchestrationCompatibilityEvidence callerAuthority?: OrchestrationCompatibilityCallerAuthority - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined /** Preserve legacy callers that treated a missing pane as an ordinary fence. */ requireStablePane?: boolean /** Only skip attestation when the caller does it after the legacy-takeover check (run-use); omitting that reopens impersonation. */ diff --git a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts index da0075d287c..dc79fd4eef1 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts @@ -60,6 +60,7 @@ export async function deliverWorkerDispatchPreamble(args: { dispatchId: args.dispatchId, taskSpec: args.taskSpec, coordinatorHandle: orcaSessionIdOrHandle(args.coordinatorHandle, args.db), + coordinatedFrom: args.db.getPeerRun(args.runId)?.home_label, // Its mailbox stays keyed by the handle; its commands name its Orca session ID, which binds to it. workerHandle: structuredSession && isOrcaSessionId(structuredSession.identity.sessionId) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts b/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts index a2d53e01de2..6adb724d6f9 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts @@ -1,5 +1,6 @@ import type { OrcaRuntimeService } from '../../../../orca-runtime' import { OrchestrationError } from '../../../../orchestration/orchestration-error' +import { isPeerCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import { isStructuredWorkerHandle } from '../../../../../../shared/orchestration/workers/structured-worker-handle' import type { OrchestrationPartyIdentity as OrchestrationCallerIdentity, @@ -27,8 +28,9 @@ export async function assertExplicitWorkerTerminalUsable(args: { const targetPane = runtime.getTerminalPaneKey(terminal) const callerPane = coordinator?.paneKey ?? runtime.getTerminalPaneKey(from) // Structured coordinators may have no showable terminal; compare their handle/pane identity, or session address for handle-less callers. + // A paired Orca's coordinator has no terminal here either; its address is its handle. const coordinatorHandle = - coordinator?.terminalHandle === null + coordinator?.terminalHandle === null || isPeerCaller(coordinator) ? coordinator.address : isStructuredWorkerHandle(from) ? from diff --git a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts index 5d26db76cb2..50ee585d311 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts @@ -64,6 +64,7 @@ export function createLocalWorkerStartDispatch( requiredCreatorDispatchId: helperCreator?.id, taskId: existingTask?.id, taskSpec: params.spec, + adoptTaskId: params.adoptTaskId, taskTitle: params.taskTitle, taskDeps, // A helper's Task sits under its creator's Task, so the Run's Task tree shows who started it. diff --git a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts index 1af01cf6710..6d7446b10f6 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts @@ -11,7 +11,7 @@ import type { RunRow, TaskRow } from '../../../../orchestration/db/orchestration-records' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { OrchestrationPartyIdentity as OrchestrationCallerIdentity } from '../../../../../../shared/orchestration/mailbox/orchestration-party-identity' import { resolveDispatchCallerWorktreeId } from '../orchestration-caller-workspace' import { resolveWorkerStartModeOnHost } from '../structured-workers/orchestration-worker-start-mode' @@ -48,7 +48,7 @@ export async function startLocalWorker(args: { bindRun: () => RunRow taskDeps: string[] | undefined coordinator: OrchestrationCallerIdentity | null - callerSession?: OrchestrationSessionCaller + callerSession?: OrchestrationEntryCaller existingTask?: TaskRow orchestrationMutation?: WorkerStartMutation /** Settings-driven; the executing host still gets to refuse below. */ diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts index b506a17124f..c8083a27648 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts @@ -251,6 +251,7 @@ async function projectWorkerListPageWithFilteredSnapshot( dispatchStatus: row.dispatchStatus, agentTerminalHandle: row.agentTerminalHandle, terminalState: row.terminalState, + coordinatedFrom: row.coordinatedFrom, resource: row.resource ? exposeWorkerTerminalResource(row.resource) : null, // Emit only derived ownership; resource identity/release fields already live on the row. projection: { diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts index 3df8bd1e22f..d54f92f4c16 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts @@ -4,7 +4,7 @@ import { liveDispatchCreatorEvaluation } from '../../../../orchestration/db/disp import { selectExistingWorkerStartRun } from './worker-start-run-authority' import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' import { OrchestrationError } from '../../../../orchestration/orchestration-error' -import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' +import type { OrchestrationEntryCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import type { OrchestrationPartyIdentity as OrchestrationCallerIdentity } from '../../../../../../shared/orchestration/mailbox/orchestration-party-identity' import type { DispatchContextRow, @@ -27,7 +27,7 @@ export function resolveWorkerStartRun( args: { coordinator: OrchestrationCallerIdentity | null from: string - callerSession: OrchestrationSessionCaller | undefined + callerSession: OrchestrationEntryCaller | undefined runId: string | undefined /** Objective for a Run this start may create; absent when it names an existing Task. */ objective: string | undefined @@ -120,7 +120,7 @@ export function qualifyLiveWorkerFollowUp( runtime: OrcaRuntimeService, coordinator: OrchestrationCallerIdentity | null, from: string, - session: OrchestrationSessionCaller | undefined, + session: OrchestrationEntryCaller | undefined, runId: string, followUp: Parameters>[1] ) { diff --git a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts index dcf2e63ef94..2538b27521b 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts @@ -40,13 +40,15 @@ export const ORCHESTRATION_WORKER_START_METHODS = [ access: { kind: 'writes', caller: 'from', + server: 'on', targets: { task: 'task', parent: 'task', deps: 'task', run: 'run', retryOf: 'dispatch', - terminal: 'worker' + terminal: 'worker', + adoptTaskId: 'task' } }, requires: ['accounts'], @@ -81,6 +83,26 @@ export async function startWorker( { effectsApplied: false } ) } + if (params.adoptTaskId !== undefined && orchestrationCaller?.kind !== 'peer') { + throw new OrchestrationError( + 'invalid_argument', + 'Only the paired Orca coordinating a Run names its Task IDs. No effects were applied.', + { effectsApplied: false } + ) + } + const placement = params.worktree ?? 'current' + if ( + orchestrationCaller?.kind === 'peer' && + !followUp && + !params.terminal && + /^(current|new-child)$/.test(placement) + ) { + throw new OrchestrationError( + 'invalid_argument', + `--on requires an exact remote worktree selector or new-top-level: ${orchestrationCaller.peerRun.home_label} has no workspace on this server for --worktree ${placement}. No effects were applied.`, + { effectsApplied: false } + ) + } const readinessTimeoutMs = resolveWorkerStartReadinessTimeoutMs(params.timeoutMs) const db = runtime.getOrchestrationDb() const coordinator = resolveOrchestrationCaller(runtime, { diff --git a/src/main/runtime/rpc/orchestration-mutation-executor.test.ts b/src/main/runtime/rpc/orchestration-mutation-executor.test.ts index 2bda0a49813..c3e7a5466b8 100644 --- a/src/main/runtime/rpc/orchestration-mutation-executor.test.ts +++ b/src/main/runtime/rpc/orchestration-mutation-executor.test.ts @@ -50,6 +50,7 @@ function createHarness() { vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('window-1:leaf-prompt') return { db, + runtime, executor: new OrchestrationMutationExecutor(runtime), bindTerminal: (next: { generation: number; processIncarnation: string }) => { binding.mockReturnValue({ ptyId: 'pty-prompt', ...next }) @@ -236,3 +237,27 @@ describe('worker start mutation coalescing', () => { expect(invoke).toHaveBeenCalledOnce() }) }) + +describe('a stop or Run cancel interrupted by a restart', () => { + it.each(['orchestration.workerStop', 'orchestration.peerRunCancel'])( + '%s runs again on a retry with the same request ID instead of answering operation_unknown', + async (method) => { + const harness = createHarness() + const params = method === 'orchestration.workerStop' ? { dispatch: 'ctx_1' } : {} + // The first attempt's receipt is pending when this runtime goes away. + void harness.executor.run( + workerStartRequest(method, 'stop-1.1:ctx_1', params), + params, + () => new Promise(() => {}) + ) + const restarted = new OrchestrationMutationExecutor(harness.runtime) + + const invoke = vi.fn(async () => ({ remaining: 0 })) + await expect( + restarted.run(workerStartRequest(method, 'stop-1.1:ctx_1', params), params, invoke) + ).resolves.toMatchObject({ remaining: 0 }) + expect(invoke).toHaveBeenCalledOnce() + harness.db.close() + } + ) +}) diff --git a/src/main/runtime/rpc/orchestration-mutation-executor.ts b/src/main/runtime/rpc/orchestration-mutation-executor.ts index 2569a8d0e6e..a20faeb6649 100644 --- a/src/main/runtime/rpc/orchestration-mutation-executor.ts +++ b/src/main/runtime/rpc/orchestration-mutation-executor.ts @@ -3,7 +3,7 @@ import { isTerminalPromptMutation } from '../../../shared/orchestration/orchestration-rpc-contract' import type { OrcaRuntimeService } from '../orca-runtime' -import type { OrcaSessionId } from '../../../shared/orca-session-address' +import type { OrchestrationEntryCaller } from '../orchestration/addressing/orchestration-caller-identity' import { OrchestrationError } from '../orchestration/orchestration-error' import type { RpcRequest } from './core' import { @@ -32,6 +32,14 @@ export type DurableMutationInvocation = { replayedReceipt?: unknown } +/** Mutations whose pending receipt a restart re-enters, so a stop or cancel still finishes rather + * than answering operation_unknown: each re-runs its ordinary owner, which is safe to repeat. */ +const RESUMABLE_PENDING_METHODS: ReadonlySet = new Set([ + 'orchestration.workerRelease', + 'orchestration.workerStop', + 'orchestration.peerRunCancel' +]) + type InFlightMutation = { method: string payloadHash: string @@ -48,8 +56,8 @@ export class OrchestrationMutationExecutor { params: unknown, invoke: (mutation?: DurableMutationInvocation) => unknown, callerFingerprintOverride?: string, - /** The resolved session's Orca session id; it joins the payload so another caller cannot replay it. */ - callerOrcaSessionId?: OrcaSessionId + /** The resolved caller; its identity joins the payload so another caller cannot replay it. */ + caller?: OrchestrationEntryCaller ): Promise { const requestId = request.orchestrationRequestId if (!requestId || !isDurableMutation(request.method, params)) { @@ -57,12 +65,7 @@ export class OrchestrationMutationExecutor { } const callerFingerprint = callerFingerprintOverride ?? this.getLocalAuthenticatedCallerFingerprint() - const stableParams = replayStableCallerParams( - this.runtime, - params, - callerOrcaSessionId, - request.method - ) + const stableParams = replayStableCallerParams(this.runtime, params, caller, request.method) const basePayloadHash = hashCanonical({ method: request.method, params: stableParams }) const key = `${callerFingerprint}:${requestId}` const db = this.runtime.getOrchestrationDb() @@ -134,7 +137,7 @@ export class OrchestrationMutationExecutor { isResumablePendingWorkerDone(request.method, params, begun.row.receipt) const resumedPendingMutation = begun.disposition === 'pending' && - (request.method === 'orchestration.workerRelease' || resumedPendingWorkerDone) + (RESUMABLE_PENDING_METHODS.has(request.method) || resumedPendingWorkerDone) if (begun.disposition === 'completed') { const active = this.inFlight.get(key) @@ -191,7 +194,7 @@ export class OrchestrationMutationExecutor { { requestId } ) } - if (request.method !== 'orchestration.workerRelease' && !resumedPendingWorkerDone) { + if (!resumedPendingMutation) { const recovery = getPendingWorkerStartRecovery(request.method, begun.row.receipt) throw new OrchestrationError( 'operation_unknown', diff --git a/src/main/runtime/rpc/orchestration-mutation-receipt.ts b/src/main/runtime/rpc/orchestration-mutation-receipt.ts index 03cf9ebd9d7..ed20925ed9a 100644 --- a/src/main/runtime/rpc/orchestration-mutation-receipt.ts +++ b/src/main/runtime/rpc/orchestration-mutation-receipt.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto' import { isTerminalPromptMutation } from '../../../shared/orchestration/orchestration-rpc-contract' import { parsePaneKey } from '../../../shared/workspace-layout/stable-pane-id' import type { OrcaRuntimeService } from '../orca-runtime' -import type { OrcaSessionId } from '../../../shared/orca-session-address' +import type { OrchestrationEntryCaller } from '../orchestration/addressing/orchestration-caller-identity' export const EFFECT_FREE_WORKER_DONE_CHECKPOINT = JSON.stringify({ pending: { effectFree: 'worker_done' } @@ -15,11 +15,13 @@ export type MutationReplayNudge = | { kind: 'federation'; runId?: string } const CALLER_ORCA_SESSION_ID_KEY = '__orcaCallerOrcaSessionId' +// A paired Orca's address names its conversation, so one Run's receipt never answers another's. +const PEER_CALLER_KEY = '__orcaPeerCaller' export function replayStableCallerParams( runtime: OrcaRuntimeService, params: unknown, - callerOrcaSessionId?: OrcaSessionId, + caller?: OrchestrationEntryCaller, method?: string ): unknown { if (!params || typeof params !== 'object' || Array.isArray(params)) { @@ -27,9 +29,12 @@ export function replayStableCallerParams( } const source = params as Record // Absent for terminal callers, so their payload hashes are unchanged. - const result: Record = callerOrcaSessionId - ? { ...source, [CALLER_ORCA_SESSION_ID_KEY]: callerOrcaSessionId } - : { ...source } + const result: Record = + caller?.kind === 'peer' + ? { ...source, [PEER_CALLER_KEY]: caller.address } + : caller?.orcaSessionId + ? { ...source, [CALLER_ORCA_SESSION_ID_KEY]: caller.orcaSessionId } + : { ...source } if (method === 'orchestration.federationImport' && Array.isArray(result.items)) { result.items = result.items.map((item: Record) => { const { answerSenderReport: _answer, ...canonical } = item diff --git a/src/main/runtime/rpc/orchestration-peer-caller.ts b/src/main/runtime/rpc/orchestration-peer-caller.ts new file mode 100644 index 00000000000..f57053b9d88 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-peer-caller.ts @@ -0,0 +1,72 @@ +import type { OrcaRuntimeService } from '../orca-runtime' +import type { RpcMethod, RpcRequest, RpcStreamingMethod } from './core' +import type { RpcDispatchStreamingOptions } from './dispatcher-stream-options' +import { OrchestrationError } from '../orchestration/orchestration-error' +import { isGeneratedId } from '../orchestration/db/generated-id' +import { RemoteOwnedPeerRun } from '../../../shared/orchestration/remote-owned-wire' +import { + peerCoordinatorCaller, + type OrchestrationPeerCaller +} from '../orchestration/addressing/orchestration-caller-identity' + +const NO_EFFECTS = { effectsApplied: false } as const + +/** + * A call a paired Orca makes as the coordinator of one of its Runs here. Admitted from the + * authenticated runtime pairing, never from a param, and synchronously: its Run is written in the + * frame's first turn, so a later frame of the same conversation (a cancel) always finds it. + * The declared caller field then names the peer, as a session's names the session. + */ +export function admitOrchestrationPeerCall( + runtime: OrcaRuntimeService, + request: RpcRequest, + method: RpcMethod | RpcStreamingMethod, + options: RpcDispatchStreamingOptions | undefined +): { request: RpcRequest; caller: OrchestrationPeerCaller } | undefined { + if (request.orchestrationPeerRun === undefined) { + return undefined + } + const access = method.orchestrationAccess + if (!access) { + throw new OrchestrationError( + 'invalid_argument', + `${request.method} is not an orchestration method, so it takes no peer Run. No effects were applied.`, + NO_EFFECTS + ) + } + const fingerprint = options?.authenticatedCallerFingerprint + if (options?.clientKind !== 'runtime' || options.pairedDeviceId === undefined || !fingerprint) { + throw new OrchestrationError( + 'mail_requires_server_pairing', + 'Only a paired computer can coordinate a Run on this server. No effects were applied.', + NO_EFFECTS + ) + } + const peerRun = RemoteOwnedPeerRun.safeParse(request.orchestrationPeerRun) + if (!peerRun.success || !isGeneratedId(peerRun.data.conversationId, 'peerconv')) { + throw new OrchestrationError( + 'invalid_argument', + 'A peer Run needs its Run, conversation ID and label. No effects were applied.', + NO_EFFECTS + ) + } + const caller = peerCoordinatorCaller( + runtime + .getOrchestrationDb() + .admitPeerRun({ peerFingerprint: fingerprint, peerRun: peerRun.data }) + ) + // Terminal attestation and pane keys name callers on the host that has them, never this one. + const peerRequest = { ...request, orchestrationCompatibilityEvidence: undefined } + const params: unknown = request.params + if (!params || typeof params !== 'object' || Array.isArray(params)) { + return { request: peerRequest, caller } + } + const bound: Record = { ...params } + if (access.callerPane) { + delete bound[access.callerPane] + } + if (access.caller) { + bound[access.caller] = caller.address + } + return { request: { ...peerRequest, params: bound }, caller } +} diff --git a/src/main/runtime/rpc/orchestration-session-caller.ts b/src/main/runtime/rpc/orchestration-session-caller.ts index 10a82436696..160cf758f1e 100644 --- a/src/main/runtime/rpc/orchestration-session-caller.ts +++ b/src/main/runtime/rpc/orchestration-session-caller.ts @@ -11,7 +11,10 @@ import { ORCA_SESSION_ADDRESS_PREFIX } from '../../../shared/orca-session-addres import { isOrcaSessionId, parseOrcaSessionAddress } from '../../../shared/orca-session-address' import { ORCHESTRATION_SESSION_CALLER_ERROR_CODES as CODES } from '../../../shared/orchestration/orchestration-session-caller-codes' import type { OrcaRuntimeService } from '../orca-runtime' -import type { OrchestrationSessionCaller } from '../orchestration/addressing/orchestration-caller-identity' +import type { + OrchestrationEntryCaller, + OrchestrationSessionCaller +} from '../orchestration/addressing/orchestration-caller-identity' import { OrchestrationError } from '../orchestration/orchestration-error' import { canonicalOrcaSessionId } from '../orchestration/addressing/session-lineage/canonical-orca-session-id' import type { OrchestrationDb } from '../orchestration/db/orchestration-db' @@ -36,7 +39,7 @@ export type OrchestrationRequestRoute = { export type ResolvedOrchestrationRequest = { /** The request with its declared caller bound to the session and its evidence reduced to it. */ request: RpcRequest - caller?: OrchestrationSessionCaller + caller?: OrchestrationEntryCaller } const NO_EFFECTS = { effectsApplied: false } as const diff --git a/src/main/runtime/rpc/rpc-request-admission-prologue.ts b/src/main/runtime/rpc/rpc-request-admission-prologue.ts index ee6560cea66..fa0deb52116 100644 --- a/src/main/runtime/rpc/rpc-request-admission-prologue.ts +++ b/src/main/runtime/rpc/rpc-request-admission-prologue.ts @@ -23,6 +23,7 @@ import { resolveOrchestrationSessionCaller, type ResolvedOrchestrationRequest } from './orchestration-session-caller' +import { admitOrchestrationPeerCall } from './orchestration-peer-caller' type RpcRequestAdmissionDeps = { runtime: OrcaRuntimeService @@ -59,6 +60,13 @@ export function admitRpcRequest( return reject(migrationFence) } + let peer: ReturnType + try { + peer = admitOrchestrationPeerCall(runtime, rawRequest, method, options) + } catch (error) { + return reject(mapDispatcherError(rawRequest, meta, error)) + } + const admitResolved = ({ request, caller }: ResolvedOrchestrationRequest) => { const parsedParams = parseRpcRequestParams(request, method, meta) if (parsedParams.error) { @@ -76,6 +84,9 @@ export function admitRpcRequest( } return { method, request, caller, params: parsedParams.value } } + if (peer) { + return admitResolved(peer) + } // Resolve once before parse and unary/streaming branching so both see the same caller. const callerField = method.orchestrationAccess?.caller return needsOrchestrationCallerResolution(rawRequest, callerField) diff --git a/src/shared/orchestration/orchestration-rpc-contract.ts b/src/shared/orchestration/orchestration-rpc-contract.ts index 40872534b1d..43828627849 100644 --- a/src/shared/orchestration/orchestration-rpc-contract.ts +++ b/src/shared/orchestration/orchestration-rpc-contract.ts @@ -40,7 +40,8 @@ const ORCHESTRATION_MUTATION_METHODS = new Set([ 'orchestration.federationImport', 'orchestration.federationStop', 'orchestration.federationRelease', - 'orchestration.federationRetain' + 'orchestration.federationRetain', + 'orchestration.peerRunCancel' ]) const RETIRED_ORCHESTRATION_METHODS = new Set(['orchestration.run', 'orchestration.runStop']) diff --git a/src/shared/orchestration/remote-owned-wire.ts b/src/shared/orchestration/remote-owned-wire.ts new file mode 100644 index 00000000000..d52d253b10c --- /dev/null +++ b/src/shared/orchestration/remote-owned-wire.ts @@ -0,0 +1,79 @@ +import { z } from 'zod' + +// What a paired server and the Orca that coordinates a Run on it exchange about that Run's workers +// (contract-b1-b3). The server owns the workers; these carry its facts home, in its order. + +/** The coordinator's Run a call is about, and the conversation its queue items come home under. */ +export const RemoteOwnedPeerRun = z.object({ + runId: z.string().min(1).max(256), + conversationId: z.string().min(1).max(256), + label: z.string().min(1).max(256) +}) +export type RemoteOwnedPeerRun = z.infer + +export const REMOTE_OWNED_WIRE_VERSION = 1 + +const Header = { + v: z.literal(REMOTE_OWNED_WIRE_VERSION), + id: z.string().min(1), + homeRunId: z.string().min(1), + peerRunId: z.string().min(1) +} + +/** A final message filed in the peer Run's mailbox, with the Dispatch and Task its sender worked for. */ +export const RemoteOwnedWorkerMail = z.object({ + ...Header, + message: z.object({ + from: z.string().min(1), + type: z.enum([ + 'status', + 'dispatch', + 'worker_done', + 'merge_ready', + 'escalation', + 'handoff', + 'decision_gate', + 'question' + ]), + subject: z.string(), + body: z.string(), + payload: z.string().nullable(), + priority: z.enum(['normal', 'high', 'urgent']), + threadId: z.string().nullable() + }), + dispatchId: z.string().min(1).nullable(), + taskId: z.string().min(1).nullable() +}) +export type RemoteOwnedWorkerMail = z.infer + +/** A peer-Run Dispatch, its worker or its Task changed state on the server, or the server's reset removed it. */ +export const RemoteOwnedWorkerState = z.object({ + ...Header, + dispatch: z.object({ + id: z.string().min(1), + taskId: z.string().min(1), + status: z.enum(['pending', 'dispatched', 'completed', 'failed', 'circuit_broken']), + workerState: z.string().min(1).nullable(), + lastFailure: z.string().nullable(), + agent: z.string().nullable() + }), + task: z.object({ + status: z.enum(['pending', 'ready', 'dispatched', 'completed', 'failed', 'blocked']), + result: z.string().nullable() + }), + /** The server's report owner settled this change through its unobserved-prompt correction. */ + correctedUnobservedPrompt: z.boolean(), + removed: z.object({ reason: z.string().min(1) }).nullable() +}) +export type RemoteOwnedWorkerState = z.infer + +export const REMOTE_OWNED_ITEM_KINDS = { + worker_mail: RemoteOwnedWorkerMail, + worker_state: RemoteOwnedWorkerState +} as const + +/** Params of orchestration.peerRunCancel; the Run comes from the call's peer-Run envelope. */ +export const PeerRunCancelParams = z.object({}).strict() + +/** Dispatches in the peer Run the server has not proven terminal after running the ordinary stop on each. */ +export const PeerRunCancelResult = z.object({ remaining: z.number().int().nonnegative() }) diff --git a/src/shared/remote-runtime/remote-runtime-memory-limits.ts b/src/shared/remote-runtime/remote-runtime-memory-limits.ts index c35e01b1f0a..d0ffb5a95c9 100644 --- a/src/shared/remote-runtime/remote-runtime-memory-limits.ts +++ b/src/shared/remote-runtime/remote-runtime-memory-limits.ts @@ -84,6 +84,7 @@ export function serializeRemoteRuntimeRpcRequest(args: { orchestrationCapability: args.envelope?.orchestrationCapability, orchestrationContractVersion: args.envelope?.orchestrationContractVersion, orchestrationRequestId: args.envelope?.orchestrationRequestId, + orchestrationPeerRun: args.envelope?.orchestrationPeerRun, compatibilityInvocationId: args.envelope?.compatibilityInvocationId, orchestrationCompatibilityEvidence: args.envelope?.orchestrationCompatibilityEvidence }) diff --git a/src/shared/rpc-contract/orchestration-params.ts b/src/shared/rpc-contract/orchestration-params.ts index cedac9622cb..a86e95b8198 100644 --- a/src/shared/rpc-contract/orchestration-params.ts +++ b/src/shared/rpc-contract/orchestration-params.ts @@ -74,7 +74,9 @@ export const ReplyParams = z.object({ export const InboxParams = z.object({ limit: OptionalFiniteNumber, // Filter inbox by handle to agree with check --all. - terminal: OptionalString + terminal: OptionalString, + // Only this Run's mail; a paired Orca's inbox is always narrowed to its Run. + run: OptionalString }) export const TaskCreateParams = z.object({ diff --git a/src/shared/rpc-contract/orchestration-worker-start-params.ts b/src/shared/rpc-contract/orchestration-worker-start-params.ts index 128c8a46a8e..1cbdeb8853c 100644 --- a/src/shared/rpc-contract/orchestration-worker-start-params.ts +++ b/src/shared/rpc-contract/orchestration-worker-start-params.ts @@ -31,7 +31,9 @@ export const WorkerStartParams = z effort: OptionalWorkerLaunchPreference, retryOf: OptionalString, timeoutMs: OptionalFiniteNumber, - devMode: z.boolean().optional() + devMode: z.boolean().optional(), + /** Only from the paired Orca coordinating this Run: its own ID for the Task this start creates or reopens. */ + adoptTaskId: OptionalString }) .superRefine((params, ctx) => { if (!params.task && !params.spec) { diff --git a/src/shared/rpc-contract/rpc-params-catalog.generated.ts b/src/shared/rpc-contract/rpc-params-catalog.generated.ts index d68a27482b2..60356b6a064 100644 --- a/src/shared/rpc-contract/rpc-params-catalog.generated.ts +++ b/src/shared/rpc-contract/rpc-params-catalog.generated.ts @@ -41,6 +41,7 @@ import { } from '../orcad-peer-desktops-contract' import { OrcadTerminalCensusParamsSchema } from '../orcad-terminal-census' import { CrewSubscribeParams } from '../orchestration/orchestration-crew' +import { PeerRunCancelParams } from '../orchestration/remote-owned-wire' import { RegisterDeviceKeyParamsSchema } from '../remote-runtime/pairing/device-key-registration-contract' import { PairingCreateOfferParamsSchema } from '../remote-runtime/pairing/pairing-create-offer-contract' import { ExchangePairingTicketParamsSchema } from '../remote-runtime/pairing/ticket-exchange-contract' @@ -1371,6 +1372,7 @@ export const RPC_PARAMS_BY_METHOD = { 'orchestration.peerMailAck': PeerMailAckParams, 'orchestration.peerMailDeliver': PeerMailDeliverParams, 'orchestration.peerMailPull': PeerMailPullParams, + 'orchestration.peerRunCancel': PeerRunCancelParams, 'orchestration.reply': ReplyParams, 'orchestration.requestShow': RequestShowParams, 'orchestration.reset': ResetParams, diff --git a/src/shared/runtime-rpc-envelope.ts b/src/shared/runtime-rpc-envelope.ts index 5eb43892d1a..46c09295183 100644 --- a/src/shared/runtime-rpc-envelope.ts +++ b/src/shared/runtime-rpc-envelope.ts @@ -1,6 +1,7 @@ // Share envelopes across CLI/desktop/non-Electron clients without client-to-client imports. import { z } from 'zod' import type { OrchestrationCompatibilityEvidence } from './orchestration/orchestration-compatibility-evidence' +import type { RemoteOwnedPeerRun } from './orchestration/remote-owned-wire' // Independently updated peers may add envelope fields; strip unknown fields while validating known ones. const MetaSuccess = z @@ -76,6 +77,8 @@ export type RuntimeOrchestrationEnvelope = { orchestrationCapability?: string orchestrationContractVersion?: number orchestrationRequestId?: string + /** A paired Orca acting as the coordinator of one of its Runs on this server. */ + orchestrationPeerRun?: RemoteOwnedPeerRun compatibilityInvocationId?: string orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence } From d5759e67a2f2183b911d16887107f9463aa44b93 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:48:45 +0000 Subject: [PATCH 04/38] fix(browser): keep unsent page notes across restarts and ask before closing a tab that has them (#40851) * fix(browser): keep unsent page notes across restarts and ask before closing a tab that has them Unsent browser notes lived only in the renderer store and hydration reset them, so a reload, restart or tab close lost them. Each saved browser page now carries its unsent notes, hydration restores them, and every close path (tab strip, keyboard, CLI, phone) asks before discarding them. A page script's window.close() can't discard them. * fix(browser): keep tabs with unsent notes open on bulk close, like pinned tabs --- .../browser-page-window-close.test.ts | 15 +++- .../host-guest/browser-page-window-close.ts | 3 +- .../layout/close/PinnedTabCloseDialog.tsx | 55 ++++++++----- .../use-terminal-bulk-close-actions.ts | 8 +- .../ipc-events-close-routing-test-harness.ts | 1 + .../browser-request-ipc-bridge.owner.test.ts | 1 + .../ipc-events/browser-request-ipc-bridge.ts | 44 +++-------- .../browser-tab-close-with-reply.ts | 52 +++++++++++++ .../ipc-events/session-tab-ipc-bridge.ts | 7 +- src/renderer/src/i18n/locales/en.json | 5 +- .../workspace-session-browser-tabs.ts | 24 ++++-- .../persistence/workspace-session-patch.ts | 9 ++- .../persistence/workspace-session.ts | 5 +- .../workspace/tabs/workspace-tab-commands.ts | 9 ++- .../annotations/browser-page-annotation.ts | 13 ++++ ...browser-annotation-session-restore.test.ts | 76 ++++++++++++++++++ .../hydration/browser-hydration-state.ts | 23 ++++-- .../browser-workspace-doc-location.test.ts | 1 + .../tabs/closing/pinned-tab-close-guard.ts | 16 ++-- .../slices/tabs/pinned-tab-close-confirm.ts | 6 +- .../workspace-layout-frame-store-view.ts | 3 +- .../browser-page-annotation-schema.ts | 77 +++++++++++++++++++ .../languages/protected-copy-keys.json | 2 + .../workspace-session-browser-schema.ts | 5 +- .../workspace-session-state-types.ts | 6 +- 25 files changed, 374 insertions(+), 92 deletions(-) create mode 100644 src/renderer/src/hooks/ipc-events/browser-tab-close-with-reply.ts create mode 100644 src/renderer/src/store/slices/browser/hydration/browser-annotation-session-restore.test.ts create mode 100644 src/shared/browser/annotations/browser-page-annotation-schema.ts diff --git a/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.test.ts b/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.test.ts index 257250b024a..7f10e7912b4 100644 --- a/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.test.ts +++ b/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.test.ts @@ -12,9 +12,15 @@ vi.mock('@/lib/workspace/tabs/workspace-tab-commands', () => ({ import { handleGuestWindowCloseMessage } from './browser-page-window-close' import { BROWSER_GUEST_WINDOW_CLOSE_CHANNEL } from '../../../../../shared/browser-guest-window-close' -function seedTab(pageId: string, isPinned = false, siblingPageIds: string[] = []): void { +function seedTab( + pageId: string, + isPinned = false, + siblingPageIds: string[] = [], + browserAnnotationsByPageId: Record = {} +): void { storeState.value = { closeBrowserPage: closeBrowserPageMock, + browserAnnotationsByPageId, browserTabsByWorktree: { 'wt-1': [{ id: 'ws-1' }] }, browserPagesByWorkspace: { 'ws-1': [{ id: pageId }, ...siblingPageIds.map((id) => ({ id }))] }, unifiedTabsByWorktree: { @@ -40,6 +46,13 @@ describe('window.close() from a browser page', () => { expect(dispatchMock).not.toHaveBeenCalled() }) + it('never discards unsent notes', () => { + seedTab('page-n', false, [], { 'page-n': [{ id: 'note-1' }] }) + closeFrom('page-n', { back: false, forward: false }) + expect(dispatchMock).not.toHaveBeenCalled() + expect(closeBrowserPageMock).not.toHaveBeenCalled() + }) + it('closes only the calling page when its Orca tab holds other pages', () => { seedTab('page-e', false, ['page-sibling']) closeFrom('page-e', { back: false, forward: false }) diff --git a/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.ts b/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.ts index 452f602c2b1..5b37f60164e 100644 --- a/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.ts +++ b/src/renderer/src/components/browser-pane/host-guest/browser-page-window-close.ts @@ -32,7 +32,8 @@ export function handleGuestWindowCloseMessage( const tab = (state.unifiedTabsByWorktree[owner.worktreeId] ?? []).find( (candidate) => candidate.contentType === 'browser' && candidate.entityId === owner.workspaceId ) - if (tab?.isPinned) { + // A script may not discard what the user pinned or left unsent; the page's notes are all it closes. + if (tab?.isPinned || (state.browserAnnotationsByPageId[pageId]?.length ?? 0) > 0) { return } pagesOpenedByPage.delete(pageId) diff --git a/src/renderer/src/components/terminal-pane/layout/close/PinnedTabCloseDialog.tsx b/src/renderer/src/components/terminal-pane/layout/close/PinnedTabCloseDialog.tsx index 576c72f73ea..36b9280f7da 100644 --- a/src/renderer/src/components/terminal-pane/layout/close/PinnedTabCloseDialog.tsx +++ b/src/renderer/src/components/terminal-pane/layout/close/PinnedTabCloseDialog.tsx @@ -25,6 +25,7 @@ export default function PinnedTabCloseDialog(): React.JSX.Element { const [previousRequest, setPreviousRequest] = useState(request) const tabLabel = request?.tabLabel.trim() + const unsentNotes = request?.unsentNotes === true // Reset new requests before paint without reviving a cancelled request’s checkbox state. if (request !== previousRequest) { @@ -53,16 +54,26 @@ export default function PinnedTabCloseDialog(): React.JSX.Element { - {t( - 'auto.components.terminal.pane.PinnedTabCloseDialog.6c190f295a', - 'Close pinned tab?' - )} + {unsentNotes + ? t( + 'auto.components.terminal.pane.PinnedTabCloseDialog.unsent_notes_title', + 'Discard unsent notes?' + ) + : t( + 'auto.components.terminal.pane.PinnedTabCloseDialog.6c190f295a', + 'Close pinned tab?' + )} - {t( - 'auto.components.terminal.pane.PinnedTabCloseDialog.0d1963f4a6', - 'This tab is pinned. Are you sure you want to close it?' - )} + {unsentNotes + ? t( + 'auto.components.terminal.pane.PinnedTabCloseDialog.unsent_notes_description', + "This tab has notes you haven't sent. Closing it discards them." + ) + : t( + 'auto.components.terminal.pane.PinnedTabCloseDialog.0d1963f4a6', + 'This tab is pinned. Are you sure you want to close it?' + )} {tabLabel ? ( @@ -70,19 +81,21 @@ export default function PinnedTabCloseDialog(): React.JSX.Element { {tabLabel}

) : null} -
- setDontAskAgain(checked === true)} - /> - -
+ {unsentNotes ? null : ( +
+ setDontAskAgain(checked === true)} + /> + +
+ )} + + + + + {openLabel} + + + {expanded && ( + + + + )} +
+ {actions} +
+
+ {showSummaryRow && ( + + {branch && ( + + + + {branch} + )} - - {(branch || !!ahead || !!behind) && ( - - {branch && ( - - - {branch} - - )} - {!!ahead && ( - - - {ahead} - - )} - {!!behind && ( - - - {behind} + + {!expanded && } + {!expanded && changeCount > 0 && ( + + {t( + 'auto.components.rightSidebar.FolderWorkspaceSourceControl.changeCount', + '{{count}} changes', + { count: changeCount, defaultValue_one: '{{count}} change' } + )} )} - )} - - - - - - {openLabel} - + + )} ) } diff --git a/src/renderer/src/components/right-sidebar/source-control/listing/section-header.tsx b/src/renderer/src/components/right-sidebar/source-control/listing/section-header.tsx index e6b95041d57..00a38832642 100644 --- a/src/renderer/src/components/right-sidebar/source-control/listing/section-header.tsx +++ b/src/renderer/src/components/right-sidebar/source-control/listing/section-header.tsx @@ -3,6 +3,7 @@ import { ChevronDown } from 'lucide-react' import { cn } from '@/lib/class-names' import { Button } from '@/components/ui/button' import { useTranslate } from '@/i18n/use-translate' +import { useSourceControlScope } from '../panel/worktree-scope' export function SectionHeader({ label, @@ -23,22 +24,27 @@ export function SectionHeader({ actions?: React.ReactNode }): React.JSX.Element { const t = useTranslate() + // Folder cards keep the gap above the first file equal to the one below the last. + const compact = useSourceControlScope() !== null // Share the rounded container so hover styling spans the whole row. return ( -
+
-
{actions}
+
+ {actions} +
) diff --git a/src/renderer/src/components/right-sidebar/source-control/listing/uncommitted-sections.tsx b/src/renderer/src/components/right-sidebar/source-control/listing/uncommitted-sections.tsx index c80dea85b36..6c77ceaf81a 100644 --- a/src/renderer/src/components/right-sidebar/source-control/listing/uncommitted-sections.tsx +++ b/src/renderer/src/components/right-sidebar/source-control/listing/uncommitted-sections.tsx @@ -14,6 +14,7 @@ import { import { ActionButton } from './action-button' import { SectionHeader } from './section-header' import { SourceControlSectionFileList } from './section-file-list' +import { useSourceControlScope } from '../panel/worktree-scope' import { getSourceControlSectionViewAction, type SourceControlDisplaySection, @@ -89,6 +90,7 @@ export function SourceControlUncommittedSections(props: { requestDiscardEntry: (entry: GitStatusEntry) => void diffCommentCountByPath: Map }): React.JSX.Element { + const embedded = useSourceControlScope() !== null const t = useTranslate() return ( <> @@ -176,35 +178,37 @@ export function SourceControlUncommittedSections(props: { } /> {!isCollapsed && ( - +
+ +
)}
) diff --git a/src/renderer/src/components/right-sidebar/source-control/listing/use-selection.ts b/src/renderer/src/components/right-sidebar/source-control/listing/use-selection.ts index 9888f31b182..9cc0b17c0d0 100644 --- a/src/renderer/src/components/right-sidebar/source-control/listing/use-selection.ts +++ b/src/renderer/src/components/right-sidebar/source-control/listing/use-selection.ts @@ -190,7 +190,8 @@ export function useSourceControlSelection({ return } - const container = containerRef.current + const panel = containerRef.current + const container = panel?.closest('[data-source-control-worktree]') ?? panel const target = e.target if (!container || !(target instanceof Node) || container.contains(target)) { return diff --git a/src/renderer/src/components/right-sidebar/source-control/panel/branch-context-row.tsx b/src/renderer/src/components/right-sidebar/source-control/panel/branch-context-row.tsx index af4712d35a9..940a009f98c 100644 --- a/src/renderer/src/components/right-sidebar/source-control/panel/branch-context-row.tsx +++ b/src/renderer/src/components/right-sidebar/source-control/panel/branch-context-row.tsx @@ -103,6 +103,7 @@ export function SourceControlBranchContextRow({ headControl, manualReviewUrl, branchLineTotal, + reviewControl, onChangeBaseRef, onRetry }: { @@ -112,6 +113,7 @@ export function SourceControlBranchContextRow({ headControl?: React.ReactNode manualReviewUrl?: string | null branchLineTotal?: GitBranchLineTotal | null + reviewControl?: React.ReactNode onChangeBaseRef: () => void onRetry: () => void }): React.JSX.Element | null { @@ -124,8 +126,11 @@ export function SourceControlBranchContextRow({ return null } return ( -
- {headControl ?? } +
+ + {headControl ?? } + + {reviewControl}
) } @@ -159,7 +164,12 @@ export function SourceControlBranchContextRow({ onChangeBaseRef={onChangeBaseRef} changeBaseTitle={changeBaseTitle} leading={