From 32a53d3ec6f6cfb378c96a1b0a5b919eb91c8d8a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:07:29 -0700 Subject: [PATCH] feat(native-chat): image messages wait in the queue like text (#28655) * feat(native-chat): image messages wait in the queue like text A message with images now queues behind the running turn and the cards ahead of it, instead of skipping the queue. Cards show an image icon and count, an image-only card names its image, and editing a card's text keeps its images. The host publishes no card holding an image to a client that has not advertised agent-session.queued-image-cards.v1, since released clients edit by copying a card's text and deleting the card. * test(native-chat): an image follow-up queues like text * fix(native-chat): project queued images before phone frame limits * fix(native-chat): preserve image sends from older clients * Clarify direct text recovery at send admission * fix(native-chat): expose queued image names to keyboard users --- .../desktop-renderer-runtime-capabilities.ts | 3 + .../journal-unsent-send-hold.ts | 4 +- .../agent-session-subscriber-frame-fields.ts | 7 +- ...structured-agent-session-host-mutations.ts | 2 + .../structured-agent-session-host.ts | 12 +- ...ured-agent-session-queued-messages.test.ts | 36 +++-- ...tructured-agent-session-queued-messages.ts | 16 +- .../structured-agent-session-queued-send.ts | 1 + .../structured-agent-session-subscribers.ts | 24 ++- ...nt-session-queued-image-capability.test.ts | 146 ++++++++++++++++++ ...d-agent-session-queued-image-capability.ts | 66 ++++++++ ...ctured-agent-session-send-compatibility.ts | 9 +- ...tured-agent-session-wire-contracts.test.ts | 63 ++++++++ .../rpc/methods/structured-agent-session.ts | 11 +- .../NativeChatQueuedMessageCard.tsx | 59 ++++++- .../NativeChatQueuedMessageEditor.tsx | 7 +- .../NativeChatTranscriptChrome.tsx | 9 +- .../native-chat/native-chat-image-label.ts | 16 ++ ...ctured-agent-session-queue-request.test.ts | 12 +- .../structured-agent-session-queue-request.ts | 11 +- .../structured-agent-session-queued-cards.ts | 14 +- ...se-structured-agent-session-sends.test.tsx | 8 +- .../use-structured-agent-session-sends.ts | 2 +- .../src/i18n/en-runtime-required.json | 4 + src/renderer/src/i18n/locales/en.json | 3 + src/renderer/src/i18n/locales/es.json | 3 + src/renderer/src/i18n/locales/fr.json | 3 + src/renderer/src/i18n/locales/ja.json | 3 + src/renderer/src/i18n/locales/ko.json | 3 + src/renderer/src/i18n/locales/zh.json | 3 + .../web/web-runtime-client-capabilities.ts | 3 + .../src/web/web-runtime-client.test.ts | 2 + .../agent-session-queued-message-wire.ts | 3 +- ...tron-remote-runtime-client-capabilities.ts | 3 + src/shared/runtime-capability-constants.ts | 5 + 35 files changed, 488 insertions(+), 88 deletions(-) create mode 100644 src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.test.ts create mode 100644 src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.ts create mode 100644 src/renderer/src/components/native-chat/native-chat-image-label.ts diff --git a/src/main/ipc/desktop-renderer-runtime-capabilities.ts b/src/main/ipc/desktop-renderer-runtime-capabilities.ts index a1df52b5e26..72af9f0e33f 100644 --- a/src/main/ipc/desktop-renderer-runtime-capabilities.ts +++ b/src/main/ipc/desktop-renderer-runtime-capabilities.ts @@ -4,6 +4,7 @@ import { AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY, AGENT_SESSION_SNAPSHOT_CATCH_UP_CLIENT_CAPABILITY, AGENT_SESSION_TURN_ITEM_CAPABILITY, CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, @@ -48,6 +49,8 @@ export const DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES: readonly RuntimeCapab AGENT_SESSION_TURN_ITEM_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_CHILD_VIEWS_CAPABILITY, + // Its queued cards show their images, and Edit keeps them. + AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, AGENT_SESSION_OPTIONAL_MODEL_CLIENT_CAPABILITY, diff --git a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts index ad3f6a11d55..08a8f91cfa8 100644 --- a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts +++ b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts @@ -1,6 +1,6 @@ // What becomes of a send the host accepted and can no longer hand over: an earlier host process // quit or crashed first (settled at the next open), or the chat is closing. Either way the agent -// provably never got it. A person's message (typed, or a launch's first prompt) is kept as a card +// provably never got it. A person's text (typed, or a launch's first prompt) is kept as a card // at the head of the queue, an ordinary one: like every card the chat closed with, it waits for the // chat's next turn (`queued-message-pause.ts`). Everything else is rejected as before, because // something else re-derives it or the person re-runs it. The submission itself is always rejected, @@ -31,7 +31,7 @@ export type UnsentSendHold = /** The body an unsent send is kept with, or null when it is rejected instead: * - a card's own hand-off: its rejection already returns the card (`rejectedDraftSettlement`); * - `/compact` and other commands: a command in flight is not resumed, the person re-runs it; - * - an image: cards are text-only; + * - a direct image: it retains the existing rejected-send recovery instead of becoming a card; * - a send not from a person: orchestration mail (the mailbox re-sends it), a dispatch preamble * or a restart continuation (their owners re-derive them), a kind this build does not know, and * a row with no source that is not a person's (it could be either). */ diff --git a/src/main/native-chat/agent-session-wire/agent-session-subscriber-frame-fields.ts b/src/main/native-chat/agent-session-wire/agent-session-subscriber-frame-fields.ts index 91c13ed8b58..1bd6383bcd0 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-subscriber-frame-fields.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-subscriber-frame-fields.ts @@ -18,8 +18,9 @@ export type SubscriberFieldState = { sessionId: string /** Every frame to fit a phone's bridge frame; see `boundedFrames`. */ boundedFrames: boolean + projectQueuePublication?: (publication: QueuePublication) => QueuePublication commands?: AgentSessionSlashCommand[] | null - /** The last queue publication actually SENT. */ + /** Canonical publication last sent, before reader projection. */ queuePublication?: QueuePublication /** Fingerprint of the roster last SENT; absent until this subscriber's first frame. */ backgroundTasks?: string @@ -70,7 +71,9 @@ export function buildSubscriberFrame( event.type !== 'end' && (event.type !== 'batch' || queued !== subscriber.queuePublication) const backgroundTasks = subscriberBackgroundTasks(hooks, subscriber, event) - const sideQueued = attachedQueued ? queued : undefined + const sideQueued = attachedQueued + ? (subscriber.projectQueuePublication?.(queued) ?? queued) + : undefined return { frames: subscriber.boundedFrames ? boundedFrames(event, { queued: sideQueued, backgroundTasks }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index 42d668033cc..31de6353827 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -69,6 +69,8 @@ export function sendStructuredAgentSessionTurn( body: AgentJournalMessageItem retryUnknown?: true delivery?: 'queue-if-active' + /** Host-local: whether the sending client can manage an image card. */ + queuedImageCards?: boolean options?: Readonly> /** Host-local, set only by the client-facing `agentSession.send` RPC (the * renderer's launch prompt included): recorded as the submission's `client` diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 0d7aff07571..766a91fae5d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -29,11 +29,7 @@ import { type StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' import { flushStructuredAgentSessionHost } from './structured-agent-session-host-teardown' -import type { - StructuredAgentSessionCaller, - StructuredAgentSessionHostDeps, - StructuredAgentSessionReveal -} from './structured-agent-session-host-types' +import type * as HostTypes from './structured-agent-session-host-types' import { StructuredAgentSessionEventRecovery } from './structured-agent-session-event-recovery' import { StructuredAgentSessionBackgroundTaskChannel } from './structured-agent-session-background-task-channel' import { StructuredAgentSessionClientDelivery } from './structured-agent-session-client-delivery' @@ -88,7 +84,7 @@ export class StructuredAgentSessionHost { /** Public because the RPC surface addresses it directly; see the restart-resume collaborator. */ readonly restartResume: StructuredAgentSessionRestartResume - constructor(readonly deps: StructuredAgentSessionHostDeps) { + constructor(readonly deps: HostTypes.StructuredAgentSessionHostDeps) { // Every collaborator reads this copy, so a logger that throws cannot fail what it reports. this.deps = deps = sessionLogger.withNeverThrowingLogger(deps) this.clientDelivery.watchAtRestCommands(deps.adapter) @@ -239,7 +235,7 @@ export class StructuredAgentSessionHost { this.restore.restoreReadableSessions(sessionIds) /** Make one persisted session addressable again; see `structured-agent-session-reveal`. */ - revealSession = (sessionId: string): Promise => + revealSession = (sessionId: string): Promise => reveal.revealStructuredAgentSession(this.deps, sessionId, this.lifetime.conversation) private serialize = this.tasks.serialize.bind(this.tasks) @@ -313,7 +309,7 @@ export class StructuredAgentSessionHost { changeThreadGoal = this.mutations.changeThreadGoal readOptions = this.mutations.readOptions - rewind = (caller: StructuredAgentSessionCaller, params: AgentSessionRewindParams) => + rewind = (caller: HostTypes.StructuredAgentSessionCaller, params: AgentSessionRewindParams) => rewindStructuredAgentSession(this.mutationContext(), this.attachContext(), caller, params) conversationCommand = (...args: Parameters) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts index 6276a780f4b..b00a1e63752 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts @@ -103,25 +103,27 @@ describe('accept', () => { expect(await drafts()).toHaveLength(1) }) - it('routes an image send to the immediate path even while working (text-only v1)', async () => { - await workingSend() - const body = { - kind: 'message' as const, - role: 'user' as const, - blocks: [{ type: 'image-ref' as const, path: '/tmp/shot.png' }] + it('queues an image message in line behind text, and hands it over with its image', async () => { + const working = await workingSend() + const text = await send('text first', 'queue-if-active').result + const body = hostTestMessage('and look at this') + body.blocks.push({ type: 'image-ref', path: '/tmp/shot.png' }) + const fields = { body, delivery: 'queue-if-active' as const } + const envelopeFields = rig.envelope(fields, 'agentSession.send', hostTestOperationId()) + const image = await host.send(CALLER, { envelope: envelopeFields, ...fields, userSend: true }) + if (!text.ok || !('queued' in text.value) || !image.ok || !('queued' in image.value)) { + throw new Error('expected queued receipts') } - const clientOperationId = hostTestOperationId() - const result = await host.send(CALLER, { - envelope: rig.envelope( - { body, delivery: 'queue-if-active' }, - 'agentSession.send', - clientOperationId - ), - body, - delivery: 'queue-if-active' + const [textId, imageId] = [text.value.queued.messageId, image.value.queued.messageId] + await settleAccepted(working, 'a') + await eventually(async () => expect(await rig.handoff(textId)).toBeDefined()) + expect(await rig.handoff(imageId)).toBeUndefined() + await settleAccepted(await rig.handoffId(textId), 'b') + await eventually(async () => { + const handedOver = await rig.handoffId(imageId) + const sent = dispatch.mock.calls.find(([input]) => input.clientMessageId === handedOver) + expect(sent?.[0].body.blocks).toEqual(body.blocks) }) - expect(result).toMatchObject({ ok: true, value: { submission: expect.anything() } }) - expect(await drafts()).toHaveLength(0) }) it('a send without the delivery field never queues, whatever the session is doing', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 061592e3163..99d276c64ca 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -37,11 +37,6 @@ import type { StructuredAgentSessionLogger } from './structured-agent-session-lo import { agentSessionAttachmentExpiredRefusal } from './structured-agent-session-turns' import { isAgentSessionAttachmentExpiredError } from '../agent-session-attachments/agent-session-attachment-claims' -/** Text-only v1: any image block routes to the immediate path. */ -export function queuedMessageBodyIsTextOnly(body: AgentJournalMessageItem): boolean { - return body.blocks.every((block) => block.type === 'text') -} - /** Waiting, not held on its own, and not positioned behind a returned card or a * card the queue's pause holds: the queue never reorders. The admission rule * (§accept) and the drain's selection both read it; only the drain's (`automatic`) stops at a @@ -142,8 +137,8 @@ export function shouldQueueStructuredAgentSessionSend( /** * The accept branch: a capable send while the session is working (or behind an * actionable backlog) becomes a draft instead of a submission. Returns null for - * the immediate path — an incapable client, an image body (text-only v1), a - * replayed id the journal already answers, or an idle session. + * the immediate path — an incapable client, a replayed id the journal already + * answers, or an idle session. */ export async function maybeQueueStructuredAgentSessionSend( context: { @@ -157,6 +152,7 @@ export async function maybeQueueStructuredAgentSessionSend( envelope: { clientOperationId: string } body: AgentJournalMessageItem delivery?: 'queue-if-active' + queuedImageCards?: boolean options?: Readonly> /** A person's send at a chat surface: every attachment it names must still be stored. */ userSend?: true @@ -169,7 +165,11 @@ export async function maybeQueueStructuredAgentSessionSend( | null > { const clientMessageId = params.envelope.clientOperationId - if (params.delivery !== 'queue-if-active' || !queuedMessageBodyIsTextOnly(params.body)) { + if ( + params.delivery !== 'queue-if-active' || + (params.queuedImageCards === false && + params.body.blocks.some((block) => block.type === 'image-ref')) + ) { return null } // Asked again with no ledger answer: a send this host queued answers as its replay would — diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-send.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-send.ts index 7467a2afe84..3a21c9a9b5e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-send.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-send.ts @@ -14,6 +14,7 @@ export async function runQueueableStructuredAgentSessionSend( envelope: { clientOperationId: string } body: AgentJournalMessageItem delivery?: 'queue-if-active' + queuedImageCards?: boolean options?: Readonly> userSend?: true personsMessage?: true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index 19dbfacf462..49f8479232b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -17,7 +17,8 @@ import { backgroundTaskFingerprint, emitSubscriberFrame, subscriberCommandsChanged, - type SubscriberFieldHooks + type SubscriberFieldHooks, + type SubscriberFieldState } from './agent-session-subscriber-frame-fields' import type { QueuePublication } from './structured-agent-session-queued-publication' import { deliverToSubscriber } from './agent-session-subscriber-catch-up' @@ -38,6 +39,7 @@ export type AgentSessionSubscribeInput = { cursor?: AgentJournalCursor /** Every frame to fit a phone's bridge frame; the RPC sets it for every phone. */ boundedFrames?: boolean + projectQueuePublication?: SubscriberFieldState['projectQueuePublication'] readsCurrentRequests?: boolean /** False for desktops that need exact replay to keep send echoes and current requests. */ snapshotCatchUp?: boolean @@ -50,9 +52,10 @@ export type Subscriber = CurrentRequestsSubscriber & { cursor: AgentJournalCursor fence: number boundedFrames: boolean + projectQueuePublication?: SubscriberFieldState['projectQueuePublication'] snapshotCatchUp?: boolean commands?: AgentSessionSlashCommand[] | null - /** The last draft list actually SENT — never advanced on a page that withheld + /** Canonical publication last sent — never advanced on a page that withheld * it, or the final replacement would be suppressed by the identity dedup. */ queuePublication?: QueuePublication /** Fingerprint of the background-task roster last SENT. */ @@ -93,17 +96,9 @@ export class AgentSessionSubscribers { return this.bySession.get(sessionId)?.size ?? 0 } - open(input: { - id: string - sessionId: string - journal: AgentSessionJournal - fence: number - emit: AgentSessionSubscriberEmit - cursor?: AgentJournalCursor - boundedFrames?: boolean - readsCurrentRequests?: boolean - snapshotCatchUp?: boolean - }): () => void { + open( + input: AgentSessionSubscribeInput & { journal: AgentSessionJournal; fence: number } + ): () => void { const liveCursor = input.journal.cursor() const subscriber: Subscriber = { id: input.id, @@ -111,8 +106,9 @@ export class AgentSessionSubscribers { emit: input.emit, cursor: input.cursor ?? { epoch: liveCursor.epoch, sequence: 0 }, fence: input.fence, - readsCurrentRequests: input.readsCurrentRequests === true, boundedFrames: input.boundedFrames === true, + projectQueuePublication: input.projectQueuePublication, + readsCurrentRequests: input.readsCurrentRequests === true, snapshotCatchUp: input.snapshotCatchUp !== false } const session = this.bySession.get(input.sessionId) ?? new Map() diff --git a/src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.test.ts new file mode 100644 index 00000000000..8b21a00b8bc --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.test.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, expect, it } from 'vitest' +import type { + AgentSessionQueuedMessage, + AgentSessionSubscribeEvent +} from '../../../../shared/agent-session-wire' +import { + buildSubscriberFrame, + type SubscriberFieldState +} from '../../../native-chat/agent-session-wire/agent-session-subscriber-frame-fields' +import { boundedQueuePublication } from '../../../native-chat/agent-session-wire/agent-session-bounded-queue' +import { DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES } from '../../../ipc/desktop-renderer-runtime-capabilities' +import type { AgentSessionSubscribeInput } from '../../../native-chat/agent-session-wire/structured-agent-session-subscribers' +import { + call, + clearStructuredHostStub, + hostCalls, + installStructuredHostStub, + SESSION, + STRUCTURED_CLIENT, + STRUCTURED_MOBILE_CLIENT +} from './structured-agent-session-rpc.test-fixture' + +beforeEach(installStructuredHostStub) +afterEach(clearStructuredHostStub) + +function card(messageId: string, position: number, image: boolean): AgentSessionQueuedMessage { + return { + messageId, + position, + state: 'waiting', + body: { + kind: 'message', + role: 'user', + blocks: [ + { type: 'text', text: messageId }, + ...(image ? [{ type: 'image-ref' as const, path: '/tmp/shot.png' }] : []) + ] + } + } +} + +const TEXT = card('text', 2, false) +const IMAGE = card('image', 1, true) +const DESKTOP = { + clientKind: 'runtime' as const, + clientCapabilities: [...DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES] +} + +it('withholds image cards from a reader that predates them, on history and the stream', async () => { + const queue = { + queuedMessages: [IMAGE, TEXT], + queuePause: { reason: 'stopped' as const }, + nextQueuedMessageId: 'image' + } + hostCalls.history.mockReturnValue({ ok: true, page: { items: [], ...queue } }) + hostCalls.subscribe.mockImplementation((input: AgentSessionSubscribeInput) => { + const event: AgentSessionSubscribeEvent = { + type: 'batch', + sessionId: SESSION, + batch: { + cursor: { epoch: 'a', sequence: 0 }, + items: [], + removedItemIds: [], + submissions: [] + } + } + for (const frame of buildSubscriberFrame( + { readQueuePublication: () => queue }, + { + sessionId: SESSION, + boundedFrames: input.boundedFrames === true, + projectQueuePublication: input.projectQueuePublication + }, + event, + false + ).frames) { + input.emit(frame) + } + return () => {} + }) + const read = async (client: Parameters[2]) => [ + await call('agentSession.history', { sessionId: SESSION, direction: 'tail' }, client), + await call('agentSession.subscribe', { sessionId: SESSION }, client) + ] + // The local renderer is a negotiated reader too: its own list must carry the capability. + for (const client of [DESKTOP, undefined]) { + const [history, stream] = await read(client) + expect(history).toMatchObject({ result: { page: queue } }) + expect(stream).toMatchObject({ result: queue }) + } + for (const client of [STRUCTURED_CLIENT, STRUCTURED_MOBILE_CLIENT]) { + const [history, stream] = await read(client) + // Positions, the pause over a card it sees, and the next card's id stay as the host published. + const withheld = { ...queue, queuedMessages: [TEXT] } + expect(history).toMatchObject({ result: { page: withheld } }) + expect(stream).toMatchObject({ result: withheld }) + } + + // With no card left to show, no pause either: its Resume would send only what it cannot see. + hostCalls.history.mockReturnValue({ + ok: true, + page: { items: [], ...queue, queuedMessages: [IMAGE] } + }) + expect( + await call('agentSession.history', { sessionId: SESSION, direction: 'tail' }, STRUCTURED_CLIENT) + ).toMatchObject({ result: { page: { queuedMessages: [], queuePause: null } } }) +}) + +it('publishes phone text behind hidden images before bounding, and deduplicates the canonical queue', async () => { + const images = Array.from({ length: 3 }, (_, index) => { + const image = card(`image-${index}`, index + 1, true) + image.body.blocks.unshift({ type: 'text', text: 'x'.repeat(200 * 1024) }) + return image + }) + const text = card('text', images.length + 1, false) + const queue = { + queuedMessages: [...images, text], + queuePause: { reason: 'stopped' as const }, + nextQueuedMessageId: null + } + expect(boundedQueuePublication(queue).queuedMessages).not.toContain(text) + hostCalls.subscribe.mockImplementation((input: AgentSessionSubscribeInput) => { + const subscriber: SubscriberFieldState = { + sessionId: SESSION, + boundedFrames: input.boundedFrames === true, + projectQueuePublication: input.projectQueuePublication + } + const event: AgentSessionSubscribeEvent = { + type: 'batch', + sessionId: SESSION, + batch: { cursor: { epoch: 'a', sequence: 0 }, items: [], removedItemIds: [], submissions: [] } + } + const context = { readQueuePublication: () => queue } + const built = buildSubscriberFrame(context, subscriber, event, false) + for (const frame of built.frames) { + input.emit(frame) + } + expect(built.queued).toBe(queue) + subscriber.queuePublication = built.queued + expect(buildSubscriberFrame(context, subscriber, event, false).attachedQueued).toBe(false) + return () => {} + }) + expect( + await call('agentSession.subscribe', { sessionId: SESSION }, STRUCTURED_MOBILE_CLIENT) + ).toMatchObject({ result: { queuedMessages: [text] } }) +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.ts b/src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.ts new file mode 100644 index 00000000000..e44fa8079f5 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-queued-image-capability.ts @@ -0,0 +1,66 @@ +// A reader that predates image cards shows a card's text only, and edits by copying that text and +// deleting the card, which would drop its images. It is never handed a card holding an image, at +// the RPC boundary only: the journal, the drain and every in-process reader keep the whole queue. + +import type { + AgentSessionHistoryResult, + AgentSessionQueuedMessage, + AgentSessionQueuePublicationFields +} from '../../../../shared/agent-session-wire' +import type { QueuePublication } from '../../../native-chat/agent-session-wire/structured-agent-session-queued-publication' +import { AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY } from '../../../../shared/protocol-version' +import type { RpcContext } from '../core' + +type QueuedImageReader = Pick + +type QueueFields = AgentSessionQueuePublicationFields & { + queuedMessages?: AgentSessionQueuedMessage[] | null +} + +export function readsQueuedImageCards(ctx: QueuedImageReader): boolean { + // An in-process caller is this build; only a negotiated client can predate image cards. + return ( + ctx.clientKind === undefined || + ctx.clientCapabilities?.includes(AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY) === true + ) +} + +function holdsImage(card: AgentSessionQueuedMessage): boolean { + return card.body.blocks.some((block) => block.type === 'image-ref') +} + +/** The fields to replace, or null when nothing is withheld. Positions and the next card's id stay + * as published, as when a phone's frame trims the list. A pause over no card the reader sees would + * offer a Resume that sends nothing it shows. */ +function withoutImageCards( + fields: QueueFields +): + | (Pick & Partial>) + | null { + const cards = fields.queuedMessages + if (!cards?.some(holdsImage)) { + return null + } + const queuedMessages = cards.filter((card) => !holdsImage(card)) + return { + queuedMessages, + ...(queuedMessages.length === 0 && fields.queuePause !== undefined ? { queuePause: null } : {}) + } +} + +export function projectQueuedImageCardHistory( + result: AgentSessionHistoryResult, + ctx: QueuedImageReader +): AgentSessionHistoryResult { + const page = readsQueuedImageCards(ctx) ? null : withoutImageCards(result.page) + return page ? { ...result, page: { ...result.page, ...page } } : result +} + +/** Applied before subscriber frame sizing; the host's canonical publication stays unchanged. */ +export function projectQueuedImageCardPublication( + publication: QueuePublication, + ctx: QueuedImageReader +): QueuePublication { + const fields = readsQueuedImageCards(ctx) ? null : withoutImageCards(publication) + return fields ? { ...publication, ...fields } : publication +} diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts index 7f6460bdc55..3130c8988e9 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts @@ -7,6 +7,7 @@ import type { StructuredAgentSessionHost } from '../../../native-chat/agent-sess import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import type { RpcContext } from '../core' import { requireStructuredSessionHost, structuredCallerFor } from './structured-agent-session-gate' +import { readsQueuedImageCards } from './structured-agent-session-queued-image-capability' /** * A send answers once the host accepts it. A client that predates that answer cannot show a @@ -20,8 +21,12 @@ export async function sendStructuredAgentSessionForClient( context: RpcContext ) { const host = requireStructuredSessionHost(context, params.envelope.sessionId) - // A client's own send: a restart or a close keeps it as a card if it never reached the agent. - const result = await host.send(structuredCallerFor(context), { ...params, userSend: true }) + // Person provenance lets undelivered text recover as an ordinary queued card. + const result = await host.send(structuredCallerFor(context), { + ...params, + userSend: true, + queuedImageCards: readsQueuedImageCards(context) + }) const capabilities = context.clientCapabilities ?? [] if ( !result.ok || diff --git a/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts index cc4702432ab..5a2abbecedb 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts @@ -28,6 +28,16 @@ import { STRUCTURED_CLIENT, STRUCTURED_MOBILE_CLIENT } from './structured-agent-session-rpc.test-fixture' +import { + createQueuedMessageTestRig, + eventually +} from '../../../native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture' +import { + HOST_TEST_SESSION as HOST_SESSION, + hostTestMessage, + hostTestOperationId +} from '../../../native-chat/agent-session-wire/structured-agent-session-host-test-data' +import { holdDelivery } from '../../../native-chat/agent-session-wire/structured-agent-session-delivery-hold.test-fixture' beforeEach(installStructuredHostStub) afterEach(clearStructuredHostStub) @@ -425,6 +435,59 @@ describe('agentSession.send reply timing', () => { } expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled() }) + + it('keeps an old phone image send immediate and never recovers its interrupted send as a hidden card', async () => { + const rig = await createQueuedMessageTestRig() + hostCalls.send.mockImplementation(rig.host.send.bind(rig.host)) + const imageSend = () => { + const body = hostTestMessage('from an old phone') + body.blocks.push({ type: 'image-ref', path: '/tmp/shot.png' }) + const fields = { body, delivery: 'queue-if-active' as const } + const id = hostTestOperationId() + return { id, request: { envelope: rig.envelope(fields, 'agentSession.send', id), ...fields } } + } + let release = () => {} + try { + const working = await rig.workingSend() + const image = imageSend() + expect( + await call('agentSession.send', image.request, STRUCTURED_MOBILE_CLIENT) + ).toMatchObject({ + ok: true, + result: { value: { submission: { clientMessageId: image.id } } } + }) + expect(await rig.drafts()).toEqual([]) + await rig.settleAccepted(working, 'working') + await eventually(async () => { + expect((await rig.submission(image.id))?.handedOverAt).toBeDefined() + expect( + rig.dispatch.mock.calls.find(([input]) => input.clientMessageId === image.id)?.[0].body + ).toEqual(image.request.body) + }) + await rig.settleAccepted(image.id, 'image') + + const hold = holdDelivery() + release = hold.release + const interrupted = imageSend() + expect( + await call('agentSession.send', interrupted.request, STRUCTURED_MOBILE_CLIENT) + ).toMatchObject({ ok: true, result: { value: { submission: expect.anything() } } }) + await hold.held + const closed = rig.host.close(HOST_SESSION, 'user-close') + release() + await closed + rig.crashRestartHostProcess() + expect(await rig.drafts()).toEqual([]) + expect(await rig.submission(interrupted.id)).toMatchObject({ + source: { kind: 'user' }, + dispatchState: 'rejected' + }) + expect(await rig.submission(interrupted.id)).not.toHaveProperty('keptAsQueuedMessageId') + } finally { + release() + await rig.dispose() + } + }) }) function pendingSendResult() { diff --git a/src/main/runtime/rpc/methods/structured-agent-session.ts b/src/main/runtime/rpc/methods/structured-agent-session.ts index e0c92cc03f4..cef46e353a7 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.ts @@ -13,6 +13,10 @@ import { projectBackgroundTaskEvent, projectBackgroundTaskHistory } from './structured-agent-session-background-task-capability' +import { + projectQueuedImageCardPublication, + projectQueuedImageCardHistory +} from './structured-agent-session-queued-image-capability' import { projectTurnItemEvent, projectTurnItemHistory @@ -167,7 +171,10 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ handler: async (params, ctx) => { const host = await requireInstalledHost(ctx, params.sessionId) return projectTurnItemHistory( - projectBackgroundTaskHistory(await host.history(params), ctx), + projectQueuedImageCardHistory( + projectBackgroundTaskHistory(await host.history(params), ctx), + ctx + ), ctx, host.sessionAgent(params.sessionId) ) @@ -192,6 +199,8 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ readsCurrentRequests: clientReadsCurrentRequests(ctx), id: subscriptionId, sessionId: params.sessionId, + projectQueuePublication: (publication) => + projectQueuedImageCardPublication(publication, ctx), emit: (event) => emit( projectTurnItemEvent( diff --git a/src/renderer/src/components/native-chat/NativeChatQueuedMessageCard.tsx b/src/renderer/src/components/native-chat/NativeChatQueuedMessageCard.tsx index d543119a812..a489e532206 100644 --- a/src/renderer/src/components/native-chat/NativeChatQueuedMessageCard.tsx +++ b/src/renderer/src/components/native-chat/NativeChatQueuedMessageCard.tsx @@ -3,6 +3,7 @@ import { AlertCircle, ChevronDown, CornerDownRight, + Image as ImageIcon, ListEnd, MoreHorizontal, Pencil, @@ -37,6 +38,7 @@ import type { AgentSessionFailureFact } from '../../../../shared/agent-session-f import { agentSessionWriteNotDoneParts } from '../../../../shared/agent-session-refusal-notice' import { NativeChatQueuedMessageEditor } from './NativeChatQueuedMessageEditor' import type { QueuedMessageInlineEditor } from './use-structured-agent-session-queued-edit' +import { nativeChatImageLabel } from './native-chat-image-label' /** The visible caption under the text; the default waiting hold needs none. */ export function queuedMessageCardCaption( @@ -101,6 +103,23 @@ export function queuedMessageCardCaption( } } +function queuedMessageImageCount(count: number): string { + return translate('components.native-chat.queuedMessages.imageCount', '{{count}} images', { + count + }) +} + +/** The card's line: its text, or for a card of images alone, the image's name or their count. */ +function queuedMessageCardLine(card: QueuedMessageCard): string { + const images = card.images ?? [] + if (card.text.trim() || images.length === 0) { + return card.text + } + return images.length === 1 + ? nativeChatImageLabel(images[0]) + : queuedMessageImageCount(images.length) +} + /** Steer's or Send's label and tooltip (`queuedMessageCardSteers`). */ export function queuedMessageCardSendNow(card: QueuedMessageCard): { /** Steer's ↳, or Send's paper plane. */ @@ -163,6 +182,7 @@ export function NativeChatQueuedMessageCard({ onTurnOffQueueing?: () => void }): React.JSX.Element { const caption = queuedMessageCardCaption(card, agentName, statedFailures) + const line = queuedMessageCardLine(card) const returned = card.state === 'returned' const sendNow = queuedMessageCardSendNow(card) const isMac = isMacPlatform() @@ -182,7 +202,11 @@ export function NativeChatQueuedMessageCard({ data-queued-message-state={card.state} className="px-2.5 py-1.5" > - + : null} + /> ) } @@ -201,8 +225,8 @@ export function NativeChatQueuedMessageCard({
{sender} {expanded ? null : ( -

- {card.text} +

+ {line}

)} {caption ? ( @@ -217,6 +241,7 @@ export function NativeChatQueuedMessageCard({

) : null}
+ {card.images ? : null} {expanded || clipped ? ( - {card.text} + {line}

) : null} ) } +/** Images are named in the tooltip, not drawn: the row stays one line. */ +function QueuedMessageImageCount({ + images +}: { + images: NonNullable +}): React.JSX.Element { + return ( + + + + + {images.length} + + + + {images.map(nativeChatImageLabel).join(', ')} + + + ) +} + function QueuedMessageExpandToggle({ expanded, controls, diff --git a/src/renderer/src/components/native-chat/NativeChatQueuedMessageEditor.tsx b/src/renderer/src/components/native-chat/NativeChatQueuedMessageEditor.tsx index 71754511ed0..c7f9497a3f1 100644 --- a/src/renderer/src/components/native-chat/NativeChatQueuedMessageEditor.tsx +++ b/src/renderer/src/components/native-chat/NativeChatQueuedMessageEditor.tsx @@ -11,11 +11,14 @@ import type { QueuedMessageInlineEditor } from './use-structured-agent-session-q * not the composer, so it stays usable while a question or approval holds the composer's slot. */ export function NativeChatQueuedMessageEditor({ editor, - sender + sender, + images }: { editor: QueuedMessageInlineEditor /** The card's "From " line, kept beside the label. */ sender?: ReactNode + /** The card's images, which saving the text keeps. */ + images?: ReactNode }): React.JSX.Element { const field = useRef(null) useLayoutEffect(() => { @@ -49,11 +52,11 @@ export function NativeChatQueuedMessageEditor({
{sender}
) : null} +
{images}