feat(native-chat): image messages wait in the queue like text (#28655)

* feat(native-chat): image messages wait in the queue like text

A message with images now queues behind the running turn and the cards
ahead of it, instead of skipping the queue. Cards show an image icon and
count, an image-only card names its image, and editing a card's text keeps
its images. The host publishes no card holding an image to a client that
has not advertised agent-session.queued-image-cards.v1, since released
clients edit by copying a card's text and deleting the card.

* test(native-chat): an image follow-up queues like text

* fix(native-chat): project queued images before phone frame limits

* fix(native-chat): preserve image sends from older clients

* Clarify direct text recovery at send admission

* fix(native-chat): expose queued image names to keyboard users
This commit is contained in:
Brennan Benson
2026-10-17 00:07:29 -07:00
committed by GitHub
parent 58f1f7de1e
commit 32a53d3ec6
35 changed files with 488 additions and 88 deletions
@@ -4,6 +4,7 @@ import {
AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY,
AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY,
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
AGENT_SESSION_SNAPSHOT_CATCH_UP_CLIENT_CAPABILITY,
AGENT_SESSION_TURN_ITEM_CAPABILITY,
CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY,
@@ -48,6 +49,8 @@ export const DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES: readonly RuntimeCapab
AGENT_SESSION_TURN_ITEM_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_CHILD_VIEWS_CAPABILITY,
// Its queued cards show their images, and Edit keeps them.
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY,
CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY,
AGENT_SESSION_OPTIONAL_MODEL_CLIENT_CAPABILITY,
@@ -1,6 +1,6 @@
// What becomes of a send the host accepted and can no longer hand over: an earlier host process
// quit or crashed first (settled at the next open), or the chat is closing. Either way the agent
// provably never got it. A person's message (typed, or a launch's first prompt) is kept as a card
// provably never got it. A person's text (typed, or a launch's first prompt) is kept as a card
// at the head of the queue, an ordinary one: like every card the chat closed with, it waits for the
// chat's next turn (`queued-message-pause.ts`). Everything else is rejected as before, because
// something else re-derives it or the person re-runs it. The submission itself is always rejected,
@@ -31,7 +31,7 @@ export type UnsentSendHold =
/** The body an unsent send is kept with, or null when it is rejected instead:
* - a card's own hand-off: its rejection already returns the card (`rejectedDraftSettlement`);
* - `/compact` and other commands: a command in flight is not resumed, the person re-runs it;
* - an image: cards are text-only;
* - a direct image: it retains the existing rejected-send recovery instead of becoming a card;
* - a send not from a person: orchestration mail (the mailbox re-sends it), a dispatch preamble
* or a restart continuation (their owners re-derive them), a kind this build does not know, and
* a row with no source that is not a person's (it could be either). */
@@ -18,8 +18,9 @@ export type SubscriberFieldState = {
sessionId: string
/** Every frame to fit a phone's bridge frame; see `boundedFrames`. */
boundedFrames: boolean
projectQueuePublication?: (publication: QueuePublication) => QueuePublication
commands?: AgentSessionSlashCommand[] | null
/** The last queue publication actually SENT. */
/** Canonical publication last sent, before reader projection. */
queuePublication?: QueuePublication
/** Fingerprint of the roster last SENT; absent until this subscriber's first frame. */
backgroundTasks?: string
@@ -70,7 +71,9 @@ export function buildSubscriberFrame(
event.type !== 'end' &&
(event.type !== 'batch' || queued !== subscriber.queuePublication)
const backgroundTasks = subscriberBackgroundTasks(hooks, subscriber, event)
const sideQueued = attachedQueued ? queued : undefined
const sideQueued = attachedQueued
? (subscriber.projectQueuePublication?.(queued) ?? queued)
: undefined
return {
frames: subscriber.boundedFrames
? boundedFrames(event, { queued: sideQueued, backgroundTasks })
@@ -69,6 +69,8 @@ export function sendStructuredAgentSessionTurn(
body: AgentJournalMessageItem
retryUnknown?: true
delivery?: 'queue-if-active'
/** Host-local: whether the sending client can manage an image card. */
queuedImageCards?: boolean
options?: Readonly<Record<string, string>>
/** Host-local, set only by the client-facing `agentSession.send` RPC (the
* renderer's launch prompt included): recorded as the submission's `client`
@@ -29,11 +29,7 @@ import {
type StructuredAgentSessionMutationContext
} from './structured-agent-session-host-mutations'
import { flushStructuredAgentSessionHost } from './structured-agent-session-host-teardown'
import type {
StructuredAgentSessionCaller,
StructuredAgentSessionHostDeps,
StructuredAgentSessionReveal
} from './structured-agent-session-host-types'
import type * as HostTypes from './structured-agent-session-host-types'
import { StructuredAgentSessionEventRecovery } from './structured-agent-session-event-recovery'
import { StructuredAgentSessionBackgroundTaskChannel } from './structured-agent-session-background-task-channel'
import { StructuredAgentSessionClientDelivery } from './structured-agent-session-client-delivery'
@@ -88,7 +84,7 @@ export class StructuredAgentSessionHost {
/** Public because the RPC surface addresses it directly; see the restart-resume collaborator. */
readonly restartResume: StructuredAgentSessionRestartResume
constructor(readonly deps: StructuredAgentSessionHostDeps) {
constructor(readonly deps: HostTypes.StructuredAgentSessionHostDeps) {
// Every collaborator reads this copy, so a logger that throws cannot fail what it reports.
this.deps = deps = sessionLogger.withNeverThrowingLogger(deps)
this.clientDelivery.watchAtRestCommands(deps.adapter)
@@ -239,7 +235,7 @@ export class StructuredAgentSessionHost {
this.restore.restoreReadableSessions(sessionIds)
/** Make one persisted session addressable again; see `structured-agent-session-reveal`. */
revealSession = (sessionId: string): Promise<StructuredAgentSessionReveal> =>
revealSession = (sessionId: string): Promise<HostTypes.StructuredAgentSessionReveal> =>
reveal.revealStructuredAgentSession(this.deps, sessionId, this.lifetime.conversation)
private serialize = this.tasks.serialize.bind(this.tasks)
@@ -313,7 +309,7 @@ export class StructuredAgentSessionHost {
changeThreadGoal = this.mutations.changeThreadGoal
readOptions = this.mutations.readOptions
rewind = (caller: StructuredAgentSessionCaller, params: AgentSessionRewindParams) =>
rewind = (caller: HostTypes.StructuredAgentSessionCaller, params: AgentSessionRewindParams) =>
rewindStructuredAgentSession(this.mutationContext(), this.attachContext(), caller, params)
conversationCommand = (...args: Parameters<StructuredConversationCommandController['run']>) =>
@@ -103,25 +103,27 @@ describe('accept', () => {
expect(await drafts()).toHaveLength(1)
})
it('routes an image send to the immediate path even while working (text-only v1)', async () => {
await workingSend()
const body = {
kind: 'message' as const,
role: 'user' as const,
blocks: [{ type: 'image-ref' as const, path: '/tmp/shot.png' }]
it('queues an image message in line behind text, and hands it over with its image', async () => {
const working = await workingSend()
const text = await send('text first', 'queue-if-active').result
const body = hostTestMessage('and look at this')
body.blocks.push({ type: 'image-ref', path: '/tmp/shot.png' })
const fields = { body, delivery: 'queue-if-active' as const }
const envelopeFields = rig.envelope(fields, 'agentSession.send', hostTestOperationId())
const image = await host.send(CALLER, { envelope: envelopeFields, ...fields, userSend: true })
if (!text.ok || !('queued' in text.value) || !image.ok || !('queued' in image.value)) {
throw new Error('expected queued receipts')
}
const clientOperationId = hostTestOperationId()
const result = await host.send(CALLER, {
envelope: rig.envelope(
{ body, delivery: 'queue-if-active' },
'agentSession.send',
clientOperationId
),
body,
delivery: 'queue-if-active'
const [textId, imageId] = [text.value.queued.messageId, image.value.queued.messageId]
await settleAccepted(working, 'a')
await eventually(async () => expect(await rig.handoff(textId)).toBeDefined())
expect(await rig.handoff(imageId)).toBeUndefined()
await settleAccepted(await rig.handoffId(textId), 'b')
await eventually(async () => {
const handedOver = await rig.handoffId(imageId)
const sent = dispatch.mock.calls.find(([input]) => input.clientMessageId === handedOver)
expect(sent?.[0].body.blocks).toEqual(body.blocks)
})
expect(result).toMatchObject({ ok: true, value: { submission: expect.anything() } })
expect(await drafts()).toHaveLength(0)
})
it('a send without the delivery field never queues, whatever the session is doing', async () => {
@@ -37,11 +37,6 @@ import type { StructuredAgentSessionLogger } from './structured-agent-session-lo
import { agentSessionAttachmentExpiredRefusal } from './structured-agent-session-turns'
import { isAgentSessionAttachmentExpiredError } from '../agent-session-attachments/agent-session-attachment-claims'
/** Text-only v1: any image block routes to the immediate path. */
export function queuedMessageBodyIsTextOnly(body: AgentJournalMessageItem): boolean {
return body.blocks.every((block) => block.type === 'text')
}
/** Waiting, not held on its own, and not positioned behind a returned card or a
* card the queue's pause holds: the queue never reorders. The admission rule
* (§accept) and the drain's selection both read it; only the drain's (`automatic`) stops at a
@@ -142,8 +137,8 @@ export function shouldQueueStructuredAgentSessionSend(
/**
* The accept branch: a capable send while the session is working (or behind an
* actionable backlog) becomes a draft instead of a submission. Returns null for
* the immediate path — an incapable client, an image body (text-only v1), a
* replayed id the journal already answers, or an idle session.
* the immediate path — an incapable client, a replayed id the journal already
* answers, or an idle session.
*/
export async function maybeQueueStructuredAgentSessionSend(
context: {
@@ -157,6 +152,7 @@ export async function maybeQueueStructuredAgentSessionSend(
envelope: { clientOperationId: string }
body: AgentJournalMessageItem
delivery?: 'queue-if-active'
queuedImageCards?: boolean
options?: Readonly<Record<string, string>>
/** A person's send at a chat surface: every attachment it names must still be stored. */
userSend?: true
@@ -169,7 +165,11 @@ export async function maybeQueueStructuredAgentSessionSend(
| null
> {
const clientMessageId = params.envelope.clientOperationId
if (params.delivery !== 'queue-if-active' || !queuedMessageBodyIsTextOnly(params.body)) {
if (
params.delivery !== 'queue-if-active' ||
(params.queuedImageCards === false &&
params.body.blocks.some((block) => block.type === 'image-ref'))
) {
return null
}
// Asked again with no ledger answer: a send this host queued answers as its replay would —
@@ -14,6 +14,7 @@ export async function runQueueableStructuredAgentSessionSend(
envelope: { clientOperationId: string }
body: AgentJournalMessageItem
delivery?: 'queue-if-active'
queuedImageCards?: boolean
options?: Readonly<Record<string, string>>
userSend?: true
personsMessage?: true
@@ -17,7 +17,8 @@ import {
backgroundTaskFingerprint,
emitSubscriberFrame,
subscriberCommandsChanged,
type SubscriberFieldHooks
type SubscriberFieldHooks,
type SubscriberFieldState
} from './agent-session-subscriber-frame-fields'
import type { QueuePublication } from './structured-agent-session-queued-publication'
import { deliverToSubscriber } from './agent-session-subscriber-catch-up'
@@ -38,6 +39,7 @@ export type AgentSessionSubscribeInput = {
cursor?: AgentJournalCursor
/** Every frame to fit a phone's bridge frame; the RPC sets it for every phone. */
boundedFrames?: boolean
projectQueuePublication?: SubscriberFieldState['projectQueuePublication']
readsCurrentRequests?: boolean
/** False for desktops that need exact replay to keep send echoes and current requests. */
snapshotCatchUp?: boolean
@@ -50,9 +52,10 @@ export type Subscriber = CurrentRequestsSubscriber & {
cursor: AgentJournalCursor
fence: number
boundedFrames: boolean
projectQueuePublication?: SubscriberFieldState['projectQueuePublication']
snapshotCatchUp?: boolean
commands?: AgentSessionSlashCommand[] | null
/** The last draft list actually SENT — never advanced on a page that withheld
/** Canonical publication last sent — never advanced on a page that withheld
* it, or the final replacement would be suppressed by the identity dedup. */
queuePublication?: QueuePublication
/** Fingerprint of the background-task roster last SENT. */
@@ -93,17 +96,9 @@ export class AgentSessionSubscribers {
return this.bySession.get(sessionId)?.size ?? 0
}
open(input: {
id: string
sessionId: string
journal: AgentSessionJournal
fence: number
emit: AgentSessionSubscriberEmit
cursor?: AgentJournalCursor
boundedFrames?: boolean
readsCurrentRequests?: boolean
snapshotCatchUp?: boolean
}): () => void {
open(
input: AgentSessionSubscribeInput & { journal: AgentSessionJournal; fence: number }
): () => void {
const liveCursor = input.journal.cursor()
const subscriber: Subscriber = {
id: input.id,
@@ -111,8 +106,9 @@ export class AgentSessionSubscribers {
emit: input.emit,
cursor: input.cursor ?? { epoch: liveCursor.epoch, sequence: 0 },
fence: input.fence,
readsCurrentRequests: input.readsCurrentRequests === true,
boundedFrames: input.boundedFrames === true,
projectQueuePublication: input.projectQueuePublication,
readsCurrentRequests: input.readsCurrentRequests === true,
snapshotCatchUp: input.snapshotCatchUp !== false
}
const session = this.bySession.get(input.sessionId) ?? new Map<string, Subscriber>()
@@ -0,0 +1,146 @@
import { afterEach, beforeEach, expect, it } from 'vitest'
import type {
AgentSessionQueuedMessage,
AgentSessionSubscribeEvent
} from '../../../../shared/agent-session-wire'
import {
buildSubscriberFrame,
type SubscriberFieldState
} from '../../../native-chat/agent-session-wire/agent-session-subscriber-frame-fields'
import { boundedQueuePublication } from '../../../native-chat/agent-session-wire/agent-session-bounded-queue'
import { DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES } from '../../../ipc/desktop-renderer-runtime-capabilities'
import type { AgentSessionSubscribeInput } from '../../../native-chat/agent-session-wire/structured-agent-session-subscribers'
import {
call,
clearStructuredHostStub,
hostCalls,
installStructuredHostStub,
SESSION,
STRUCTURED_CLIENT,
STRUCTURED_MOBILE_CLIENT
} from './structured-agent-session-rpc.test-fixture'
beforeEach(installStructuredHostStub)
afterEach(clearStructuredHostStub)
function card(messageId: string, position: number, image: boolean): AgentSessionQueuedMessage {
return {
messageId,
position,
state: 'waiting',
body: {
kind: 'message',
role: 'user',
blocks: [
{ type: 'text', text: messageId },
...(image ? [{ type: 'image-ref' as const, path: '/tmp/shot.png' }] : [])
]
}
}
}
const TEXT = card('text', 2, false)
const IMAGE = card('image', 1, true)
const DESKTOP = {
clientKind: 'runtime' as const,
clientCapabilities: [...DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES]
}
it('withholds image cards from a reader that predates them, on history and the stream', async () => {
const queue = {
queuedMessages: [IMAGE, TEXT],
queuePause: { reason: 'stopped' as const },
nextQueuedMessageId: 'image'
}
hostCalls.history.mockReturnValue({ ok: true, page: { items: [], ...queue } })
hostCalls.subscribe.mockImplementation((input: AgentSessionSubscribeInput) => {
const event: AgentSessionSubscribeEvent = {
type: 'batch',
sessionId: SESSION,
batch: {
cursor: { epoch: 'a', sequence: 0 },
items: [],
removedItemIds: [],
submissions: []
}
}
for (const frame of buildSubscriberFrame(
{ readQueuePublication: () => queue },
{
sessionId: SESSION,
boundedFrames: input.boundedFrames === true,
projectQueuePublication: input.projectQueuePublication
},
event,
false
).frames) {
input.emit(frame)
}
return () => {}
})
const read = async (client: Parameters<typeof call>[2]) => [
await call('agentSession.history', { sessionId: SESSION, direction: 'tail' }, client),
await call('agentSession.subscribe', { sessionId: SESSION }, client)
]
// The local renderer is a negotiated reader too: its own list must carry the capability.
for (const client of [DESKTOP, undefined]) {
const [history, stream] = await read(client)
expect(history).toMatchObject({ result: { page: queue } })
expect(stream).toMatchObject({ result: queue })
}
for (const client of [STRUCTURED_CLIENT, STRUCTURED_MOBILE_CLIENT]) {
const [history, stream] = await read(client)
// Positions, the pause over a card it sees, and the next card's id stay as the host published.
const withheld = { ...queue, queuedMessages: [TEXT] }
expect(history).toMatchObject({ result: { page: withheld } })
expect(stream).toMatchObject({ result: withheld })
}
// With no card left to show, no pause either: its Resume would send only what it cannot see.
hostCalls.history.mockReturnValue({
ok: true,
page: { items: [], ...queue, queuedMessages: [IMAGE] }
})
expect(
await call('agentSession.history', { sessionId: SESSION, direction: 'tail' }, STRUCTURED_CLIENT)
).toMatchObject({ result: { page: { queuedMessages: [], queuePause: null } } })
})
it('publishes phone text behind hidden images before bounding, and deduplicates the canonical queue', async () => {
const images = Array.from({ length: 3 }, (_, index) => {
const image = card(`image-${index}`, index + 1, true)
image.body.blocks.unshift({ type: 'text', text: 'x'.repeat(200 * 1024) })
return image
})
const text = card('text', images.length + 1, false)
const queue = {
queuedMessages: [...images, text],
queuePause: { reason: 'stopped' as const },
nextQueuedMessageId: null
}
expect(boundedQueuePublication(queue).queuedMessages).not.toContain(text)
hostCalls.subscribe.mockImplementation((input: AgentSessionSubscribeInput) => {
const subscriber: SubscriberFieldState = {
sessionId: SESSION,
boundedFrames: input.boundedFrames === true,
projectQueuePublication: input.projectQueuePublication
}
const event: AgentSessionSubscribeEvent = {
type: 'batch',
sessionId: SESSION,
batch: { cursor: { epoch: 'a', sequence: 0 }, items: [], removedItemIds: [], submissions: [] }
}
const context = { readQueuePublication: () => queue }
const built = buildSubscriberFrame(context, subscriber, event, false)
for (const frame of built.frames) {
input.emit(frame)
}
expect(built.queued).toBe(queue)
subscriber.queuePublication = built.queued
expect(buildSubscriberFrame(context, subscriber, event, false).attachedQueued).toBe(false)
return () => {}
})
expect(
await call('agentSession.subscribe', { sessionId: SESSION }, STRUCTURED_MOBILE_CLIENT)
).toMatchObject({ result: { queuedMessages: [text] } })
})
@@ -0,0 +1,66 @@
// A reader that predates image cards shows a card's text only, and edits by copying that text and
// deleting the card, which would drop its images. It is never handed a card holding an image, at
// the RPC boundary only: the journal, the drain and every in-process reader keep the whole queue.
import type {
AgentSessionHistoryResult,
AgentSessionQueuedMessage,
AgentSessionQueuePublicationFields
} from '../../../../shared/agent-session-wire'
import type { QueuePublication } from '../../../native-chat/agent-session-wire/structured-agent-session-queued-publication'
import { AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY } from '../../../../shared/protocol-version'
import type { RpcContext } from '../core'
type QueuedImageReader = Pick<RpcContext, 'clientKind' | 'clientCapabilities'>
type QueueFields = AgentSessionQueuePublicationFields & {
queuedMessages?: AgentSessionQueuedMessage[] | null
}
export function readsQueuedImageCards(ctx: QueuedImageReader): boolean {
// An in-process caller is this build; only a negotiated client can predate image cards.
return (
ctx.clientKind === undefined ||
ctx.clientCapabilities?.includes(AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY) === true
)
}
function holdsImage(card: AgentSessionQueuedMessage): boolean {
return card.body.blocks.some((block) => block.type === 'image-ref')
}
/** The fields to replace, or null when nothing is withheld. Positions and the next card's id stay
* as published, as when a phone's frame trims the list. A pause over no card the reader sees would
* offer a Resume that sends nothing it shows. */
function withoutImageCards(
fields: QueueFields
):
| (Pick<QueuePublication, 'queuedMessages'> & Partial<Pick<QueuePublication, 'queuePause'>>)
| null {
const cards = fields.queuedMessages
if (!cards?.some(holdsImage)) {
return null
}
const queuedMessages = cards.filter((card) => !holdsImage(card))
return {
queuedMessages,
...(queuedMessages.length === 0 && fields.queuePause !== undefined ? { queuePause: null } : {})
}
}
export function projectQueuedImageCardHistory(
result: AgentSessionHistoryResult,
ctx: QueuedImageReader
): AgentSessionHistoryResult {
const page = readsQueuedImageCards(ctx) ? null : withoutImageCards(result.page)
return page ? { ...result, page: { ...result.page, ...page } } : result
}
/** Applied before subscriber frame sizing; the host's canonical publication stays unchanged. */
export function projectQueuedImageCardPublication(
publication: QueuePublication,
ctx: QueuedImageReader
): QueuePublication {
const fields = readsQueuedImageCards(ctx) ? null : withoutImageCards(publication)
return fields ? { ...publication, ...fields } : publication
}
@@ -7,6 +7,7 @@ import type { StructuredAgentSessionHost } from '../../../native-chat/agent-sess
import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement'
import type { RpcContext } from '../core'
import { requireStructuredSessionHost, structuredCallerFor } from './structured-agent-session-gate'
import { readsQueuedImageCards } from './structured-agent-session-queued-image-capability'
/**
* A send answers once the host accepts it. A client that predates that answer cannot show a
@@ -20,8 +21,12 @@ export async function sendStructuredAgentSessionForClient(
context: RpcContext
) {
const host = requireStructuredSessionHost(context, params.envelope.sessionId)
// A client's own send: a restart or a close keeps it as a card if it never reached the agent.
const result = await host.send(structuredCallerFor(context), { ...params, userSend: true })
// Person provenance lets undelivered text recover as an ordinary queued card.
const result = await host.send(structuredCallerFor(context), {
...params,
userSend: true,
queuedImageCards: readsQueuedImageCards(context)
})
const capabilities = context.clientCapabilities ?? []
if (
!result.ok ||
@@ -28,6 +28,16 @@ import {
STRUCTURED_CLIENT,
STRUCTURED_MOBILE_CLIENT
} from './structured-agent-session-rpc.test-fixture'
import {
createQueuedMessageTestRig,
eventually
} from '../../../native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture'
import {
HOST_TEST_SESSION as HOST_SESSION,
hostTestMessage,
hostTestOperationId
} from '../../../native-chat/agent-session-wire/structured-agent-session-host-test-data'
import { holdDelivery } from '../../../native-chat/agent-session-wire/structured-agent-session-delivery-hold.test-fixture'
beforeEach(installStructuredHostStub)
afterEach(clearStructuredHostStub)
@@ -425,6 +435,59 @@ describe('agentSession.send reply timing', () => {
}
expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled()
})
it('keeps an old phone image send immediate and never recovers its interrupted send as a hidden card', async () => {
const rig = await createQueuedMessageTestRig()
hostCalls.send.mockImplementation(rig.host.send.bind(rig.host))
const imageSend = () => {
const body = hostTestMessage('from an old phone')
body.blocks.push({ type: 'image-ref', path: '/tmp/shot.png' })
const fields = { body, delivery: 'queue-if-active' as const }
const id = hostTestOperationId()
return { id, request: { envelope: rig.envelope(fields, 'agentSession.send', id), ...fields } }
}
let release = () => {}
try {
const working = await rig.workingSend()
const image = imageSend()
expect(
await call('agentSession.send', image.request, STRUCTURED_MOBILE_CLIENT)
).toMatchObject({
ok: true,
result: { value: { submission: { clientMessageId: image.id } } }
})
expect(await rig.drafts()).toEqual([])
await rig.settleAccepted(working, 'working')
await eventually(async () => {
expect((await rig.submission(image.id))?.handedOverAt).toBeDefined()
expect(
rig.dispatch.mock.calls.find(([input]) => input.clientMessageId === image.id)?.[0].body
).toEqual(image.request.body)
})
await rig.settleAccepted(image.id, 'image')
const hold = holdDelivery()
release = hold.release
const interrupted = imageSend()
expect(
await call('agentSession.send', interrupted.request, STRUCTURED_MOBILE_CLIENT)
).toMatchObject({ ok: true, result: { value: { submission: expect.anything() } } })
await hold.held
const closed = rig.host.close(HOST_SESSION, 'user-close')
release()
await closed
rig.crashRestartHostProcess()
expect(await rig.drafts()).toEqual([])
expect(await rig.submission(interrupted.id)).toMatchObject({
source: { kind: 'user' },
dispatchState: 'rejected'
})
expect(await rig.submission(interrupted.id)).not.toHaveProperty('keptAsQueuedMessageId')
} finally {
release()
await rig.dispose()
}
})
})
function pendingSendResult() {
@@ -13,6 +13,10 @@ import {
projectBackgroundTaskEvent,
projectBackgroundTaskHistory
} from './structured-agent-session-background-task-capability'
import {
projectQueuedImageCardPublication,
projectQueuedImageCardHistory
} from './structured-agent-session-queued-image-capability'
import {
projectTurnItemEvent,
projectTurnItemHistory
@@ -167,7 +171,10 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [
handler: async (params, ctx) => {
const host = await requireInstalledHost(ctx, params.sessionId)
return projectTurnItemHistory(
projectBackgroundTaskHistory(await host.history(params), ctx),
projectQueuedImageCardHistory(
projectBackgroundTaskHistory(await host.history(params), ctx),
ctx
),
ctx,
host.sessionAgent(params.sessionId)
)
@@ -192,6 +199,8 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [
readsCurrentRequests: clientReadsCurrentRequests(ctx),
id: subscriptionId,
sessionId: params.sessionId,
projectQueuePublication: (publication) =>
projectQueuedImageCardPublication(publication, ctx),
emit: (event) =>
emit(
projectTurnItemEvent(
@@ -3,6 +3,7 @@ import {
AlertCircle,
ChevronDown,
CornerDownRight,
Image as ImageIcon,
ListEnd,
MoreHorizontal,
Pencil,
@@ -37,6 +38,7 @@ import type { AgentSessionFailureFact } from '../../../../shared/agent-session-f
import { agentSessionWriteNotDoneParts } from '../../../../shared/agent-session-refusal-notice'
import { NativeChatQueuedMessageEditor } from './NativeChatQueuedMessageEditor'
import type { QueuedMessageInlineEditor } from './use-structured-agent-session-queued-edit'
import { nativeChatImageLabel } from './native-chat-image-label'
/** The visible caption under the text; the default waiting hold needs none. */
export function queuedMessageCardCaption(
@@ -101,6 +103,23 @@ export function queuedMessageCardCaption(
}
}
function queuedMessageImageCount(count: number): string {
return translate('components.native-chat.queuedMessages.imageCount', '{{count}} images', {
count
})
}
/** The card's line: its text, or for a card of images alone, the image's name or their count. */
function queuedMessageCardLine(card: QueuedMessageCard): string {
const images = card.images ?? []
if (card.text.trim() || images.length === 0) {
return card.text
}
return images.length === 1
? nativeChatImageLabel(images[0])
: queuedMessageImageCount(images.length)
}
/** Steer's or Send's label and tooltip (`queuedMessageCardSteers`). */
export function queuedMessageCardSendNow(card: QueuedMessageCard): {
/** Steer's ↳, or Send's paper plane. */
@@ -163,6 +182,7 @@ export function NativeChatQueuedMessageCard({
onTurnOffQueueing?: () => void
}): React.JSX.Element {
const caption = queuedMessageCardCaption(card, agentName, statedFailures)
const line = queuedMessageCardLine(card)
const returned = card.state === 'returned'
const sendNow = queuedMessageCardSendNow(card)
const isMac = isMacPlatform()
@@ -182,7 +202,11 @@ export function NativeChatQueuedMessageCard({
data-queued-message-state={card.state}
className="px-2.5 py-1.5"
>
<NativeChatQueuedMessageEditor editor={editor} sender={sender} />
<NativeChatQueuedMessageEditor
editor={editor}
sender={sender}
images={card.images ? <QueuedMessageImageCount images={card.images} /> : null}
/>
</li>
)
}
@@ -201,8 +225,8 @@ export function NativeChatQueuedMessageCard({
<div className="min-w-0 flex-1">
{sender}
{expanded ? null : (
<p ref={measureLine} className="truncate text-sm" title={card.text}>
{card.text}
<p ref={measureLine} className="truncate text-sm" title={line}>
{line}
</p>
)}
{caption ? (
@@ -217,6 +241,7 @@ export function NativeChatQueuedMessageCard({
</p>
) : null}
</div>
{card.images ? <QueuedMessageImageCount images={card.images} /> : null}
{expanded || clipped ? (
<QueuedMessageExpandToggle
expanded={expanded}
@@ -320,13 +345,39 @@ export function NativeChatQueuedMessageCard({
id={textId}
className="scrollbar-sleek mt-1 max-h-60 overflow-y-auto whitespace-pre-wrap break-words pl-5.5 text-sm"
>
{card.text}
{line}
</p>
) : null}
</li>
)
}
/** Images are named in the tooltip, not drawn: the row stays one line. */
function QueuedMessageImageCount({
images
}: {
images: NonNullable<QueuedMessageCard['images']>
}): React.JSX.Element {
return (
<Tooltip>
<TooltipTrigger asChild>
<span
role="img"
tabIndex={0}
aria-label={queuedMessageImageCount(images.length)}
className="flex shrink-0 items-center gap-0.5 text-xs text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
>
<ImageIcon className="size-3.5" aria-hidden />
<span aria-hidden>{images.length}</span>
</span>
</TooltipTrigger>
<TooltipContent side="top" sideOffset={4}>
{images.map(nativeChatImageLabel).join(', ')}
</TooltipContent>
</Tooltip>
)
}
function QueuedMessageExpandToggle({
expanded,
controls,
@@ -11,11 +11,14 @@ import type { QueuedMessageInlineEditor } from './use-structured-agent-session-q
* not the composer, so it stays usable while a question or approval holds the composer's slot. */
export function NativeChatQueuedMessageEditor({
editor,
sender
sender,
images
}: {
editor: QueuedMessageInlineEditor
/** The card's "From <name>" line, kept beside the label. */
sender?: ReactNode
/** The card's images, which saving the text keeps. */
images?: ReactNode
}): React.JSX.Element {
const field = useRef<HTMLTextAreaElement>(null)
useLayoutEffect(() => {
@@ -49,11 +52,11 @@ export function NativeChatQueuedMessageEditor({
<div className="min-w-0">{sender}</div>
</>
) : null}
<div className="ml-auto flex items-center">{images}</div>
<Button
type="button"
variant="ghost"
size="icon-xs"
className="ml-auto"
aria-label={translate(
'components.native-chat.queuedMessages.cancelEditing',
'Cancel editing'
@@ -18,6 +18,7 @@ import {
import type { RuntimeFileOperationArgs } from '@/runtime/runtime-file-client-types'
import { copyableNativeChatImageSrc, keepPreviewOpenForChatMenu } from './native-chat-image-copy'
import { isNativeChatPastedImagePath } from './native-chat-image-paste'
import { nativeChatImageLabel } from './native-chat-image-label'
import { chatImageAccess } from '@/lib/local-file-access'
type VisibilityListener = (isVisible: boolean) => void
@@ -125,13 +126,7 @@ function TranscriptImagePreview({
)
const displaySrc = external && leaseActive ? source : localSrc
const copySrc = copyableNativeChatImageSrc(displaySrc)
const label =
block.alt?.trim() ||
(block.path && isNativeChatPastedImagePath(block.path)
? translate('components.native-chat.composer.pastedImageLabel', 'Pasted image')
: block.path
? basename(block.path)
: 'Image')
const label = nativeChatImageLabel(block)
const viewImageLabel = translate('components.native-chat.composer.viewAttachment', 'View image')
const fallback = (
<div
@@ -0,0 +1,16 @@
import { translate } from '@/i18n/i18n'
import { basename } from '@/lib/path'
import type { NativeChatImageRefBlock } from '../../../../shared/native-chat-types'
import { isNativeChatPastedImagePath } from './native-chat-image-paste'
/** What a chat image is called where it is not drawn: its alt text, else its file's name. */
export function nativeChatImageLabel(block: NativeChatImageRefBlock): string {
return (
block.alt?.trim() ||
(block.path && isNativeChatPastedImagePath(block.path)
? translate('components.native-chat.composer.pastedImageLabel', 'Pasted image')
: block.path
? basename(block.path)
: 'Image')
)
}
@@ -4,23 +4,17 @@ import {
structuredAgentSessionQueueRequest
} from './structured-agent-session-queue-request'
const image = [{ path: '/tmp/a.png', previewUri: '/tmp/a.png' }]
it('asks the host to queue only a text send, to a host that queues, with queueing on', () => {
expect(structuredAgentSessionQueueRequest({ capability: 'supported', enabled: true }, [])).toBe(
it('asks the host to queue only a host that queues, with queueing on', () => {
expect(structuredAgentSessionQueueRequest({ capability: 'supported', enabled: true })).toBe(
'queue-if-active'
)
// Its queue takes only text: an image goes out at once.
expect(
structuredAgentSessionQueueRequest({ capability: 'supported', enabled: true }, image)
).toBeUndefined()
// Queueing off, or a host that does not queue or has not said yet: a plain send.
for (const queue of [
{ capability: 'supported' as const, enabled: false },
{ capability: 'unsupported' as const, enabled: true },
{ capability: 'unknown' as const, enabled: true }
]) {
expect(structuredAgentSessionQueueRequest(queue, [])).toBeUndefined()
expect(structuredAgentSessionQueueRequest(queue)).toBeUndefined()
expect(structuredAgentSessionNewSendsQueue(queue)).toBe(false)
}
expect(structuredAgentSessionNewSendsQueue({ capability: 'supported', enabled: true })).toBe(true)
@@ -1,4 +1,3 @@
import type { StructuredAgentSessionAttachment } from '../../../../shared/structured-agent-session-send-mutation'
import type { StructuredAgentSessionHostCapabilityState } from '@/runtime/structured-agent-session-host-capability'
export type StructuredAgentSessionQueueDelivery = {
@@ -13,13 +12,9 @@ export function structuredAgentSessionNewSendsQueue(
return queue.capability === 'supported' && queue.enabled
}
/** Whether the host holds a send as a card while the agent works: only text, which is all its
* queue takes. */
/** Whether the host holds a send, images included, as a card while the agent works. */
export function structuredAgentSessionQueueRequest(
queue: StructuredAgentSessionQueueDelivery,
attachments: readonly StructuredAgentSessionAttachment[]
queue: StructuredAgentSessionQueueDelivery
): 'queue-if-active' | undefined {
return structuredAgentSessionNewSendsQueue(queue) && attachments.length === 0
? 'queue-if-active'
: undefined
return structuredAgentSessionNewSendsQueue(queue) ? 'queue-if-active' : undefined
}
@@ -3,6 +3,7 @@
import type { UnreadAgentSessionFailureFact } from '../../../../shared/agent-session-failure'
import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types'
import type { NativeChatImageRefBlock } from '../../../../shared/native-chat-types'
import type {
AgentSessionQueuedMessage,
AgentSessionQueuePause
@@ -31,8 +32,10 @@ export type QueuedMessageCardHold =
export type QueuedMessageCard = {
messageId: string
position: number
/** The draft's text blocks joined; drafts are text-only in v1. */
/** The draft's text blocks joined. */
text: string
/** The draft's images, which an edit of its text keeps. */
images?: readonly NativeChatImageRefBlock[]
state: 'waiting' | 'returned'
hold: QueuedMessageCardHold
/** A conversation command such as /compact: it never steers into a running turn. */
@@ -51,6 +54,13 @@ function queuedMessageCardText(body: AgentSessionQueuedMessage['body']): string
return body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])).join('\n')
}
function queuedMessageCardImages(
body: AgentSessionQueuedMessage['body']
): Pick<QueuedMessageCard, 'images'> {
const images = body.blocks.flatMap((block) => (block.type === 'image-ref' ? [block] : []))
return images.length > 0 ? { images } : {}
}
/**
* Cards in queue order. A waiting card is hidden once a submission hands it off
* (`queuedMessageId`) and that hand-off is live: on a multi-page catch-up the shrunk
@@ -87,6 +97,7 @@ export function projectQueuedMessageCards(
messageId: message.messageId,
position: message.position,
text: queuedMessageCardText(message.body),
...queuedMessageCardImages(message.body),
state: message.state,
hold,
...(message.body.command !== undefined
@@ -148,6 +159,7 @@ export function sendingQueuedMessageCards(
// After every card the host holds, in send order.
position: Number.MAX_SAFE_INTEGER - entries.length + index,
text: queuedMessageCardText(entry.body),
...queuedMessageCardImages(entry.body),
state: 'waiting',
hold: 'sending'
}))
@@ -23,7 +23,13 @@ it.each([
['a text follow-up while the agent works', QUEUEING, true, [], 'queued'],
['the same follow-up when the agent is idle', QUEUEING, false, [], true],
['with the queue setting off', { capability: 'supported', enabled: false }, true, [], true],
['an image, which never queues', QUEUEING, true, [{ path: '/a.png', previewUri: '/a.png' }], true]
[
'an image follow-up, which queues like text',
QUEUEING,
true,
[{ path: '/a.png', previewUri: '/a.png' }],
'queued'
]
] as const)(
'answers whether %s waits as a queued card',
(_case, queue, isWorking, images, admission) => {
@@ -98,7 +98,7 @@ export function useStructuredAgentSessionSends(args: {
if (!text.trim() && attachments.length === 0) {
return false
}
const delivery = structuredAgentSessionQueueRequest(queue, attachments)
const delivery = structuredAgentSessionQueueRequest(queue)
const options = sendOptions?.()
const sent = sendStructuredAgentSessionMessage({
sessionId,
+4
View File
@@ -2990,6 +2990,10 @@
"question": {
"other": "Other…"
},
"queuedMessages": {
"imageCount_one": "{{count}} image",
"imageCount_other": "{{count}} images"
},
"receipt": {
"cancelled": "Cancelled",
"cancelledBy": "Cancelled on {{device}}",
+3
View File
@@ -18931,6 +18931,9 @@
"editFailed": "This message can't be edited.",
"editGone": "This message was already sent or removed. Your edit is in the chat box.",
"editGonePrompt": "This message was already sent or removed. Your edit will be in the chat box after you answer the agent.",
"imageCount": "{{count}} images",
"imageCount_one": "{{count}} image",
"imageCount_other": "{{count}} images",
"collapse": "Collapse",
"expand": "Expand",
"queued": "Queued",
+3
View File
@@ -18516,6 +18516,9 @@
"editFailed": "Este mensaje no se puede editar.",
"editGone": "Este mensaje ya se envió o se eliminó. Tu edición está en el cuadro de chat.",
"editGonePrompt": "Este mensaje ya se envió o se eliminó. Tu edición estará en el cuadro de chat cuando respondas al agente.",
"imageCount": "{{count}} imágenes",
"imageCount_one": "{{count}} imagen",
"imageCount_other": "{{count}} imágenes",
"collapse": "Contraer",
"expand": "Expandir",
"queued": "En cola",
+3
View File
@@ -18768,6 +18768,9 @@
"editFailed": "Ce message ne peut pas être modifié.",
"editGone": "Ce message a déjà été envoyé ou supprimé. Votre modification se trouve dans le champ de discussion.",
"editGonePrompt": "Ce message a déjà été envoyé ou supprimé. Votre modification sera dans le champ de discussion une fois que vous aurez répondu à l'agent.",
"imageCount": "{{count}} images",
"imageCount_one": "{{count}} image",
"imageCount_other": "{{count}} images",
"collapse": "Réduire",
"expand": "Développer",
"queued": "En file d'attente",
+3
View File
@@ -18699,6 +18699,9 @@
"editFailed": "このメッセージは編集できません。",
"editGone": "このメッセージはすでに送信または削除されています。編集したテキストはチャット入力欄にあります。",
"editGonePrompt": "このメッセージはすでに送信または削除されています。Agent に回答すると、編集したテキストがチャット入力欄に表示されます。",
"imageCount": "画像 {{count}} 枚",
"imageCount_one": "画像 {{count}} 枚",
"imageCount_other": "画像 {{count}} 枚",
"collapse": "折りたたむ",
"expand": "展開",
"queued": "キュー待ち",
+3
View File
@@ -18699,6 +18699,9 @@
"editFailed": "이 메시지는 편집할 수 없습니다.",
"editGone": "이 메시지는 이미 전송되었거나 삭제되었습니다. 편집한 내용은 채팅 입력란에 있습니다.",
"editGonePrompt": "이 메시지는 이미 전송되었거나 삭제되었습니다. 에이전트에 답하면 편집한 내용이 채팅 입력란에 표시됩니다.",
"imageCount": "이미지 {{count}}개",
"imageCount_one": "이미지 {{count}}개",
"imageCount_other": "이미지 {{count}}개",
"collapse": "접기",
"expand": "펼치기",
"queued": "대기 중",
+3
View File
@@ -18664,6 +18664,9 @@
"editFailed": "此消息无法编辑。",
"editGone": "此消息已发送或移除。你编辑的内容已放入聊天输入框。",
"editGonePrompt": "此消息已发送或移除。回答智能体后,你编辑的内容会出现在聊天输入框中。",
"imageCount": "{{count}} 张图片",
"imageCount_one": "{{count}} 张图片",
"imageCount_other": "{{count}} 张图片",
"collapse": "收起",
"expand": "展开",
"queued": "已排队",
@@ -2,6 +2,7 @@ import { AGENT_SESSION_CURRENT_REQUESTS_CLIENT_CAPABILITY } from '../../../share
import {
AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY,
AGENT_SESSION_BOUNDARY_RUNTIME_CAPABILITY,
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
AGENT_STATUS_RESTORED_UNCONFIRMED_RUNTIME_CAPABILITY,
REPO_SEARCH_QUALIFIED_REFS_RUNTIME_CAPABILITY,
AGENT_SESSION_TURN_ITEM_CAPABILITY,
@@ -23,6 +24,8 @@ export const WEB_RUNTIME_CLIENT_CAPABILITIES = [
AGENT_SESSION_CURRENT_REQUESTS_CLIENT_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_CHILD_VIEWS_CAPABILITY,
AGENT_SESSION_TURN_ITEM_CAPABILITY,
// The desktop's chat components: queued cards show their images, and Edit keeps them.
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
AGENT_SESSION_SNAPSHOT_CATCH_UP_CLIENT_CAPABILITY,
SESSION_TAB_CLOSE_INTENT_RUNTIME_CAPABILITY,
SESSION_TABS_SPLIT_GROUP_PLACEMENT_RUNTIME_CAPABILITY,
@@ -19,6 +19,7 @@ import {
AGENT_SESSION_TURN_ITEM_CAPABILITY,
AGENT_SESSION_SNAPSHOT_CATCH_UP_CLIENT_CAPABILITY,
AGENT_SESSION_BOUNDARY_RUNTIME_CAPABILITY,
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
AGENT_STATUS_RESTORED_UNCONFIRMED_RUNTIME_CAPABILITY,
REPO_SEARCH_QUALIFIED_REFS_RUNTIME_CAPABILITY,
SESSION_TAB_CLOSE_INTENT_RUNTIME_CAPABILITY,
@@ -133,6 +134,7 @@ describe('WebRuntimeClient', () => {
AGENT_SESSION_CURRENT_REQUESTS_CLIENT_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_CHILD_VIEWS_CAPABILITY,
AGENT_SESSION_TURN_ITEM_CAPABILITY,
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
AGENT_SESSION_SNAPSHOT_CATCH_UP_CLIENT_CAPABILITY,
SESSION_TAB_CLOSE_INTENT_RUNTIME_CAPABILITY,
SESSION_TABS_SPLIT_GROUP_PLACEMENT_RUNTIME_CAPABILITY,
@@ -32,7 +32,8 @@ export type AgentSessionQueuedMessagesResumeResult = {
}
/** One draft the host holds for this conversation, published whole-list on the
* subscribe stream and on history pages. Text-only v1. */
* subscribe stream and on history pages. A card holding an image goes only to a reader that
* advertises `agent-session.queued-image-cards.v1`. */
export type AgentSessionQueuedMessage = {
messageId: string
position: number
@@ -3,6 +3,7 @@ import { TERMINAL_IMAGE_VIEWER_CAPABILITY } from './terminal-image-viewer-capabi
import {
AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY,
AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY,
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
AGENT_SESSION_SNAPSHOT_CATCH_UP_CLIENT_CAPABILITY,
AGENT_STATUS_RESTORED_UNCONFIRMED_RUNTIME_CAPABILITY,
BROWSER_CLIENT_HOST_RUNTIME_CAPABILITY,
@@ -34,6 +35,8 @@ export const ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [
STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY,
CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY,
AGENT_SESSION_OPTIONAL_MODEL_CLIENT_CAPABILITY,
// Same renderer: its queued cards show their images, and Edit keeps them.
AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY,
// Same renderer: it reads each host's `agentSession.agents` and renders any agent listed there.
STRUCTURED_AGENT_SESSION_REGISTERED_AGENTS_RUNTIME_CAPABILITY,
PI_STRUCTURED_DIALOGS_RUNTIME_CAPABILITY,
@@ -166,6 +166,11 @@ export const AGENT_SESSION_QUEUED_MESSAGES_RUNTIME_CAPABILITY =
// A client offers Edit only when the host advertises it; there is no copy-and-delete fallback.
export const AGENT_SESSION_QUEUED_MESSAGE_EDIT_RUNTIME_CAPABILITY =
'agent-session.queued-message-edit.v1' as const
// Why: a client advertising this shows a queued card's images and edits its text in place, keeping
// them. Released readers show only a card's text and edit by copying it and deleting the card, which
// would drop the images, so the host publishes no card holding an image to a reader without it.
export const AGENT_SESSION_QUEUED_IMAGE_CARDS_CLIENT_CAPABILITY =
'agent-session.queued-image-cards.v1' as const
// Why: `agentSession.conversationCommand`'s params are strict, so an older host rejects
// `delivery`. A host advertising this holds a /compact sent while the agent works as a queued
// card instead of refusing it. Clients ask only when queued-messages.v1 is advertised too: