diff --git a/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.test.ts b/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.test.ts index 991c6be217e..0003a0d2c34 100644 --- a/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.test.ts @@ -51,6 +51,15 @@ describe('nativeChatSentPrompts', () => { expect(nativeChatSentPrompts({ messages, withoutImageMarkers: true })).toEqual([ { id: '2', prompt: 'fix' } ]) + // A marker the user typed, with no image of that number, is their text. + const typed = [ + message('3', 'user', '[Image #1]'), + message('4', 'user', '', { blocks: [image, { type: 'text', text: 'not [Image #2]' }] }) + ] + expect(nativeChatSentPrompts({ messages: typed, withoutImageMarkers: true })).toEqual([ + { id: '3', prompt: '[Image #1]' }, + { id: '4', prompt: 'not [Image #2]' } + ]) // A terminal chat's transcript text is the agent's own, markers included. expect(nativeChatSentPrompts({ messages }).map(({ prompt }) => prompt)).toEqual([ '[Image #1]', diff --git a/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.ts b/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.ts index ff34383ac52..f568168b647 100644 --- a/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.ts +++ b/src/renderer/src/components/native-chat/native-chat-sent-prompt-history.ts @@ -76,7 +76,15 @@ export function nativeChatSentPrompts(source: NativeChatRecallSource): NativeCha push(`command:${command.id}`, command.command) } const text = deriveNativeChatRowContent(message.blocks).markdown - push(message.id, source.withoutImageMarkers ? stripSentImagePromptMarkers(text) : text) + push( + message.id, + source.withoutImageMarkers + ? stripSentImagePromptMarkers( + text, + message.blocks.filter((block) => block.type === 'image-ref').length + ) + : text + ) } for (const command of commands.slice(nextCommand)) { push(`command:${command.id}`, command.command) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.test.tsx index 8438c2b3445..35a0f60a50e 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.test.tsx @@ -231,11 +231,18 @@ describe('inline queued editor', () => { }) it('moves typing into the chat box without image markers, since the card keeps its images', async () => { - const hook = harness() + const withImage: AgentSessionQueuedMessage = { + ...message, + body: { + ...message.body, + blocks: [{ type: 'image-ref', path: '/tmp/a.png' }, ...message.body.blocks] + } + } + const hook = harness(true, { messages: [withImage] }) await begin(hook) - act(() => hook.result.current.editor?.change('see [Image #1] again')) + act(() => hook.result.current.editor?.change('see [Image #1] not [Image #2]')) hook.rerender({ messages: [] }) - expect(readNativeChatDraftCache(SCOPE)).toBe('see again') + expect(readNativeChatDraftCache(SCOPE)).toBe('see not [Image #2]') }) it('a clean edit whose card leaves just closes; an unloaded list proves nothing', async () => { diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.ts index f9b2b506068..7e44592eed8 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-edit.ts @@ -64,7 +64,13 @@ type Edit = { /** Unsaved typing goes to the chat box without the card's images, so no marker may name one. */ function handOffEditText(edit: Edit): void { - appendNativeChatDraftCache(edit.draftKey, stripSentImagePromptMarkers(edit.text)) + appendNativeChatDraftCache( + edit.draftKey, + stripSentImagePromptMarkers( + edit.text, + edit.body.blocks.filter((block) => block.type === 'image-ref').length + ) + ) } export function useStructuredAgentSessionQueuedEdit(args: { diff --git a/src/shared/native-chat-image-transcript-markers.test.ts b/src/shared/native-chat-image-transcript-markers.test.ts index 337a304e6d9..a88a1cf490e 100644 --- a/src/shared/native-chat-image-transcript-markers.test.ts +++ b/src/shared/native-chat-image-transcript-markers.test.ts @@ -316,6 +316,15 @@ describe('composer image chips at send and back', () => { ).toBe(`${chip('r-3')} then ${chip('r-1')}, not [Image #3]`) }) + it('strips only the markers its images have, leaving a typed one past them', () => { + expect(stripSentImagePromptMarkers('[Image #1] then [Image #2], not [Image #3]', 2)).toBe( + 'then , not [Image #3]' + ) + expect(stripSentImagePromptMarkers('Explain the literal token [Image #1]', 0)).toBe( + 'Explain the literal token [Image #1]' + ) + }) + it('leaves a quoted file path that reads like a marker as typed, back or stripped', () => { const files = [ formatNativeChatFileReference('/repo/[Image #1].md'), @@ -326,7 +335,7 @@ describe('composer image chips at send and back', () => { expect(promptMarkersToImageReferences(`[Image #1] in ${file}`, ['r-1'])).toBe( `${chip('r-1')} in ${file}` ) - expect(stripSentImagePromptMarkers(`[Image #1] in ${file} [Image #1]`)).toBe(`in ${file}`) + expect(stripSentImagePromptMarkers(`[Image #1] in ${file} [Image #1]`, 1)).toBe(`in ${file}`) } }) diff --git a/src/shared/native-chat-image-transcript-markers.ts b/src/shared/native-chat-image-transcript-markers.ts index bc850c2404a..7c3b04fa2f5 100644 --- a/src/shared/native-chat-image-transcript-markers.ts +++ b/src/shared/native-chat-image-transcript-markers.ts @@ -86,11 +86,14 @@ export function promptMarkersToImageReferences(text: string, imageIds: readonly }) } -/** A structured chat's sent text without the `[Image #k]` markers its send added. */ -export function stripSentImagePromptMarkers(text: string): string { +/** A structured chat's sent text without the `[Image #k]` markers its send added for its + * `imageCount` images. A number past its images is left as typed. */ +export function stripSentImagePromptMarkers(text: string, imageCount: number): string { return withoutStrippedMarkerSpace( text, - replaceSentImagePromptMarkers(text, () => '') + replaceSentImagePromptMarkers(text, (number, marker) => + number >= 1 && number <= imageCount ? '' : marker + ) ) } diff --git a/src/shared/structured-agent-session-first-prompt.spec.ts b/src/shared/structured-agent-session-first-prompt.spec.ts index 9b0c6d4a3c3..2bac3e6aae7 100644 --- a/src/shared/structured-agent-session-first-prompt.spec.ts +++ b/src/shared/structured-agent-session-first-prompt.spec.ts @@ -84,12 +84,31 @@ describe('first structured chat prompt', () => { }) it('names a chat by what was asked, not by where its images sat', () => { + const withImages = (text: string, images: number, itemId: string) => + message('', { + itemId, + body: { + kind: 'message', + role: 'user', + blocks: [ + ...Array.from({ length: images }, () => ({ + type: 'image-ref' as const, + path: '/tmp/a.png' + })), + { type: 'text', text } + ] + } + }) expect( firstStructuredAgentSessionPrompt([ - message('[Image #1] '), - message('compare [Image #1] with [Image #2]', { itemId: 'second' }) + withImages('[Image #1] ', 1, 'first'), + withImages('compare [Image #1] with [Image #2]', 2, 'second') ]) ).toBe('compare with') + // A marker the user typed, with no image of that number, is their text. + expect(firstStructuredAgentSessionPrompt([message('Explain [Image #1]')])).toBe( + 'Explain [Image #1]' + ) }) it('skips punctuation-only entries in favor of the first text request', () => { diff --git a/src/shared/structured-agent-session-first-prompt.ts b/src/shared/structured-agent-session-first-prompt.ts index 0c4c8fe90b1..1ef24ef5fbc 100644 --- a/src/shared/structured-agent-session-first-prompt.ts +++ b/src/shared/structured-agent-session-first-prompt.ts @@ -24,6 +24,7 @@ export function firstStructuredAgentSessionPrompt( continue } let prompt = '' + const imageCount = body.blocks.filter((block) => block.type === 'image-ref').length for (const block of body.blocks) { if (block.type === 'text') { if (prompt) { @@ -31,7 +32,7 @@ export function firstStructuredAgentSessionPrompt( } // A title names what was asked; `[Image #N]` only says where an image sat. prompt += sliceAtCodeUnitLimit( - stripSentImagePromptMarkers(block.text), + stripSentImagePromptMarkers(block.text, imageCount), STRUCTURED_CHAT_NAME_PROMPT_LIMIT - prompt.length ) if (prompt.length >= STRUCTURED_CHAT_NAME_PROMPT_LIMIT) {