diff --git a/mobile/src/platform/mobile-picked-file-name.ts b/mobile/src/platform/mobile-picked-file-name.ts index 51db5f28c16..a44ad5cbc4b 100644 --- a/mobile/src/platform/mobile-picked-file-name.ts +++ b/mobile/src/platform/mobile-picked-file-name.ts @@ -22,7 +22,7 @@ export function mobilePickedFileFallbackName(mime: string, index: number): strin return extension ? `attachment-${index}.${extension}` : `attachment-${index}` } -// Images keep the host's orca-paste-*.png name, which transcript readers recognize as a pasted image. +// Images take the host's orca-paste-* name, which transcript readers recognize as a pasted image. export function isPickedImageMime(mime: string | null | undefined): boolean { return mime?.trim().toLowerCase().startsWith('image/') ?? false } diff --git a/mobile/src/platform/native-media-device.test.ts b/mobile/src/platform/native-media-device.test.ts index d3f9cdb5c6a..f819b68ff55 100644 --- a/mobile/src/platform/native-media-device.test.ts +++ b/mobile/src/platform/native-media-device.test.ts @@ -15,6 +15,7 @@ const expo = vi.hoisted(() => ({ vi.mock('expo-clipboard', () => ({ getImageAsync: expo.getImageAsync })) vi.mock('expo-document-picker', () => ({ getDocumentAsync: expo.getDocumentAsync })) vi.mock('expo-image-picker', () => ({ + UIImagePickerPreferredAssetRepresentationMode: { Compatible: 'compatible' }, launchImageLibraryAsync: expo.launchImageLibraryAsync, requestMediaLibraryPermissionsAsync: expo.requestMediaLibraryPermissionsAsync })) @@ -92,7 +93,9 @@ describe('how the pickers are launched', () => { expect.objectContaining({ mediaTypes: ['images'], base64: false, - allowsMultipleSelection: false + allowsMultipleSelection: false, + // Agents can't read the HEIC iOS hands back by default. + preferredAssetRepresentationMode: 'compatible' }) ) // Omit selectionLimit for single picks; zero means unlimited. diff --git a/mobile/src/platform/native-media-device.ts b/mobile/src/platform/native-media-device.ts index bf02e063c6f..184369e0c48 100644 --- a/mobile/src/platform/native-media-device.ts +++ b/mobile/src/platform/native-media-device.ts @@ -51,7 +51,10 @@ export function nativeMediaDeviceDeps(registry: MediaHandleRegistry): NativeMedi base64: false, allowsMultipleSelection: multiple, ...(multiple ? { selectionLimit: limit, orderedSelection: true } : {}), - quality: 1 + quality: 1, + // iOS otherwise hands back HEIC as-is, which agents can't read; Compatible gives JPEG. + preferredAssetRepresentationMode: + ImagePicker.UIImagePickerPreferredAssetRepresentationMode.Compatible }), // DocumentPicker has no selection cap; enforce remaining room again before staging and minting. launchFiles: ({ multiple }) => diff --git a/mobile/src/platform/use-native-device-verbs.test.tsx b/mobile/src/platform/use-native-device-verbs.test.tsx index e9d3dbc4ab9..7f46fb91bb4 100644 --- a/mobile/src/platform/use-native-device-verbs.test.tsx +++ b/mobile/src/platform/use-native-device-verbs.test.tsx @@ -44,6 +44,7 @@ vi.mock('expo-keep-awake', () => ({ } })) vi.mock('expo-image-picker', () => ({ + UIImagePickerPreferredAssetRepresentationMode: { Compatible: 'compatible' }, launchImageLibraryAsync: device.picker.launchImageLibraryAsync })) vi.mock('expo-file-system', () => ({ diff --git a/mobile/src/session/mobile-image-source-picker.test.ts b/mobile/src/session/mobile-image-source-picker.test.ts index b37c2619a5e..a2c5f1d7189 100644 --- a/mobile/src/session/mobile-image-source-picker.test.ts +++ b/mobile/src/session/mobile-image-source-picker.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { CLIPBOARD_IMAGE_MAX_SOURCE_BYTES } from '../../../src/shared/clipboard-image' vi.mock('expo-image-picker', () => ({ + UIImagePickerPreferredAssetRepresentationMode: { Compatible: 'compatible' }, requestMediaLibraryPermissionsAsync: vi.fn(), launchImageLibraryAsync: vi.fn() })) @@ -78,7 +79,9 @@ describe('pickMobileImage', () => { base64: Buffer.from(bytes).toString('base64'), uri: 'file:///x.jpg' }) - expect(launchLibrary).toHaveBeenCalledWith(expect.objectContaining({ base64: false })) + expect(launchLibrary).toHaveBeenCalledWith( + expect.objectContaining({ base64: false, preferredAssetRepresentationMode: 'compatible' }) + ) expect(fetchSpy).not.toHaveBeenCalled() expect(file.close).toHaveBeenCalledTimes(1) }) diff --git a/mobile/src/session/mobile-image-source-picker.ts b/mobile/src/session/mobile-image-source-picker.ts index 5e284f32d47..2ba991c06f5 100644 --- a/mobile/src/session/mobile-image-source-picker.ts +++ b/mobile/src/session/mobile-image-source-picker.ts @@ -90,7 +90,10 @@ async function* pickFromLibrary( base64: false, allowsMultipleSelection: multiple, ...(multiple ? { selectionLimit: 0, orderedSelection: true } : {}), - quality: 1 + quality: 1, + // iOS otherwise hands back HEIC as-is, which agents can't read; Compatible gives JPEG. + preferredAssetRepresentationMode: + ImagePicker.UIImagePickerPreferredAssetRepresentationMode.Compatible }) if (result.canceled) { return diff --git a/src/main/native-chat/transcript-line-decoders-grok.ts b/src/main/native-chat/transcript-line-decoders-grok.ts index 005b19a2bb3..516f36f1a6c 100644 --- a/src/main/native-chat/transcript-line-decoders-grok.ts +++ b/src/main/native-chat/transcript-line-decoders-grok.ts @@ -6,6 +6,7 @@ import { timestampMs } from '../ai-vault/session-scanner-values' import { claudeContentBlocks, toolResultOutput } from './transcript-record-blocks' +import { AGENT_SESSION_PASTED_IMAGE_SUFFIX_SOURCE } from '../../shared/agent-session-attachments' export function decodeGrokTranscriptLine( line: string, @@ -209,11 +210,16 @@ function normalizeGrokUserQueryBlock(block: NativeChatBlock): NativeChatBlock[] ] } +const GROK_PASTED_IMAGE_QUERY = new RegExp( + String.raw`^((?:[a-z]:[\\/]|\/|[\\/]{2}[^\\/\r\n]+[\\/][^\\/\r\n]+[\\/])(?:.*?[\\/])?orca-paste-[^\\/\r\n]+?` + + AGENT_SESSION_PASTED_IMAGE_SUFFIX_SOURCE + + String.raw`)([\s\S]*)$`, + 'i' +) + function splitGrokPastedImageQuery(text: string): { path: string; query: string } | null { // Grok 0.2.93 concatenates clipboard temp paths with prompts; recover the attachment separately. - const match = text.match( - /^((?:[a-z]:[\\/]|\/|[\\/]{2}[^\\/\r\n]+[\\/][^\\/\r\n]+[\\/])(?:.*?[\\/])?orca-paste-[^\\/\r\n]+?\.png)([\s\S]*)$/i - ) + const match = text.match(GROK_PASTED_IMAGE_QUERY) if (!match?.[1]) { return null } diff --git a/src/main/native-chat/transcript-line-decoders.grok.test.ts b/src/main/native-chat/transcript-line-decoders.grok.test.ts index 2adff4dff64..c7212961120 100644 --- a/src/main/native-chat/transcript-line-decoders.grok.test.ts +++ b/src/main/native-chat/transcript-line-decoders.grok.test.ts @@ -36,6 +36,7 @@ describe('decodeGrokTranscriptLine', () => { it.each([ 'C:\\Users\\me\\AppData\\Local\\Temp\\orca-paste-1783675302563-2207c073-535f-4b83-a181-61127c8bbd68.png', '/tmp/orca-paste-1783675302563-2207c073-535f-4b83-a181-61127c8bbd68.png', + '/tmp/orca-paste-1783675302563-2207c073-535f-4b83-a181-61127c8bbd68.jpg', 'C:\\orca-paste-1783675302563-2207c073-535f-4b83-a181-61127c8bbd68.png', '/orca-paste-1783675302563-2207c073-535f-4b83-a181-61127c8bbd68.png', '\\\\server\\temp\\orca-paste-1783675302563-2207c073-535f-4b83-a181-61127c8bbd68.png' diff --git a/src/main/window/clipboard-image-temp-file.test.ts b/src/main/window/clipboard-image-temp-file.test.ts index 4b7a191290c..d8467a8d744 100644 --- a/src/main/window/clipboard-image-temp-file.test.ts +++ b/src/main/window/clipboard-image-temp-file.test.ts @@ -32,6 +32,13 @@ describe('saveClipboardImageBufferAsTempFile', () => { expect(writeFileMock).toHaveBeenCalledWith(savedPath, Buffer.from([1, 2, 3])) }) + it('names a JPEG phone pick by its bytes, not as a .png', async () => { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1]) + const savedPath = await saveClipboardImageBufferAsTempFile(jpeg) + + expect(basename(savedPath)).toMatch(/^orca-paste-\d+-uuid-1\.jpg$/) + }) + it('writes a terminal-backed composer paste into the never-swept paste folder', async () => { const savedPath = await saveClipboardImageBufferAsTempFile(Buffer.from([1, 2, 3]), { forNativeChatDraft: true diff --git a/src/main/window/clipboard-image-temp-file.ts b/src/main/window/clipboard-image-temp-file.ts index ba1184013d8..7db3762a728 100644 --- a/src/main/window/clipboard-image-temp-file.ts +++ b/src/main/window/clipboard-image-temp-file.ts @@ -10,11 +10,12 @@ import { sanitizeAgentSessionAttachmentName, type AgentSessionAttachmentClipboardTarget } from '../../shared/agent-session-attachments' +import { detectImageFileSignature } from '../image-file-signature' import { nativeChatPasteFolder } from './native-chat-paste-files' export type SaveClipboardImageAsTempFileArgs = { connectionId?: string | null - /** A phone-picked file's own name; without it the paste is saved as a .png. */ + /** A phone-picked file's own name; without it the paste is named by its image bytes. */ fileName?: string runtimeEnvironmentId?: string | null /** With `runtimeEnvironmentId`: store the image as an attachment of this structured chat. */ @@ -49,7 +50,15 @@ export async function saveClipboardImageBufferAsTempFile( // A picked file keeps its own name, so the paste gets a folder to stay unique. const tempPath = args?.fileName ? path.join(folder, `orca-paste-${randomUUID()}`, shellInertFileName(args.fileName)) - : path.join(folder, agentSessionPastedImageName(Date.now(), randomUUID())) + : path.join( + folder, + // Phone library picks are often JPEG; agents trust the extension over the bytes. + agentSessionPastedImageName( + Date.now(), + randomUUID(), + detectImageFileSignature(buffer)?.extension + ) + ) if (args?.fileName) { await fs.mkdir(path.dirname(tempPath), { recursive: true }) } diff --git a/src/shared/agent-session-attachments.ts b/src/shared/agent-session-attachments.ts index 4250c55a744..17ff10e39a8 100644 --- a/src/shared/agent-session-attachments.ts +++ b/src/shared/agent-session-attachments.ts @@ -122,9 +122,20 @@ export type AgentSessionAttachmentClipboardTarget = Omit< 'environmentId' > -/** `orca-paste-…png`, the name every pasted image gets, so it reads as "Pasted image". */ -export function agentSessionPastedImageName(now: number, unique: string): string { - return `orca-paste-${now}-${unique}.png` +/** Extensions a pasted image's name can carry; the host picks one from the image's bytes. */ +const AGENT_SESSION_PASTED_IMAGE_EXTENSIONS = ['png', 'jpg', 'gif', 'webp'] as const +type AgentSessionPastedImageExtension = (typeof AGENT_SESSION_PASTED_IMAGE_EXTENSIONS)[number] + +/** Matches the `.png`, `.jpg`… suffix of a pasted image's name. */ +export const AGENT_SESSION_PASTED_IMAGE_SUFFIX_SOURCE = `\\.(?:${AGENT_SESSION_PASTED_IMAGE_EXTENSIONS.join('|')})` + +/** `orca-paste-…`, the name every pasted image gets, so it reads as "Pasted image". */ +export function agentSessionPastedImageName( + now: number, + unique: string, + extension: AgentSessionPastedImageExtension = 'png' +): string { + return `orca-paste-${now}-${unique}.${extension}` } /** `orca-paste-…txt`, the name a long text paste saved as a file gets, so it reads as "Pasted text". */ diff --git a/src/shared/native-chat-image-label.ts b/src/shared/native-chat-image-label.ts index a9e7d6ab443..c519d4f220e 100644 --- a/src/shared/native-chat-image-label.ts +++ b/src/shared/native-chat-image-label.ts @@ -1,3 +1,4 @@ +import { AGENT_SESSION_PASTED_IMAGE_SUFFIX_SOURCE } from './agent-session-attachments' import { getRuntimePathBasename } from './cross-platform-path' import type { NativeChatImageRefBlock } from './native-chat/native-chat-types' @@ -11,10 +12,15 @@ export const NATIVE_CHAT_IMAGE_LABEL_ENGLISH: NativeChatImageLabelCopy = { imageCount: (count) => `${count} images` } +const PASTED_IMAGE_NAME = new RegExp( + `^orca-paste-.+${AGENT_SESSION_PASTED_IMAGE_SUFFIX_SOURCE}$`, + 'i' +) + /** Use friendly UI labels for Orca clipboard-image temp filenames. */ export function isNativeChatPastedImagePath(path: string): boolean { const base = path.split(/[\\/]/).findLast(Boolean) ?? path - return /^orca-paste-.+\.png$/i.test(base) + return PASTED_IMAGE_NAME.test(base) } /** What a chat image is called where it is not drawn: its alt text, else its file's name. */