diff --git a/config/scripts/ci/sharding/ci-shard-timings.json b/config/scripts/ci/sharding/ci-shard-timings.json index 82ec3977005..ac292a29c6b 100644 --- a/config/scripts/ci/sharding/ci-shard-timings.json +++ b/config/scripts/ci/sharding/ci-shard-timings.json @@ -1932,7 +1932,7 @@ "src/main/host/secret-protection-report.test.ts": 59, "src/main/i18n/main-i18n-lazy-locale.test.ts": 518, "src/main/i18n/system-language.test.ts": 358, - "src/main/media/image-file-signature.test.ts": 47, + "src/shared/image-file-signature.test.ts": 47, "src/main/credentials/integration-state-file-cache.test.ts": 47, "src/main/credentials/integration-token-file-cache.test.ts": 152, "src/main/ipc/account-sign-in-link-opener.test.ts": 128, diff --git a/mobile/src/files/attachments/mobile-agent-image-fit.test.ts b/mobile/src/files/attachments/mobile-agent-image-fit.test.ts new file mode 100644 index 00000000000..8ea5b6ab89f --- /dev/null +++ b/mobile/src/files/attachments/mobile-agent-image-fit.test.ts @@ -0,0 +1,73 @@ +import { Buffer } from 'node:buffer' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { fitMobileAgentImageBase64 } from './mobile-agent-image-fit' + +// Node has no image decoder or canvas: the fake decodes to fixed dimensions and records each encode. +function stubRaster(decoded: { + width: number + height: number +}): { width: number; height: number; type: string }[] { + const encodes: { width: number; height: number; type: string }[] = [] + vi.stubGlobal( + 'Image', + class { + src = '' + naturalWidth = decoded.width + naturalHeight = decoded.height + decode(): Promise { + return Promise.resolve() + } + } + ) + vi.stubGlobal('document', { + createElement: () => { + const canvas = { + width: 0, + height: 0, + getContext: () => ({ fillRect: () => {}, drawImage: () => {} }), + toDataURL: (type: string) => { + encodes.push({ width: canvas.width, height: canvas.height, type }) + return 'data:image/jpeg;base64,/9j/4AAQ' + } + } + return canvas + } + }) + return encodes +} + +function base64Of(header: number[] | string): string { + const bytes = typeof header === 'string' ? Buffer.from(header, 'latin1') : Buffer.from(header) + return Buffer.concat([bytes, Buffer.alloc(64)]).toString('base64') +} + +const PNG = base64Of([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) +const JPEG = base64Of([0xff, 0xd8, 0xff, 0xe0]) +const HEIC = base64Of('\0\0\0\x18ftypheic') + +describe('fitMobileAgentImageBase64', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('re-encodes a full-size photo as a 2000 px JPEG', async () => { + const encodes = stubRaster({ width: 4032, height: 3024 }) + + expect(await fitMobileAgentImageBase64(JPEG)).toBe('/9j/4AAQ') + expect(encodes).toEqual([{ width: 2000, height: 1500, type: 'image/jpeg' }]) + }) + + it('passes a small PNG through unchanged', async () => { + const encodes = stubRaster({ width: 1200, height: 800 }) + + expect(await fitMobileAgentImageBase64(PNG)).toBe(PNG) + expect(encodes).toEqual([]) + }) + + it('converts a small image agents cannot read to JPEG', async () => { + const encodes = stubRaster({ width: 1200, height: 800 }) + + expect(await fitMobileAgentImageBase64(HEIC)).toBe('/9j/4AAQ') + expect(encodes).toEqual([{ width: 1200, height: 800, type: 'image/jpeg' }]) + }) +}) diff --git a/mobile/src/files/attachments/mobile-agent-image-fit.ts b/mobile/src/files/attachments/mobile-agent-image-fit.ts new file mode 100644 index 00000000000..5c49afb7774 --- /dev/null +++ b/mobile/src/files/attachments/mobile-agent-image-fit.ts @@ -0,0 +1,80 @@ +import { + AGENT_IMAGE_MAX_BYTES, + AGENT_IMAGE_MAX_LONG_EDGE, + clipboardImageSizeWithinEdge, + encodeClipboardImageWithinLimit, + type ClipboardImageDimensions +} from '../../../../src/shared/clipboard-image' +import { base64DecodedByteLength } from '../../../../src/shared/agent-session/agent-session-attachments' +import { detectImageFileSignature } from '../../../../src/shared/image-file-signature' +import { decodeBase64Bytes } from '../../transport/encryption/base64-byte-codec' +import type { PickedMobileImage } from '../../platform/media-picker-contract' + +// Page-only: decodes and re-encodes through the WebView's canvas. + +const DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i +const SIGNATURE_BASE64_CHARS = 16 +const JPEG_QUALITY = 0.85 + +function encodeJpegBase64(image: HTMLImageElement, size: ClipboardImageDimensions): string { + const canvas = document.createElement('canvas') + canvas.width = size.width + canvas.height = size.height + const context = canvas.getContext('2d') + if (context === null) { + throw new Error('Failed to encode image') + } + // JPEG has no alpha, so transparent pixels would otherwise turn black. + context.fillStyle = '#ffffff' + context.fillRect(0, 0, size.width, size.height) + context.imageSmoothingQuality = 'high' + context.drawImage(image, 0, 0, size.width, size.height) + const encoded = canvas.toDataURL('image/jpeg', JPEG_QUALITY).replace(DATA_URL_PREFIX_RE, '') + if (encoded.length === 0) { + throw new Error('Failed to encode image') + } + return encoded +} + +/** Shrink a photo to what agents accept; small images in formats they read pass through unchanged. */ +export async function fitMobileAgentImageBase64(data: string, mimeHint?: string): Promise { + const base64 = data.replace(DATA_URL_PREFIX_RE, '') + const signature = detectImageFileSignature( + decodeBase64Bytes(base64.slice(0, SIGNATURE_BASE64_CHARS)) + ) + const image = new Image() + image.src = `data:${signature?.mimeType ?? mimeHint ?? 'image/png'};base64,${base64}` + try { + await image.decode() + } catch (error) { + // A format agents read uploads as picked; anything else would reach them as an unreadable .png. + if (signature !== null) { + return base64 + } + throw error + } + const source = { width: image.naturalWidth, height: image.naturalHeight } + const size = clipboardImageSizeWithinEdge(source, AGENT_IMAGE_MAX_LONG_EDGE) + if ( + signature !== null && + size.width === source.width && + size.height === source.height && + base64DecodedByteLength(base64) <= AGENT_IMAGE_MAX_BYTES + ) { + return base64 + } + return encodeClipboardImageWithinLimit( + size, + (target) => encodeJpegBase64(image, target), + base64DecodedByteLength, + AGENT_IMAGE_MAX_BYTES + ) +} + +/** Files-app picks that aren't images carry a fileName and upload as-is. */ +export async function fitMobileAgentImage(image: PickedMobileImage): Promise { + if (image.fileName !== undefined) { + return image + } + return { ...image, base64: await fitMobileAgentImageBase64(image.base64, image.mime) } +} diff --git a/mobile/src/files/attachments/mobile-clipboard-image-downscale.ts b/mobile/src/files/attachments/mobile-clipboard-image-downscale.ts deleted file mode 100644 index 810ec138ccb..00000000000 --- a/mobile/src/files/attachments/mobile-clipboard-image-downscale.ts +++ /dev/null @@ -1,32 +0,0 @@ -// Keep resize arithmetic free of upload RPC dependencies so browser raster checks can import it alone. - -// Why: PNG bytes don't scale exactly with pixel area, so undershoot the target on -// each pass and let the bounded retry converge instead of distorting in one shot. -const MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY = 0.85 - -export type MobileClipboardImageResizer = ( - source: string, - target: { width: number; height: number } -) => Promise<{ data: string; width: number; height: number }> - -export function computeMobileClipboardImageDownscale( - base64Length: number, - width: number, - height: number, - maxBase64Length: number -): { width: number; height: number } | null { - if (base64Length <= maxBase64Length) { - return null - } - if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { - return null - } - const scale = Math.sqrt(maxBase64Length / base64Length) * MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY - const nextWidth = Math.max(1, Math.floor(width * scale)) - const nextHeight = Math.max(1, Math.floor(height * scale)) - // Guard against a no-op shrink (already 1px) so the retry loop can't spin forever. - if (nextWidth >= width && nextHeight >= height) { - return null - } - return { width: nextWidth, height: nextHeight } -} diff --git a/mobile/src/files/attachments/mobile-clipboard-image-resize.ts b/mobile/src/files/attachments/mobile-clipboard-image-resize.ts deleted file mode 100644 index 8e59ddb032a..00000000000 --- a/mobile/src/files/attachments/mobile-clipboard-image-resize.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { MobileClipboardImageResizer } from './mobile-clipboard-image-downscale' - -// Keep PNG output: upload retries measure its base64 size and terminal paste assumes PNG. Browser tests exercise a real raster under the shipped CSP. - -const CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i -const RESIZED_MIME = 'image/png' - -export const resizeMobileClipboardImage: MobileClipboardImageResizer = async (source, target) => { - const base64 = source.replace(CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE, '') - const image = new Image() - image.src = `data:${RESIZED_MIME};base64,${base64}` - // decode rejects unreadable sources; an onload-only promise could remain pending forever. - await image.decode() - const canvas = document.createElement('canvas') - canvas.width = target.width - canvas.height = target.height - const context = canvas.getContext('2d') - if (context === null) { - throw new Error('Failed to encode resized clipboard image') - } - context.drawImage(image, 0, 0, canvas.width, canvas.height) - const resized = canvas.toDataURL(RESIZED_MIME).replace(CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE, '') - // Why: empty base64 would pass the downstream base64 check and upload a corrupt image, so fail - // loudly here instead of silently sending an invalid payload. - if (resized.length === 0) { - throw new Error('Failed to encode resized clipboard image') - } - // Return dimensions from the encoded canvas so metadata and bytes describe the same raster. - return { data: resized, width: canvas.width, height: canvas.height } -} diff --git a/mobile/src/files/attachments/mobile-clipboard-image.ts b/mobile/src/files/attachments/mobile-clipboard-image.ts index aaac9ba4f2f..1dfd393a413 100644 --- a/mobile/src/files/attachments/mobile-clipboard-image.ts +++ b/mobile/src/files/attachments/mobile-clipboard-image.ts @@ -1,10 +1,6 @@ import { isWindowsAbsolutePathLike } from '../../../../src/shared/cross-platform-path' import { formatImageDropPasteText } from '../../../../src/shared/image-path-paste' import { formatAgentImagePath } from '../../../../src/shared/agent-image-paste' -import { - computeMobileClipboardImageDownscale, - type MobileClipboardImageResizer -} from './mobile-clipboard-image-downscale' import { MOBILE_CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS } from './mobile-clipboard-image-upload-chunk' import '../../transport/connection/stable-logical-rpc-client' import { isLogicalClientCutoverError } from '../../transport/connection/logical-client-cutover-error' @@ -21,7 +17,6 @@ export const MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS = 24 * 1024 * 1024 export const MOBILE_CLIPBOARD_IMAGE_SINGLE_FRAME_FALLBACK_BASE64_CHARS = 256 * 1024 const MOBILE_CLIPBOARD_IMAGE_UPLOAD_CUTOVER_MAX_RETRIES = 1 -const MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS = 3 const DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/ @@ -37,39 +32,6 @@ export function normalizeMobileClipboardImageBase64(data: string): string { return contentBase64 } -export type MobileClipboardImage = { - data: string - size: { width: number; height: number } -} - -// Retry bounded downscales; any remaining overflow reaches the existing too-large check. -export async function prepareMobileClipboardImageBase64( - image: MobileClipboardImage, - resize: MobileClipboardImageResizer, - maxBase64Length: number = MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS -): Promise { - let data = image.data - let width = image.size.width - let height = image.size.height - for (let attempt = 0; attempt < MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS; attempt += 1) { - const contentLength = data.replace(DATA_URL_PREFIX_RE, '').length - const target = computeMobileClipboardImageDownscale( - contentLength, - width, - height, - maxBase64Length - ) - if (!target) { - return data - } - const resized = await resize(data, target) - data = resized.data - width = resized.width - height = resized.height - } - return data -} - export async function saveMobileClipboardImageAsTempFile( client: MobileClipboardImageRpcSender, imageData: string, diff --git a/mobile/src/files/attachments/mobile-image-attachment.ts b/mobile/src/files/attachments/mobile-image-attachment.ts index e876db4978e..259d2816731 100644 --- a/mobile/src/files/attachments/mobile-image-attachment.ts +++ b/mobile/src/files/attachments/mobile-image-attachment.ts @@ -17,6 +17,7 @@ export type AttachMobileImageDeps = { readonly pickImage: (source: MobileImageSource) => Promise // Start spinner only after selection; the OS picker may remain open a long time. readonly onUploadStart?: () => void + readonly fitImage?: (image: PickedMobileImage) => Promise readonly beforeTerminalSend?: (terminal: string) => Promise } @@ -30,14 +31,16 @@ export async function attachMobileImageToTerminal( getConnectionId, pickImage, onUploadStart, + fitImage, beforeTerminalSend }: AttachMobileImageDeps ): Promise { - const picked = await pickImage(source) - if (!picked) { + const pickedImage = await pickImage(source) + if (!pickedImage) { return false } onUploadStart?.() + const picked = fitImage ? await fitImage(pickedImage) : pickedImage const connectionId = await getConnectionId() const imagePath = await saveMobileClipboardImageAsTempFile(client, picked.base64, { connectionId, diff --git a/mobile/src/files/attachments/use-mobile-image-attachment.ts b/mobile/src/files/attachments/use-mobile-image-attachment.ts index fe8648ef372..450a50e4975 100644 --- a/mobile/src/files/attachments/use-mobile-image-attachment.ts +++ b/mobile/src/files/attachments/use-mobile-image-attachment.ts @@ -8,6 +8,7 @@ import { type MobileImageSource } from '../../platform/media-picker-contract' import { attachMobileImageToTerminal } from './mobile-image-attachment' +import { fitMobileAgentImage } from './mobile-agent-image-fit' type CurrentRef = { readonly current: T @@ -68,6 +69,7 @@ export function useMobileImageAttachment({ getConnectionId: getActiveWorktreeConnectionId, pickImage: picker.pickImage, onUploadStart: () => setIsAttaching(true), + fitImage: fitMobileAgentImage, beforeTerminalSend }) if (sent) { diff --git a/mobile/src/platform/media-picker-contract.ts b/mobile/src/platform/media-picker-contract.ts index f74ed63fe71..6108257ba3e 100644 --- a/mobile/src/platform/media-picker-contract.ts +++ b/mobile/src/platform/media-picker-contract.ts @@ -11,6 +11,8 @@ export type PickedMobileImage = { readonly uri?: string // Non-image Files-app picks keep their name so the host saves the file under it. readonly fileName?: string + // Decode hint for formats with no byte signature the phone recognizes (SVG, HEIC). + readonly mime?: string } export type PickedMobileFile = { diff --git a/mobile/src/platform/media-picker.web.ts b/mobile/src/platform/media-picker.web.ts index 67437d14ec9..a41d502bf5f 100644 --- a/mobile/src/platform/media-picker.web.ts +++ b/mobile/src/platform/media-picker.web.ts @@ -23,8 +23,12 @@ import { isPickedImageMime, mobilePickedFileFallbackName } from './mobile-picked // Refusals reject; only user cancellation returns an empty selection. // The page cannot load a native cache URI; the composer falls back to an inline data URI. -function pickedImage(base64: string, fileName: string | undefined): PickedMobileImage { - return { base64, fileName } +function pickedImage( + base64: string, + fileName: string | undefined, + mime: string +): PickedMobileImage { + return { base64, fileName, mime } } // Older shells lack nameMedia; derive a filename from MIME instead. @@ -58,7 +62,7 @@ async function* readPicked( // Release before yielding because callers may stop after one image and strand scarce handles. unread.shift() await releaseStagedMedia(verbs, item.handle) - yield pickedImage(base64, fileName) + yield pickedImage(base64, fileName, item.mime) } } finally { await releaseAllStagedMedia(verbs, unread) diff --git a/mobile/src/session/native-chat/composer/mobile-native-chat-composer-paste.ts b/mobile/src/session/native-chat/composer/mobile-native-chat-composer-paste.ts index 614a13cbf01..f4708ea1c3a 100644 --- a/mobile/src/session/native-chat/composer/mobile-native-chat-composer-paste.ts +++ b/mobile/src/session/native-chat/composer/mobile-native-chat-composer-paste.ts @@ -7,12 +7,6 @@ import { isNativeChatLongPaste } from '../../../../../src/shared/native-chat/inp import type { ClipboardReader } from '../../../platform/clipboard' import type { PickedMobileImage } from '../../../platform/media-picker-contract' import { encodeBase64Bytes } from '../../../transport/encryption/base64-byte-codec' -import { - MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS, - normalizeMobileClipboardImageBase64, - prepareMobileClipboardImageBase64 -} from '../../../files/attachments/mobile-clipboard-image' -import { resizeMobileClipboardImage } from '../../../files/attachments/mobile-clipboard-image-resize' /** One paste's image: the event's file, or null when WebKit left the bytes out and the shell clipboard holds them. */ export type MobilePastedImages = { readonly file: File | null } @@ -22,45 +16,21 @@ type PasteEvent = ClipboardEventDataLike & { preventDefault: () => void } -async function sizedForUpload(data: string, width: number, height: number): Promise { - const prepared = await prepareMobileClipboardImageBase64( - { data, size: { width, height } }, - resizeMobileClipboardImage - ) - return normalizeMobileClipboardImageBase64(prepared) -} - -async function readEventImage(file: Blob): Promise { - const base64 = encodeBase64Bytes(new Uint8Array(await file.arrayBuffer())) - const uri = `data:${file.type};base64,${base64}` - // Agents read only these formats; anything else (HEIC, TIFF) is re-encoded as PNG. - const readable = /^image\/(?:png|jpeg|gif|webp)$/.test(file.type) - if (readable && base64.length <= MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS) { - return { base64, uri } - } - const image = new Image() - image.src = uri - await image.decode() - const size = { width: image.naturalWidth, height: image.naturalHeight } - const source = readable ? base64 : (await resizeMobileClipboardImage(base64, size)).data - return { base64: await sizedForUpload(source, size.width, size.height) } -} - -/** Sized for upload; the clipboard reader is the uploader's, so the composer needs no shell bridge. */ +/** Raw bytes; the uploader fits them for agents. The clipboard reader is the uploader's, so the composer needs no shell bridge. */ export async function readMobilePastedImages( { file }: MobilePastedImages, clipboard: Pick ): Promise { if (file) { - return [await readEventImage(file)] + return [ + { base64: encodeBase64Bytes(new Uint8Array(await file.arrayBuffer())), mime: file.type } + ] } if (!(await clipboard.contents()).image) { return [] } const image = await clipboard.readImage() - return image - ? [{ base64: await sizedForUpload(image.data, image.size.width, image.size.height) }] - : [] + return image ? [{ base64: image.data }] : [] } /** Text pastes stay native unless long; images attach as chips like Photos picks. A disabled composer leaves pastes to the field. */ diff --git a/mobile/src/session/native-chat/composer/mobile-native-chat-image-attachment.ts b/mobile/src/session/native-chat/composer/mobile-native-chat-image-attachment.ts index 0aeebfd47af..31fd7ceaed1 100644 --- a/mobile/src/session/native-chat/composer/mobile-native-chat-image-attachment.ts +++ b/mobile/src/session/native-chat/composer/mobile-native-chat-image-attachment.ts @@ -35,6 +35,8 @@ export type UploadNativeChatImagesDeps = { | AsyncIterable | Promise | AsyncIterable> readonly onUploadStart?: () => void + // Injected so this module stays free of the page's canvas. + readonly fitImage?: (image: PickedMobileImage) => Promise /** Retains each completed upload if a later image in the same selection fails. */ readonly onImageUploaded?: (image: Omit) => void } @@ -48,6 +50,7 @@ export async function uploadMobileNativeChatImages( getConnectionId, pickImages, onUploadStart, + fitImage, onImageUploaded }: UploadNativeChatImagesDeps ): Promise[]> { @@ -57,7 +60,7 @@ export async function uploadMobileNativeChatImages( if (canContinue?.() === false) { return uploaded } - for await (const image of picked) { + for await (const pickedImage of picked) { if (canContinue?.() === false) { break } @@ -68,6 +71,7 @@ export async function uploadMobileNativeChatImages( if (canContinue?.() === false) { break } + const image = fitImage ? await fitImage(pickedImage) : pickedImage const path = await saveMobileClipboardImageAsTempFile(client, image.base64, { connectionId }) const previewUri = image.uri ?? `data:image/png;base64,${image.base64}` const result = { diff --git a/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-attachments.test.ts b/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-attachments.test.ts index cdae2e0d6e9..79badaa3ec9 100644 --- a/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-attachments.test.ts +++ b/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-attachments.test.ts @@ -16,6 +16,10 @@ vi.mock('../../../../../src/shared/non-secure-context-uuid', () => ({ createNonSecureContextUuid: () => `img-${++ids.next}` })) vi.mock('../../../platform/media-picker', () => ({ useMediaPicker: () => ({ pickImages: pick }) })) +// The canvas fit needs a WebView; this test is about the attaching counter. +vi.mock('../../../files/attachments/mobile-agent-image-fit', () => ({ + fitMobileAgentImage: async (image: unknown) => image +})) function ok(id: string, result: unknown): RpcSuccess { return { id, ok: true, result, _meta: { runtimeId: 'r' } } diff --git a/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-upload.ts b/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-upload.ts index 2aaea4368f8..9e3fd851eef 100644 --- a/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-upload.ts +++ b/mobile/src/session/native-chat/composer/use-mobile-native-chat-image-upload.ts @@ -22,6 +22,7 @@ import { type MobilePastedImages } from './mobile-native-chat-composer-paste' import { useClipboardReader } from '../../../platform/clipboard' +import { fitMobileAgentImage } from '../../../files/attachments/mobile-agent-image-fit' type CurrentRef = { readonly current: T } type UploadedNativeChatImage = Omit @@ -99,7 +100,8 @@ export function useMobileNativeChatImageUpload(args: { started = true attachingCount.current += 1 setIsAttaching(true) - } + }, + fitImage: fitMobileAgentImage }) } catch (error) { uploadError = error diff --git a/mobile/src/terminal/session/use-mobile-terminal-paste.ts b/mobile/src/terminal/session/use-mobile-terminal-paste.ts index fd0ef30d0ef..8d61401c0b3 100644 --- a/mobile/src/terminal/session/use-mobile-terminal-paste.ts +++ b/mobile/src/terminal/session/use-mobile-terminal-paste.ts @@ -12,10 +12,9 @@ import type { RpcClient } from '../../transport/rpc/rpc-client-contract' import type { ConnectionState } from '../../transport/connection/connection-state' import { buildMobileTerminalImagePastePayload, - prepareMobileClipboardImageBase64, saveMobileClipboardImageAsTempFile } from '../../files/attachments/mobile-clipboard-image' -import { resizeMobileClipboardImage } from '../../files/attachments/mobile-clipboard-image-resize' +import { fitMobileAgentImageBase64 } from '../../files/attachments/mobile-agent-image-fit' // Match xterm's paste(): newlines become CR, and DECSET 2004 frames the text with ESC made inert. function buildMobileTerminalClipboardTextPayload( @@ -99,7 +98,7 @@ export function useMobileTerminalPaste({ return } const connectionId = await getActiveWorktreeConnectionId() - const base64 = await prepareMobileClipboardImageBase64(image, resizeMobileClipboardImage) + const base64 = await fitMobileAgentImageBase64(image.data) const imagePath = await saveMobileClipboardImageAsTempFile(client, base64, { connectionId }) diff --git a/src/main/ipc/filesystem/filesystem-runtime-upload-staging.ts b/src/main/ipc/filesystem/filesystem-runtime-upload-staging.ts index cb8ef6369e7..f701b84607e 100644 --- a/src/main/ipc/filesystem/filesystem-runtime-upload-staging.ts +++ b/src/main/ipc/filesystem/filesystem-runtime-upload-staging.ts @@ -7,7 +7,10 @@ import { constants } from 'node:fs' import { lstat, open, readdir, realpath, type FileHandle } from 'node:fs/promises' import { basename, extname, isAbsolute, join, relative, resolve, sep } from 'node:path' import { classifyImportSource } from './filesystem-import-source-stat' -import { detectImageFileSignature, IMAGE_SIGNATURE_BYTES } from '../../media/image-file-signature' +import { + detectImageFileSignature, + IMAGE_SIGNATURE_BYTES +} from '../../../shared/image-file-signature' import type { StagedRuntimeUploadEntry as StagedExternalImportEntry, StagedRuntimeUploadSource as StagedExternalImportSource diff --git a/src/main/pi/rpc/prompts/rpc-prompt.ts b/src/main/pi/rpc/prompts/rpc-prompt.ts index d8b7d1f0fb1..6903b49b8b5 100644 --- a/src/main/pi/rpc/prompts/rpc-prompt.ts +++ b/src/main/pi/rpc/prompts/rpc-prompt.ts @@ -10,7 +10,7 @@ import { import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types' import type { NativeChatImageRefBlock } from '../../../../shared/native-chat/native-chat-types' import { agentInlineImageMimeType } from '../../../../shared/agent-inline-image-types' -import { detectImageFileSignature } from '../../../media/image-file-signature' +import { detectImageFileSignature } from '../../../../shared/image-file-signature' import { agentSessionFailureFact, type SubmissionRejectionFact diff --git a/src/main/repository-icons/repo-icon-file-detection.ts b/src/main/repository-icons/repo-icon-file-detection.ts index 1a60db840f2..9429280c3a4 100644 --- a/src/main/repository-icons/repo-icon-file-detection.ts +++ b/src/main/repository-icons/repo-icon-file-detection.ts @@ -2,7 +2,7 @@ import { readFile, stat } from 'node:fs/promises' import { buildImageDataUri } from '../../shared/file-preview/image-data-uri' import { MAX_REPO_ICON_UPLOAD_BYTES, type RepoIcon } from '../../shared/repo-icon' import type { ExecutionHostFilesystemRoute } from '../providers/execution-host-provider-dispatch' -import { detectImageFileSignature } from '../media/image-file-signature' +import { detectImageFileSignature } from '../../shared/image-file-signature' import { extractIconHref } from './repo-icon-source-href' import { iconHrefCandidates } from './repo-icon-href-candidates' import { joinWorktreeRelativePath } from '../runtime/files/paths/runtime-relative-paths' diff --git a/src/main/window/clipboard-image-temp-file.ts b/src/main/window/clipboard-image-temp-file.ts index 869ba3622ce..b1ea02406b8 100644 --- a/src/main/window/clipboard-image-temp-file.ts +++ b/src/main/window/clipboard-image-temp-file.ts @@ -10,7 +10,7 @@ import { sanitizeAgentSessionAttachmentName, type AgentSessionAttachmentClipboardTarget } from '../../shared/agent-session/agent-session-attachments' -import { detectImageFileExtension } from '../media/image-file-signature' +import { detectImageFileExtension } from '../../shared/image-file-signature' import { nativeChatPasteFolder } from './native-chat-paste-files' export type SaveClipboardImageAsTempFileArgs = { diff --git a/src/shared/clipboard-image.ts b/src/shared/clipboard-image.ts index a7989d61f20..23b24faa8af 100644 --- a/src/shared/clipboard-image.ts +++ b/src/shared/clipboard-image.ts @@ -4,6 +4,9 @@ export const CLIPBOARD_IMAGE_MAX_SOURCE_BYTES = Math.floor( ) export const CLIPBOARD_IMAGE_MAX_PIXELS = 32 * 1024 * 1024 export const CLIPBOARD_IMAGE_TOO_LARGE_ERROR = 'Clipboard image is too large' +// Agents downscale to about this edge anyway, and Anthropic refuses images over 5 MiB of base64. +export const AGENT_IMAGE_MAX_LONG_EDGE = 2000 +export const AGENT_IMAGE_MAX_BYTES = Math.floor(((5 * 1024 * 1024) / 4) * 3) export type ClipboardImageDimensions = { height: number @@ -55,7 +58,8 @@ function scaleClipboardImageDimensions( export async function encodeClipboardImageWithinLimit( source: ClipboardImageDimensions, encode: (size: ClipboardImageDimensions) => T | Promise, - byteLength: (encoded: T) => number + byteLength: (encoded: T) => number, + maxBytes: number = CLIPBOARD_IMAGE_MAX_SOURCE_BYTES ): Promise { const { height, width } = source if (!Number.isFinite(width * height) || width <= 0 || height <= 0) { @@ -71,12 +75,12 @@ export async function encodeClipboardImageWithinLimit( for (let attempt = 0; attempt < CLIPBOARD_IMAGE_FIT_ATTEMPTS; attempt += 1) { const encoded = await encode(size) const bytes = byteLength(encoded) - if (bytes <= CLIPBOARD_IMAGE_MAX_SOURCE_BYTES) { + if (bytes <= maxBytes) { return encoded } const next = scaleClipboardImageDimensions( size, - CLIPBOARD_IMAGE_FIT_MARGIN * Math.min(1, Math.sqrt(CLIPBOARD_IMAGE_MAX_SOURCE_BYTES / bytes)) + CLIPBOARD_IMAGE_FIT_MARGIN * Math.min(1, Math.sqrt(maxBytes / bytes)) ) if (next.width === size.width && next.height === size.height) { break @@ -95,15 +99,22 @@ export type ClipboardImageThumbnail = ClipboardImageDimensions & { } /** Fit the thumbnail budget; preserve already-small dimensions so callers skip resizing. */ -export function clipboardImageThumbnailSize({ - height, - width -}: ClipboardImageDimensions): ClipboardImageDimensions { +export function clipboardImageThumbnailSize( + size: ClipboardImageDimensions +): ClipboardImageDimensions { + return clipboardImageSizeWithinEdge(size, CLIPBOARD_IMAGE_THUMBNAIL_MAX_EDGE) +} + +/** Preserve already-small dimensions so callers can skip resizing. */ +export function clipboardImageSizeWithinEdge( + { height, width }: ClipboardImageDimensions, + maxEdge: number +): ClipboardImageDimensions { const longestEdge = Math.max(width, height) - if (longestEdge <= CLIPBOARD_IMAGE_THUMBNAIL_MAX_EDGE) { + if (longestEdge <= maxEdge) { return { height, width } } - const scale = CLIPBOARD_IMAGE_THUMBNAIL_MAX_EDGE / longestEdge + const scale = maxEdge / longestEdge return { height: Math.max(1, Math.round(height * scale)), width: Math.max(1, Math.round(width * scale)) diff --git a/src/main/media/image-file-signature.test.ts b/src/shared/image-file-signature.test.ts similarity index 100% rename from src/main/media/image-file-signature.test.ts rename to src/shared/image-file-signature.test.ts diff --git a/src/main/media/image-file-signature.ts b/src/shared/image-file-signature.ts similarity index 100% rename from src/main/media/image-file-signature.ts rename to src/shared/image-file-signature.ts