mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
fix(mobile): shrink chat photos to 2000 px before upload
Phone photos went up at full size, and agents refuse images over 5 MiB of base64. Fixes #40844
This commit is contained in:
@@ -1932,7 +1932,7 @@
|
||||
"src/main/host/secret-protection-report.test.ts": 59,
|
||||
"src/main/i18n/main-i18n-lazy-locale.test.ts": 518,
|
||||
"src/main/i18n/system-language.test.ts": 358,
|
||||
"src/main/media/image-file-signature.test.ts": 47,
|
||||
"src/shared/image-file-signature.test.ts": 47,
|
||||
"src/main/credentials/integration-state-file-cache.test.ts": 47,
|
||||
"src/main/credentials/integration-token-file-cache.test.ts": 152,
|
||||
"src/main/ipc/account-sign-in-link-opener.test.ts": 128,
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { Buffer } from 'node:buffer'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { fitMobileAgentImageBase64 } from './mobile-agent-image-fit'
|
||||
|
||||
// Node has no image decoder or canvas: the fake decodes to fixed dimensions and records each encode.
|
||||
function stubRaster(decoded: {
|
||||
width: number
|
||||
height: number
|
||||
}): { width: number; height: number; type: string }[] {
|
||||
const encodes: { width: number; height: number; type: string }[] = []
|
||||
vi.stubGlobal(
|
||||
'Image',
|
||||
class {
|
||||
src = ''
|
||||
naturalWidth = decoded.width
|
||||
naturalHeight = decoded.height
|
||||
decode(): Promise<void> {
|
||||
return Promise.resolve()
|
||||
}
|
||||
}
|
||||
)
|
||||
vi.stubGlobal('document', {
|
||||
createElement: () => {
|
||||
const canvas = {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext: () => ({ fillRect: () => {}, drawImage: () => {} }),
|
||||
toDataURL: (type: string) => {
|
||||
encodes.push({ width: canvas.width, height: canvas.height, type })
|
||||
return 'data:image/jpeg;base64,/9j/4AAQ'
|
||||
}
|
||||
}
|
||||
return canvas
|
||||
}
|
||||
})
|
||||
return encodes
|
||||
}
|
||||
|
||||
function base64Of(header: number[] | string): string {
|
||||
const bytes = typeof header === 'string' ? Buffer.from(header, 'latin1') : Buffer.from(header)
|
||||
return Buffer.concat([bytes, Buffer.alloc(64)]).toString('base64')
|
||||
}
|
||||
|
||||
const PNG = base64Of([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
||||
const JPEG = base64Of([0xff, 0xd8, 0xff, 0xe0])
|
||||
const HEIC = base64Of('\0\0\0\x18ftypheic')
|
||||
|
||||
describe('fitMobileAgentImageBase64', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('re-encodes a full-size photo as a 2000 px JPEG', async () => {
|
||||
const encodes = stubRaster({ width: 4032, height: 3024 })
|
||||
|
||||
expect(await fitMobileAgentImageBase64(JPEG)).toBe('/9j/4AAQ')
|
||||
expect(encodes).toEqual([{ width: 2000, height: 1500, type: 'image/jpeg' }])
|
||||
})
|
||||
|
||||
it('passes a small PNG through unchanged', async () => {
|
||||
const encodes = stubRaster({ width: 1200, height: 800 })
|
||||
|
||||
expect(await fitMobileAgentImageBase64(PNG)).toBe(PNG)
|
||||
expect(encodes).toEqual([])
|
||||
})
|
||||
|
||||
it('converts a small image agents cannot read to JPEG', async () => {
|
||||
const encodes = stubRaster({ width: 1200, height: 800 })
|
||||
|
||||
expect(await fitMobileAgentImageBase64(HEIC)).toBe('/9j/4AAQ')
|
||||
expect(encodes).toEqual([{ width: 1200, height: 800, type: 'image/jpeg' }])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,80 @@
|
||||
import {
|
||||
AGENT_IMAGE_MAX_BYTES,
|
||||
AGENT_IMAGE_MAX_LONG_EDGE,
|
||||
clipboardImageSizeWithinEdge,
|
||||
encodeClipboardImageWithinLimit,
|
||||
type ClipboardImageDimensions
|
||||
} from '../../../../src/shared/clipboard-image'
|
||||
import { base64DecodedByteLength } from '../../../../src/shared/agent-session/agent-session-attachments'
|
||||
import { detectImageFileSignature } from '../../../../src/shared/image-file-signature'
|
||||
import { decodeBase64Bytes } from '../../transport/encryption/base64-byte-codec'
|
||||
import type { PickedMobileImage } from '../../platform/media-picker-contract'
|
||||
|
||||
// Page-only: decodes and re-encodes through the WebView's canvas.
|
||||
|
||||
const DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i
|
||||
const SIGNATURE_BASE64_CHARS = 16
|
||||
const JPEG_QUALITY = 0.85
|
||||
|
||||
function encodeJpegBase64(image: HTMLImageElement, size: ClipboardImageDimensions): string {
|
||||
const canvas = document.createElement('canvas')
|
||||
canvas.width = size.width
|
||||
canvas.height = size.height
|
||||
const context = canvas.getContext('2d')
|
||||
if (context === null) {
|
||||
throw new Error('Failed to encode image')
|
||||
}
|
||||
// JPEG has no alpha, so transparent pixels would otherwise turn black.
|
||||
context.fillStyle = '#ffffff'
|
||||
context.fillRect(0, 0, size.width, size.height)
|
||||
context.imageSmoothingQuality = 'high'
|
||||
context.drawImage(image, 0, 0, size.width, size.height)
|
||||
const encoded = canvas.toDataURL('image/jpeg', JPEG_QUALITY).replace(DATA_URL_PREFIX_RE, '')
|
||||
if (encoded.length === 0) {
|
||||
throw new Error('Failed to encode image')
|
||||
}
|
||||
return encoded
|
||||
}
|
||||
|
||||
/** Shrink a photo to what agents accept; small images in formats they read pass through unchanged. */
|
||||
export async function fitMobileAgentImageBase64(data: string, mimeHint?: string): Promise<string> {
|
||||
const base64 = data.replace(DATA_URL_PREFIX_RE, '')
|
||||
const signature = detectImageFileSignature(
|
||||
decodeBase64Bytes(base64.slice(0, SIGNATURE_BASE64_CHARS))
|
||||
)
|
||||
const image = new Image()
|
||||
image.src = `data:${signature?.mimeType ?? mimeHint ?? 'image/png'};base64,${base64}`
|
||||
try {
|
||||
await image.decode()
|
||||
} catch (error) {
|
||||
// A format agents read uploads as picked; anything else would reach them as an unreadable .png.
|
||||
if (signature !== null) {
|
||||
return base64
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const source = { width: image.naturalWidth, height: image.naturalHeight }
|
||||
const size = clipboardImageSizeWithinEdge(source, AGENT_IMAGE_MAX_LONG_EDGE)
|
||||
if (
|
||||
signature !== null &&
|
||||
size.width === source.width &&
|
||||
size.height === source.height &&
|
||||
base64DecodedByteLength(base64) <= AGENT_IMAGE_MAX_BYTES
|
||||
) {
|
||||
return base64
|
||||
}
|
||||
return encodeClipboardImageWithinLimit(
|
||||
size,
|
||||
(target) => encodeJpegBase64(image, target),
|
||||
base64DecodedByteLength,
|
||||
AGENT_IMAGE_MAX_BYTES
|
||||
)
|
||||
}
|
||||
|
||||
/** Files-app picks that aren't images carry a fileName and upload as-is. */
|
||||
export async function fitMobileAgentImage(image: PickedMobileImage): Promise<PickedMobileImage> {
|
||||
if (image.fileName !== undefined) {
|
||||
return image
|
||||
}
|
||||
return { ...image, base64: await fitMobileAgentImageBase64(image.base64, image.mime) }
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
// Keep resize arithmetic free of upload RPC dependencies so browser raster checks can import it alone.
|
||||
|
||||
// Why: PNG bytes don't scale exactly with pixel area, so undershoot the target on
|
||||
// each pass and let the bounded retry converge instead of distorting in one shot.
|
||||
const MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY = 0.85
|
||||
|
||||
export type MobileClipboardImageResizer = (
|
||||
source: string,
|
||||
target: { width: number; height: number }
|
||||
) => Promise<{ data: string; width: number; height: number }>
|
||||
|
||||
export function computeMobileClipboardImageDownscale(
|
||||
base64Length: number,
|
||||
width: number,
|
||||
height: number,
|
||||
maxBase64Length: number
|
||||
): { width: number; height: number } | null {
|
||||
if (base64Length <= maxBase64Length) {
|
||||
return null
|
||||
}
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
||||
return null
|
||||
}
|
||||
const scale = Math.sqrt(maxBase64Length / base64Length) * MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY
|
||||
const nextWidth = Math.max(1, Math.floor(width * scale))
|
||||
const nextHeight = Math.max(1, Math.floor(height * scale))
|
||||
// Guard against a no-op shrink (already 1px) so the retry loop can't spin forever.
|
||||
if (nextWidth >= width && nextHeight >= height) {
|
||||
return null
|
||||
}
|
||||
return { width: nextWidth, height: nextHeight }
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import type { MobileClipboardImageResizer } from './mobile-clipboard-image-downscale'
|
||||
|
||||
// Keep PNG output: upload retries measure its base64 size and terminal paste assumes PNG. Browser tests exercise a real raster under the shipped CSP.
|
||||
|
||||
const CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i
|
||||
const RESIZED_MIME = 'image/png'
|
||||
|
||||
export const resizeMobileClipboardImage: MobileClipboardImageResizer = async (source, target) => {
|
||||
const base64 = source.replace(CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE, '')
|
||||
const image = new Image()
|
||||
image.src = `data:${RESIZED_MIME};base64,${base64}`
|
||||
// decode rejects unreadable sources; an onload-only promise could remain pending forever.
|
||||
await image.decode()
|
||||
const canvas = document.createElement('canvas')
|
||||
canvas.width = target.width
|
||||
canvas.height = target.height
|
||||
const context = canvas.getContext('2d')
|
||||
if (context === null) {
|
||||
throw new Error('Failed to encode resized clipboard image')
|
||||
}
|
||||
context.drawImage(image, 0, 0, canvas.width, canvas.height)
|
||||
const resized = canvas.toDataURL(RESIZED_MIME).replace(CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE, '')
|
||||
// Why: empty base64 would pass the downstream base64 check and upload a corrupt image, so fail
|
||||
// loudly here instead of silently sending an invalid payload.
|
||||
if (resized.length === 0) {
|
||||
throw new Error('Failed to encode resized clipboard image')
|
||||
}
|
||||
// Return dimensions from the encoded canvas so metadata and bytes describe the same raster.
|
||||
return { data: resized, width: canvas.width, height: canvas.height }
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
import { isWindowsAbsolutePathLike } from '../../../../src/shared/cross-platform-path'
|
||||
import { formatImageDropPasteText } from '../../../../src/shared/image-path-paste'
|
||||
import { formatAgentImagePath } from '../../../../src/shared/agent-image-paste'
|
||||
import {
|
||||
computeMobileClipboardImageDownscale,
|
||||
type MobileClipboardImageResizer
|
||||
} from './mobile-clipboard-image-downscale'
|
||||
import { MOBILE_CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS } from './mobile-clipboard-image-upload-chunk'
|
||||
import '../../transport/connection/stable-logical-rpc-client'
|
||||
import { isLogicalClientCutoverError } from '../../transport/connection/logical-client-cutover-error'
|
||||
@@ -21,7 +17,6 @@ export const MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS = 24 * 1024 * 1024
|
||||
|
||||
export const MOBILE_CLIPBOARD_IMAGE_SINGLE_FRAME_FALLBACK_BASE64_CHARS = 256 * 1024
|
||||
const MOBILE_CLIPBOARD_IMAGE_UPLOAD_CUTOVER_MAX_RETRIES = 1
|
||||
const MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS = 3
|
||||
|
||||
const DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i
|
||||
const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/
|
||||
@@ -37,39 +32,6 @@ export function normalizeMobileClipboardImageBase64(data: string): string {
|
||||
return contentBase64
|
||||
}
|
||||
|
||||
export type MobileClipboardImage = {
|
||||
data: string
|
||||
size: { width: number; height: number }
|
||||
}
|
||||
|
||||
// Retry bounded downscales; any remaining overflow reaches the existing too-large check.
|
||||
export async function prepareMobileClipboardImageBase64(
|
||||
image: MobileClipboardImage,
|
||||
resize: MobileClipboardImageResizer,
|
||||
maxBase64Length: number = MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS
|
||||
): Promise<string> {
|
||||
let data = image.data
|
||||
let width = image.size.width
|
||||
let height = image.size.height
|
||||
for (let attempt = 0; attempt < MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS; attempt += 1) {
|
||||
const contentLength = data.replace(DATA_URL_PREFIX_RE, '').length
|
||||
const target = computeMobileClipboardImageDownscale(
|
||||
contentLength,
|
||||
width,
|
||||
height,
|
||||
maxBase64Length
|
||||
)
|
||||
if (!target) {
|
||||
return data
|
||||
}
|
||||
const resized = await resize(data, target)
|
||||
data = resized.data
|
||||
width = resized.width
|
||||
height = resized.height
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
export async function saveMobileClipboardImageAsTempFile(
|
||||
client: MobileClipboardImageRpcSender,
|
||||
imageData: string,
|
||||
|
||||
@@ -17,6 +17,7 @@ export type AttachMobileImageDeps = {
|
||||
readonly pickImage: (source: MobileImageSource) => Promise<PickedMobileImage | null>
|
||||
// Start spinner only after selection; the OS picker may remain open a long time.
|
||||
readonly onUploadStart?: () => void
|
||||
readonly fitImage?: (image: PickedMobileImage) => Promise<PickedMobileImage>
|
||||
readonly beforeTerminalSend?: (terminal: string) => Promise<boolean>
|
||||
}
|
||||
|
||||
@@ -30,14 +31,16 @@ export async function attachMobileImageToTerminal(
|
||||
getConnectionId,
|
||||
pickImage,
|
||||
onUploadStart,
|
||||
fitImage,
|
||||
beforeTerminalSend
|
||||
}: AttachMobileImageDeps
|
||||
): Promise<boolean> {
|
||||
const picked = await pickImage(source)
|
||||
if (!picked) {
|
||||
const pickedImage = await pickImage(source)
|
||||
if (!pickedImage) {
|
||||
return false
|
||||
}
|
||||
onUploadStart?.()
|
||||
const picked = fitImage ? await fitImage(pickedImage) : pickedImage
|
||||
const connectionId = await getConnectionId()
|
||||
const imagePath = await saveMobileClipboardImageAsTempFile(client, picked.base64, {
|
||||
connectionId,
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type MobileImageSource
|
||||
} from '../../platform/media-picker-contract'
|
||||
import { attachMobileImageToTerminal } from './mobile-image-attachment'
|
||||
import { fitMobileAgentImage } from './mobile-agent-image-fit'
|
||||
|
||||
type CurrentRef<T> = {
|
||||
readonly current: T
|
||||
@@ -68,6 +69,7 @@ export function useMobileImageAttachment({
|
||||
getConnectionId: getActiveWorktreeConnectionId,
|
||||
pickImage: picker.pickImage,
|
||||
onUploadStart: () => setIsAttaching(true),
|
||||
fitImage: fitMobileAgentImage,
|
||||
beforeTerminalSend
|
||||
})
|
||||
if (sent) {
|
||||
|
||||
@@ -11,6 +11,8 @@ export type PickedMobileImage = {
|
||||
readonly uri?: string
|
||||
// Non-image Files-app picks keep their name so the host saves the file under it.
|
||||
readonly fileName?: string
|
||||
// Decode hint for formats with no byte signature the phone recognizes (SVG, HEIC).
|
||||
readonly mime?: string
|
||||
}
|
||||
|
||||
export type PickedMobileFile = {
|
||||
|
||||
@@ -23,8 +23,12 @@ import { isPickedImageMime, mobilePickedFileFallbackName } from './mobile-picked
|
||||
// Refusals reject; only user cancellation returns an empty selection.
|
||||
|
||||
// The page cannot load a native cache URI; the composer falls back to an inline data URI.
|
||||
function pickedImage(base64: string, fileName: string | undefined): PickedMobileImage {
|
||||
return { base64, fileName }
|
||||
function pickedImage(
|
||||
base64: string,
|
||||
fileName: string | undefined,
|
||||
mime: string
|
||||
): PickedMobileImage {
|
||||
return { base64, fileName, mime }
|
||||
}
|
||||
|
||||
// Older shells lack nameMedia; derive a filename from MIME instead.
|
||||
@@ -58,7 +62,7 @@ async function* readPicked(
|
||||
// Release before yielding because callers may stop after one image and strand scarce handles.
|
||||
unread.shift()
|
||||
await releaseStagedMedia(verbs, item.handle)
|
||||
yield pickedImage(base64, fileName)
|
||||
yield pickedImage(base64, fileName, item.mime)
|
||||
}
|
||||
} finally {
|
||||
await releaseAllStagedMedia(verbs, unread)
|
||||
|
||||
@@ -7,12 +7,6 @@ import { isNativeChatLongPaste } from '../../../../../src/shared/native-chat/inp
|
||||
import type { ClipboardReader } from '../../../platform/clipboard'
|
||||
import type { PickedMobileImage } from '../../../platform/media-picker-contract'
|
||||
import { encodeBase64Bytes } from '../../../transport/encryption/base64-byte-codec'
|
||||
import {
|
||||
MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS,
|
||||
normalizeMobileClipboardImageBase64,
|
||||
prepareMobileClipboardImageBase64
|
||||
} from '../../../files/attachments/mobile-clipboard-image'
|
||||
import { resizeMobileClipboardImage } from '../../../files/attachments/mobile-clipboard-image-resize'
|
||||
|
||||
/** One paste's image: the event's file, or null when WebKit left the bytes out and the shell clipboard holds them. */
|
||||
export type MobilePastedImages = { readonly file: File | null }
|
||||
@@ -22,45 +16,21 @@ type PasteEvent = ClipboardEventDataLike<File> & {
|
||||
preventDefault: () => void
|
||||
}
|
||||
|
||||
async function sizedForUpload(data: string, width: number, height: number): Promise<string> {
|
||||
const prepared = await prepareMobileClipboardImageBase64(
|
||||
{ data, size: { width, height } },
|
||||
resizeMobileClipboardImage
|
||||
)
|
||||
return normalizeMobileClipboardImageBase64(prepared)
|
||||
}
|
||||
|
||||
async function readEventImage(file: Blob): Promise<PickedMobileImage> {
|
||||
const base64 = encodeBase64Bytes(new Uint8Array(await file.arrayBuffer()))
|
||||
const uri = `data:${file.type};base64,${base64}`
|
||||
// Agents read only these formats; anything else (HEIC, TIFF) is re-encoded as PNG.
|
||||
const readable = /^image\/(?:png|jpeg|gif|webp)$/.test(file.type)
|
||||
if (readable && base64.length <= MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS) {
|
||||
return { base64, uri }
|
||||
}
|
||||
const image = new Image()
|
||||
image.src = uri
|
||||
await image.decode()
|
||||
const size = { width: image.naturalWidth, height: image.naturalHeight }
|
||||
const source = readable ? base64 : (await resizeMobileClipboardImage(base64, size)).data
|
||||
return { base64: await sizedForUpload(source, size.width, size.height) }
|
||||
}
|
||||
|
||||
/** Sized for upload; the clipboard reader is the uploader's, so the composer needs no shell bridge. */
|
||||
/** Raw bytes; the uploader fits them for agents. The clipboard reader is the uploader's, so the composer needs no shell bridge. */
|
||||
export async function readMobilePastedImages(
|
||||
{ file }: MobilePastedImages,
|
||||
clipboard: Pick<ClipboardReader, 'readImage' | 'contents'>
|
||||
): Promise<readonly PickedMobileImage[]> {
|
||||
if (file) {
|
||||
return [await readEventImage(file)]
|
||||
return [
|
||||
{ base64: encodeBase64Bytes(new Uint8Array(await file.arrayBuffer())), mime: file.type }
|
||||
]
|
||||
}
|
||||
if (!(await clipboard.contents()).image) {
|
||||
return []
|
||||
}
|
||||
const image = await clipboard.readImage()
|
||||
return image
|
||||
? [{ base64: await sizedForUpload(image.data, image.size.width, image.size.height) }]
|
||||
: []
|
||||
return image ? [{ base64: image.data }] : []
|
||||
}
|
||||
|
||||
/** Text pastes stay native unless long; images attach as chips like Photos picks. A disabled composer leaves pastes to the field. */
|
||||
|
||||
@@ -35,6 +35,8 @@ export type UploadNativeChatImagesDeps<S = MobileImageSource> = {
|
||||
| AsyncIterable<PickedMobileImage>
|
||||
| Promise<Iterable<PickedMobileImage> | AsyncIterable<PickedMobileImage>>
|
||||
readonly onUploadStart?: () => void
|
||||
// Injected so this module stays free of the page's canvas.
|
||||
readonly fitImage?: (image: PickedMobileImage) => Promise<PickedMobileImage>
|
||||
/** Retains each completed upload if a later image in the same selection fails. */
|
||||
readonly onImageUploaded?: (image: Omit<PendingNativeChatImage, 'id'>) => void
|
||||
}
|
||||
@@ -48,6 +50,7 @@ export async function uploadMobileNativeChatImages<S>(
|
||||
getConnectionId,
|
||||
pickImages,
|
||||
onUploadStart,
|
||||
fitImage,
|
||||
onImageUploaded
|
||||
}: UploadNativeChatImagesDeps<S>
|
||||
): Promise<Omit<PendingNativeChatImage, 'id'>[]> {
|
||||
@@ -57,7 +60,7 @@ export async function uploadMobileNativeChatImages<S>(
|
||||
if (canContinue?.() === false) {
|
||||
return uploaded
|
||||
}
|
||||
for await (const image of picked) {
|
||||
for await (const pickedImage of picked) {
|
||||
if (canContinue?.() === false) {
|
||||
break
|
||||
}
|
||||
@@ -68,6 +71,7 @@ export async function uploadMobileNativeChatImages<S>(
|
||||
if (canContinue?.() === false) {
|
||||
break
|
||||
}
|
||||
const image = fitImage ? await fitImage(pickedImage) : pickedImage
|
||||
const path = await saveMobileClipboardImageAsTempFile(client, image.base64, { connectionId })
|
||||
const previewUri = image.uri ?? `data:image/png;base64,${image.base64}`
|
||||
const result = {
|
||||
|
||||
+4
@@ -16,6 +16,10 @@ vi.mock('../../../../../src/shared/non-secure-context-uuid', () => ({
|
||||
createNonSecureContextUuid: () => `img-${++ids.next}`
|
||||
}))
|
||||
vi.mock('../../../platform/media-picker', () => ({ useMediaPicker: () => ({ pickImages: pick }) }))
|
||||
// The canvas fit needs a WebView; this test is about the attaching counter.
|
||||
vi.mock('../../../files/attachments/mobile-agent-image-fit', () => ({
|
||||
fitMobileAgentImage: async (image: unknown) => image
|
||||
}))
|
||||
|
||||
function ok(id: string, result: unknown): RpcSuccess {
|
||||
return { id, ok: true, result, _meta: { runtimeId: 'r' } }
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
type MobilePastedImages
|
||||
} from './mobile-native-chat-composer-paste'
|
||||
import { useClipboardReader } from '../../../platform/clipboard'
|
||||
import { fitMobileAgentImage } from '../../../files/attachments/mobile-agent-image-fit'
|
||||
|
||||
type CurrentRef<T> = { readonly current: T }
|
||||
type UploadedNativeChatImage = Omit<PendingNativeChatImage, 'id'>
|
||||
@@ -99,7 +100,8 @@ export function useMobileNativeChatImageUpload(args: {
|
||||
started = true
|
||||
attachingCount.current += 1
|
||||
setIsAttaching(true)
|
||||
}
|
||||
},
|
||||
fitImage: fitMobileAgentImage
|
||||
})
|
||||
} catch (error) {
|
||||
uploadError = error
|
||||
|
||||
@@ -12,10 +12,9 @@ import type { RpcClient } from '../../transport/rpc/rpc-client-contract'
|
||||
import type { ConnectionState } from '../../transport/connection/connection-state'
|
||||
import {
|
||||
buildMobileTerminalImagePastePayload,
|
||||
prepareMobileClipboardImageBase64,
|
||||
saveMobileClipboardImageAsTempFile
|
||||
} from '../../files/attachments/mobile-clipboard-image'
|
||||
import { resizeMobileClipboardImage } from '../../files/attachments/mobile-clipboard-image-resize'
|
||||
import { fitMobileAgentImageBase64 } from '../../files/attachments/mobile-agent-image-fit'
|
||||
|
||||
// Match xterm's paste(): newlines become CR, and DECSET 2004 frames the text with ESC made inert.
|
||||
function buildMobileTerminalClipboardTextPayload(
|
||||
@@ -99,7 +98,7 @@ export function useMobileTerminalPaste({
|
||||
return
|
||||
}
|
||||
const connectionId = await getActiveWorktreeConnectionId()
|
||||
const base64 = await prepareMobileClipboardImageBase64(image, resizeMobileClipboardImage)
|
||||
const base64 = await fitMobileAgentImageBase64(image.data)
|
||||
const imagePath = await saveMobileClipboardImageAsTempFile(client, base64, {
|
||||
connectionId
|
||||
})
|
||||
|
||||
@@ -7,7 +7,10 @@ import { constants } from 'node:fs'
|
||||
import { lstat, open, readdir, realpath, type FileHandle } from 'node:fs/promises'
|
||||
import { basename, extname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
||||
import { classifyImportSource } from './filesystem-import-source-stat'
|
||||
import { detectImageFileSignature, IMAGE_SIGNATURE_BYTES } from '../../media/image-file-signature'
|
||||
import {
|
||||
detectImageFileSignature,
|
||||
IMAGE_SIGNATURE_BYTES
|
||||
} from '../../../shared/image-file-signature'
|
||||
import type {
|
||||
StagedRuntimeUploadEntry as StagedExternalImportEntry,
|
||||
StagedRuntimeUploadSource as StagedExternalImportSource
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import type { NativeChatImageRefBlock } from '../../../../shared/native-chat/native-chat-types'
|
||||
import { agentInlineImageMimeType } from '../../../../shared/agent-inline-image-types'
|
||||
import { detectImageFileSignature } from '../../../media/image-file-signature'
|
||||
import { detectImageFileSignature } from '../../../../shared/image-file-signature'
|
||||
import {
|
||||
agentSessionFailureFact,
|
||||
type SubmissionRejectionFact
|
||||
|
||||
@@ -2,7 +2,7 @@ import { readFile, stat } from 'node:fs/promises'
|
||||
import { buildImageDataUri } from '../../shared/file-preview/image-data-uri'
|
||||
import { MAX_REPO_ICON_UPLOAD_BYTES, type RepoIcon } from '../../shared/repo-icon'
|
||||
import type { ExecutionHostFilesystemRoute } from '../providers/execution-host-provider-dispatch'
|
||||
import { detectImageFileSignature } from '../media/image-file-signature'
|
||||
import { detectImageFileSignature } from '../../shared/image-file-signature'
|
||||
import { extractIconHref } from './repo-icon-source-href'
|
||||
import { iconHrefCandidates } from './repo-icon-href-candidates'
|
||||
import { joinWorktreeRelativePath } from '../runtime/files/paths/runtime-relative-paths'
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
sanitizeAgentSessionAttachmentName,
|
||||
type AgentSessionAttachmentClipboardTarget
|
||||
} from '../../shared/agent-session/agent-session-attachments'
|
||||
import { detectImageFileExtension } from '../media/image-file-signature'
|
||||
import { detectImageFileExtension } from '../../shared/image-file-signature'
|
||||
import { nativeChatPasteFolder } from './native-chat-paste-files'
|
||||
|
||||
export type SaveClipboardImageAsTempFileArgs = {
|
||||
|
||||
@@ -4,6 +4,9 @@ export const CLIPBOARD_IMAGE_MAX_SOURCE_BYTES = Math.floor(
|
||||
)
|
||||
export const CLIPBOARD_IMAGE_MAX_PIXELS = 32 * 1024 * 1024
|
||||
export const CLIPBOARD_IMAGE_TOO_LARGE_ERROR = 'Clipboard image is too large'
|
||||
// Agents downscale to about this edge anyway, and Anthropic refuses images over 5 MiB of base64.
|
||||
export const AGENT_IMAGE_MAX_LONG_EDGE = 2000
|
||||
export const AGENT_IMAGE_MAX_BYTES = Math.floor(((5 * 1024 * 1024) / 4) * 3)
|
||||
|
||||
export type ClipboardImageDimensions = {
|
||||
height: number
|
||||
@@ -55,7 +58,8 @@ function scaleClipboardImageDimensions(
|
||||
export async function encodeClipboardImageWithinLimit<T>(
|
||||
source: ClipboardImageDimensions,
|
||||
encode: (size: ClipboardImageDimensions) => T | Promise<T>,
|
||||
byteLength: (encoded: T) => number
|
||||
byteLength: (encoded: T) => number,
|
||||
maxBytes: number = CLIPBOARD_IMAGE_MAX_SOURCE_BYTES
|
||||
): Promise<T> {
|
||||
const { height, width } = source
|
||||
if (!Number.isFinite(width * height) || width <= 0 || height <= 0) {
|
||||
@@ -71,12 +75,12 @@ export async function encodeClipboardImageWithinLimit<T>(
|
||||
for (let attempt = 0; attempt < CLIPBOARD_IMAGE_FIT_ATTEMPTS; attempt += 1) {
|
||||
const encoded = await encode(size)
|
||||
const bytes = byteLength(encoded)
|
||||
if (bytes <= CLIPBOARD_IMAGE_MAX_SOURCE_BYTES) {
|
||||
if (bytes <= maxBytes) {
|
||||
return encoded
|
||||
}
|
||||
const next = scaleClipboardImageDimensions(
|
||||
size,
|
||||
CLIPBOARD_IMAGE_FIT_MARGIN * Math.min(1, Math.sqrt(CLIPBOARD_IMAGE_MAX_SOURCE_BYTES / bytes))
|
||||
CLIPBOARD_IMAGE_FIT_MARGIN * Math.min(1, Math.sqrt(maxBytes / bytes))
|
||||
)
|
||||
if (next.width === size.width && next.height === size.height) {
|
||||
break
|
||||
@@ -95,15 +99,22 @@ export type ClipboardImageThumbnail = ClipboardImageDimensions & {
|
||||
}
|
||||
|
||||
/** Fit the thumbnail budget; preserve already-small dimensions so callers skip resizing. */
|
||||
export function clipboardImageThumbnailSize({
|
||||
height,
|
||||
width
|
||||
}: ClipboardImageDimensions): ClipboardImageDimensions {
|
||||
export function clipboardImageThumbnailSize(
|
||||
size: ClipboardImageDimensions
|
||||
): ClipboardImageDimensions {
|
||||
return clipboardImageSizeWithinEdge(size, CLIPBOARD_IMAGE_THUMBNAIL_MAX_EDGE)
|
||||
}
|
||||
|
||||
/** Preserve already-small dimensions so callers can skip resizing. */
|
||||
export function clipboardImageSizeWithinEdge(
|
||||
{ height, width }: ClipboardImageDimensions,
|
||||
maxEdge: number
|
||||
): ClipboardImageDimensions {
|
||||
const longestEdge = Math.max(width, height)
|
||||
if (longestEdge <= CLIPBOARD_IMAGE_THUMBNAIL_MAX_EDGE) {
|
||||
if (longestEdge <= maxEdge) {
|
||||
return { height, width }
|
||||
}
|
||||
const scale = CLIPBOARD_IMAGE_THUMBNAIL_MAX_EDGE / longestEdge
|
||||
const scale = maxEdge / longestEdge
|
||||
return {
|
||||
height: Math.max(1, Math.round(height * scale)),
|
||||
width: Math.max(1, Math.round(width * scale))
|
||||
|
||||
Reference in New Issue
Block a user