diff --git a/lib/__tests__/image-shrink.test.ts b/lib/__tests__/image-shrink.test.ts index 817827e44..d9f58dc92 100644 --- a/lib/__tests__/image-shrink.test.ts +++ b/lib/__tests__/image-shrink.test.ts @@ -2,14 +2,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { shrinkImage, toBase64, MAX_SHRUNK_IMAGE_BYTES } from '../image-shrink'; // jsdom has no image decoder and no canvas: the browser's createImageBitmap -// and OffscreenCanvas are stood in for. A "picture" here is a blob holding -// JSON: its size and whether it has see-through pixels. Encoded sizes follow -// a formula the test sets, so the byte cap can be hit on purpose. +// and OffscreenCanvas are stood in for. A "picture" is a PNG header (the +// size the file declares; colour type 6 for see-through pixels). Encoded +// sizes follow a formula the test sets, so the byte cap can be hit on purpose. -interface Fake { w: number; h: number; alpha?: boolean } +interface Fake { w: number; h: number; alpha?: boolean; broken?: boolean } let bytesPer: (type: string, quality: number | undefined, w: number, h: number) => number; const encoded: { type: string; quality?: number; w: number; h: number }[] = []; const fills: string[] = []; +const decodes: unknown[] = []; +let oldBrowser = false; class FakeBitmap { constructor(readonly width: number, readonly height: number, readonly alpha: boolean) {} @@ -40,29 +42,47 @@ class FakeCanvas { } } -const picture = (fake: Fake, type = 'image/jpeg') => new Blob([JSON.stringify(fake)], { type }); +/** A PNG header declaring `w` × `h`; colour type 6 (alpha), 2, or 99 for a file that won't decode. */ +function header({ w, h, alpha, broken }: Fake): Uint8Array { + const b = new Uint8Array(33); + b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52]); + const view = new DataView(b.buffer); + view.setUint32(16, w); + view.setUint32(20, h); + b[24] = 8; + b[25] = broken ? 99 : alpha ? 6 : 2; + return b; +} +const picture = (fake: Fake, type = 'image/jpeg') => new Blob([header(fake) as BlobPart], { type }); beforeEach(() => { encoded.length = 0; fills.length = 0; + decodes.length = 0; + oldBrowser = false; bytesPer = () => 1000; vi.stubGlobal('OffscreenCanvas', FakeCanvas); - vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob, options: unknown) => { - expect(options).toEqual({ imageOrientation: 'from-image' }); - const text = await blob.text(); - if (text === 'broken') throw new DOMException('cannot decode', 'InvalidStateError'); - const fake = JSON.parse(text) as Fake; - return new FakeBitmap(fake.w, fake.h, !!fake.alpha); + vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob, options?: { resizeWidth?: number }) => { + decodes.push(options); + if (oldBrowser && options) throw new TypeError("Failed to execute 'createImageBitmap': The provided value 'from-image' is not a valid enum value"); + const b = new Uint8Array(await blob.arrayBuffer()); + const view = new DataView(b.buffer); + const w = view.getUint32(16); + const h = view.getUint32(20); + if (b[25] === 99) throw new DOMException('cannot decode', 'InvalidStateError'); + const width = options?.resizeWidth ?? w; + return new FakeBitmap(width, Math.round((h * width) / w), b[25] === 6); })); }); afterEach(() => vi.unstubAllGlobals()); describe('shrinkImage', () => { - it('shrinks the long edge to 1568 px as a JPEG at 0.85 on white', async () => { + it('shrinks the long edge to 1568 px as a JPEG at 0.85 on white, decoding at that size', async () => { const out = await shrinkImage(picture({ w: 4000, h: 3000 })); expect(out).toMatchObject({ status: 'ok', type: 'image/jpeg', width: 1568, height: 1176 }); expect(out.bytes?.length).toBe(1000); + expect(decodes).toEqual([{ imageOrientation: 'from-image', resizeWidth: 1568, resizeQuality: 'high' }]); expect(encoded).toEqual([{ type: 'image/jpeg', quality: 0.85, w: 1568, h: 1176 }]); expect(fills).toEqual(['#ffffff']); }); @@ -70,20 +90,40 @@ describe('shrinkImage', () => { it('takes a smaller long edge, and never makes an image larger', async () => { expect(await shrinkImage(picture({ w: 3000, h: 4000 }), { maxSide: 1024 })).toMatchObject({ width: 768, height: 1024 }); expect(await shrinkImage(picture({ w: 800, h: 600 }))).toMatchObject({ status: 'ok', width: 800, height: 600 }); + // Small enough already: decoded as it is. + expect(decodes.at(-1)).toEqual({ imageOrientation: 'from-image' }); }); - it('skips tracking pixels and tiny images without drawing them', async () => { + it('skips tracking pixels and tiny images without decoding them', async () => { expect(await shrinkImage(picture({ w: 1, h: 1 }, 'image/gif'))).toEqual({ status: 'tracker', width: 1, height: 1 }); expect(await shrinkImage(picture({ w: 2, h: 2 }))).toMatchObject({ status: 'tracker' }); expect(await shrinkImage(picture({ w: 40, h: 600 }))).toMatchObject({ status: 'tiny' }); expect(await shrinkImage(picture({ w: 100, h: 100 }))).toMatchObject({ status: 'tiny' }); - expect(encoded).toEqual([]); + expect(decodes).toEqual([]); expect(await shrinkImage(picture({ w: 130, h: 130 }))).toMatchObject({ status: 'ok' }); // Asked for on purpose: small ones too, but never a tracker. expect(await shrinkImage(picture({ w: 40, h: 40 }), { small: true })).toMatchObject({ status: 'ok', width: 40 }); expect(await shrinkImage(picture({ w: 1, h: 2 }), { small: true })).toMatchObject({ status: 'tracker' }); }); + it('refuses an image that declares more than 50 megapixels without decoding it', async () => { + expect(await shrinkImage(picture({ w: 20_000, h: 20_000 }, 'image/png'))).toEqual({ status: 'too_big', width: 20_000, height: 20_000 }); + expect(decodes).toEqual([]); + expect(await shrinkImage(picture({ w: 8_000, h: 6_000 }))).toMatchObject({ status: 'ok' }); + }); + + it('answers unsupported for a header it can not read, and for what the browser can not decode', async () => { + expect(await shrinkImage(new Blob(['not an image at all, just text'], { type: 'image/heic' }))).toEqual({ status: 'unsupported' }); + expect(decodes).toEqual([]); + expect(await shrinkImage(picture({ w: 300, h: 200, broken: true }))).toMatchObject({ status: 'unsupported' }); + }); + + it('decodes without options on a browser that does not know them', async () => { + oldBrowser = true; + expect(await shrinkImage(picture({ w: 600, h: 400 }))).toMatchObject({ status: 'ok', width: 600, height: 400 }); + expect(decodes).toEqual([{ imageOrientation: 'from-image' }, undefined]); + }); + it('keeps a PNG with see-through parts as PNG, and turns an opaque one into JPEG', async () => { expect(await shrinkImage(picture({ w: 600, h: 400, alpha: true }, 'image/png'))).toMatchObject({ status: 'ok', type: 'image/png' }); expect(fills).toEqual([]); @@ -102,10 +142,6 @@ describe('shrinkImage', () => { bytesPer = () => MAX_SHRUNK_IMAGE_BYTES + 1; expect(await shrinkImage(picture({ w: 2000, h: 1000 }))).toEqual({ status: 'too_big', width: 2000, height: 1000 }); }); - - it('answers unsupported for what the browser can not decode', async () => { - expect(await shrinkImage(new Blob(['broken'], { type: 'image/heic' }))).toEqual({ status: 'unsupported' }); - }); }); describe('toBase64', () => { diff --git a/lib/__tests__/image-size.test.ts b/lib/__tests__/image-size.test.ts new file mode 100644 index 000000000..f2a8f1839 --- /dev/null +++ b/lib/__tests__/image-size.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { imageDimensions } from '../image-size'; + +// The size an image declares, read from the header of each format. + +const bytes = (...parts: (number[] | string)[]) => new Uint8Array(parts.flatMap((p) => (typeof p === 'string' ? [...p].map((c) => c.charCodeAt(0)) : p))); +const be32 = (n: number) => [(n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255]; +const le16 = (n: number) => [n & 255, (n >>> 8) & 255]; +const le24 = (n: number) => [n & 255, (n >>> 8) & 255, (n >>> 16) & 255]; +const le32 = (n: number) => [...le16(n & 0xffff), ...le16(n >>> 16)]; + +describe('imageDimensions', () => { + it('reads PNG and GIF', () => { + expect(imageDimensions(bytes([0x89], 'PNG', [13, 10, 26, 10, 0, 0, 0, 13], 'IHDR', be32(40_000), be32(30_000), [8, 2, 0, 0, 0]))).toEqual({ width: 40_000, height: 30_000 }); + expect(imageDimensions(bytes('GIF89a', le16(320), le16(200), [0, 0, 0]))).toEqual({ width: 320, height: 200 }); + }); + + it('reads a JPEG frame header after an EXIF block', () => { + const app1 = [0xff, 0xe1, 0x00, 0x10, ...new Array(14).fill(0)]; + const sof = [0xff, 0xc2, 0x00, 0x11, 8, 0x0b, 0xb8, 0x0f, 0xa0, 3, ...new Array(9).fill(0)]; + expect(imageDimensions(bytes([0xff, 0xd8], app1, sof))).toEqual({ width: 4000, height: 3000 }); + // No frame header before the image data: unknown. + expect(imageDimensions(bytes([0xff, 0xd8], app1, [0xff, 0xda, 0, 2, 0, 0, 0, 0, 0, 0]))).toBeNull(); + }); + + it('reads the three kinds of WebP', () => { + const riff = (chunk: string, body: number[]) => bytes('RIFF', [0, 0, 0, 0], 'WEBP', chunk, [0, 0, 0, 0], body); + expect(imageDimensions(riff('VP8 ', [0, 0, 0, 0x9d, 0x01, 0x2a, ...le16(640), ...le16(480)]))).toEqual({ width: 640, height: 480 }); + // VP8L: 14 bits each of width - 1 and height - 1 after the 0x2f signature. + const w = 1023; + const h = 767; + const packed = (w - 1) | ((h - 1) << 14); + expect(imageDimensions(riff('VP8L', [0x2f, ...le32(packed), 0]))).toEqual({ width: w, height: h }); + expect(imageDimensions(riff('VP8X', [0, 0, 0, 0, ...le24(16_383), ...le24(9_999)]))).toEqual({ width: 16_384, height: 10_000 }); + }); + + it('reads BMP, and nothing from what it does not know', () => { + expect(imageDimensions(bytes('BM', new Array(16).fill(0), le32(800), le32(-600 >>> 0)))).toEqual({ width: 800, height: 600 }); + expect(imageDimensions(bytes(''))).toBeNull(); + expect(imageDimensions(new Uint8Array(0))).toBeNull(); + }); +}); diff --git a/lib/__tests__/plugin-host-attachment-image.test.ts b/lib/__tests__/plugin-host-attachment-image.test.ts index 340290ac6..d9450d96a 100644 --- a/lib/__tests__/plugin-host-attachment-image.test.ts +++ b/lib/__tests__/plugin-host-attachment-image.test.ts @@ -17,7 +17,7 @@ import { snapshotHostFeatures } from '../plugin-sandbox/host-features'; import { withAssistantRuntime } from '../plugin-sandbox/assistant/runtime'; // jsdom can't decode or draw images: createImageBitmap and OffscreenCanvas -// are stood in for. A "picture" is a blob holding its size as JSON. +// are stood in for. A "picture" is a PNG header declaring its size. function plugin(perms: string[], id = 'ai-assistant'): InstalledPlugin { return { @@ -42,6 +42,16 @@ const SIZES: Record = { 'blob-photo': { w: 3000, h: 2000 }, 'blob-logo': { w: 240, h: 160 }, 'blob-pixel': { w: 1, h: 1 }, }; +function pngHeader({ w, h }: { w: number; h: number }): Uint8Array { + const b = new Uint8Array(33); + b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52]); + new DataView(b.buffer).setUint32(16, w); + new DataView(b.buffer).setUint32(20, h); + b[24] = 8; + b[25] = 2; + return b; +} + const message = { id: 'm1', threadId: 't1', mailboxIds: { inbox: true }, keywords: {}, subject: 'Logo', receivedAt: '2026-05-01T10:00:00Z', htmlBody: [{ partId: '1', blobId: 'b-html', size: 1, type: 'text/html' }], @@ -63,13 +73,16 @@ beforeEach(() => { resetPluginRateLimits(); exif.read.mockClear(); vi.stubGlobal('OffscreenCanvas', FakeCanvas); - vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob) => { - const { w, h } = JSON.parse(await blob.text()) as { w: number; h: number }; - return { width: w, height: h, close() {} }; + vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob, options?: { resizeWidth?: number }) => { + const view = new DataView(await blob.arrayBuffer()); + const w = view.getUint32(16); + const h = view.getUint32(20); + const width = options?.resizeWidth ?? w; + return { width, height: Math.round((h * width) / w), close() {} }; })); client = { getEmail: vi.fn(async (id: string) => (id === 'm1' ? message : null)), - fetchBlobArrayBuffer: vi.fn(async (blobId: string) => new TextEncoder().encode(JSON.stringify(SIZES[blobId] ?? { w: 0, h: 0 })).buffer), + fetchBlobArrayBuffer: vi.fn(async (blobId: string) => pngHeader(SIZES[blobId] ?? { w: 0, h: 0 }).buffer), getAccountId: vi.fn(() => 'acc1'), getMailboxes: vi.fn(async () => []), }; diff --git a/lib/image-shrink.ts b/lib/image-shrink.ts index 7df8edcdf..03579d0a5 100644 --- a/lib/image-shrink.ts +++ b/lib/image-shrink.ts @@ -5,6 +5,8 @@ // has see-through parts. Images too small to show anything (tracking pixels, // spacers, icons) are not drawn at all. +import { IMAGE_HEADER_BYTES, imageDimensions } from './image-size'; + /** Default and allowed range of the long edge, in pixels. */ export const DEFAULT_IMAGE_SIDE = 1568; export const MIN_IMAGE_SIDE = 64; @@ -61,27 +63,56 @@ export function fitSize(width: number, height: number, maxSide: number): { width /** Types the browser may decode that can carry transparency. */ const ALPHA_TYPES = /^image\/(png|webp|gif|avif)$/i; +/** Most pixels an image may declare; a larger one is not decoded (its pixels alone could fill the memory). */ +export const MAX_IMAGE_PIXELS = 50_000_000; + +/** + * Decodes with the photo's EXIF orientation and, when given, a smaller + * width. Browsers before imageOrientation (or resize) throw a TypeError on + * the options: then without them. + */ +async function decode(blob: Blob, resizeWidth: number | null): Promise { + try { + return await createImageBitmap(blob, { + imageOrientation: 'from-image', + // Only the width: the browser keeps the aspect, whichever way EXIF turns the photo. + ...(resizeWidth ? { resizeWidth, resizeQuality: 'high' as const } : {}), + }); + } catch (err) { + if (err instanceof TypeError) return createImageBitmap(blob); + throw err; + } +} + /** * The image in `blob`, at most `maxSide` on its long edge and * MAX_SHRUNK_IMAGE_BYTES in size. JPEG at quality 0.85, then 0.7, then * smaller by a quarter per step until it fits; PNG when transparent. * `small`: also shrink images below the "tiny" limits (still not trackers). + * The size comes from the file's header before anything is decoded: a + * format whose header isn't read here is unsupported, more than + * MAX_IMAGE_PIXELS too_big, trackers and tiny images are never decoded. */ export async function shrinkImage(blob: Blob, { maxSide = DEFAULT_IMAGE_SIDE, small = false }: { maxSide?: number; small?: boolean } = {}): Promise { if (typeof createImageBitmap !== 'function') return { status: 'unsupported' }; + const declared = imageDimensions(new Uint8Array(await blob.slice(0, IMAGE_HEADER_BYTES).arrayBuffer())); + // HEIC, TIFF, SVG, a damaged file: no size to check, nothing to decode. + if (!declared || declared.width < 1 || declared.height < 1) return { status: 'unsupported' }; + const { width: w, height: h } = declared; + if (w * h > MAX_IMAGE_PIXELS) return { status: 'too_big', width: w, height: h }; + if (w <= TRACKER_SIDE && h <= TRACKER_SIDE) return { status: 'tracker', width: w, height: h }; + if (!small && (Math.min(w, h) < TINY_SIDE || w * h < TINY_AREA)) return { status: 'tiny', width: w, height: h }; + const side = Math.max(MIN_IMAGE_SIDE, Math.min(MAX_IMAGE_SIDE, Math.round(maxSide))); + const target = fitSize(w, h, side); let bitmap: ImageBitmap; try { - bitmap = await createImageBitmap(blob, { imageOrientation: 'from-image' }); + bitmap = await decode(blob, target.width < w ? target.width : null); } catch { - // HEIC, TIFF, a broken file: nothing the browser can show. - return { status: 'unsupported' }; + // A broken file: nothing the browser can show. + return { status: 'unsupported', width: w, height: h }; } try { - const { width: w, height: h } = bitmap; - if (w <= TRACKER_SIDE && h <= TRACKER_SIDE) return { status: 'tracker', width: w, height: h }; - if (!small && (Math.min(w, h) < TINY_SIDE || w * h < TINY_AREA)) return { status: 'tiny', width: w, height: h }; - const side = Math.max(MIN_IMAGE_SIDE, Math.min(MAX_IMAGE_SIDE, Math.round(maxSide))); - let size = fitSize(w, h, side); + let size = fitSize(bitmap.width, bitmap.height, side); let transparent: boolean | null = ALPHA_TYPES.test(blob.type) ? null : false; for (let step = 0; step < 12; step++) { const canvas = makeCanvas(size.width, size.height); diff --git a/lib/image-size.ts b/lib/image-size.ts new file mode 100644 index 000000000..6ab86e077 --- /dev/null +++ b/lib/image-size.ts @@ -0,0 +1,66 @@ +// The width and height an image file declares in its header, read before +// anything decodes it: PNG (IHDR), GIF, JPEG (the SOF segment), WebP +// (VP8, VP8L, VP8X) and BMP. A file can declare a size whose pixels alone +// would fill the memory, so callers check this first. + +export interface ImageDimensions { + width: number; + height: number; +} + +/** Bytes of a file's start that hold the size of every format read here, EXIF blocks before a JPEG's SOF included. */ +export const IMAGE_HEADER_BYTES = 512 * 1024; + +const u16be = (b: Uint8Array, at: number) => (b[at] << 8) | b[at + 1]; +const u16le = (b: Uint8Array, at: number) => b[at] | (b[at + 1] << 8); +const u24le = (b: Uint8Array, at: number) => b[at] | (b[at + 1] << 8) | (b[at + 2] << 16); +const u32be = (b: Uint8Array, at: number) => ((b[at] << 24) >>> 0) + ((b[at + 1] << 16) | (b[at + 2] << 8) | b[at + 3]); +const ascii = (b: Uint8Array, at: number, text: string) => [...text].every((c, i) => b[at + i] === c.charCodeAt(0)); + +function jpeg(b: Uint8Array): ImageDimensions | null { + let at = 2; + while (at + 9 < b.length) { + if (b[at] !== 0xff) return null; + const marker = b[at + 1]; + // Fill bytes, and markers without a length (restart markers, TEM). + if (marker === 0xff) { at++; continue; } + if ((marker >= 0xd0 && marker <= 0xd7) || marker === 0x01) { at += 2; continue; } + if (marker === 0xd9 || marker === 0xda) return null; + const length = u16be(b, at + 2); + // Start of frame: every SOF but DHT (C4), JPG (C8) and DAC (CC). + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { + return { height: u16be(b, at + 5), width: u16be(b, at + 7) }; + } + if (length < 2) return null; + at += 2 + length; + } + return null; +} + +function webp(b: Uint8Array): ImageDimensions | null { + if (b.length >= 30 && ascii(b, 12, 'VP8 ') && b[23] === 0x9d && b[24] === 0x01 && b[25] === 0x2a) { + return { width: u16le(b, 26) & 0x3fff, height: u16le(b, 28) & 0x3fff }; + } + if (ascii(b, 12, 'VP8L') && b[20] === 0x2f) { + return { + width: 1 + (((b[22] & 0x3f) << 8) | b[21]), + height: 1 + (((b[24] & 0x0f) << 10) | (b[23] << 2) | ((b[22] & 0xc0) >> 6)), + }; + } + if (b.length >= 30 && ascii(b, 12, 'VP8X')) return { width: 1 + u24le(b, 24), height: 1 + u24le(b, 27) }; + return null; +} + +/** The size the file declares, or null for a format not read here (or a damaged header). */ +export function imageDimensions(bytes: Uint8Array): ImageDimensions | null { + const b = bytes; + if (b.length >= 24 && b[0] === 0x89 && ascii(b, 1, 'PNG') && ascii(b, 12, 'IHDR')) return { width: u32be(b, 16), height: u32be(b, 20) }; + if (b.length >= 10 && ascii(b, 0, 'GIF8')) return { width: u16le(b, 6), height: u16le(b, 8) }; + if (b.length >= 4 && b[0] === 0xff && b[1] === 0xd8) return jpeg(b); + if (b.length >= 25 && ascii(b, 0, 'RIFF') && ascii(b, 8, 'WEBP')) return webp(b); + if (b.length >= 26 && ascii(b, 0, 'BM')) { + const view = new DataView(b.buffer, b.byteOffset, b.byteLength); + return { width: Math.abs(view.getInt32(18, true)), height: Math.abs(view.getInt32(22, true)) }; + } + return null; +}