fix: check an image attachment's size before decoding it, and decode on older browsers

The host reads the size a PNG, GIF, JPEG, WebP or BMP declares in its
header first: over 50 megapixels it answers too_big, trackers and tiny
images are never decoded, a format whose header it can't read is
unsupported, and large images are decoded at the target width. A browser
that throws a TypeError on createImageBitmap's options decodes without
them.

The tests' stand-in pictures are now PNG headers instead of JSON (the
code reads the header), so image-shrink.test.ts and
plugin-host-attachment-image.test.ts build them that way, and the first
shrink test expects the resize options on createImageBitmap.
This commit is contained in:
Linus Rath
2026-11-03 00:42:53 +01:00
parent f8de35dd85
commit f7a7828bfa
5 changed files with 219 additions and 31 deletions
+54 -18
View File
@@ -2,14 +2,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { shrinkImage, toBase64, MAX_SHRUNK_IMAGE_BYTES } from '../image-shrink';
// jsdom has no image decoder and no canvas: the browser's createImageBitmap
// and OffscreenCanvas are stood in for. A "picture" here is a blob holding
// JSON: its size and whether it has see-through pixels. Encoded sizes follow
// a formula the test sets, so the byte cap can be hit on purpose.
// and OffscreenCanvas are stood in for. A "picture" is a PNG header (the
// size the file declares; colour type 6 for see-through pixels). Encoded
// sizes follow a formula the test sets, so the byte cap can be hit on purpose.
interface Fake { w: number; h: number; alpha?: boolean }
interface Fake { w: number; h: number; alpha?: boolean; broken?: boolean }
let bytesPer: (type: string, quality: number | undefined, w: number, h: number) => number;
const encoded: { type: string; quality?: number; w: number; h: number }[] = [];
const fills: string[] = [];
const decodes: unknown[] = [];
let oldBrowser = false;
class FakeBitmap {
constructor(readonly width: number, readonly height: number, readonly alpha: boolean) {}
@@ -40,29 +42,47 @@ class FakeCanvas {
}
}
const picture = (fake: Fake, type = 'image/jpeg') => new Blob([JSON.stringify(fake)], { type });
/** A PNG header declaring `w` × `h`; colour type 6 (alpha), 2, or 99 for a file that won't decode. */
function header({ w, h, alpha, broken }: Fake): Uint8Array {
const b = new Uint8Array(33);
b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52]);
const view = new DataView(b.buffer);
view.setUint32(16, w);
view.setUint32(20, h);
b[24] = 8;
b[25] = broken ? 99 : alpha ? 6 : 2;
return b;
}
const picture = (fake: Fake, type = 'image/jpeg') => new Blob([header(fake) as BlobPart], { type });
beforeEach(() => {
encoded.length = 0;
fills.length = 0;
decodes.length = 0;
oldBrowser = false;
bytesPer = () => 1000;
vi.stubGlobal('OffscreenCanvas', FakeCanvas);
vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob, options: unknown) => {
expect(options).toEqual({ imageOrientation: 'from-image' });
const text = await blob.text();
if (text === 'broken') throw new DOMException('cannot decode', 'InvalidStateError');
const fake = JSON.parse(text) as Fake;
return new FakeBitmap(fake.w, fake.h, !!fake.alpha);
vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob, options?: { resizeWidth?: number }) => {
decodes.push(options);
if (oldBrowser && options) throw new TypeError("Failed to execute 'createImageBitmap': The provided value 'from-image' is not a valid enum value");
const b = new Uint8Array(await blob.arrayBuffer());
const view = new DataView(b.buffer);
const w = view.getUint32(16);
const h = view.getUint32(20);
if (b[25] === 99) throw new DOMException('cannot decode', 'InvalidStateError');
const width = options?.resizeWidth ?? w;
return new FakeBitmap(width, Math.round((h * width) / w), b[25] === 6);
}));
});
afterEach(() => vi.unstubAllGlobals());
describe('shrinkImage', () => {
it('shrinks the long edge to 1568 px as a JPEG at 0.85 on white', async () => {
it('shrinks the long edge to 1568 px as a JPEG at 0.85 on white, decoding at that size', async () => {
const out = await shrinkImage(picture({ w: 4000, h: 3000 }));
expect(out).toMatchObject({ status: 'ok', type: 'image/jpeg', width: 1568, height: 1176 });
expect(out.bytes?.length).toBe(1000);
expect(decodes).toEqual([{ imageOrientation: 'from-image', resizeWidth: 1568, resizeQuality: 'high' }]);
expect(encoded).toEqual([{ type: 'image/jpeg', quality: 0.85, w: 1568, h: 1176 }]);
expect(fills).toEqual(['#ffffff']);
});
@@ -70,20 +90,40 @@ describe('shrinkImage', () => {
it('takes a smaller long edge, and never makes an image larger', async () => {
expect(await shrinkImage(picture({ w: 3000, h: 4000 }), { maxSide: 1024 })).toMatchObject({ width: 768, height: 1024 });
expect(await shrinkImage(picture({ w: 800, h: 600 }))).toMatchObject({ status: 'ok', width: 800, height: 600 });
// Small enough already: decoded as it is.
expect(decodes.at(-1)).toEqual({ imageOrientation: 'from-image' });
});
it('skips tracking pixels and tiny images without drawing them', async () => {
it('skips tracking pixels and tiny images without decoding them', async () => {
expect(await shrinkImage(picture({ w: 1, h: 1 }, 'image/gif'))).toEqual({ status: 'tracker', width: 1, height: 1 });
expect(await shrinkImage(picture({ w: 2, h: 2 }))).toMatchObject({ status: 'tracker' });
expect(await shrinkImage(picture({ w: 40, h: 600 }))).toMatchObject({ status: 'tiny' });
expect(await shrinkImage(picture({ w: 100, h: 100 }))).toMatchObject({ status: 'tiny' });
expect(encoded).toEqual([]);
expect(decodes).toEqual([]);
expect(await shrinkImage(picture({ w: 130, h: 130 }))).toMatchObject({ status: 'ok' });
// Asked for on purpose: small ones too, but never a tracker.
expect(await shrinkImage(picture({ w: 40, h: 40 }), { small: true })).toMatchObject({ status: 'ok', width: 40 });
expect(await shrinkImage(picture({ w: 1, h: 2 }), { small: true })).toMatchObject({ status: 'tracker' });
});
it('refuses an image that declares more than 50 megapixels without decoding it', async () => {
expect(await shrinkImage(picture({ w: 20_000, h: 20_000 }, 'image/png'))).toEqual({ status: 'too_big', width: 20_000, height: 20_000 });
expect(decodes).toEqual([]);
expect(await shrinkImage(picture({ w: 8_000, h: 6_000 }))).toMatchObject({ status: 'ok' });
});
it('answers unsupported for a header it can not read, and for what the browser can not decode', async () => {
expect(await shrinkImage(new Blob(['not an image at all, just text'], { type: 'image/heic' }))).toEqual({ status: 'unsupported' });
expect(decodes).toEqual([]);
expect(await shrinkImage(picture({ w: 300, h: 200, broken: true }))).toMatchObject({ status: 'unsupported' });
});
it('decodes without options on a browser that does not know them', async () => {
oldBrowser = true;
expect(await shrinkImage(picture({ w: 600, h: 400 }))).toMatchObject({ status: 'ok', width: 600, height: 400 });
expect(decodes).toEqual([{ imageOrientation: 'from-image' }, undefined]);
});
it('keeps a PNG with see-through parts as PNG, and turns an opaque one into JPEG', async () => {
expect(await shrinkImage(picture({ w: 600, h: 400, alpha: true }, 'image/png'))).toMatchObject({ status: 'ok', type: 'image/png' });
expect(fills).toEqual([]);
@@ -102,10 +142,6 @@ describe('shrinkImage', () => {
bytesPer = () => MAX_SHRUNK_IMAGE_BYTES + 1;
expect(await shrinkImage(picture({ w: 2000, h: 1000 }))).toEqual({ status: 'too_big', width: 2000, height: 1000 });
});
it('answers unsupported for what the browser can not decode', async () => {
expect(await shrinkImage(new Blob(['broken'], { type: 'image/heic' }))).toEqual({ status: 'unsupported' });
});
});
describe('toBase64', () => {
+42
View File
@@ -0,0 +1,42 @@
import { describe, expect, it } from 'vitest';
import { imageDimensions } from '../image-size';
// The size an image declares, read from the header of each format.
const bytes = (...parts: (number[] | string)[]) => new Uint8Array(parts.flatMap((p) => (typeof p === 'string' ? [...p].map((c) => c.charCodeAt(0)) : p)));
const be32 = (n: number) => [(n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255];
const le16 = (n: number) => [n & 255, (n >>> 8) & 255];
const le24 = (n: number) => [n & 255, (n >>> 8) & 255, (n >>> 16) & 255];
const le32 = (n: number) => [...le16(n & 0xffff), ...le16(n >>> 16)];
describe('imageDimensions', () => {
it('reads PNG and GIF', () => {
expect(imageDimensions(bytes([0x89], 'PNG', [13, 10, 26, 10, 0, 0, 0, 13], 'IHDR', be32(40_000), be32(30_000), [8, 2, 0, 0, 0]))).toEqual({ width: 40_000, height: 30_000 });
expect(imageDimensions(bytes('GIF89a', le16(320), le16(200), [0, 0, 0]))).toEqual({ width: 320, height: 200 });
});
it('reads a JPEG frame header after an EXIF block', () => {
const app1 = [0xff, 0xe1, 0x00, 0x10, ...new Array(14).fill(0)];
const sof = [0xff, 0xc2, 0x00, 0x11, 8, 0x0b, 0xb8, 0x0f, 0xa0, 3, ...new Array(9).fill(0)];
expect(imageDimensions(bytes([0xff, 0xd8], app1, sof))).toEqual({ width: 4000, height: 3000 });
// No frame header before the image data: unknown.
expect(imageDimensions(bytes([0xff, 0xd8], app1, [0xff, 0xda, 0, 2, 0, 0, 0, 0, 0, 0]))).toBeNull();
});
it('reads the three kinds of WebP', () => {
const riff = (chunk: string, body: number[]) => bytes('RIFF', [0, 0, 0, 0], 'WEBP', chunk, [0, 0, 0, 0], body);
expect(imageDimensions(riff('VP8 ', [0, 0, 0, 0x9d, 0x01, 0x2a, ...le16(640), ...le16(480)]))).toEqual({ width: 640, height: 480 });
// VP8L: 14 bits each of width - 1 and height - 1 after the 0x2f signature.
const w = 1023;
const h = 767;
const packed = (w - 1) | ((h - 1) << 14);
expect(imageDimensions(riff('VP8L', [0x2f, ...le32(packed), 0]))).toEqual({ width: w, height: h });
expect(imageDimensions(riff('VP8X', [0, 0, 0, 0, ...le24(16_383), ...le24(9_999)]))).toEqual({ width: 16_384, height: 10_000 });
});
it('reads BMP, and nothing from what it does not know', () => {
expect(imageDimensions(bytes('BM', new Array(16).fill(0), le32(800), le32(-600 >>> 0)))).toEqual({ width: 800, height: 600 });
expect(imageDimensions(bytes('<svg width="100000" height="100000">'))).toBeNull();
expect(imageDimensions(new Uint8Array(0))).toBeNull();
});
});
@@ -17,7 +17,7 @@ import { snapshotHostFeatures } from '../plugin-sandbox/host-features';
import { withAssistantRuntime } from '../plugin-sandbox/assistant/runtime';
// jsdom can't decode or draw images: createImageBitmap and OffscreenCanvas
// are stood in for. A "picture" is a blob holding its size as JSON.
// are stood in for. A "picture" is a PNG header declaring its size.
function plugin(perms: string[], id = 'ai-assistant'): InstalledPlugin {
return {
@@ -42,6 +42,16 @@ const SIZES: Record<string, { w: number; h: number }> = {
'blob-photo': { w: 3000, h: 2000 }, 'blob-logo': { w: 240, h: 160 }, 'blob-pixel': { w: 1, h: 1 },
};
function pngHeader({ w, h }: { w: number; h: number }): Uint8Array {
const b = new Uint8Array(33);
b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52]);
new DataView(b.buffer).setUint32(16, w);
new DataView(b.buffer).setUint32(20, h);
b[24] = 8;
b[25] = 2;
return b;
}
const message = {
id: 'm1', threadId: 't1', mailboxIds: { inbox: true }, keywords: {}, subject: 'Logo', receivedAt: '2026-05-01T10:00:00Z',
htmlBody: [{ partId: '1', blobId: 'b-html', size: 1, type: 'text/html' }],
@@ -63,13 +73,16 @@ beforeEach(() => {
resetPluginRateLimits();
exif.read.mockClear();
vi.stubGlobal('OffscreenCanvas', FakeCanvas);
vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob) => {
const { w, h } = JSON.parse(await blob.text()) as { w: number; h: number };
return { width: w, height: h, close() {} };
vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob, options?: { resizeWidth?: number }) => {
const view = new DataView(await blob.arrayBuffer());
const w = view.getUint32(16);
const h = view.getUint32(20);
const width = options?.resizeWidth ?? w;
return { width, height: Math.round((h * width) / w), close() {} };
}));
client = {
getEmail: vi.fn(async (id: string) => (id === 'm1' ? message : null)),
fetchBlobArrayBuffer: vi.fn(async (blobId: string) => new TextEncoder().encode(JSON.stringify(SIZES[blobId] ?? { w: 0, h: 0 })).buffer),
fetchBlobArrayBuffer: vi.fn(async (blobId: string) => pngHeader(SIZES[blobId] ?? { w: 0, h: 0 }).buffer),
getAccountId: vi.fn(() => 'acc1'),
getMailboxes: vi.fn(async () => []),
};
+39 -8
View File
@@ -5,6 +5,8 @@
// has see-through parts. Images too small to show anything (tracking pixels,
// spacers, icons) are not drawn at all.
import { IMAGE_HEADER_BYTES, imageDimensions } from './image-size';
/** Default and allowed range of the long edge, in pixels. */
export const DEFAULT_IMAGE_SIDE = 1568;
export const MIN_IMAGE_SIDE = 64;
@@ -61,27 +63,56 @@ export function fitSize(width: number, height: number, maxSide: number): { width
/** Types the browser may decode that can carry transparency. */
const ALPHA_TYPES = /^image\/(png|webp|gif|avif)$/i;
/** Most pixels an image may declare; a larger one is not decoded (its pixels alone could fill the memory). */
export const MAX_IMAGE_PIXELS = 50_000_000;
/**
* Decodes with the photo's EXIF orientation and, when given, a smaller
* width. Browsers before imageOrientation (or resize) throw a TypeError on
* the options: then without them.
*/
async function decode(blob: Blob, resizeWidth: number | null): Promise<ImageBitmap> {
try {
return await createImageBitmap(blob, {
imageOrientation: 'from-image',
// Only the width: the browser keeps the aspect, whichever way EXIF turns the photo.
...(resizeWidth ? { resizeWidth, resizeQuality: 'high' as const } : {}),
});
} catch (err) {
if (err instanceof TypeError) return createImageBitmap(blob);
throw err;
}
}
/**
* The image in `blob`, at most `maxSide` on its long edge and
* MAX_SHRUNK_IMAGE_BYTES in size. JPEG at quality 0.85, then 0.7, then
* smaller by a quarter per step until it fits; PNG when transparent.
* `small`: also shrink images below the "tiny" limits (still not trackers).
* The size comes from the file's header before anything is decoded: a
* format whose header isn't read here is unsupported, more than
* MAX_IMAGE_PIXELS too_big, trackers and tiny images are never decoded.
*/
export async function shrinkImage(blob: Blob, { maxSide = DEFAULT_IMAGE_SIDE, small = false }: { maxSide?: number; small?: boolean } = {}): Promise<ShrinkResult> {
if (typeof createImageBitmap !== 'function') return { status: 'unsupported' };
const declared = imageDimensions(new Uint8Array(await blob.slice(0, IMAGE_HEADER_BYTES).arrayBuffer()));
// HEIC, TIFF, SVG, a damaged file: no size to check, nothing to decode.
if (!declared || declared.width < 1 || declared.height < 1) return { status: 'unsupported' };
const { width: w, height: h } = declared;
if (w * h > MAX_IMAGE_PIXELS) return { status: 'too_big', width: w, height: h };
if (w <= TRACKER_SIDE && h <= TRACKER_SIDE) return { status: 'tracker', width: w, height: h };
if (!small && (Math.min(w, h) < TINY_SIDE || w * h < TINY_AREA)) return { status: 'tiny', width: w, height: h };
const side = Math.max(MIN_IMAGE_SIDE, Math.min(MAX_IMAGE_SIDE, Math.round(maxSide)));
const target = fitSize(w, h, side);
let bitmap: ImageBitmap;
try {
bitmap = await createImageBitmap(blob, { imageOrientation: 'from-image' });
bitmap = await decode(blob, target.width < w ? target.width : null);
} catch {
// HEIC, TIFF, a broken file: nothing the browser can show.
return { status: 'unsupported' };
// A broken file: nothing the browser can show.
return { status: 'unsupported', width: w, height: h };
}
try {
const { width: w, height: h } = bitmap;
if (w <= TRACKER_SIDE && h <= TRACKER_SIDE) return { status: 'tracker', width: w, height: h };
if (!small && (Math.min(w, h) < TINY_SIDE || w * h < TINY_AREA)) return { status: 'tiny', width: w, height: h };
const side = Math.max(MIN_IMAGE_SIDE, Math.min(MAX_IMAGE_SIDE, Math.round(maxSide)));
let size = fitSize(w, h, side);
let size = fitSize(bitmap.width, bitmap.height, side);
let transparent: boolean | null = ALPHA_TYPES.test(blob.type) ? null : false;
for (let step = 0; step < 12; step++) {
const canvas = makeCanvas(size.width, size.height);
+66
View File
@@ -0,0 +1,66 @@
// The width and height an image file declares in its header, read before
// anything decodes it: PNG (IHDR), GIF, JPEG (the SOF segment), WebP
// (VP8, VP8L, VP8X) and BMP. A file can declare a size whose pixels alone
// would fill the memory, so callers check this first.
export interface ImageDimensions {
width: number;
height: number;
}
/** Bytes of a file's start that hold the size of every format read here, EXIF blocks before a JPEG's SOF included. */
export const IMAGE_HEADER_BYTES = 512 * 1024;
const u16be = (b: Uint8Array, at: number) => (b[at] << 8) | b[at + 1];
const u16le = (b: Uint8Array, at: number) => b[at] | (b[at + 1] << 8);
const u24le = (b: Uint8Array, at: number) => b[at] | (b[at + 1] << 8) | (b[at + 2] << 16);
const u32be = (b: Uint8Array, at: number) => ((b[at] << 24) >>> 0) + ((b[at + 1] << 16) | (b[at + 2] << 8) | b[at + 3]);
const ascii = (b: Uint8Array, at: number, text: string) => [...text].every((c, i) => b[at + i] === c.charCodeAt(0));
function jpeg(b: Uint8Array): ImageDimensions | null {
let at = 2;
while (at + 9 < b.length) {
if (b[at] !== 0xff) return null;
const marker = b[at + 1];
// Fill bytes, and markers without a length (restart markers, TEM).
if (marker === 0xff) { at++; continue; }
if ((marker >= 0xd0 && marker <= 0xd7) || marker === 0x01) { at += 2; continue; }
if (marker === 0xd9 || marker === 0xda) return null;
const length = u16be(b, at + 2);
// Start of frame: every SOF but DHT (C4), JPG (C8) and DAC (CC).
if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
return { height: u16be(b, at + 5), width: u16be(b, at + 7) };
}
if (length < 2) return null;
at += 2 + length;
}
return null;
}
function webp(b: Uint8Array): ImageDimensions | null {
if (b.length >= 30 && ascii(b, 12, 'VP8 ') && b[23] === 0x9d && b[24] === 0x01 && b[25] === 0x2a) {
return { width: u16le(b, 26) & 0x3fff, height: u16le(b, 28) & 0x3fff };
}
if (ascii(b, 12, 'VP8L') && b[20] === 0x2f) {
return {
width: 1 + (((b[22] & 0x3f) << 8) | b[21]),
height: 1 + (((b[24] & 0x0f) << 10) | (b[23] << 2) | ((b[22] & 0xc0) >> 6)),
};
}
if (b.length >= 30 && ascii(b, 12, 'VP8X')) return { width: 1 + u24le(b, 24), height: 1 + u24le(b, 27) };
return null;
}
/** The size the file declares, or null for a format not read here (or a damaged header). */
export function imageDimensions(bytes: Uint8Array): ImageDimensions | null {
const b = bytes;
if (b.length >= 24 && b[0] === 0x89 && ascii(b, 1, 'PNG') && ascii(b, 12, 'IHDR')) return { width: u32be(b, 16), height: u32be(b, 20) };
if (b.length >= 10 && ascii(b, 0, 'GIF8')) return { width: u16le(b, 6), height: u16le(b, 8) };
if (b.length >= 4 && b[0] === 0xff && b[1] === 0xd8) return jpeg(b);
if (b.length >= 25 && ascii(b, 0, 'RIFF') && ascii(b, 8, 'WEBP')) return webp(b);
if (b.length >= 26 && ascii(b, 0, 'BM')) {
const view = new DataView(b.buffer, b.byteOffset, b.byteLength);
return { width: Math.abs(view.getInt32(18, true)), height: Math.abs(view.getInt32(22, true)) };
}
return null;
}