From bda658bada443e0dd5d6536f4eb58ba2e90442aa Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:58:16 +0100 Subject: [PATCH] feat: tell plugins the Content-ID and alt text of a message's inline images EmailReadView.attachments gain cid and alt for images the HTML shows inline. plugin-projection.test.ts now expects the cid on its inline logo, since the view carries it. --- .../plugin-assistant-b-projection.test.ts | 49 +++++++++++++++ lib/__tests__/plugin-projection.test.ts | 2 +- lib/plugin-projection.ts | 3 + lib/plugin-sandbox/assistant/b.projection.ts | 59 +++++++++++++++++++ lib/plugin-sandbox/assistant/b.types.ts | 8 +++ lib/plugin-types.ts | 6 +- 6 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 lib/__tests__/plugin-assistant-b-projection.test.ts create mode 100644 lib/plugin-sandbox/assistant/b.projection.ts diff --git a/lib/__tests__/plugin-assistant-b-projection.test.ts b/lib/__tests__/plugin-assistant-b-projection.test.ts new file mode 100644 index 000000000..1fe7c5843 --- /dev/null +++ b/lib/__tests__/plugin-assistant-b-projection.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest'; +import type { Email } from '../jmap/types'; +import { emailToReadView } from '../plugin-projection'; +import { cidAltTexts, normaliseCid } from '../plugin-sandbox/assistant/b.projection'; + +// An inline image of a message names its Content-ID and the alt text the +// sender wrote for it, so the assistant can name the picture. + +const html = [ + '

Hi Alice, here is the logo idea:

', + '

Orange logo with a stripe

', + "", + '', + 'tracker', +].join(''); + +const email = { + id: 'm1', threadId: 't1', mailboxIds: { inbox: true }, keywords: {}, subject: 'Logo', receivedAt: '2026-05-01T10:00:00Z', + htmlBody: [{ partId: '1', blobId: 'b-html', size: 1, type: 'text/html' }], + bodyValues: { 1: { value: html } }, + attachments: [ + { partId: '2', blobId: 'blob-logo', name: 'logo.png', type: 'image/png', size: 10, disposition: 'inline', cid: '' }, + { partId: '3', blobId: 'blob-chart', name: 'chart.png', type: 'image/png', size: 10, cid: 'chart@aa' }, + { partId: '4', blobId: 'blob-plain', name: 'plain.png', type: 'image/png', size: 10, cid: 'plain@aa' }, + { partId: '5', blobId: 'blob-pdf', name: 'agenda.pdf', type: 'application/pdf', size: 10, disposition: 'attachment' }, + ], +} as unknown as Email; + +describe('cid and alt on EmailReadView.attachments', () => { + it('gives inline images their Content-ID and alt text, and leaves plain attachments alone', () => { + const view = emailToReadView(email); + expect(view.attachments).toEqual([ + { blobId: 'blob-logo', name: 'logo.png', type: 'image/png', size: 10, inline: true, cid: 'logo1@aa', alt: 'Orange logo with a stripe' }, + { blobId: 'blob-chart', name: 'chart.png', type: 'image/png', size: 10, inline: true, cid: 'chart@aa', alt: 'Sales & costs' }, + { blobId: 'blob-plain', name: 'plain.png', type: 'image/png', size: 10, inline: true, cid: 'plain@aa' }, + { blobId: 'blob-pdf', name: 'agenda.pdf', type: 'application/pdf', size: 10, inline: false }, + ]); + }); + + it('reads alt texts only of cid images, never of remote ones', () => { + expect([...cidAltTexts(email).entries()]).toEqual([['logo1@aa', 'Orange logo with a stripe'], ['chart@aa', 'Sales & costs']]); + expect(normaliseCid('')).toBe('logo1@aa'); + }); + + it('copes with a message without HTML', () => { + const plain = { ...email, htmlBody: [], bodyValues: {} } as unknown as Email; + expect(emailToReadView(plain).attachments[0]).toEqual({ blobId: 'blob-logo', name: 'logo.png', type: 'image/png', size: 10, inline: true, cid: 'logo1@aa' }); + }); +}); diff --git a/lib/__tests__/plugin-projection.test.ts b/lib/__tests__/plugin-projection.test.ts index 750025aa4..7f35f894d 100644 --- a/lib/__tests__/plugin-projection.test.ts +++ b/lib/__tests__/plugin-projection.test.ts @@ -57,7 +57,7 @@ describe('emailToReadView attachments', () => { expect(view.attachments).toEqual([ { blobId: 'b-pdf', name: 'invoice.pdf', type: 'application/pdf', size: 1234, inline: false }, - { blobId: 'b-img', name: 'logo.png', type: 'image/png', size: 99, inline: true }, + { blobId: 'b-img', name: 'logo.png', type: 'image/png', size: 99, inline: true, cid: 'logo@x' }, ]); }); }); diff --git a/lib/plugin-projection.ts b/lib/plugin-projection.ts index 7f2ec5aa9..5407d7f4a 100644 --- a/lib/plugin-projection.ts +++ b/lib/plugin-projection.ts @@ -12,6 +12,7 @@ import type { import type { EmailTemplate } from '@/lib/template-types'; import { generateEmailSource } from '@/lib/email-source'; import { isServerSetKeyword } from '@/lib/jmap/server-keywords'; +import { attachmentExtras, cidAltTexts } from '@/lib/plugin-sandbox/assistant/b.projection'; // Resolve a message's plain-text body from its JMAP body parts. Plugins that // translate or scan content need the real body, not just the short `preview` @@ -52,6 +53,7 @@ export function plainTextFromEmail(email: Email): string { * carry it, others only the caller knows (see EmailReadView.accountId). */ export function emailToReadView(email: Email, accountId = email.sourceAccountId): EmailReadView { + const alts = cidAltTexts(email); return { id: email.id, ...(accountId ? { accountId } : {}), @@ -73,6 +75,7 @@ export function emailToReadView(email: Email, accountId = email.sourceAccountId) type: part.type || 'application/octet-stream', size: part.size || 0, inline: part.disposition === 'inline' || !!part.cid, + ...attachmentExtras(part, alts), })), preview: email.preview || '', text: plainTextFromEmail(email), diff --git a/lib/plugin-sandbox/assistant/b.projection.ts b/lib/plugin-sandbox/assistant/b.projection.ts new file mode 100644 index 000000000..f4c338e2d --- /dev/null +++ b/lib/plugin-sandbox/assistant/b.projection.ts @@ -0,0 +1,59 @@ +// Area b of the assistant's host APIs: what the message view adds to each +// attachment. An image the HTML shows inline (``) gets its +// Content-ID and the alt text the sender wrote for it, so the assistant can +// name a picture without looking at it. + +import type { Email } from '@/lib/jmap/types'; +import type { AttachmentExtrasB } from './b.types'; + +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }; + +function decodeEntities(value: string): string { + return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, name: string) => { + if (name[0] === '#') { + const code = name[1] === 'x' || name[1] === 'X' ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10); + return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : whole; + } + return ENTITIES[name.toLowerCase()] ?? whole; + }); +} + +/** The value of one attribute of a tag's source, or null. */ +function attribute(tag: string, name: string): string | null { + const m = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i').exec(tag); + if (!m) return null; + return decodeEntities(m[1] ?? m[2] ?? m[3] ?? ''); +} + +/** "", "logo1@aa", "cid:logo1%40aa" → "logo1@aa" (lower case, for comparing). */ +export function normaliseCid(value: string): string { + let id = value.trim().replace(/^cid:/i, '').replace(/^<|>$/g, ''); + try { id = decodeURIComponent(id); } catch { /* keep it as written */ } + return id.toLowerCase(); +} + +/** Content-ID → alt text, from the `…` tags of the message's HTML. */ +export function cidAltTexts(email: Pick): Map { + const out = new Map(); + const values = email.bodyValues ?? {}; + for (const part of email.htmlBody ?? []) { + const html = values[part.partId]?.value; + if (typeof html !== 'string') continue; + for (const [tag] of html.matchAll(/]*>/gi)) { + const src = attribute(tag, 'src'); + if (!src || !/^cid:/i.test(src.trim())) continue; + const alt = (attribute(tag, 'alt') ?? attribute(tag, 'title') ?? '').replace(/\s+/g, ' ').trim(); + const cid = normaliseCid(src); + if (alt && !out.has(cid)) out.set(cid, alt.slice(0, 500)); + } + } + return out; +} + +/** `cid` and `alt` of one attachment, for EmailReadView.attachments (empty for a plain attachment). */ +export function attachmentExtras(part: { cid?: string | null }, alts: Map): AttachmentExtrasB { + if (!part.cid) return {}; + const cid = part.cid.replace(/^<|>$/g, ''); + const alt = alts.get(normaliseCid(cid)); + return { cid, ...(alt ? { alt } : {}) }; +} diff --git a/lib/plugin-sandbox/assistant/b.types.ts b/lib/plugin-sandbox/assistant/b.types.ts index 443f07894..e6c99e253 100644 --- a/lib/plugin-sandbox/assistant/b.types.ts +++ b/lib/plugin-sandbox/assistant/b.types.ts @@ -10,6 +10,14 @@ export interface AreaHostFeaturesB { attachmentText?: boolean; } +/** What EmailReadView.attachments adds for an image the message shows inline. */ +export interface AttachmentExtrasB { + /** The Content-ID the message's HTML refers to (``), without angle brackets. */ + cid?: string; + /** The alt text of that image in the HTML, when the sender wrote one. */ + alt?: string; +} + /** * What api.email.getAttachmentText answers. `status`: ok (`text` holds the * text), too_big (over 20 MB, or an Office file that unpacks to more than diff --git a/lib/plugin-types.ts b/lib/plugin-types.ts index a122af6c6..644e5e6aa 100644 --- a/lib/plugin-types.ts +++ b/lib/plugin-types.ts @@ -1,6 +1,7 @@ import type { CredentialSpec } from './plugin-secrets/credentials'; import type { CatalogSlotName } from './plugin-slot-catalog'; import { ASSISTANT_PERMISSIONS, type AssistantHostFeatures } from './plugin-sandbox/assistant/types'; +import type { AttachmentExtrasB } from './plugin-sandbox/assistant/b.types'; // Plugin & Theme system types @@ -419,9 +420,10 @@ export interface EmailReadView { /** * The message's attachments. `blobId` is what `email.getAttachment` * takes. Inline images are included and marked. Empty until the host has - * loaded the message's body structure. + * loaded the message's body structure. An image the HTML shows inline + * also has its `cid` and, when the sender wrote one, its `alt` text. */ - attachments: { blobId: string; name: string; type: string; size: number; inline: boolean }[]; + attachments: ({ blobId: string; name: string; type: string; size: number; inline: boolean } & AttachmentExtrasB)[]; preview: string; keywords: string[]; /** Full plain-text body of the message (HTML-only bodies are stripped to