From bda99e1df887a547fcba7a1f523e524e1c290eb7 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:53:05 +0100 Subject: [PATCH] feat: let plugins read the text of attachments api.email.getAttachmentText reads PDF (pdf.js), Word, Excel, PowerPoint and OpenDocument files (own reader over fflate and DOMParser, unpacking capped at 50 MB), text, CSV, HTML, calendar and address files, and answers with a status for what it can't read. The extraction lives in lib/attachment-text so Files can use it too. The sandbox runtime now copies every flag the host sends into api.host.features; before, it kept only the three core flags, so the assistant's flags never reached a plugin. The host bridge assertions in plugin-host-features.test.ts compare against the core flags plus the assistant's (withAssistant), since the init payload and host-change now carry attachmentText too. --- .../plugin-assistant-b-runtime.test.ts | 64 ++++ .../plugin-host-attachment-text.test.ts | 128 +++++++ lib/__tests__/plugin-host-features.test.ts | 8 +- .../__tests__/attachment-text.test.ts | 226 +++++++++++ .../__tests__/pdf-real.test.ts | 39 ++ lib/attachment-text/index.ts | 106 ++++++ lib/attachment-text/office.ts | 359 ++++++++++++++++++ lib/attachment-text/pdf.ts | 79 ++++ lib/attachment-text/plain.ts | 142 +++++++ lib/attachment-text/types.ts | 54 +++ lib/plugin-sandbox/assistant/b.runtime.ts | 18 +- lib/plugin-sandbox/assistant/b.ts | 58 ++- lib/plugin-sandbox/assistant/b.types.ts | 37 +- lib/plugin-sandbox/runtime.tsx | 6 + 14 files changed, 1310 insertions(+), 14 deletions(-) create mode 100644 lib/__tests__/plugin-assistant-b-runtime.test.ts create mode 100644 lib/__tests__/plugin-host-attachment-text.test.ts create mode 100644 lib/attachment-text/__tests__/attachment-text.test.ts create mode 100644 lib/attachment-text/__tests__/pdf-real.test.ts create mode 100644 lib/attachment-text/index.ts create mode 100644 lib/attachment-text/office.ts create mode 100644 lib/attachment-text/pdf.ts create mode 100644 lib/attachment-text/plain.ts create mode 100644 lib/attachment-text/types.ts diff --git a/lib/__tests__/plugin-assistant-b-runtime.test.ts b/lib/__tests__/plugin-assistant-b-runtime.test.ts new file mode 100644 index 000000000..7433da72e --- /dev/null +++ b/lib/__tests__/plugin-assistant-b-runtime.test.ts @@ -0,0 +1,64 @@ +/** + * The AI assistant's host-call flags (lib/plugin-sandbox/assistant) reach + * the plugin: api.host.features in the sandbox carries them from init and + * follows host-change, next to the core flags. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import React from 'react'; +import { cleanup, render, waitFor } from '@testing-library/react'; + +vi.mock('@/stores/toast-store', () => ({ + toast: { success: vi.fn(), error: vi.fn(), info: vi.fn(), warning: vi.fn() }, +})); + +const HOST_ORIGIN = window.location.origin; + +function post(source: unknown, data: unknown): void { + const ev = new MessageEvent('message', { data, origin: HOST_ORIGIN }); + Object.defineProperty(ev, 'source', { value: source }); + window.dispatchEvent(ev); +} + +describe('the assistant flags in the sandbox runtime', () => { + let hostWindow: { postMessage: ReturnType }; + const realParent = Object.getOwnPropertyDescriptor(window, 'parent'); + const g = globalThis as { __pluginApi?: { host: { features: Record } }; __seenAtActivate?: boolean }; + + beforeEach(() => { + vi.resetModules(); + hostWindow = { postMessage: vi.fn() }; + Object.defineProperty(window, 'parent', { configurable: true, get: () => hostWindow as unknown as Window }); + vi.stubGlobal('ResizeObserver', class { observe() {} disconnect() {} }); + }); + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + delete g.__pluginApi; + delete g.__seenAtActivate; + if (realParent) Object.defineProperty(window, 'parent', realParent); + }); + + it('come with init and follow host-change', async () => { + const bundle = `module.exports = { activate: function (api) { + globalThis.__seenAtActivate = api.host.features.attachmentText; + globalThis.__pluginApi = api; + } };`; + const { SandboxRuntime } = await import('@/lib/plugin-sandbox/runtime'); + render(React.createElement(SandboxRuntime)); + post(hostWindow, { + type: 'init', + payload: { + mode: 'background', pluginId: 'x', tier: 'privileged', locale: 'en', + manifest: { id: 'x', version: '1.0.0', permissions: [], settings: {} }, + code: bundle, + hostFeatures: { externalSenderLabel: false, externalRecipientWarning: false, noThirdPartyTraffic: false, attachmentText: true }, + }, + }); + await waitFor(() => expect(g.__pluginApi).toBeDefined()); + expect(g.__seenAtActivate).toBe(true); + const kept = g.__pluginApi!.host.features; + post(hostWindow, { type: 'host-change', features: { externalSenderLabel: true, externalRecipientWarning: false, noThirdPartyTraffic: false, attachmentText: 'yes' } }); + // Booleans only: anything but true is false. + expect(kept).toMatchObject({ externalSenderLabel: true, attachmentText: false }); + }, 30_000); +}); diff --git a/lib/__tests__/plugin-host-attachment-text.test.ts b/lib/__tests__/plugin-host-attachment-text.test.ts new file mode 100644 index 000000000..00d0a5dac --- /dev/null +++ b/lib/__tests__/plugin-host-attachment-text.test.ts @@ -0,0 +1,128 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { strToU8, zipSync } from 'fflate'; +import type { InstalledPlugin } from '../plugin-types'; +import type { IJMAPClient } from '../jmap/client-interface'; +import type { Email } from '../jmap/types'; +import { useAuthStore } from '@/stores/auth-store'; +import { useEmailStore } from '@/stores/email-store'; + +vi.mock('@/stores/toast-store', () => ({ + toast: { success: vi.fn(), error: vi.fn(), info: vi.fn(), warning: vi.fn() }, +})); + +import { dispatchApiCall, PERM_PER_METHOD } from '../plugin-sandbox/host-api'; +import { resetPluginRateLimits } from '../plugin-sandbox/host-api-input'; +import { snapshotHostFeatures } from '../plugin-sandbox/host-features'; +import { withAssistantRuntime } from '../plugin-sandbox/assistant/runtime'; + +function plugin(perms: string[], id = 'ai-assistant'): InstalledPlugin { + return { + id, name: 'AI Assistant', version: '2.2.0', author: 'test', description: '', + type: 'hook', permissions: perms, grantedPermissions: perms, entrypoint: 'index.js', + enabled: true, status: 'running', settings: {}, + }; +} + +const ASSISTANT = plugin(['email:read', 'email:blob-read']); +const call = (p: InstalledPlugin, ...args: unknown[]) => dispatchApiCall(p, 'email.getAttachmentText', args); + +const notes = { partId: '2', blobId: 'blob-notes', name: 'workshop-notes.txt', type: 'text/plain', size: 40 }; +const minutes = { partId: '3', blobId: 'blob-docx', name: 'minutes.docx', type: 'application/octet-stream', size: 300 }; +const photo = { partId: '4', blobId: 'blob-jpg', name: 'photo.jpg', type: 'image/jpeg', size: 3 }; +const old = { partId: '5', blobId: 'blob-doc', name: 'old.doc', type: 'application/msword', size: 8 }; +const huge = { partId: '6', blobId: 'blob-big', name: 'scan.pdf', type: 'application/pdf', size: 20 * 1024 * 1024 + 1 }; + +const message = { + id: 'm1', threadId: 't1', blobId: 'blob-raw', size: 4, mailboxIds: { inbox: true }, keywords: {}, + subject: 'Workshop', receivedAt: '2026-05-01T10:00:00Z', attachments: [notes, minutes, photo, old, huge], +} as unknown as Email; + +const DOCX = zipSync({ + 'word/document.xml': strToU8('Launch moves to 12 December.'), +}); +const BLOBS: Record = { + 'blob-notes': strToU8('Workshop notes\n\n1. Launch date moves to 12 December.\n'), + 'blob-docx': DOCX, + 'blob-jpg': new Uint8Array([0xff, 0xd8, 0xff]), + 'blob-doc': new Uint8Array([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]), +}; + +let client: Record>; + +beforeEach(() => { + resetPluginRateLimits(); + client = { + getEmail: vi.fn(async (id: string, accountId?: string) => (id === 'm1' && (!accountId || accountId === 'shared-acc') ? message : null)), + fetchBlobArrayBuffer: vi.fn(async (blobId: string) => { + const bytes = BLOBS[blobId]; + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + }), + getAccountId: vi.fn(() => 'acc1'), + getMailboxes: vi.fn(async () => []), + }; + useAuthStore.setState({ client: client as unknown as IJMAPClient }); + useEmailStore.setState({ selectedEmail: null, emails: [], mailboxes: [], selectedMailbox: '' }); +}); + +afterEach(() => { + useAuthStore.setState({ client: null }); +}); + +describe('api.email.getAttachmentText', () => { + it('is behind email:blob-read, offered in the runtime and announced in the host features', async () => { + expect(PERM_PER_METHOD['email.getAttachmentText']).toBe('email:blob-read'); + await expect(call(plugin(['email:read']), 'm1', 'blob-notes')).rejects.toThrow(/lacks permission "email:blob-read"/); + expect(client.fetchBlobArrayBuffer).not.toHaveBeenCalled(); + expect(snapshotHostFeatures()).toMatchObject({ attachmentText: true }); + const calls: unknown[][] = []; + const api = withAssistantRuntime({} as { email?: { getAttachmentText?: (...a: unknown[]) => unknown } }, async (method, args) => { calls.push([method, args]); return null; }); + await api.email?.getAttachmentText?.('m1', 'blob-notes', { maxChars: 10 }); + expect(calls).toEqual([['email.getAttachmentText', ['m1', 'blob-notes', { maxChars: 10 }]]]); + }); + + it('returns the text of a text attachment', async () => { + const out = await call(ASSISTANT, 'm1', 'blob-notes'); + expect(out).toEqual({ + name: 'workshop-notes.txt', type: 'text/plain', size: BLOBS['blob-notes'].byteLength, + format: 'text', status: 'ok', text: 'Workshop notes\n\n1. Launch date moves to 12 December.', truncated: false, + }); + expect(client.fetchBlobArrayBuffer).toHaveBeenCalledWith('blob-notes', 'workshop-notes.txt', 'text/plain'); + }); + + it('reads an Office file by its name and cuts at maxChars', async () => { + expect(await call(ASSISTANT, 'm1', 'blob-docx')).toMatchObject({ format: 'docx', status: 'ok', text: 'Launch moves to 12 December.' }); + expect(await call(ASSISTANT, 'm1', 'blob-docx', { maxChars: 12 })).toMatchObject({ text: 'Launch moves', truncated: true }); + }); + + it('answers with a status for what it can not read', async () => { + expect(await call(ASSISTANT, 'm1', 'blob-jpg')).toMatchObject({ status: 'unsupported', text: '' }); + expect(await call(ASSISTANT, 'm1', 'blob-doc')).toMatchObject({ status: 'unsupported', format: 'other' }); + }); + + it('answers too_big for an attachment over 20 MB without downloading it', async () => { + expect(await call(ASSISTANT, 'm1', 'blob-big')).toMatchObject({ name: 'scan.pdf', format: 'pdf', status: 'too_big', text: '', truncated: false }); + expect(client.fetchBlobArrayBuffer).not.toHaveBeenCalled(); + }); + + it('refuses a blob that is not an attachment of that message, and a message it can not find', async () => { + await expect(call(ASSISTANT, 'm1', 'blob-raw')).rejects.toThrow(/not an attachment of this message/); + await expect(call(ASSISTANT, 'm1', 'someone-elses-blob')).rejects.toThrow(/not an attachment/); + await expect(call(ASSISTANT, 'missing', 'blob-notes')).rejects.toThrow(/no message with id "missing"/); + // An account this login doesn't show is not asked. + await expect(call(ASSISTANT, 'm1', 'blob-notes', { accountId: 'stranger' })).rejects.toThrow(/no message with id/); + expect(client.fetchBlobArrayBuffer).not.toHaveBeenCalled(); + }); + + it('checks its options', async () => { + await expect(call(ASSISTANT, 'm1', 'blob-notes', { maxChars: 100_001 })).rejects.toThrow(/maxChars must be a whole number from 1 to 100000/); + await expect(call(ASSISTANT, 'm1', 'blob-notes', { pages: 3 })).rejects.toThrow(/unknown property "pages"/); + await expect(call(ASSISTANT, 'm1', 7)).rejects.toThrow(/blobId must be a string/); + }); + + it('allows 30 reads a minute per plugin', async () => { + for (let i = 0; i < 30; i++) await call(ASSISTANT, 'm1', 'blob-notes'); + await expect(call(ASSISTANT, 'm1', 'blob-notes')).rejects.toThrow(/too many reads/); + // Another plugin has its own count. + await expect(call(plugin(['email:blob-read'], 'other'), 'm1', 'blob-notes')).resolves.toMatchObject({ status: 'ok' }); + }); +}); diff --git a/lib/__tests__/plugin-host-features.test.ts b/lib/__tests__/plugin-host-features.test.ts index 9e0dc0abf..03e75577a 100644 --- a/lib/__tests__/plugin-host-features.test.ts +++ b/lib/__tests__/plugin-host-features.test.ts @@ -85,21 +85,21 @@ describe('host bridge', () => { const plugin = { id: 'p', name: 'P', version: '1.0.0', permissions: [], settings: {}, settingsSchema: {} } as never; const instance = createBackgroundInstance({ plugin, tier: 'privileged', code: '', locale: 'en' }); const pending = (instance as unknown as { pendingInit: { hostFeatures?: unknown } | null }).pendingInit; - expect(pending?.hostFeatures).toEqual({ externalSenderLabel: true, externalRecipientWarning: false, noThirdPartyTraffic: false }); + expect(pending?.hostFeatures).toEqual(withAssistant({ externalSenderLabel: true, externalRecipientWarning: false, noThirdPartyTraffic: false })); // Before init has gone out a change lands in the payload ... setExternalMail({}); - expect(pending?.hostFeatures).toEqual({ externalSenderLabel: false, externalRecipientWarning: false, noThirdPartyTraffic: false }); + expect(pending?.hostFeatures).toEqual(withAssistant({ externalSenderLabel: false, externalRecipientWarning: false, noThirdPartyTraffic: false })); // ... after it, as a host-change message. const sent: unknown[] = []; (instance as unknown as { pendingInit: unknown }).pendingInit = null; (instance as unknown as { send: (m: unknown) => void }).send = (m) => sent.push(m); setExternalMail({ recipientWarning: true }); - expect(sent).toEqual([{ type: 'host-change', features: { externalSenderLabel: false, externalRecipientWarning: true, noThirdPartyTraffic: false } }]); + expect(sent).toEqual([{ type: 'host-change', features: withAssistant({ externalSenderLabel: false, externalRecipientWarning: true, noThirdPartyTraffic: false }) }]); // The admin turns third-party traffic off: running plugins hear it. setNoThirdPartyTraffic(true); - expect(sent[1]).toEqual({ type: 'host-change', features: { externalSenderLabel: false, externalRecipientWarning: true, noThirdPartyTraffic: true } }); + expect(sent[1]).toEqual({ type: 'host-change', features: withAssistant({ externalSenderLabel: false, externalRecipientWarning: true, noThirdPartyTraffic: true }) }); setNoThirdPartyTraffic(false); instance.destroy(); }, 30_000); diff --git a/lib/attachment-text/__tests__/attachment-text.test.ts b/lib/attachment-text/__tests__/attachment-text.test.ts new file mode 100644 index 000000000..798081cd3 --- /dev/null +++ b/lib/attachment-text/__tests__/attachment-text.test.ts @@ -0,0 +1,226 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { strToU8, zipSync, type Zippable } from 'fflate'; +import { extractText, setPdfLoader, textFormatOf, type PdfLib } from '@/lib/attachment-text'; +import { officeText } from '@/lib/attachment-text/office'; + +// Office files are ZIPs of XML: the fixtures are built here, as small as a +// real file of each kind can be while still holding the parts read. + +const zip = (files: Record): Uint8Array => { + const entries: Zippable = {}; + for (const [name, value] of Object.entries(files)) entries[name] = typeof value === 'string' ? strToU8(value) : value; + return zipSync(entries, { level: 6 }); +}; + +const W = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'; +const DOCX = zip({ + '[Content_Types].xml': '', + 'word/document.xml': ` + Minutes of the meeting + Budget: 4,500 EUR9,000 EUR + WhoWhat + CarolPress release + `, +}); + +const S = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'; +const xlsxFiles = (rows = 2) => ({ + 'xl/workbook.xml': ``, + 'xl/_rels/workbook.xml.rels': '', + 'xl/sharedStrings.xml': `monthvisitorsAugust, late`, + 'xl/worksheets/sheet1.xml': ` + 01 + ${Array.from({ length: rows - 1 }, (_, i) => `2${1200 + i}1`).join('')} + `, + 'xl/worksheets/sheet2.xml': ``, +}); + +const P = 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"'; +const slide = (texts: string[]) => `${texts.map((t) => `${t}`).join('')}`; +const PPTX = zip({ + 'ppt/presentation.xml': ``, + 'ppt/slides/slide10.xml': slide(['Thanks']), + 'ppt/slides/slide2.xml': slide(['Launch plan', 'Codeword PELICAN']), + 'ppt/slides/slide1.xml': slide(['Welcome']), +}); + +const O = 'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"'; +const odf = (body: string, extra: Record = {}) => zip({ + mimetype: 'application/vnd.oasis.opendocument.text', + 'META-INF/manifest.xml': '', + 'content.xml': `${body}`, + ...extra, +}); +const ODT = odf('ReportTwospacesand abreaka footnote'); +const ODS = odf(` + itemcost + ads, print4500 + +`); +const ODP = odf('Hello slideSecond'); + +const text = (s: string) => new TextEncoder().encode(s); + +afterEach(() => setPdfLoader()); + +describe('textFormatOf', () => { + it('goes by the extension first, then the media type', () => { + expect(textFormatOf('notes.TXT', 'application/octet-stream')).toBe('text'); + expect(textFormatOf('report', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')).toBe('docx'); + expect(textFormatOf('x.ods', '')).toBe('ods'); + expect(textFormatOf('', 'text/x-python')).toBe('text'); + expect(textFormatOf('old.doc', 'application/msword')).toBe('other'); + expect(textFormatOf('photo.jpg', 'image/jpeg')).toBe('other'); + }); +}); + +describe('plain formats', () => { + it('reads text in UTF-8 and Windows-1252 and keeps CSV as it is', async () => { + expect(await extractText(text('Grüße\r\nfrom Bob'), { name: 'a.txt' })).toMatchObject({ format: 'text', status: 'ok', text: 'Grüße\nfrom Bob', truncated: false }); + expect((await extractText(new Uint8Array([0x47, 0x72, 0xfc, 0xdf, 0x65]), { name: 'a.txt' })).text).toBe('Grüße'); + expect(await extractText(text('month,visitors\nAugust,1200\n'), { name: 'figures.csv' })).toMatchObject({ format: 'csv', text: 'month,visitors\nAugust,1200' }); + }); + + it('turns HTML into text', async () => { + const out = await extractText(text('

Agenda

Launch plan

logo'), { type: 'text/html' }); + expect(out.status).toBe('ok'); + expect(out.text).toContain('Agenda'); + expect(out.text).toContain('Launch plan'); + expect(out.text).toContain('[Image: logo]'); + expect(out.text).not.toContain('alert'); + }); + + it('writes one line per event of a calendar file', async () => { + const ics = [ + 'BEGIN:VCALENDAR', 'METHOD:REQUEST', 'BEGIN:VEVENT', 'SUMMARY:Launch review\\, part 2', + 'DTSTART;TZID=Europe/Berlin:20261105T090000', 'DTEND;TZID=Europe/Berlin:20261105T100000', + 'LOCATION:Room 4', 'ORGANIZER;CN="Bob, the boss":mailto:bob@example.org', 'DESCRIPTION:Bring the', ' figures', 'END:VEVENT', + 'BEGIN:VEVENT', 'SUMMARY:Holiday', 'DTSTART;VALUE=DATE:20261224', 'END:VEVENT', 'END:VCALENDAR', + ].join('\r\n'); + const out = await extractText(text(ics), { name: 'invite.ics' }); + expect(out.text).toBe([ + 'event: Launch review, part 2, 2026-11-05 09:00 (Europe/Berlin), 2026-11-05 10:00 (Europe/Berlin), Room 4, organiser Bob, the boss ', + ' Bring thefigures', + 'event: Holiday, 2026-12-24', + ].join('\n')); + }); + + it('writes name, addresses, phones and organisation of an address card', async () => { + const vcf = 'BEGIN:VCARD\r\nVERSION:3.0\r\nFN:Carol Example\r\nN:Example;Carol;;;\r\nEMAIL;TYPE=work:carol@example.org\r\nTEL;TYPE=cell:+49 170 123\r\nORG:Example GmbH\r\nEND:VCARD\r\n'; + const out = await extractText(text(vcf), { name: 'carol.vcf' }); + expect(out.status).toBe('ok'); + expect(out.text).toContain('contact: Carol Example'); + expect(out.text).toContain('email: carol@example.org'); + expect(out.text).toContain('phone: +49 170 123'); + expect(out.text).toContain('organisation: Example GmbH'); + }); + + it('cuts at maxChars and says so', async () => { + const out = await extractText(text('word '.repeat(100)), { name: 'a.txt', maxChars: 42 }); + expect(out.truncated).toBe(true); + expect(out.text.length).toBeLessThanOrEqual(42); + expect(out.text.endsWith('word')).toBe(true); + }); + + it('answers no_text for an empty file and unsupported for binary bytes and images', async () => { + expect((await extractText(text(' \n'), { name: 'a.txt' })).status).toBe('no_text'); + expect((await extractText(new Uint8Array([1, 0, 2, 0]), { name: 'a.txt' })).status).toBe('unsupported'); + expect((await extractText(new Uint8Array([0xff, 0xd8, 0xff]), { name: 'a.jpg', type: 'image/jpeg' })).status).toBe('unsupported'); + }); + + it('refuses a file over 20 MB without reading it', async () => { + expect(await extractText(new Uint8Array(20 * 1024 * 1024 + 1), { name: 'a.txt' })).toMatchObject({ status: 'too_big', text: '' }); + }); +}); + +describe('Office files', () => { + it('reads a docx: paragraphs, tabs, tables, no deleted text', async () => { + const out = await extractText(DOCX, { name: 'minutes.docx' }); + expect(out).toMatchObject({ format: 'docx', status: 'ok' }); + expect(out.text).toBe('Minutes of the meeting\nBudget: \t4,500 EUR\nWho | What\nCarol | Press release'); + }); + + it('reads an xlsx sheet by sheet as CSV rows, shared strings resolved, columns kept', async () => { + const out = await extractText(zip(xlsxFiles()), { name: 'figures.xlsx' }); + expect(out).toMatchObject({ format: 'xlsx', status: 'ok', truncated: false, sheets: ['Figures', 'Empty'] }); + expect(out.text).toBe('Sheet: Figures\nmonth,visitors\n"August, late",,1200,TRUE\n\nSheet: Empty'); + }); + + it('reads at most 2,000 rows of a sheet', async () => { + const out = await extractText(zip(xlsxFiles(2005)), { name: 'big.xlsx', maxChars: 100_000 }); + expect(out.truncated).toBe(true); + expect(out.text.split('\n').filter((l) => l.startsWith('"August')).length).toBe(1999); + }); + + it('reads a pptx slide by slide in their order', async () => { + const out = await extractText(PPTX, { name: 'deck.pptx' }); + expect(out.text).toBe('Slide 1\nWelcome\n\nSlide 2\nLaunch plan\nCodeword PELICAN\n\nSlide 3\nThanks'); + }); + + it('reads odt, ods and odp', async () => { + expect((await extractText(ODT, { name: 'r.odt' })).text).toBe('Report\nTwo spaces\tand a\nbreak'); + const ods = await extractText(ODS, { name: 'b.ods' }); + expect(ods).toMatchObject({ status: 'ok', sheets: ['Budget'] }); + expect(ods.text).toBe('Sheet: Budget\nitem,cost\n"ads, print",4500,4500\n"ads, print",4500,4500'); + expect((await extractText(ODP, { name: 'p.odp' })).text).toBe('Slide 1\nHello slide\n\nSlide 2\nSecond'); + }); + + it('refuses a ZIP bomb: stops unpacking past the cap', async () => { + const bomb = zip({ 'word/document.xml': `${'A'.repeat(3 * 1024 * 1024)}` }); + expect(bomb.byteLength).toBeLessThan(64 * 1024); + expect(await officeText(bomb, 'docx', 1024 * 1024)).toMatchObject({ status: 'too_big', text: '' }); + // The same file under the real cap is read. + expect((await officeText(bomb, 'docx')).status).toBe('ok'); + }); + + it('answers unsupported for the legacy formats and encrypted for a protected file', async () => { + const cfb = new Uint8Array([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]); + expect((await extractText(cfb, { name: 'old.doc', type: 'application/msword' })).status).toBe('unsupported'); + expect((await extractText(cfb, { name: 'old.xls' })).status).toBe('unsupported'); + expect((await extractText(cfb, { name: 'secret.docx' })).status).toBe('encrypted'); + const odfLocked = odf('', { 'META-INF/manifest.xml': '' }); + expect((await extractText(odfLocked, { name: 'secret.odt' })).status).toBe('encrypted'); + }); + + it('answers failed for a file that is not a ZIP', async () => { + expect((await extractText(text('not a zip at all'), { name: 'broken.docx' })).status).toBe('failed'); + }); +}); + +describe('PDF', () => { + const fakePdf = (pages: string[][], error?: Error): PdfLib => ({ + getDocument: () => ({ + promise: error ? Promise.reject(error) : Promise.resolve({ + numPages: pages.length, + getPage: async (n: number) => ({ + getTextContent: async () => ({ items: pages[n - 1].map((str, i, all) => ({ str, hasEOL: i < all.length - 1 })) }), + }), + }), + destroy: async () => {}, + }), + }); + + it('reads the text of each page', async () => { + setPdfLoader(async () => fakePdf([['Workshop agenda', '09:30 Launch plan (codeword PELICAN)'], ['11:00 Budget and the rest of the day']])); + const out = await extractText(text('%PDF-1.4'), { name: 'agenda.pdf' }); + expect(out).toMatchObject({ format: 'pdf', status: 'ok', pages: 2, truncated: false }); + expect(out.text).toBe('Page 1\nWorkshop agenda\n09:30 Launch plan (codeword PELICAN)\n\nPage 2\n11:00 Budget and the rest of the day'); + }); + + it('reads at most 50 pages', async () => { + setPdfLoader(async () => fakePdf(Array.from({ length: 60 }, (_, i) => [`This is page number ${i + 1} of the report`]))); + const out = await extractText(text('%PDF'), { name: 'long.pdf', maxChars: 100_000 }); + expect(out).toMatchObject({ status: 'ok', pages: 60, truncated: true }); + expect(out.text).toContain('page number 50 '); + expect(out.text).not.toContain('page number 51 '); + }); + + it('says a scan has no text, and a protected PDF is encrypted', async () => { + setPdfLoader(async () => fakePdf([[''], ['3'], ['']])); + expect(await extractText(text('%PDF'), { name: 'scan.pdf' })).toMatchObject({ status: 'no_text', pages: 3, text: '' }); + setPdfLoader(async () => fakePdf([], Object.assign(new Error('No password given'), { name: 'PasswordException' }))); + expect((await extractText(text('%PDF'), { name: 'locked.pdf' })).status).toBe('encrypted'); + setPdfLoader(async () => fakePdf([], new Error('Invalid PDF structure'))); + expect((await extractText(text('%PDF'), { name: 'broken.pdf' })).status).toBe('failed'); + }); +}); diff --git a/lib/attachment-text/__tests__/pdf-real.test.ts b/lib/attachment-text/__tests__/pdf-real.test.ts new file mode 100644 index 000000000..15b3819c1 --- /dev/null +++ b/lib/attachment-text/__tests__/pdf-real.test.ts @@ -0,0 +1,39 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { extractText, setPdfLoader, type PdfLib } from '@/lib/attachment-text'; + +// The PDF the test rig seeds: one page, Helvetica, a few lines of text. +function pdf(lines: string[]): Uint8Array { + const content = `BT /F1 12 Tf 72 720 Td ${lines.map((l, i) => `${i ? '0 -16 Td ' : ''}(${l.replace(/[()\\]/g, (m) => '\\' + m)}) Tj`).join(' ')} ET`; + const objs = [ + '<< /Type /Catalog /Pages 2 0 R >>', + '<< /Type /Pages /Kids [3 0 R] /Count 1 >>', + '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>', + `<< /Length ${content.length} >>\nstream\n${content}\nendstream`, + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>', + ]; + let out = '%PDF-1.4\n'; + const offs: number[] = []; + objs.forEach((o, i) => { offs.push(out.length); out += `${i + 1} 0 obj\n${o}\nendobj\n`; }); + const xref = out.length; + out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n${offs.map((o) => `${String(o).padStart(10, '0')} 00000 n \n`).join('')}trailer\n<< /Size ${objs.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return new TextEncoder().encode(out); +} + +afterEach(() => setPdfLoader()); + +describe('PDF with the real pdf.js', () => { + it('reads the text of a one-page PDF', async () => { + // The app loads the browser build with its worker file; vitest runs in + // Node, which needs the legacy build (its worker runs in the same thread). + setPdfLoader(async () => { + const lib = await import('pdfjs-dist/legacy/build/pdf.mjs'); + // pdf.js takes an already loaded worker module from here. + // @ts-expect-error -- pdf.js ships no types for the worker module + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker = await import('pdfjs-dist/legacy/build/pdf.worker.mjs'); + return lib as unknown as PdfLib; + }); + const out = await extractText(pdf(['Workshop agenda', '09:30 Launch plan (codeword PELICAN)']), { name: 'agenda.pdf' }); + expect(out).toMatchObject({ status: 'ok', pages: 1 }); + expect(out.text).toContain('codeword PELICAN'); + }, 30_000); +}); diff --git a/lib/attachment-text/index.ts b/lib/attachment-text/index.ts new file mode 100644 index 000000000..afe26eea4 --- /dev/null +++ b/lib/attachment-text/index.ts @@ -0,0 +1,106 @@ +// The text of an attachment or a file, for a reader that needs words rather +// than layout (the AI assistant's plugin, Files). Picks the format by the +// file name, then the media type, and hands the bytes to its reader: +// plain.ts (text, CSV, HTML, calendar files, address cards), pdf.ts +// (pdf.js), office.ts (docx, xlsx, pptx, odt, ods, odp). Never throws: a +// file that can't be read answers with a status. + +import { decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain'; +import { pdfText } from './pdf'; +import { officeText } from './office'; +import { + DEFAULT_TEXT_CHARS, MAX_TEXT_CHARS, MAX_TEXT_SOURCE_BYTES, + type ExtractOptions, type ExtractedText, type TextFormat, +} from './types'; + +export * from './types'; +export { setPdfLoader, type PdfLib } from './pdf'; + +const BY_EXTENSION: Record = { + txt: 'text', text: 'text', md: 'text', markdown: 'text', log: 'text', json: 'text', xml: 'text', + yaml: 'text', yml: 'text', ini: 'text', eml: 'text', tsv: 'csv', csv: 'csv', + htm: 'html', html: 'html', xhtml: 'html', ics: 'ics', ical: 'ics', ifb: 'ics', vcf: 'vcf', vcard: 'vcf', + pdf: 'pdf', docx: 'docx', docm: 'docx', dotx: 'docx', xlsx: 'xlsx', xlsm: 'xlsx', xltx: 'xlsx', + pptx: 'pptx', pptm: 'pptx', potx: 'pptx', odt: 'odt', ott: 'odt', ods: 'ods', ots: 'ods', odp: 'odp', otp: 'odp', +}; + +const BY_TYPE: Record = { + 'text/plain': 'text', 'text/markdown': 'text', 'application/json': 'text', 'application/xml': 'text', 'text/xml': 'text', + 'text/csv': 'csv', 'text/tab-separated-values': 'csv', 'text/html': 'html', 'application/xhtml+xml': 'html', + 'text/calendar': 'ics', 'application/ics': 'ics', 'text/vcard': 'vcf', 'text/x-vcard': 'vcf', 'text/directory': 'vcf', + 'application/pdf': 'pdf', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx', + 'application/vnd.oasis.opendocument.text': 'odt', + 'application/vnd.oasis.opendocument.spreadsheet': 'ods', + 'application/vnd.oasis.opendocument.presentation': 'odp', +}; + +/** The format to read a file as: by its name's extension, else its media type, else 'other'. */ +export function textFormatOf(name = '', type = ''): TextFormat { + const ext = /\.([a-z0-9]+)$/i.exec(name.trim())?.[1]?.toLowerCase(); + if (ext && BY_EXTENSION[ext]) return BY_EXTENSION[ext]; + const mediaType = type.split(';')[0].trim().toLowerCase(); + if (BY_TYPE[mediaType]) return BY_TYPE[mediaType]; + // Other text types (text/x-python, text/rfc822-headers, ...); not text/html, handled above. + if (mediaType.startsWith('text/')) return 'text'; + return 'other'; +} + +/** Cut `text` at `maxChars`, at a line or word end when one is near. */ +export function clipText(text: string, maxChars: number): { text: string; truncated: boolean } { + if (text.length <= maxChars) return { text, truncated: false }; + const cut = text.slice(0, maxChars); + const at = Math.max(cut.lastIndexOf('\n'), cut.lastIndexOf(' ')); + return { text: (at > maxChars * 0.9 ? cut.slice(0, at) : cut).trimEnd(), truncated: true }; +} + +const empty = (format: TextFormat, status: ExtractedText['status']): ExtractedText => ({ format, status, text: '', truncated: false }); + +async function read(bytes: Uint8Array, format: TextFormat): Promise> { + switch (format) { + case 'text': + case 'csv': + if (!looksLikeText(bytes)) return { status: 'unsupported', text: '', truncated: false }; + return { status: 'ok', text: decodeText(bytes).replace(/\r\n?/g, '\n'), truncated: false }; + case 'html': + return { status: 'ok', text: htmlText(decodeText(bytes)), truncated: false }; + case 'ics': + return { status: 'ok', text: icsToText(decodeText(bytes)), truncated: false }; + case 'vcf': + return { status: 'ok', text: vcfText(decodeText(bytes)), truncated: false }; + case 'pdf': + return pdfText(bytes); + case 'docx': case 'xlsx': case 'pptx': case 'odt': case 'ods': case 'odp': + return officeText(bytes, format); + default: + return { status: 'unsupported', text: '', truncated: false }; + } +} + +/** + * The text of a file's bytes. Statuses: too_big over MAX_TEXT_SOURCE_BYTES + * (or an Office file that unpacks to more than 50 MB), unsupported for + * images, archives and the legacy .doc/.xls/.ppt, no_text for a scanned + * PDF or a file with nothing in it, encrypted, failed. + */ +export async function extractText(data: ArrayBuffer | Uint8Array, options: ExtractOptions = {}): Promise { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + const format = textFormatOf(options.name, options.type); + if (bytes.byteLength > MAX_TEXT_SOURCE_BYTES) return empty(format, 'too_big'); + const maxChars = Math.max(1, Math.min(MAX_TEXT_CHARS, Math.floor(options.maxChars ?? DEFAULT_TEXT_CHARS))); + // A legacy Office file (and anything else) without a reader. + if (format === 'other') return empty(format, 'unsupported'); + let result: Omit; + try { + result = await read(bytes, format); + } catch { + return empty(format, 'failed'); + } + if (result.status !== 'ok') return { ...result, format, text: '' }; + const text = result.text.trim(); + if (!text) return { ...result, format, status: 'no_text', text: '', truncated: false }; + const clipped = clipText(text, maxChars); + return { ...result, format, text: clipped.text, truncated: result.truncated || clipped.truncated }; +} diff --git a/lib/attachment-text/office.ts b/lib/attachment-text/office.ts new file mode 100644 index 000000000..8f4dad20f --- /dev/null +++ b/lib/attachment-text/office.ts @@ -0,0 +1,359 @@ +// Text out of Office files: Word, Excel and PowerPoint (docx, xlsx, pptx) +// and OpenDocument (odt, ods, odp). All are ZIP archives of XML; only the +// entries holding text are unpacked (fflate, through the mail import's ZIP +// reader) and read with DOMParser. No layout, no styles: the reader of the +// text is a language model. +// +// The sizes in a ZIP's headers are whatever the sender wrote, so the bytes +// actually unpacked are counted, and reading stops past MAX_INFLATED_BYTES +// for the whole file (a ZIP bomb answers too_big). + +import { listZipEntries, streamZipEntry, ZipFormatError, type ZipEntry } from '@/lib/mail-import/zip-reader'; +import { MAX_INFLATED_BYTES, MAX_SHEET_ROWS, MAX_SHEETS, type ExtractedText, type TextFormat } from './types'; + +export type OfficeFormat = Extract; +type Result = Omit; + +/** The file unpacks to more than the cap. */ +class TooBig extends Error {} +/** The file is protected by a password. */ +class Encrypted extends Error {} + +/** Inflated chunks at most this big per step (a deflate block expands about 1000 times). */ +const INPUT_CHUNK = 16 * 1024; + +/** Reads entries of one archive, counting every byte unpacked against one cap. */ +class Unpacker { + private used = 0; + constructor(private readonly file: Blob, readonly entries: ZipEntry[], private readonly cap: number) {} + + has(name: string): boolean { + return this.entries.some((e) => e.name === name); + } + + async bytes(name: string): Promise { + const entry = this.entries.find((e) => e.name === name && !e.directory); + if (!entry) return null; + if (entry.encrypted) throw new Encrypted(); + const chunks: Uint8Array[] = []; + let total = 0; + for await (const chunk of streamZipEntry(this.file, entry, INPUT_CHUNK)) { + total += chunk.byteLength; + this.used += chunk.byteLength; + if (this.used > this.cap) throw new TooBig(); + chunks.push(chunk); + } + const out = new Uint8Array(total); + let at = 0; + for (const chunk of chunks) { + out.set(chunk, at); + at += chunk.byteLength; + } + return out; + } + + async xml(name: string): Promise { + const data = await this.bytes(name); + if (!data) return null; + const doc = new DOMParser().parseFromString(new TextDecoder('utf-8').decode(data), 'application/xml'); + if (doc.getElementsByTagName('parsererror').length > 0) throw new ZipFormatError(`${name} is not XML`); + return doc; + } +} + +// ─── XML helpers (by local name, whatever the prefix) ──────── + +function kids(el: Element, local?: string): Element[] { + return Array.from(el.children).filter((c) => !local || c.localName === local); +} + +function all(root: Document | Element, local: string): Element[] { + return Array.from(root.getElementsByTagNameNS('*', local)); +} + +function attr(el: Element, local: string): string | null { + for (const a of Array.from(el.attributes)) if (a.localName === local) return a.value; + return null; +} + +const tidy = (lines: string[]) => lines.map((l) => l.replace(/[ \t]+$/g, '')).join('\n').replace(/\n{3,}/g, '\n\n').trim(); + +/** A cell for CSV-like rows: quoted when it holds a comma, a quote or a line break. */ +function csvCell(value: string): string { + return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value; +} + +function csvRow(cells: string[]): string { + let end = cells.length; + while (end > 0 && !cells[end - 1]) end--; + return cells.slice(0, end).map(csvCell).join(','); +} + +// ─── Word (docx) ───────────────────────────────────────────── + +function wordRuns(el: Element): string { + let out = ''; + for (const c of Array.from(el.children)) { + switch (c.localName) { + case 't': out += c.textContent ?? ''; break; + case 'tab': out += '\t'; break; + case 'br': case 'cr': out += '\n'; break; + // Deleted text of tracked changes, field codes: not part of what the reader sees. + case 'delText': case 'instrText': case 'del': break; + default: out += wordRuns(c); + } + } + return out; +} + +function wordBlocks(el: Element, out: string[]): void { + for (const c of Array.from(el.children)) { + if (c.localName === 'p') out.push(wordRuns(c)); + else if (c.localName === 'tbl') { + for (const row of kids(c, 'tr')) { + out.push(kids(row, 'tc').map((cell) => all(cell, 'p').map(wordRuns).join(' ').trim()).join(' | ')); + } + out.push(''); + } else if (c.localName !== 'sectPr') wordBlocks(c, out); + } +} + +async function docx(zip: Unpacker): Promise { + const doc = await zip.xml('word/document.xml'); + if (!doc) return { status: 'failed', text: '', truncated: false }; + const body = all(doc, 'body')[0] ?? doc.documentElement; + const lines: string[] = []; + wordBlocks(body, lines); + return { status: 'ok', text: tidy(lines), truncated: false }; +} + +// ─── Excel (xlsx) ──────────────────────────────────────────── + +/** "BC12" → 54 (0-based column of the cell reference). */ +function columnOf(ref: string | null): number | null { + const m = /^([A-Z]+)\d*$/i.exec(ref ?? ''); + if (!m) return null; + let n = 0; + for (const ch of m[1].toUpperCase()) n = n * 26 + (ch.charCodeAt(0) - 64); + return n - 1; +} + +/** rId → target path inside the archive, from a .rels file next to `base`. */ +async function relations(zip: Unpacker, relsPath: string, baseDir: string): Promise> { + const rels = await zip.xml(relsPath); + const map = new Map(); + if (!rels) return map; + for (const rel of all(rels, 'Relationship')) { + const id = rel.getAttribute('Id'); + const target = rel.getAttribute('Target'); + if (!id || !target) continue; + map.set(id, target.startsWith('/') ? target.slice(1) : normalisePath(`${baseDir}/${target}`)); + } + return map; +} + +function normalisePath(path: string): string { + const parts: string[] = []; + for (const part of path.split('/')) { + if (part === '..') parts.pop(); + else if (part && part !== '.') parts.push(part); + } + return parts.join('/'); +} + +async function xlsx(zip: Unpacker): Promise { + const book = await zip.xml('xl/workbook.xml'); + if (!book) return { status: 'failed', text: '', truncated: false }; + const rels = await relations(zip, 'xl/_rels/workbook.xml.rels', 'xl'); + const shared: string[] = []; + const strings = await zip.xml('xl/sharedStrings.xml'); + if (strings) { + for (const si of all(strings, 'si')) { + // Phonetic runs (rPh) are reading aids, not the cell's text. + shared.push(all(si, 't').filter((t) => t.parentElement?.localName !== 'rPh').map((t) => t.textContent ?? '').join('')); + } + } + const sheets = all(book, 'sheet'); + const names: string[] = []; + const blocks: string[] = []; + let truncated = sheets.length > MAX_SHEETS; + for (const sheet of sheets.slice(0, MAX_SHEETS)) { + const name = sheet.getAttribute('name') ?? `Sheet ${names.length + 1}`; + const rid = attr(sheet, 'id'); + const path = rid ? rels.get(rid) : undefined; + const doc = path ? await zip.xml(path) : null; + names.push(name); + const rows: string[] = []; + if (doc) { + const rowEls = all(doc, 'row'); + if (rowEls.length > MAX_SHEET_ROWS) truncated = true; + for (const row of rowEls.slice(0, MAX_SHEET_ROWS)) { + const cells: string[] = []; + for (const c of kids(row, 'c')) { + const col = columnOf(c.getAttribute('r')) ?? cells.length; + if (col > 1000) continue; + const type = c.getAttribute('t'); + const v = kids(c, 'v')[0]?.textContent ?? ''; + let value: string; + if (type === 's') value = shared[Number(v)] ?? ''; + else if (type === 'inlineStr') value = all(c, 't').map((t) => t.textContent ?? '').join(''); + else if (type === 'b') value = v === '1' ? 'TRUE' : v === '0' ? 'FALSE' : v; + else value = v; + while (cells.length < col) cells.push(''); + cells[col] = value; + } + const line = csvRow(cells); + if (line) rows.push(line); + } + } + blocks.push(`Sheet: ${name}\n${rows.join('\n')}`.trim()); + } + return { status: 'ok', text: blocks.join('\n\n'), truncated, sheets: names }; +} + +// ─── PowerPoint (pptx) ─────────────────────────────────────── + +async function pptx(zip: Unpacker): Promise { + const slides = zip.entries + .map((e) => ({ name: e.name, n: Number(/^ppt\/slides\/slide(\d+)\.xml$/.exec(e.name)?.[1]) })) + .filter((s) => Number.isFinite(s.n)) + .sort((a, b) => a.n - b.n); + if (!slides.length && !zip.has('ppt/presentation.xml')) return { status: 'failed', text: '', truncated: false }; + const blocks: string[] = []; + for (const [i, slide] of slides.entries()) { + const doc = await zip.xml(slide.name); + if (!doc) continue; + const paras = all(doc, 'p').map((p) => all(p, 't').map((t) => t.textContent ?? '').join('').trim()).filter(Boolean); + blocks.push([`Slide ${i + 1}`, ...paras].join('\n')); + } + return { status: 'ok', text: blocks.join('\n\n'), truncated: false }; +} + +// ─── OpenDocument (odt, ods, odp) ──────────────────────────── + +/** A paragraph's text: spans, spaces (text:s), tabs and line breaks; notes left out. */ +function odfRuns(el: Element): string { + let out = ''; + for (const node of Array.from(el.childNodes)) { + if (node.nodeType === 3) { out += node.nodeValue ?? ''; continue; } + if (node.nodeType !== 1) continue; + const c = node as Element; + switch (c.localName) { + case 's': out += ' '.repeat(Math.min(100, Math.max(1, Number(attr(c, 'c')) || 1))); break; + case 'tab': out += '\t'; break; + case 'line-break': out += '\n'; break; + case 'note': case 'annotation': break; + default: out += odfRuns(c); + } + } + return out; +} + +/** Rows of a table:table as cell texts; repeated rows and cells expanded, but only up to the caps. */ +function odfTableRows(table: Element, maxRows: number): { rows: string[][]; more: boolean } { + const rows: string[][] = []; + let more = false; + for (const row of all(table, 'table-row')) { + // Only rows of this table, not of a table nested in a cell. + let owner: Element | null = row.parentElement; + while (owner && owner.localName !== 'table') owner = owner.parentElement; + if (owner !== table) continue; + const cells: string[] = []; + for (const cell of kids(row).filter((c) => c.localName === 'table-cell' || c.localName === 'covered-table-cell')) { + const text = all(cell, 'p').map(odfRuns).join('\n').trim(); + const repeat = Math.max(1, Number(attr(cell, 'number-columns-repeated')) || 1); + // A run of empty cells to the sheet's edge says "repeated 16384 times". + const times = text ? Math.min(repeat, 100) : Math.min(repeat, 1000 - cells.length); + for (let i = 0; i < times && cells.length < 1000; i++) cells.push(text); + } + const repeatRows = Math.max(1, Number(attr(row, 'number-rows-repeated')) || 1); + const empty = cells.every((c) => !c); + for (let i = 0; i < (empty ? 1 : Math.min(repeatRows, maxRows)); i++) { + if (rows.length >= maxRows) { more = true; break; } + rows.push(cells); + } + if (more) break; + } + return { rows, more }; +} + +function odfBlocks(el: Element, out: string[]): void { + for (const c of Array.from(el.children)) { + if (c.localName === 'p' || c.localName === 'h') out.push(odfRuns(c)); + else if (c.localName === 'table') { + for (const row of odfTableRows(c, MAX_SHEET_ROWS).rows) { + const line = row.filter(Boolean).join(' | '); + if (line) out.push(line); + } + out.push(''); + } else if (c.localName !== 'tracked-changes' && c.localName !== 'sequence-decls') odfBlocks(c, out); + } +} + +async function odf(zip: Unpacker, format: 'odt' | 'ods' | 'odp'): Promise { + const manifest = await zip.bytes('META-INF/manifest.xml'); + if (manifest && /encryption-data/.test(new TextDecoder('utf-8').decode(manifest))) throw new Encrypted(); + const doc = await zip.xml('content.xml'); + if (!doc) return { status: 'failed', text: '', truncated: false }; + if (format === 'odt') { + const body = all(doc, 'text')[0]; + const lines: string[] = []; + if (body) odfBlocks(body, lines); + return { status: 'ok', text: tidy(lines), truncated: false }; + } + if (format === 'ods') { + const sheet = all(doc, 'spreadsheet')[0]; + const tables = sheet ? kids(sheet, 'table') : []; + const names: string[] = []; + const blocks: string[] = []; + let truncated = tables.length > MAX_SHEETS; + for (const table of tables.slice(0, MAX_SHEETS)) { + const name = attr(table, 'name') ?? `Sheet ${names.length + 1}`; + names.push(name); + const { rows, more } = odfTableRows(table, MAX_SHEET_ROWS); + if (more) truncated = true; + const lines = rows.map(csvRow).filter(Boolean); + blocks.push(`Sheet: ${name}\n${lines.join('\n')}`.trim()); + } + return { status: 'ok', text: blocks.join('\n\n'), truncated, sheets: names }; + } + const pres = all(doc, 'presentation')[0]; + const blocks: string[] = []; + for (const [i, page] of (pres ? kids(pres, 'page') : []).entries()) { + const paras = all(page, 'p').map((p) => odfRuns(p).trim()).filter(Boolean); + blocks.push([`Slide ${i + 1}`, ...paras].join('\n')); + } + return { status: 'ok', text: blocks.join('\n\n'), truncated: false }; +} + +// ─── Entry point ───────────────────────────────────────────── + +const CFB_MAGIC = [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]; + +/** True for an OLE compound file: legacy .doc/.xls/.ppt, or an Office file with a password. */ +export function isCompoundFile(bytes: Uint8Array): boolean { + return CFB_MAGIC.every((b, i) => bytes[i] === b); +} + +/** + * The text of an Office file. A password-protected docx, xlsx or pptx is a + * compound file, not a ZIP: encrypted. `cap` (tests) replaces MAX_INFLATED_BYTES. + */ +export async function officeText(bytes: Uint8Array, format: OfficeFormat, cap = MAX_INFLATED_BYTES): Promise { + if (isCompoundFile(bytes)) return { status: 'encrypted', text: '', truncated: false }; + try { + const file = new Blob([bytes as BlobPart]); + const zip = new Unpacker(file, await listZipEntries(file), cap); + switch (format) { + case 'docx': return await docx(zip); + case 'xlsx': return await xlsx(zip); + case 'pptx': return await pptx(zip); + default: return await odf(zip, format); + } + } catch (err) { + if (err instanceof TooBig) return { status: 'too_big', text: '', truncated: false }; + if (err instanceof Encrypted) return { status: 'encrypted', text: '', truncated: false }; + if (err instanceof ZipFormatError && /Encrypted entry/.test(err.message)) return { status: 'encrypted', text: '', truncated: false }; + return { status: 'failed', text: '', truncated: false }; + } +} diff --git a/lib/attachment-text/pdf.ts b/lib/attachment-text/pdf.ts new file mode 100644 index 000000000..eabf9a925 --- /dev/null +++ b/lib/attachment-text/pdf.ts @@ -0,0 +1,79 @@ +// Text out of a PDF with pdf.js, the same library and worker the Files +// viewer loads. Imported on demand, so nothing is fetched until a PDF is +// read. Runs in the app page (the worker is a same-origin file; the CSP +// refuses blob: workers). + +import { liteMountedAssetUrl } from '@/lib/lite'; +import { MAX_PDF_PAGES, MIN_PDF_CHARS_PER_PAGE, type ExtractedText } from './types'; + +/** The part of pdf.js used here (tests pass a stand-in). */ +export interface PdfLib { + getDocument(src: { data: Uint8Array }): { promise: Promise; destroy(): Promise }; +} +interface PdfDoc { + numPages: number; + getPage(n: number): Promise<{ getTextContent(): Promise<{ items: unknown[] }> }>; +} + +async function loadPdfJs(): Promise { + const pdfjs = await import('pdfjs-dist'); + pdfjs.GlobalWorkerOptions.workerSrc = liteMountedAssetUrl(new URL( + 'pdfjs-dist/build/pdf.worker.min.mjs', + import.meta.url, + ).toString()); + return pdfjs as unknown as PdfLib; +} + +let loader: () => Promise = loadPdfJs; + +/** Swap the pdf.js loader (tests). Without an argument the real one comes back. */ +export function setPdfLoader(next?: () => Promise): void { + loader = next ?? loadPdfJs; +} + +/** The text items of one page as lines: a new line where pdf.js marks one. */ +function pageText(items: unknown[]): string { + let out = ''; + for (const item of items) { + if (!item || typeof item !== 'object' || !('str' in item)) continue; + const { str, hasEOL } = item as { str: unknown; hasEOL?: unknown }; + if (typeof str === 'string') out += str; + if (hasEOL === true) out += '\n'; + else if (typeof str === 'string' && str && !/\s$/.test(str)) out += ' '; + } + return out.replace(/[ \t]+\n/g, '\n').replace(/[ \t]{2,}/g, ' ').trim(); +} + +/** + * The text of at most MAX_PDF_PAGES pages, a "Page n" line before each. + * no_text when the pages hold fewer than MIN_PDF_CHARS_PER_PAGE characters + * on average (a scan); encrypted for a PDF that needs a password. + */ +export async function pdfText(bytes: Uint8Array): Promise> { + const lib = await loader(); + // pdf.js may take over the buffer: give it a copy. + const task = lib.getDocument({ data: bytes.slice() }); + try { + const doc = await task.promise; + const pages = doc.numPages; + const read = Math.min(pages, MAX_PDF_PAGES); + const parts: string[] = []; + let chars = 0; + for (let n = 1; n <= read; n++) { + const page = await doc.getPage(n); + const text = pageText((await page.getTextContent()).items); + chars += text.replace(/\s/g, '').length; + if (text) parts.push(read > 1 ? `Page ${n}\n${text}` : text); + } + if (read === 0 || chars / read < MIN_PDF_CHARS_PER_PAGE) { + return { status: 'no_text', text: '', truncated: false, pages }; + } + return { status: 'ok', text: parts.join('\n\n'), truncated: pages > read, pages }; + } catch (err) { + const name = (err as { name?: unknown } | null)?.name; + if (name === 'PasswordException') return { status: 'encrypted', text: '', truncated: false }; + return { status: 'failed', text: '', truncated: false }; + } finally { + void task.destroy().catch(() => {}); + } +} diff --git a/lib/attachment-text/plain.ts b/lib/attachment-text/plain.ts new file mode 100644 index 000000000..e4dd451df --- /dev/null +++ b/lib/attachment-text/plain.ts @@ -0,0 +1,142 @@ +// Text out of the formats that already are text: plain text, CSV, HTML, +// calendar files (ICS) and address cards (VCF). + +import { htmlToPlainText } from '@/lib/html-to-text'; +import { parseVCard } from '@/lib/vcard'; +import type { ContactCard } from '@/lib/jmap/types'; + +/** + * Bytes as text: UTF-8 (with or without a byte order mark), UTF-16 with a + * byte order mark, else Windows-1252, which never fails. + */ +export function decodeText(bytes: Uint8Array): string { + if (bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2)); + if (bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2)); + try { + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: false }).decode(bytes); + } catch { + return new TextDecoder('windows-1252').decode(bytes); + } +} + +/** True when the bytes look like text rather than a binary file (no NUL bytes in the first 4 KB). */ +export function looksLikeText(bytes: Uint8Array): boolean { + const head = bytes.subarray(0, 4096); + if (head[0] === 0xff && head[1] === 0xfe) return true; + if (head[0] === 0xfe && head[1] === 0xff) return true; + return !head.includes(0); +} + +/** The readable text of an HTML file: no scripts, styles or head. */ +export function htmlText(html: string): string { + const body = html.replace(/<(script|style|noscript|template|head)\b[\s\S]*?<\/\1\s*>/gi, ' '); + return htmlToPlainText(body, { paragraphSpacing: true, mailMarkup: true, imageAlt: (alt) => `[Image: ${alt}]` }); +} + +// ─── ICS ───────────────────────────────────────────────────── + +interface IcsLine { name: string; params: Record; value: string } + +function unfold(text: string): string[] { + return text.replace(/\r\n?/g, '\n').replace(/\n[ \t]/g, '').split('\n'); +} + +function parseIcsLine(line: string): IcsLine | null { + // NAME;PARAM=a;PARAM="b:c":value. A colon inside a quoted parameter is not the separator. + let quoted = false; + let colon = -1; + for (let i = 0; i < line.length; i++) { + const ch = line[i]; + if (ch === '"') quoted = !quoted; + else if (ch === ':' && !quoted) { colon = i; break; } + } + if (colon < 0) return null; + const [name, ...rawParams] = line.slice(0, colon).split(';'); + const params: Record = {}; + for (const p of rawParams) { + const eq = p.indexOf('='); + if (eq > 0) params[p.slice(0, eq).toUpperCase()] = p.slice(eq + 1).replace(/^"|"$/g, ''); + } + return { name: name.toUpperCase(), params, value: line.slice(colon + 1) }; +} + +function icsText(value: string): string { + return value.replace(/\\[nN]/g, ' ').replace(/\\([,;\\])/g, '$1').trim(); +} + +/** 20261105T090000Z → "2026-11-05 09:00 UTC"; with TZID the zone in brackets; a date alone stays a date. */ +function icsWhen(line: IcsLine | undefined): string { + if (!line) return ''; + const m = /^(\d{4})(\d{2})(\d{2})(?:T(\d{2})(\d{2})\d{0,2}(Z?))?$/.exec(line.value.trim()); + if (!m) return line.value.trim(); + const date = `${m[1]}-${m[2]}-${m[3]}`; + if (!m[4]) return date; + const zone = m[6] ? ' UTC' : line.params.TZID ? ` (${line.params.TZID})` : ''; + return `${date} ${m[4]}:${m[5]}${zone}`; +} + +function organiser(line: IcsLine | undefined): string { + if (!line) return ''; + const address = line.value.replace(/^mailto:/i, '').trim(); + const name = line.params.CN?.trim(); + return name && address ? `${name} <${address}>` : (name || address); +} + +/** One line per event and task: "event: title, start, end, location, organiser …". */ +export function icsToText(ics: string): string { + const out: string[] = []; + let current: Map | null = null; + let kind = ''; + for (const raw of unfold(ics)) { + const line = parseIcsLine(raw); + if (!line) continue; + if (line.name === 'BEGIN' && /^(VEVENT|VTODO)$/i.test(line.value.trim())) { + current = new Map(); + kind = line.value.trim().toUpperCase() === 'VTODO' ? 'task' : 'event'; + continue; + } + if (line.name === 'END' && current && /^(VEVENT|VTODO)$/i.test(line.value.trim())) { + const parts = [ + icsText(current.get('SUMMARY')?.value ?? '') || '(no title)', + icsWhen(current.get('DTSTART')), + icsWhen(current.get('DTEND') ?? current.get('DUE')), + icsText(current.get('LOCATION')?.value ?? ''), + ]; + const org = organiser(current.get('ORGANIZER')); + if (org) parts.push(`organiser ${org}`); + out.push(`${kind}: ${parts.filter(Boolean).join(', ')}`); + const description = icsText(current.get('DESCRIPTION')?.value ?? ''); + if (description) out.push(` ${description}`); + current = null; + continue; + } + if (current && !current.has(line.name)) current.set(line.name, line); + } + return out.join('\n'); +} + +// ─── VCF ───────────────────────────────────────────────────── + +function cardName(card: ContactCard): string { + const full = card.name?.full?.trim(); + if (full) return full; + return (card.name?.components ?? []) + .filter((c) => c.kind !== 'separator') + .map((c) => c.value) + .join(' ') + .trim(); +} + +/** One block per card: name, email addresses, phone numbers, organisation. */ +export function vcfText(vcf: string): string { + return parseVCard(vcf).map((card) => { + const lines = [`contact: ${cardName(card) || '(no name)'}`]; + const emails = Object.values(card.emails ?? {}).map((e) => e.address).filter(Boolean); + const phones = Object.values(card.phones ?? {}).map((p) => p.number).filter(Boolean); + const orgs = Object.values(card.organizations ?? {}).map((o) => o.name).filter(Boolean); + if (emails.length) lines.push(` email: ${emails.join(', ')}`); + if (phones.length) lines.push(` phone: ${phones.join(', ')}`); + if (orgs.length) lines.push(` organisation: ${orgs.join(', ')}`); + return lines.join('\n'); + }).join('\n'); +} diff --git a/lib/attachment-text/types.ts b/lib/attachment-text/types.ts new file mode 100644 index 000000000..03e107bbd --- /dev/null +++ b/lib/attachment-text/types.ts @@ -0,0 +1,54 @@ +// Shapes and limits of the text extraction for attachments and files. +// Pure: no imports from the app, so the plugin host and Files can share it. + +/** What kind of file the text came from. */ +export type TextFormat = + | 'text' | 'csv' | 'html' | 'ics' | 'vcf' | 'pdf' + | 'docx' | 'xlsx' | 'pptx' | 'odt' | 'ods' | 'odp' | 'other'; + +/** + * ok: `text` holds the text. too_big: over the size limit, not read. + * unsupported: a format without a reader (images, legacy .doc/.xls/.ppt, + * archives). no_text: nothing to read, e.g. a scanned PDF. encrypted: + * protected by a password. failed: the file is damaged or reading broke. + */ +export type TextStatus = 'ok' | 'too_big' | 'unsupported' | 'no_text' | 'encrypted' | 'failed'; + +export interface ExtractedText { + format: TextFormat; + status: TextStatus; + /** The text, '' unless status is ok. */ + text: string; + /** True when the text was cut at maxChars, or a PDF or sheet had more than was read. */ + truncated: boolean; + /** PDFs: the number of pages in the file. */ + pages?: number; + /** Spreadsheets: the names of the sheets read. */ + sheets?: string[]; +} + +export interface ExtractOptions { + /** File name; its extension picks the format first. */ + name?: string; + /** Media type, used when the name says nothing. */ + type?: string; + /** Longest text returned (default DEFAULT_TEXT_CHARS, at most MAX_TEXT_CHARS). */ + maxChars?: number; +} + +/** Text returned when the caller doesn't ask for a length. */ +export const DEFAULT_TEXT_CHARS = 20_000; +/** Longest text one read returns. */ +export const MAX_TEXT_CHARS = 100_000; +/** Largest file read for its text. */ +export const MAX_TEXT_SOURCE_BYTES = 20 * 1024 * 1024; +/** Pages of a PDF read at most. */ +export const MAX_PDF_PAGES = 50; +/** Fewer characters per page than this on average: the PDF has no text layer. */ +export const MIN_PDF_CHARS_PER_PAGE = 20; +/** Sheets of a spreadsheet read at most. */ +export const MAX_SHEETS = 5; +/** Rows per sheet read at most. */ +export const MAX_SHEET_ROWS = 2_000; +/** Bytes an Office file may unpack to, all entries read together (zip bombs). */ +export const MAX_INFLATED_BYTES = 50 * 1024 * 1024; diff --git a/lib/plugin-sandbox/assistant/b.runtime.ts b/lib/plugin-sandbox/assistant/b.runtime.ts index b9e5f1850..ab4aae3c1 100644 --- a/lib/plugin-sandbox/assistant/b.runtime.ts +++ b/lib/plugin-sandbox/assistant/b.runtime.ts @@ -1,8 +1,22 @@ // Area b of the assistant's host APIs, inside the plugin sandbox (see README.md). import type { CallApi, RuntimeAdditions } from './runtime'; +import type { AttachmentTextOptions, AttachmentTextView } from './b.types'; + +/** Reads that download and unpack a file get the network timeout. */ +const READ_TIMEOUT_MS = 120_000; /** The methods this area adds to the plugin's api, by namespace. */ -export function runtime(_callApi: CallApi): RuntimeAdditions { - return {}; +export function runtime(callApi: CallApi): RuntimeAdditions { + return { + email: { + /** + * The text of one attachment of a message (email:blob-read, 30 a + * minute): PDF, Word, Excel, PowerPoint, OpenDocument, text, CSV, + * HTML, calendar and address files. See AttachmentTextView.status. + */ + getAttachmentText: (emailId: string, blobId: string, options?: AttachmentTextOptions) => + callApi('email.getAttachmentText', [emailId, blobId, options], READ_TIMEOUT_MS) as Promise, + }, + }; } diff --git a/lib/plugin-sandbox/assistant/b.ts b/lib/plugin-sandbox/assistant/b.ts index ea6ac7e71..28f6f0ae8 100644 --- a/lib/plugin-sandbox/assistant/b.ts +++ b/lib/plugin-sandbox/assistant/b.ts @@ -1,18 +1,64 @@ -// Area b of the assistant's host APIs, host side (see README.md). +// Area b of the assistant's host APIs, host side (see README.md): more +// context for the assistant. The text of an attachment. import type { InstalledPlugin, Permission } from '../../plugin-types'; import { NOT_MINE } from './not-mine'; -import type { AreaHostFeaturesB } from './b.types'; +import type { AreaHostFeaturesB, AttachmentTextView } from './b.types'; +import { readNamedEmail } from '../host-api-mail'; +import { readId, readInteger, readObject, sessionClient, withinRate } from '../host-api-input'; +import { DEFAULT_TEXT_CHARS, MAX_TEXT_CHARS, MAX_TEXT_SOURCE_BYTES, extractText, textFormatOf } from '@/lib/attachment-text'; /** Method → permission, joined into PERM_PER_METHOD. */ -export const PERMS: Readonly> = {}; +export const PERMS: Readonly> = { + 'email.getAttachmentText': 'email:blob-read', +}; + +/** Attachment reads (text) a plugin may make per minute. */ +export const MAX_ATTACHMENT_TEXT_PER_MINUTE = 30; + +/** + * The text of one attachment of one message. The blob has to be an + * attachment of that message, as for api.email.getAttachment; the message + * is found the way api.email.get finds it (the account the plugin names, + * the list's row, the signed-in account). + */ +export async function doEmailGetAttachmentText( + plugin: InstalledPlugin, + emailIdArg: unknown, + blobIdArg: unknown, + optionsArg?: unknown, +): Promise { + const method = 'email.getAttachmentText'; + const emailId = readId(emailIdArg, method, 'emailId'); + const blobId = readId(blobIdArg, method, 'blobId'); + const opts = readObject(optionsArg, method, 'options', ['maxChars', 'accountId'], true); + const maxChars = opts.maxChars === undefined ? DEFAULT_TEXT_CHARS : readInteger(opts.maxChars, method, 'maxChars', 1, MAX_TEXT_CHARS); + if (!withinRate(method, plugin.id, MAX_ATTACHMENT_TEXT_PER_MINUTE)) throw new Error(`${method}: too many reads, try again in a minute`); + const client = sessionClient(method); + const found = await readNamedEmail(client, emailId, method, opts.accountId); + if (!found) throw new Error(`${method}: no message with id "${emailId}"`); + const part = (found.email.attachments ?? []).find((a) => a.blobId === blobId); + if (!part) throw new Error(`${method}: that blob is not an attachment of this message`); + const name = part.name || 'attachment'; + const type = part.type || 'application/octet-stream'; + const base = { name, type, size: part.size || 0 }; + if ((part.size || 0) > MAX_TEXT_SOURCE_BYTES) { + return { ...base, format: textFormatOf(name, type), status: 'too_big', text: '', truncated: false }; + } + const data = await client.fetchBlobArrayBuffer(blobId, part.name || undefined, part.type || undefined, ...(found.accountId ? [found.accountId] : [])); + const result = await extractText(data, { name, type, maxChars }); + return { ...base, size: data.byteLength, ...result }; +} /** Runs a method of this area; NOT_MINE for any other. */ -export async function dispatch(_plugin: InstalledPlugin, _method: string, _args: unknown[]): Promise { - return NOT_MINE; +export async function dispatch(plugin: InstalledPlugin, method: string, args: unknown[]): Promise { + switch (method) { + case 'email.getAttachmentText': return doEmailGetAttachmentText(plugin, args[0], args[1], args[2]); + default: return NOT_MINE; + } } /** This area's flags in api.host.features, right now. */ export function features(): AreaHostFeaturesB { - return {}; + return { attachmentText: true }; } diff --git a/lib/plugin-sandbox/assistant/b.types.ts b/lib/plugin-sandbox/assistant/b.types.ts index bfbb3b790..443f07894 100644 --- a/lib/plugin-sandbox/assistant/b.types.ts +++ b/lib/plugin-sandbox/assistant/b.types.ts @@ -4,5 +4,38 @@ /** Permissions this area adds to ALL_PERMISSIONS. */ export const PERMISSIONS = [] as const; -/** Flags this area adds to api.host.features. */ -export interface AreaHostFeaturesB {} +/** Flags this area adds to api.host.features (missing on older webmails: read them as `=== true`). */ +export interface AreaHostFeaturesB { + /** api.email.getAttachmentText: the text of an attachment (PDF, Office, text, calendar and address files). */ + attachmentText?: boolean; +} + +/** + * What api.email.getAttachmentText answers. `status`: ok (`text` holds the + * text), too_big (over 20 MB, or an Office file that unpacks to more than + * 50 MB), unsupported (images, archives, legacy .doc/.xls/.ppt), no_text + * (a scanned PDF, an empty file), encrypted, failed. + */ +export interface AttachmentTextView { + name: string; + type: string; + /** Size of the attachment in bytes. */ + size: number; + format: 'text' | 'csv' | 'html' | 'ics' | 'vcf' | 'pdf' | 'docx' | 'xlsx' | 'pptx' | 'odt' | 'ods' | 'odp' | 'other'; + status: 'ok' | 'too_big' | 'unsupported' | 'no_text' | 'encrypted' | 'failed'; + /** The text, '' unless status is ok. */ + text: string; + /** True when the text was cut at maxChars, or a PDF or spreadsheet had more than was read. */ + truncated: boolean; + /** PDFs: the number of pages in the file (at most 50 are read). */ + pages?: number; + /** Spreadsheets: the names of the sheets read (at most 5, 2,000 rows each). */ + sheets?: string[]; +} + +export interface AttachmentTextOptions { + /** Longest text returned, 1 to 100,000 characters (default 20,000). */ + maxChars?: number; + /** The account the message is in (EmailSummary.accountId), for shared and group mail. */ + accountId?: string; +} diff --git a/lib/plugin-sandbox/runtime.tsx b/lib/plugin-sandbox/runtime.tsx index 63b0b361a..64e65bb8a 100644 --- a/lib/plugin-sandbox/runtime.tsx +++ b/lib/plugin-sandbox/runtime.tsx @@ -161,6 +161,12 @@ function applyHostFeatures(next: HostFeaturesView | undefined): void { hostFeaturesState.externalRecipientWarning = next.externalRecipientWarning === true; // A host older than 1.20 doesn't send it, and never let such a plugin run in the mode. hostFeaturesState.noThirdPartyTraffic = next.noThirdPartyTraffic === true; + // The AI assistant's host calls (lib/plugin-sandbox/assistant): every other flag the host sends. + for (const [key, value] of Object.entries(next)) { + if (key !== 'externalSenderLabel' && key !== 'externalRecipientWarning' && key !== 'noThirdPartyTraffic') { + (hostFeaturesState as unknown as Record)[key] = value === true; + } + } for (const listener of hostFeatureListeners) { try { listener(hostFeaturesState); } catch (err) { console.error('[plugin] host feature listener failed', err); } }