diff --git a/lib/attachment-text/__tests__/plain.test.ts b/lib/attachment-text/__tests__/plain.test.ts new file mode 100644 index 000000000..9cce490d2 --- /dev/null +++ b/lib/attachment-text/__tests__/plain.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { extractText } from '@/lib/attachment-text'; + +// HTML and calendar files: reading takes time in step with the file, and an +// event is described by its own properties only. + +const text = (s: string) => new TextEncoder().encode(s); + +describe('HTML', () => { + it('reads an HTML file full of unclosed tags in a moment', async () => { + // 400 KB of "
Agenda${' { + const started = performance.now(); + const out = await extractText(text(`Body
tpl'), { name: 'a.html' }); + expect(out.text).toBe('Body'); + }); + + it('reads at most 5 million characters of HTML, and says so', async () => { + // Little text, a lot of source: only the cap can make this truncated. + const out = await extractText(text(`Body
`), { name: 'huge.html' }); + expect(out).toMatchObject({ status: 'ok', text: 'Body', truncated: true }); + }); +}); + +describe('calendar files', () => { + it('ignores what an alarm inside an event says', async () => { + const ics = [ + 'BEGIN:VCALENDAR', 'BEGIN:VEVENT', 'SUMMARY:Dentist', 'DTSTART:20261105T090000Z', + 'BEGIN:VALARM', 'ACTION:DISPLAY', 'DESCRIPTION:Reminder', 'LOCATION:Nowhere', 'END:VALARM', + 'LOCATION:Main street 1', 'END:VEVENT', 'END:VCALENDAR', + ].join('\r\n'); + expect((await extractText(text(ics), { name: 'a.ics' })).text).toBe('event: Dentist, 2026-11-05 09:00 UTC, Main street 1'); + }); +}); diff --git a/lib/attachment-text/index.ts b/lib/attachment-text/index.ts index 3d51e404f..3764a9822 100644 --- a/lib/attachment-text/index.ts +++ b/lib/attachment-text/index.ts @@ -5,7 +5,7 @@ // (pdf.js), office.ts (docx, xlsx, pptx, odt, ods, odp). Never throws: a // file that can't be read answers with a status. -import { decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain'; +import { MAX_HTML_CHARS, decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain'; import { pdfText } from './pdf'; import { officeText } from './office'; import { @@ -68,7 +68,11 @@ async function read(bytes: Uint8Array, format: TextFormat, budget: number): Prom if (!looksLikeText(bytes)) return { status: 'unsupported', text: '', truncated: false }; return { status: 'ok', text: decodeText(bytes).replace(/\r\n?/g, '\n'), truncated: false }; case 'html': - return { status: 'ok', text: htmlText(decodeText(bytes)), truncated: false }; + { + const html = decodeText(bytes); + const cut = html.length > MAX_HTML_CHARS; + return { status: 'ok', text: htmlText(cut ? html.slice(0, MAX_HTML_CHARS) : html), truncated: cut }; + } case 'ics': return { status: 'ok', text: icsToText(decodeText(bytes)), truncated: false }; case 'vcf': diff --git a/lib/attachment-text/plain.ts b/lib/attachment-text/plain.ts index e4dd451df..008098939 100644 --- a/lib/attachment-text/plain.ts +++ b/lib/attachment-text/plain.ts @@ -27,10 +27,18 @@ export function looksLikeText(bytes: Uint8Array): boolean { return !head.includes(0); } -/** The readable text of an HTML file: no scripts, styles or head. */ +/** Characters of HTML read at most; the rest of a larger file is left out. */ +export const MAX_HTML_CHARS = 5_000_000; + +/** + * The readable text of an HTML file: its body without scripts, styles and + * templates. Parsed with DOMParser (linear in the file, whatever its tags); + * list markers left out, since numbering them is slow on long lists. + */ export function htmlText(html: string): string { - const body = html.replace(/<(script|style|noscript|template|head)\b[\s\S]*?<\/\1\s*>/gi, ' '); - return htmlToPlainText(body, { paragraphSpacing: true, mailMarkup: true, imageAlt: (alt) => `[Image: ${alt}]` }); + const doc = new DOMParser().parseFromString(html, 'text/html'); + for (const el of Array.from(doc.querySelectorAll('script, style, noscript, template'))) el.remove(); + return htmlToPlainText(doc.body?.innerHTML ?? '', { paragraphSpacing: true, imageAlt: (alt) => `[Image: ${alt}]` }); } // ─── ICS ───────────────────────────────────────────────────── @@ -87,9 +95,16 @@ export function icsToText(ics: string): string { const out: string[] = []; let current: Map