diff --git a/lib/attachment-text/__tests__/plain.test.ts b/lib/attachment-text/__tests__/plain.test.ts new file mode 100644 index 000000000..9cce490d2 --- /dev/null +++ b/lib/attachment-text/__tests__/plain.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { extractText } from '@/lib/attachment-text'; + +// HTML and calendar files: reading takes time in step with the file, and an +// event is described by its own properties only. + +const text = (s: string) => new TextEncoder().encode(s); + +describe('HTML', () => { + it('reads an HTML file full of unclosed tags in a moment', async () => { + // 400 KB of "Agenda

${' { + const started = performance.now(); + const out = await extractText(text(`
    ${'
  1. item
  2. '.repeat(30_000)}
`), { name: 'list.html', maxChars: 1_000 }); + expect(performance.now() - started).toBeLessThan(3_000); + expect(out.text.startsWith('item')).toBe(true); + }); + + it('leaves out scripts, styles, templates and the head', async () => { + const out = await extractText(text('T

Body

'), { name: 'a.html' }); + expect(out.text).toBe('Body'); + }); + + it('reads at most 5 million characters of HTML, and says so', async () => { + // Little text, a lot of source: only the cap can make this truncated. + const out = await extractText(text(`

Body

`), { name: 'huge.html' }); + expect(out).toMatchObject({ status: 'ok', text: 'Body', truncated: true }); + }); +}); + +describe('calendar files', () => { + it('ignores what an alarm inside an event says', async () => { + const ics = [ + 'BEGIN:VCALENDAR', 'BEGIN:VEVENT', 'SUMMARY:Dentist', 'DTSTART:20261105T090000Z', + 'BEGIN:VALARM', 'ACTION:DISPLAY', 'DESCRIPTION:Reminder', 'LOCATION:Nowhere', 'END:VALARM', + 'LOCATION:Main street 1', 'END:VEVENT', 'END:VCALENDAR', + ].join('\r\n'); + expect((await extractText(text(ics), { name: 'a.ics' })).text).toBe('event: Dentist, 2026-11-05 09:00 UTC, Main street 1'); + }); +}); diff --git a/lib/attachment-text/index.ts b/lib/attachment-text/index.ts index 3d51e404f..3764a9822 100644 --- a/lib/attachment-text/index.ts +++ b/lib/attachment-text/index.ts @@ -5,7 +5,7 @@ // (pdf.js), office.ts (docx, xlsx, pptx, odt, ods, odp). Never throws: a // file that can't be read answers with a status. -import { decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain'; +import { MAX_HTML_CHARS, decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain'; import { pdfText } from './pdf'; import { officeText } from './office'; import { @@ -68,7 +68,11 @@ async function read(bytes: Uint8Array, format: TextFormat, budget: number): Prom if (!looksLikeText(bytes)) return { status: 'unsupported', text: '', truncated: false }; return { status: 'ok', text: decodeText(bytes).replace(/\r\n?/g, '\n'), truncated: false }; case 'html': - return { status: 'ok', text: htmlText(decodeText(bytes)), truncated: false }; + { + const html = decodeText(bytes); + const cut = html.length > MAX_HTML_CHARS; + return { status: 'ok', text: htmlText(cut ? html.slice(0, MAX_HTML_CHARS) : html), truncated: cut }; + } case 'ics': return { status: 'ok', text: icsToText(decodeText(bytes)), truncated: false }; case 'vcf': diff --git a/lib/attachment-text/plain.ts b/lib/attachment-text/plain.ts index e4dd451df..008098939 100644 --- a/lib/attachment-text/plain.ts +++ b/lib/attachment-text/plain.ts @@ -27,10 +27,18 @@ export function looksLikeText(bytes: Uint8Array): boolean { return !head.includes(0); } -/** The readable text of an HTML file: no scripts, styles or head. */ +/** Characters of HTML read at most; the rest of a larger file is left out. */ +export const MAX_HTML_CHARS = 5_000_000; + +/** + * The readable text of an HTML file: its body without scripts, styles and + * templates. Parsed with DOMParser (linear in the file, whatever its tags); + * list markers left out, since numbering them is slow on long lists. + */ export function htmlText(html: string): string { - const body = html.replace(/<(script|style|noscript|template|head)\b[\s\S]*?<\/\1\s*>/gi, ' '); - return htmlToPlainText(body, { paragraphSpacing: true, mailMarkup: true, imageAlt: (alt) => `[Image: ${alt}]` }); + const doc = new DOMParser().parseFromString(html, 'text/html'); + for (const el of Array.from(doc.querySelectorAll('script, style, noscript, template'))) el.remove(); + return htmlToPlainText(doc.body?.innerHTML ?? '', { paragraphSpacing: true, imageAlt: (alt) => `[Image: ${alt}]` }); } // ─── ICS ───────────────────────────────────────────────────── @@ -87,9 +95,16 @@ export function icsToText(ics: string): string { const out: string[] = []; let current: Map | null = null; let kind = ''; + // Inside a component of the event (an alarm): its properties aren't the event's. + let nested = 0; for (const raw of unfold(ics)) { const line = parseIcsLine(raw); if (!line) continue; + if (current && line.name === 'BEGIN') { nested++; continue; } + if (current && nested > 0) { + if (line.name === 'END') nested--; + continue; + } if (line.name === 'BEGIN' && /^(VEVENT|VTODO)$/i.test(line.value.trim())) { current = new Map(); kind = line.value.trim().toUpperCase() === 'VTODO' ? 'task' : 'event';