fix: read HTML and calendar attachments in time with their size, without alarm texts

This commit is contained in:
Linus Rath
2026-11-02 23:24:34 +01:00
parent 7c97eb2691
commit 7013a8cf52
3 changed files with 70 additions and 5 deletions
@@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest';
import { extractText } from '@/lib/attachment-text';
// HTML and calendar files: reading takes time in step with the file, and an
// event is described by its own properties only.
const text = (s: string) => new TextEncoder().encode(s);
describe('HTML', () => {
it('reads an HTML file full of unclosed tags in a moment', async () => {
// 400 KB of "<head": a regular expression over the source took 8.6 s for this.
const started = performance.now();
const out = await extractText(text(`<p>Agenda</p>${'<head'.repeat(80_000)}`), { name: 'odd.html' });
expect(performance.now() - started).toBeLessThan(3_000);
expect(out.status).toBe('ok');
});
it('reads a long numbered list in a moment', async () => {
const started = performance.now();
const out = await extractText(text(`<ol>${'<li>item</li>'.repeat(30_000)}</ol>`), { name: 'list.html', maxChars: 1_000 });
expect(performance.now() - started).toBeLessThan(3_000);
expect(out.text.startsWith('item')).toBe(true);
});
it('leaves out scripts, styles, templates and the head', async () => {
const out = await extractText(text('<html><head><title>T</title><style>p{}</style></head><body><p>Body</p><script>alert(1)</script><noscript>ns</noscript><template>tpl</template></body></html>'), { name: 'a.html' });
expect(out.text).toBe('Body');
});
it('reads at most 5 million characters of HTML, and says so', async () => {
// Little text, a lot of source: only the cap can make this truncated.
const out = await extractText(text(`<p>Body</p><!--${'x'.repeat(5_100_000)}-->`), { name: 'huge.html' });
expect(out).toMatchObject({ status: 'ok', text: 'Body', truncated: true });
});
});
describe('calendar files', () => {
it('ignores what an alarm inside an event says', async () => {
const ics = [
'BEGIN:VCALENDAR', 'BEGIN:VEVENT', 'SUMMARY:Dentist', 'DTSTART:20261105T090000Z',
'BEGIN:VALARM', 'ACTION:DISPLAY', 'DESCRIPTION:Reminder', 'LOCATION:Nowhere', 'END:VALARM',
'LOCATION:Main street 1', 'END:VEVENT', 'END:VCALENDAR',
].join('\r\n');
expect((await extractText(text(ics), { name: 'a.ics' })).text).toBe('event: Dentist, 2026-11-05 09:00 UTC, Main street 1');
});
});
+6 -2
View File
@@ -5,7 +5,7 @@
// (pdf.js), office.ts (docx, xlsx, pptx, odt, ods, odp). Never throws: a
// file that can't be read answers with a status.
import { decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain';
import { MAX_HTML_CHARS, decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain';
import { pdfText } from './pdf';
import { officeText } from './office';
import {
@@ -68,7 +68,11 @@ async function read(bytes: Uint8Array, format: TextFormat, budget: number): Prom
if (!looksLikeText(bytes)) return { status: 'unsupported', text: '', truncated: false };
return { status: 'ok', text: decodeText(bytes).replace(/\r\n?/g, '\n'), truncated: false };
case 'html':
return { status: 'ok', text: htmlText(decodeText(bytes)), truncated: false };
{
const html = decodeText(bytes);
const cut = html.length > MAX_HTML_CHARS;
return { status: 'ok', text: htmlText(cut ? html.slice(0, MAX_HTML_CHARS) : html), truncated: cut };
}
case 'ics':
return { status: 'ok', text: icsToText(decodeText(bytes)), truncated: false };
case 'vcf':
+18 -3
View File
@@ -27,10 +27,18 @@ export function looksLikeText(bytes: Uint8Array): boolean {
return !head.includes(0);
}
/** The readable text of an HTML file: no scripts, styles or head. */
/** Characters of HTML read at most; the rest of a larger file is left out. */
export const MAX_HTML_CHARS = 5_000_000;
/**
* The readable text of an HTML file: its body without scripts, styles and
* templates. Parsed with DOMParser (linear in the file, whatever its tags);
* list markers left out, since numbering them is slow on long lists.
*/
export function htmlText(html: string): string {
const body = html.replace(/<(script|style|noscript|template|head)\b[\s\S]*?<\/\1\s*>/gi, ' ');
return htmlToPlainText(body, { paragraphSpacing: true, mailMarkup: true, imageAlt: (alt) => `[Image: ${alt}]` });
const doc = new DOMParser().parseFromString(html, 'text/html');
for (const el of Array.from(doc.querySelectorAll('script, style, noscript, template'))) el.remove();
return htmlToPlainText(doc.body?.innerHTML ?? '', { paragraphSpacing: true, imageAlt: (alt) => `[Image: ${alt}]` });
}
// ─── ICS ─────────────────────────────────────────────────────
@@ -87,9 +95,16 @@ export function icsToText(ics: string): string {
const out: string[] = [];
let current: Map<string, IcsLine> | null = null;
let kind = '';
// Inside a component of the event (an alarm): its properties aren't the event's.
let nested = 0;
for (const raw of unfold(ics)) {
const line = parseIcsLine(raw);
if (!line) continue;
if (current && line.name === 'BEGIN') { nested++; continue; }
if (current && nested > 0) {
if (line.name === 'END') nested--;
continue;
}
if (line.name === 'BEGIN' && /^(VEVENT|VTODO)$/i.test(line.value.trim())) {
current = new Map();
kind = line.value.trim().toUpperCase() === 'VTODO' ? 'task' : 'event';