mirror of
https://github.com/bulwarkmail/webmail.git
synced 2026-11-06 00:15:32 +00:00
fix: read HTML and calendar attachments in time with their size, without alarm texts
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { extractText } from '@/lib/attachment-text';
|
||||
|
||||
// HTML and calendar files: reading takes time in step with the file, and an
|
||||
// event is described by its own properties only.
|
||||
|
||||
const text = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
describe('HTML', () => {
|
||||
it('reads an HTML file full of unclosed tags in a moment', async () => {
|
||||
// 400 KB of "<head": a regular expression over the source took 8.6 s for this.
|
||||
const started = performance.now();
|
||||
const out = await extractText(text(`<p>Agenda</p>${'<head'.repeat(80_000)}`), { name: 'odd.html' });
|
||||
expect(performance.now() - started).toBeLessThan(3_000);
|
||||
expect(out.status).toBe('ok');
|
||||
});
|
||||
|
||||
it('reads a long numbered list in a moment', async () => {
|
||||
const started = performance.now();
|
||||
const out = await extractText(text(`<ol>${'<li>item</li>'.repeat(30_000)}</ol>`), { name: 'list.html', maxChars: 1_000 });
|
||||
expect(performance.now() - started).toBeLessThan(3_000);
|
||||
expect(out.text.startsWith('item')).toBe(true);
|
||||
});
|
||||
|
||||
it('leaves out scripts, styles, templates and the head', async () => {
|
||||
const out = await extractText(text('<html><head><title>T</title><style>p{}</style></head><body><p>Body</p><script>alert(1)</script><noscript>ns</noscript><template>tpl</template></body></html>'), { name: 'a.html' });
|
||||
expect(out.text).toBe('Body');
|
||||
});
|
||||
|
||||
it('reads at most 5 million characters of HTML, and says so', async () => {
|
||||
// Little text, a lot of source: only the cap can make this truncated.
|
||||
const out = await extractText(text(`<p>Body</p><!--${'x'.repeat(5_100_000)}-->`), { name: 'huge.html' });
|
||||
expect(out).toMatchObject({ status: 'ok', text: 'Body', truncated: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('calendar files', () => {
|
||||
it('ignores what an alarm inside an event says', async () => {
|
||||
const ics = [
|
||||
'BEGIN:VCALENDAR', 'BEGIN:VEVENT', 'SUMMARY:Dentist', 'DTSTART:20261105T090000Z',
|
||||
'BEGIN:VALARM', 'ACTION:DISPLAY', 'DESCRIPTION:Reminder', 'LOCATION:Nowhere', 'END:VALARM',
|
||||
'LOCATION:Main street 1', 'END:VEVENT', 'END:VCALENDAR',
|
||||
].join('\r\n');
|
||||
expect((await extractText(text(ics), { name: 'a.ics' })).text).toBe('event: Dentist, 2026-11-05 09:00 UTC, Main street 1');
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,7 @@
|
||||
// (pdf.js), office.ts (docx, xlsx, pptx, odt, ods, odp). Never throws: a
|
||||
// file that can't be read answers with a status.
|
||||
|
||||
import { decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain';
|
||||
import { MAX_HTML_CHARS, decodeText, htmlText, icsToText, looksLikeText, vcfText } from './plain';
|
||||
import { pdfText } from './pdf';
|
||||
import { officeText } from './office';
|
||||
import {
|
||||
@@ -68,7 +68,11 @@ async function read(bytes: Uint8Array, format: TextFormat, budget: number): Prom
|
||||
if (!looksLikeText(bytes)) return { status: 'unsupported', text: '', truncated: false };
|
||||
return { status: 'ok', text: decodeText(bytes).replace(/\r\n?/g, '\n'), truncated: false };
|
||||
case 'html':
|
||||
return { status: 'ok', text: htmlText(decodeText(bytes)), truncated: false };
|
||||
{
|
||||
const html = decodeText(bytes);
|
||||
const cut = html.length > MAX_HTML_CHARS;
|
||||
return { status: 'ok', text: htmlText(cut ? html.slice(0, MAX_HTML_CHARS) : html), truncated: cut };
|
||||
}
|
||||
case 'ics':
|
||||
return { status: 'ok', text: icsToText(decodeText(bytes)), truncated: false };
|
||||
case 'vcf':
|
||||
|
||||
@@ -27,10 +27,18 @@ export function looksLikeText(bytes: Uint8Array): boolean {
|
||||
return !head.includes(0);
|
||||
}
|
||||
|
||||
/** The readable text of an HTML file: no scripts, styles or head. */
|
||||
/** Characters of HTML read at most; the rest of a larger file is left out. */
|
||||
export const MAX_HTML_CHARS = 5_000_000;
|
||||
|
||||
/**
|
||||
* The readable text of an HTML file: its body without scripts, styles and
|
||||
* templates. Parsed with DOMParser (linear in the file, whatever its tags);
|
||||
* list markers left out, since numbering them is slow on long lists.
|
||||
*/
|
||||
export function htmlText(html: string): string {
|
||||
const body = html.replace(/<(script|style|noscript|template|head)\b[\s\S]*?<\/\1\s*>/gi, ' ');
|
||||
return htmlToPlainText(body, { paragraphSpacing: true, mailMarkup: true, imageAlt: (alt) => `[Image: ${alt}]` });
|
||||
const doc = new DOMParser().parseFromString(html, 'text/html');
|
||||
for (const el of Array.from(doc.querySelectorAll('script, style, noscript, template'))) el.remove();
|
||||
return htmlToPlainText(doc.body?.innerHTML ?? '', { paragraphSpacing: true, imageAlt: (alt) => `[Image: ${alt}]` });
|
||||
}
|
||||
|
||||
// ─── ICS ─────────────────────────────────────────────────────
|
||||
@@ -87,9 +95,16 @@ export function icsToText(ics: string): string {
|
||||
const out: string[] = [];
|
||||
let current: Map<string, IcsLine> | null = null;
|
||||
let kind = '';
|
||||
// Inside a component of the event (an alarm): its properties aren't the event's.
|
||||
let nested = 0;
|
||||
for (const raw of unfold(ics)) {
|
||||
const line = parseIcsLine(raw);
|
||||
if (!line) continue;
|
||||
if (current && line.name === 'BEGIN') { nested++; continue; }
|
||||
if (current && nested > 0) {
|
||||
if (line.name === 'END') nested--;
|
||||
continue;
|
||||
}
|
||||
if (line.name === 'BEGIN' && /^(VEVENT|VTODO)$/i.test(line.value.trim())) {
|
||||
current = new Map();
|
||||
kind = line.value.trim().toUpperCase() === 'VTODO' ? 'task' : 'event';
|
||||
|
||||
Reference in New Issue
Block a user