From 5f2ef4c1c3c35a37c90a0bbe5152110524a09657 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:30:14 +0100 Subject: [PATCH 1/7] feat: read booking markup from mail HTML without a browser DOM --- .../__tests__/html-parser.test.ts | 83 +++ src/lib/structured-mail/html-parser.ts | 536 ++++++++++++++++++ 2 files changed, 619 insertions(+) create mode 100644 src/lib/structured-mail/__tests__/html-parser.test.ts create mode 100644 src/lib/structured-mail/html-parser.ts diff --git a/src/lib/structured-mail/__tests__/html-parser.test.ts b/src/lib/structured-mail/__tests__/html-parser.test.ts new file mode 100644 index 00000000..39913c9f --- /dev/null +++ b/src/lib/structured-mail/__tests__/html-parser.test.ts @@ -0,0 +1,83 @@ +// The app's stand-in for DOMParser (html-parser.ts): the tree it builds must +// answer extract.ts's DOM calls the way a browser document does. +import { describe, expect, it } from 'vitest'; +import { decodeCharacterReferences, parseHtmlDocument } from '../html-parser'; + +const parse = (html: string) => parseHtmlDocument(html); +const tags = (nodes: ArrayLike) => Array.from(nodes).map((el) => el.tagName.toLowerCase()); + +describe('parseHtmlDocument', () => { + it('nests elements, closes void elements and ignores stray end tags', () => { + const doc = parse('
x
y

z

after'); + const div = doc.getElementById('a')!; + expect(tags(div.querySelectorAll('*' as string))).toEqual(['span', 'br', 'img', 'p']); + expect(doc.querySelector('i')!.parentElement).toBeNull(); + expect(div.textContent).toBe('xyz'); + }); + + it('closes the elements HTML closes for you', () => { + const doc = parse('

para

block
ab
c
'); + expect(doc.getElementById('two')!.parentElement!.tagName).toBe('UL'); + expect(doc.getElementById('one')!.textContent).toBe('1'); + // A block start tag ends the open paragraph. + expect(doc.getElementById('d')!.parentElement).toBeNull(); + expect(doc.getElementById('c2')!.parentElement).toBe(doc.getElementById('c1')!.parentElement); + expect(doc.getElementById('c3')!.parentElement).not.toBe(doc.getElementById('c1')!.parentElement); + }); + + it('reads a script verbatim up to its end tag, markup and entities included', () => { + const json = '{"name":"&","x":1}'; + const doc = parse(`

ok

`); + const script = doc.querySelector('script')!; + expect(script.textContent).toBe(json); + expect(doc.querySelectorAll('b')).toHaveLength(0); + expect(doc.getElementById('after')!.parentElement!.tagName).toBe('DIV'); + }); + + it('decodes character references in text and attribute values, as the browser does', () => { + const doc = parse('Café €5 😀 ©2026 &unknown;'); + const link = doc.getElementById('l')!; + expect(link.getAttribute('href')).toBe('https://x.example/?a=1&b=2©=3'); + expect(link.getAttribute('title')).toBe('"Müller"'); + expect(link.textContent).toBe('Café €5 \u{1F600} ©2026 &unknown;'); + expect(decodeCharacterReferences('€�')).toBe('€�'); + }); + + it('keeps the first of repeated attributes and matches names case-insensitively', () => { + const el = parse('
').querySelector('div')!; + expect(el.hasAttribute('itemscope')).toBe(true); + expect(el.getAttribute('ITEMTYPE')).toBe('https://schema.org/Order'); + }); + + it('answers closest and querySelectorAll for the selectors extract.ts uses', () => { + const doc = parse('
a
b'); + const quoted = 'blockquote, .gmail_quote, .moz-forward-container, #appendonsend, #divRplyFwdMsg'; + expect(doc.getElementById('in')!.closest(quoted)!.tagName).toBe('DIV'); + expect(doc.getElementById('out')!.closest(quoted)).toBeNull(); + expect(tags(doc.querySelectorAll('[id]'))).toEqual(['span', 'span']); + expect(() => doc.querySelectorAll('div span')).toThrow(/unsupported selector/); + }); + + it('matches classes case-insensitively only in a document without a doctype (quirks mode)', () => { + expect(parse('
q
').querySelector('i')!.closest('.gmail_quote')).not.toBeNull(); + expect(parse('
q
').querySelector('i')!.closest('.gmail_quote')).toBeNull(); + }); + + it('orders elements like compareDocumentPosition', () => { + const doc = parse('
x

y

z

'); + const sep = doc.getElementById('sep')!; + const FOLLOWING = 4; + expect(sep.compareDocumentPosition(doc.getElementById('after')!) & FOLLOWING).toBeTruthy(); + expect(sep.compareDocumentPosition(doc.getElementById('inside')!) & FOLLOWING).toBeTruthy(); + expect(doc.getElementById('after')!.compareDocumentPosition(sep) & FOLLOWING).toBeFalsy(); + }); + + it('drops comments, so markup inside one is not read', () => { + const doc = parse('

x

'); + expect(doc.querySelectorAll('script')).toHaveLength(0); + }); + + it('does not enter template content', () => { + expect(parse('').querySelectorAll('[itemscope]')).toHaveLength(0); + }); +}); diff --git a/src/lib/structured-mail/html-parser.ts b/src/lib/structured-mail/html-parser.ts new file mode 100644 index 00000000..fb2806b8 --- /dev/null +++ b/src/lib/structured-mail/html-parser.ts @@ -0,0 +1,536 @@ +/** + * App only (the webmail uses the browser's DOMParser): the HTML parser + * extract.ts reads booking markup with. React Native has no DOM, so this + * builds a small detached tree that answers the few DOM calls extract.ts + * makes (querySelectorAll, closest, getAttribute, hasAttribute, textContent, + * parentElement, tagName, getElementById, compareDocumentPosition) and + * nothing else. It runs no script and loads nothing: it only reads text. + * + * The tokenizer is the one src/lib/email-html.ts sanitises mail with + * (readTag and the loop in rewriteTags, at 440a2e7a), copied because + * email-html.ts keeps it private. Tree building follows the browser where it + * matters for markup in mail: void elements, raw-text elements (a JSON-LD + * script is read verbatim up to its end tag), the end tags HTML implies + * (a

closed by a block, an

  • by the next
  • , table cells and rows), + * stray end tags ignored, character references decoded in text and + * attribute values. Misnested formatting tags are not re-parented the way the + * browser's adoption agency does; that changes nothing extract.ts asks. + */ + +// --------------------------------------------------------------------------- +// tokenizer (src/lib/email-html.ts) + +interface ParsedAttr { + name: string; + /** Raw value (character references intact), or null for a boolean attribute. */ + value: string | null; +} + +interface ParsedTag { + name: string; + attrs: ParsedAttr[]; + selfClosing: boolean; + /** Index just past the closing `>`. */ + end: number; +} + +function isHtmlSpace(ch: string | undefined): boolean { + return ch === ' ' || ch === '\t' || ch === '\n' || ch === '\f' || ch === '\r'; +} + +function isAsciiAlpha(ch: string | undefined): boolean { + if (!ch) return false; + const code = ch.charCodeAt(0) | 0x20; + return code >= 97 && code <= 122; +} + +/** The browser lowercases tag and attribute names in ASCII only. */ +function asciiLower(s: string): string { + return s.replace(/[A-Z]+/g, (m) => m.toLowerCase()); +} + +/** + * Read a start or end tag whose name begins at `i`. Returns null when the + * input ends inside the tag: the browser drops such a tag and everything + * after it. + */ +function readTag(html: string, i: number): ParsedTag | null { + const n = html.length; + let j = i; + while (j < n && !isHtmlSpace(html[j]) && html[j] !== '/' && html[j] !== '>') j++; + const name = html.slice(i, j); + const attrs: ParsedAttr[] = []; + while (j < n) { + const ch = html[j]; + if (isHtmlSpace(ch)) { j++; continue; } + if (ch === '>') return { name, attrs, selfClosing: false, end: j + 1 }; + if (ch === '/') { + j++; + if (html[j] === '>') return { name, attrs, selfClosing: true, end: j + 1 }; + continue; + } + const start = j; + j++; + while (j < n && !isHtmlSpace(html[j]) && html[j] !== '/' && html[j] !== '>' && html[j] !== '=') j++; + const attrName = html.slice(start, j); + while (j < n && isHtmlSpace(html[j])) j++; + if (html[j] !== '=') { + attrs.push({ name: attrName, value: null }); + continue; + } + j++; + while (j < n && isHtmlSpace(html[j])) j++; + if (j >= n) return null; + const q = html[j]; + if (q === '"' || q === "'") { + const close = html.indexOf(q, j + 1); + if (close < 0) return null; + attrs.push({ name: attrName, value: html.slice(j + 1, close) }); + j = close + 1; + } else if (q === '>') { + attrs.push({ name: attrName, value: '' }); + } else { + const s = j; + while (j < n && !isHtmlSpace(html[j]) && html[j] !== '>') j++; + attrs.push({ name: attrName, value: html.slice(s, j) }); + } + } + return null; +} + +// --------------------------------------------------------------------------- +// character references + +const LATIN1_NAMES = ( + 'nbsp iexcl cent pound curren yen brvbar sect uml copy ordf laquo not shy reg macr deg plusmn sup2 sup3 ' + + 'acute micro para middot cedil sup1 ordm raquo frac14 frac12 frac34 iquest Agrave Aacute Acirc Atilde Auml ' + + 'Aring AElig Ccedil Egrave Eacute Ecirc Euml Igrave Iacute Icirc Iuml ETH Ntilde Ograve Oacute Ocirc Otilde ' + + 'Ouml times Oslash Ugrave Uacute Ucirc Uuml Yacute THORN szlig agrave aacute acirc atilde auml aring aelig ' + + 'ccedil egrave eacute ecirc euml igrave iacute icirc iuml eth ntilde ograve oacute ocirc otilde ouml divide ' + + 'oslash ugrave uacute ucirc uuml yacute thorn yuml' +).split(' '); + +/** Named references the browser also reads without a `;` (the legacy set). */ +const LEGACY_ENTITIES: Record = { amp: 38, lt: 60, gt: 62, quot: 34, AMP: 38, LT: 60, GT: 62, QUOT: 34 }; +LATIN1_NAMES.forEach((name, i) => { LEGACY_ENTITIES[name] = 160 + i; }); + +/** Named references that need their `;`: the ones mail commonly carries beyond Latin-1. */ +const NAMED_ENTITIES: Record = { + ...LEGACY_ENTITIES, + apos: 39, euro: 0x20ac, ndash: 0x2013, mdash: 0x2014, lsquo: 0x2018, rsquo: 0x2019, sbquo: 0x201a, + ldquo: 0x201c, rdquo: 0x201d, bdquo: 0x201e, dagger: 0x2020, Dagger: 0x2021, bull: 0x2022, hellip: 0x2026, + permil: 0x2030, prime: 0x2032, Prime: 0x2033, lsaquo: 0x2039, rsaquo: 0x203a, trade: 0x2122, + larr: 0x2190, uarr: 0x2191, rarr: 0x2192, darr: 0x2193, harr: 0x2194, rArr: 0x21d2, lArr: 0x21d0, + minus: 0x2212, ensp: 0x2002, emsp: 0x2003, thinsp: 0x2009, zwnj: 0x200c, zwj: 0x200d, lrm: 0x200e, + rlm: 0x200f, OElig: 0x152, oelig: 0x153, Scaron: 0x160, scaron: 0x161, Yuml: 0x178, fnof: 0x192, + circ: 0x2c6, tilde: 0x2dc, Alpha: 0x391, alpha: 0x3b1, Beta: 0x392, beta: 0x3b2, mu: 0x3bc, pi: 0x3c0, + Omega: 0x3a9, omega: 0x3c9, infin: 0x221e, ne: 0x2260, le: 0x2264, ge: 0x2265, asymp: 0x2248, + check: 0x2713, star: 0x2606, starf: 0x2605, hearts: 0x2665, loz: 0x25ca, NewLine: 10, Tab: 9, + colon: 58, comma: 44, period: 46, sol: 47, bsol: 92, num: 35, dollar: 36, percnt: 37, lpar: 40, + rpar: 41, ast: 42, plus: 43, equals: 61, quest: 63, commat: 64, lsqb: 91, rsqb: 93, lowbar: 95, + grave: 96, lcub: 123, rcub: 125, verbar: 124, excl: 33, +}; + +/** C1 numeric references the browser reads as windows-1252. */ +const WINDOWS_1252: Record = { + 0x80: 0x20ac, 0x82: 0x201a, 0x83: 0x192, 0x84: 0x201e, 0x85: 0x2026, 0x86: 0x2020, 0x87: 0x2021, + 0x88: 0x2c6, 0x89: 0x2030, 0x8a: 0x160, 0x8b: 0x2039, 0x8c: 0x152, 0x8e: 0x17d, 0x91: 0x2018, + 0x92: 0x2019, 0x93: 0x201c, 0x94: 0x201d, 0x95: 0x2022, 0x96: 0x2013, 0x97: 0x2014, 0x98: 0x2dc, + 0x99: 0x2122, 0x9a: 0x161, 0x9b: 0x203a, 0x9c: 0x153, 0x9e: 0x17e, 0x9f: 0x178, +}; + +function codePointText(code: number): string { + if (!Number.isFinite(code) || code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return String.fromCharCode(0xfffd); + return String.fromCodePoint(WINDOWS_1252[code] ?? code); +} + +const REFERENCE = /&(?:#[xX]([0-9a-fA-F]+);?|#([0-9]+);?|([A-Za-z][A-Za-z0-9]*);?)/g; + +/** Decode character references the way the browser does in text (or, with `attribute`, in a value). */ +export function decodeCharacterReferences(text: string, attribute = false): string { + if (!text.includes('&')) return text; + return text.replace(REFERENCE, (match: string, hex: string | undefined, dec: string | undefined, name: string | undefined, offset: number) => { + if (hex !== undefined) return codePointText(parseInt(hex, 16)); + if (dec !== undefined) return codePointText(parseInt(dec, 10)); + const withSemicolon = match.endsWith(';'); + if (withSemicolon && name && NAMED_ENTITIES[name] !== undefined) return String.fromCodePoint(NAMED_ENTITIES[name]); + // Without `;` only a legacy name counts, read as the longest one that + // prefixes the run ("©2026" is "©2026"); in a value not when an + // alphanumeric or `=` follows it ("?a=1©=2" stays a query string). + for (let len = Math.min(name?.length ?? 0, 6); len >= 2; len--) { + const candidate = name!.slice(0, len); + const code = LEGACY_ENTITIES[candidate]; + if (code === undefined) continue; + const rest = match.slice(1 + len); + if (attribute) { + const next = rest[0] ?? text[offset + match.length]; + if (next !== undefined && /[A-Za-z0-9=]/.test(next)) return match; + } + return String.fromCodePoint(code) + rest; + } + return match; + }); +} + +// --------------------------------------------------------------------------- +// tree + +const VOID_ELEMENTS = new Set([ + 'area', 'base', 'basefont', 'bgsound', 'br', 'col', 'embed', 'frame', 'hr', 'img', 'input', 'keygen', + 'link', 'meta', 'param', 'source', 'track', 'wbr', +]); + +/** Read as text up to their end tag. A DOMParser document runs no script, so