diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts index 9fab19789..3c91d7c44 100644 --- a/lib/admin/outgoing-hosts-registry.ts +++ b/lib/admin/outgoing-hosts-registry.ts @@ -170,6 +170,7 @@ export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[ { host: 'www.youtube-nocookie.com', why: 'example in a comment', files: ['lib/plugin-types.ts'] }, { host: 'bulwark-webmail.vercel.app', why: 'sample message text', files: ['stores/email-store.ts'] }, { host: 'randomuser.me', why: 'docs links in a comment of the mock server', files: ['app/api/dev-jmap/[...path]/route.ts'] }, + { host: 'schema.org', why: 'vocabulary names in booking markup, matched and never requested', files: ['lib/structured-mail/extract.ts'] }, ]; /** diff --git a/lib/structured-mail/__tests__/extract.test.ts b/lib/structured-mail/__tests__/extract.test.ts new file mode 100644 index 000000000..a6d1deec1 --- /dev/null +++ b/lib/structured-mail/__tests__/extract.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest'; +import { extractStructuredData, mayContainStructuredData } from '../extract'; + +const page = (head: string, body = '') => `${head}${body}`; +const ld = (value: unknown, type = 'application/ld+json') => ``; +const order = { '@context': 'http://schema.org', '@type': 'Order', orderNumber: '42' }; + +describe('extractStructuredData: JSON-LD', () => { + it('reads blocks in head and body, type compared loosely', () => { + const html = page(ld(order), ld({ ...order, orderNumber: '43' }, ' Application/LD+JSON ')); + expect(extractStructuredData(html).map((f) => f.item)).toEqual([order, { ...order, orderNumber: '43' }]); + }); + + it('reads a block nested inside a microdata tree', () => { + const html = page('', `
${ld(order)}
`); + expect(extractStructuredData(html).filter((f) => f.source === 'json-ld').map((f) => f.item)).toEqual([order]); + }); + + it('skips mail without any type it reads, without parsing', () => { + const html = page(ld({ '@context': 'http://schema.org', '@type': 'PromotionCard' })); + expect(mayContainStructuredData(html)).toBe(false); + expect(extractStructuredData(html)).toEqual([]); + }); + + it('ignores markup inside quoted or forwarded parts', () => { + const quoted = `
${ld(order)}
${ld(order)}
`; + const outlook = `

FYI

From: someone
${ld(order)}
`; + expect(extractStructuredData(page('', quoted))).toEqual([]); + expect(extractStructuredData(page('', outlook))).toEqual([]); + }); + + it('caps the number of blocks it reads', () => { + const html = page(Array.from({ length: 60 }, () => ld(order)).join('')); + expect(extractStructuredData(html)).toHaveLength(50); + }); + + it('runs no script and loads nothing while parsing', () => { + const html = page('', `${ld(order)}`); + extractStructuredData(html); + expect((window as unknown as Record).__structuredMailRan).toBeUndefined(); + }); +}); + +describe('extractStructuredData: microdata', () => { + it('reads values from content, href, src, datetime and text', () => { + const html = page('', ` +
+ + track + + + DHL + Express +
`); + expect(extractStructuredData(html)).toEqual([{ + source: 'microdata', + item: { + '@context': 'https://schema.org', '@type': 'ParcelDelivery', trackingNumber: '123', + trackingUrl: 'https://track.example.com/123', image: 'https://img.example.com/a.png', + expectedArrivalUntil: '2026-11-02', carrier: 'DHL Express', + }, + }]); + }); + + it('nests items, splits itemprop lists and turns repeats into arrays', () => { + const html = page('', ` +
+ +
+ + +
+
+
+
+
+
+
`); + const [{ item }] = extractStructuredData(html); + const shop = { '@type': 'Organization', name: 'Shop' }; + expect(item).toEqual({ + '@context': 'https://schema.org', '@type': 'Order', orderNumber: '7', merchant: shop, seller: shop, + orderStatus: 'http://schema.org/OrderProcessing', + acceptedOffer: [ + { '@type': 'Offer', itemOffered: { '@type': 'Product', name: 'One' } }, + { '@type': 'Offer', itemOffered: { '@type': 'Product', name: 'Two' } }, + ], + }); + }); + + it('ignores items not typed on schema.org', () => { + const html = page('', ` +
+
`); + expect(extractStructuredData(html)).toEqual([]); + }); + + it('accepts several itemtypes and keeps the schema.org ones', () => { + const html = page('', '
'); + expect(extractStructuredData(html)[0].item).toMatchObject({ '@type': 'Order', orderNumber: '1' }); + }); +}); diff --git a/lib/structured-mail/__tests__/json-repair.test.ts b/lib/structured-mail/__tests__/json-repair.test.ts new file mode 100644 index 000000000..4c2eef65c --- /dev/null +++ b/lib/structured-mail/__tests__/json-repair.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest'; +import { parseLenientJson } from '../json-repair'; + +const A = { '@type': 'FlightReservation', reservationNumber: 'A1' }; +const B = { '@type': 'FlightReservation', reservationNumber: 'B2' }; + +describe('parseLenientJson', () => { + it('parses valid JSON unchanged', () => { + expect(parseLenientJson(JSON.stringify([A, B]))).toEqual([A, B]); + }); + + it('adds the commas Eurowings leaves out between array objects', () => { + expect(parseLenientJson(`[${JSON.stringify(A)}\n${JSON.stringify(B)}]`)).toEqual([A, B]); + expect(parseLenientJson(`[${JSON.stringify(A)}${JSON.stringify(B)}]`)).toEqual([A, B]); + }); + + it('adds the brackets Volotea leaves out, with or without commas', () => { + expect(parseLenientJson(`${JSON.stringify(A)},\n${JSON.stringify(B)}`)).toEqual([A, B]); + expect(parseLenientJson(`${JSON.stringify(A)}\n${JSON.stringify(B)}`)).toEqual([A, B]); + }); + + it('drops the trailing commas Eventbrite writes', () => { + expect(parseLenientJson('{"@type": "EventReservation", "reservationNumber": "E1", }')).toEqual({ '@type': 'EventReservation', reservationNumber: 'E1' }); + expect(parseLenientJson('[{"a": [1, 2, ], }, ]')).toEqual([{ a: [1, 2] }]); + }); + + it('decodes the XML entities Airbnb encodes its JSON with', () => { + const encoded = JSON.stringify(A).replace(/"/g, '"'); + expect(parseLenientJson(encoded)).toEqual(A); + expect(parseLenientJson('{"name": "Café & Bar"}')).toEqual({ name: 'Café & Bar' }); + }); + + it('strips comment and CDATA wrappers and escapes raw newlines in strings', () => { + expect(parseLenientJson(``)).toEqual(A); + expect(parseLenientJson(``)).toEqual(A); + expect(parseLenientJson('{"name": "two\nlines"}')).toEqual({ name: 'two\nlines' }); + }); + + it('leaves commas and braces inside strings alone', () => { + expect(parseLenientJson('{"name": "a, } { b", }')).toEqual({ name: 'a, } { b' }); + }); + + it('gives up on input it cannot repair', () => { + expect(parseLenientJson('{"a": ')).toBeUndefined(); + expect(parseLenientJson('not json at all')).toBeUndefined(); + expect(parseLenientJson('')).toBeUndefined(); + expect(parseLenientJson(`[${'x'.repeat(200_001)}]`)).toBeUndefined(); + }); +}); diff --git a/lib/structured-mail/__tests__/normalize.test.ts b/lib/structured-mail/__tests__/normalize.test.ts new file mode 100644 index 000000000..d4fc14473 --- /dev/null +++ b/lib/structured-mail/__tests__/normalize.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, it } from 'vitest'; +import { normalizeItems } from '../normalize'; +import type { LodgingBooking, Shipment, TransportBooking } from '../types'; +import { address, amount, lenientDate, link, moment, orderStatus, reservationStatus } from '../values'; + +const ctx = 'http://schema.org'; +const norm = (...items: unknown[]) => normalizeItems(items.map((item) => ({ item, source: 'json-ld' as const }))); + +describe('value readers', () => { + it('reads ISO times with every offset form and computes the instant', () => { + expect(moment('2027-03-04T20:15:00-08:00')).toEqual({ date: '2027-03-04', time: '20:15:00', offsetMinutes: -480, utc: '2027-03-05T04:15:00Z' }); + expect(moment('2024-06-04T11:44:00+0200')).toEqual({ date: '2024-06-04', time: '11:44:00', offsetMinutes: 120, utc: '2024-06-04T09:44:00Z' }); + expect(moment('2026-01-01T10:00+05')).toMatchObject({ offsetMinutes: 300, utc: '2026-01-01T05:00:00Z' }); + expect(moment('2026-01-01T10:00:00.123Z')).toMatchObject({ time: '10:00:00', offsetMinutes: 0, utc: '2026-01-01T10:00:00Z' }); + expect(moment('2026-01-01T10:00:00')).toEqual({ date: '2026-01-01', time: '10:00:00' }); + expect(moment('2026-01-01')).toEqual({ date: '2026-01-01' }); + }); + + it('refuses broken times instead of guessing', () => { + expect(moment('2024-10-13CEST03:10:23')).toBeUndefined(); + expect(moment('2026-02-30T10:00:00Z')).toBeUndefined(); + expect(moment('2026-01-01T25:00:00Z')).toBeUndefined(); + expect(moment('2026-01-01T10:00:00+15:00')).toBeUndefined(); + expect(moment('next Friday')).toBeUndefined(); + }); + + it('keeps only the date of a broken value where a date is enough', () => { + expect(lenientDate('2024-10-13CEST03:10:23')).toEqual({ date: '2024-10-13' }); + expect(lenientDate('2026-03-12T12:00:00-08:00')).toEqual({ date: '2026-03-12' }); + expect(lenientDate('20241013')).toBeUndefined(); + }); + + it('reads both forms of the status enums over http and https', () => { + expect(reservationStatus('http://schema.org/ReservationConfirmed')).toBe('confirmed'); + expect(reservationStatus('https://schema.org/Confirmed')).toBe('confirmed'); + expect(reservationStatus('ReservationCancelled')).toBe('cancelled'); + expect(reservationStatus(['', 'http://schema.org/Cancelled'])).toBe('cancelled'); + expect(reservationStatus('http://schema.org/Bogus')).toBeUndefined(); + expect(orderStatus(['', 'http://schema.org/OrderProcessing'])).toBe('processing'); + expect(orderStatus('http://schema.org/OrderStatus/InTransit')).toBe('in_transit'); + expect(orderStatus('OrderDelivered')).toBe('delivered'); + expect(orderStatus('Processing')).toBe('processing'); + }); + + it('reads prices in dot and German notation and refuses ambiguous ones', () => { + expect(amount(25.99)).toBe('25.99'); + expect(amount('14.9700')).toBe('14.97'); + expect(amount('399,00 €')).toBe('399.00'); + expect(amount('1.299,00 EUR')).toBe('1299.00'); + expect(amount('$1,299.50')).toBe('1299.50'); + expect(amount('1,299')).toBeUndefined(); + }); + + it('keeps only http(s) links with a real host', () => { + expect(link('https://www.fedex.com/track/1')).toEqual({ url: 'https://www.fedex.com/track/1', host: 'fedex.com' }); + expect(link({ '@type': 'ViewAction', target: 'http://shop.example.com/o/1' })?.host).toBe('shop.example.com'); + expect(link('http://default/buchung/VK4XKD')).toBeUndefined(); + expect(link('javascript:alert(1)')).toBeUndefined(); + expect(link('https://user:pw@example.com/')).toBeUndefined(); + expect(link('http://192.168.1.1/x')).toBeUndefined(); + expect(link('http://localhost:3000/x')).toBeUndefined(); + expect(link('/relative')).toBeUndefined(); + }); + + it('reads addresses given as objects, strings and country objects', () => { + expect(address('Unter den Linden 77, Berlin')).toEqual({ street: 'Unter den Linden 77, Berlin' }); + expect(address({ streetAddress: 'Koppenstr. 3', addressLocality: 'Berlin', postalCode: '10243', addressCountry: 'de', addressRegion: '' })) + .toEqual({ street: 'Koppenstr. 3', locality: 'Berlin', postalCode: '10243', country: 'DE' }); + expect(address({ addressLocality: 'Paris', addressCountry: { '@type': 'Country', name: 'France' } })) + .toEqual({ locality: 'Paris', countryName: 'France' }); + }); +}); + +describe('normalizeItems', () => { + const flight = { + '@context': ctx, '@type': 'FlightReservation', reservationNumber: 'QX7K2M', + reservationStatus: 'http://schema.org/ReservationConfirmed', + underName: { '@type': 'Person', name: 'Erika Mustermann' }, airplaneSeat: '12C', + checkinUrl: 'https://www.lufthansa.example.com/checkin', + reservationFor: { + '@type': 'Flight', flightNumber: 'LH400', + airline: { '@type': 'Airline', name: 'Lufthansa', iataCode: 'LH' }, + departureAirport: { '@type': 'Airport', name: 'Frankfurt', iataCode: 'fra' }, departureTime: '2026-12-01T10:15:00+01:00', + arrivalAirport: { '@type': 'Airport', name: 'New York JFK', iataCode: 'JFK' }, arrivalTime: '2026-12-01T13:05:00-05:00', + }, + }; + + it('reads a flight reservation, splitting the airline code off the flight number', () => { + const [card] = norm(flight) as TransportBooking[]; + expect(card).toEqual({ + key: 'flight|ref:qx7k2m', kind: 'flight', reference: 'QX7K2M', status: 'confirmed', passengers: ['Erika Mustermann'], + provider: 'Lufthansa', links: { checkin: { url: 'https://www.lufthansa.example.com/checkin', host: 'lufthansa.example.com' } }, + source: 'json-ld', + legs: [{ + carrier: 'Lufthansa', carrierCode: 'LH', number: '400', + departure: { place: { name: 'Frankfurt', code: 'FRA' }, at: { date: '2026-12-01', time: '10:15:00', offsetMinutes: 60, utc: '2026-12-01T09:15:00Z' } }, + arrival: { place: { name: 'New York JFK', code: 'JFK' }, at: { date: '2026-12-01', time: '13:05:00', offsetMinutes: -300, utc: '2026-12-01T18:05:00Z' } }, + seats: [{ passenger: 'Erika Mustermann', seat: '12C' }], status: 'confirmed', + }], + }); + }); + + it('flattens arrays, @graph and reservationFor arrays', () => { + const two = { ...flight, reservationFor: [flight.reservationFor, { ...flight.reservationFor, flightNumber: '401' }] }; + expect(norm([flight, flight]).length).toBe(2); + expect(norm({ '@context': ctx, '@graph': [flight, flight] }).length).toBe(2); + const legs = norm(two) as TransportBooking[]; + expect(legs.map((c) => c.legs[0].number)).toEqual(['400', '401']); + }); + + it('accepts https contexts, @vocab and several @types, and drops foreign or missing contexts', () => { + expect(norm({ ...flight, '@context': 'https://schema.org/' })).toHaveLength(1); + expect(norm({ ...flight, '@context': { '@vocab': 'http://schema.org/' } })).toHaveLength(1); + expect(norm({ ...flight, '@type': ['Thing', 'FlightReservation'] })).toHaveLength(1); + expect(norm({ ...flight, '@context': 'http://example.com' })).toHaveLength(0); + const { '@context': _dropped, ...noContext } = flight; + expect(norm(noContext)).toHaveLength(0); + }); + + it('reads bus trips the way FlixBus writes them', () => { + const [card] = norm({ + '@context': ctx, '@type': 'BusReservation', reservationNumber: 3165152579, + underName: { '@type': 'Person', name: 'Max Mustermann' }, + reservationFor: { + '@type': 'BusTrip', busNumber: 'FLX30', busName: 'Berlin Ostbahnhof', busCompany: { '@type': 'Organization', name: 'Flix' }, + departureBusStop: { '@type': 'BusStop', name: 'Hannover Hauptbahnhof', address: { '@type': 'PostalAddress', addressLocality: 'Hannover', addressCountry: 'DE' } }, + departureTime: '2024-06-04T11:44:00+0200', + arrivalBusStop: { '@type': 'BusStop', name: 'Berlin Ostbahnhof' }, arrivalTime: '2024-06-04T13:44:00+0200', + }, + reservedTicket: { '@type': 'Ticket', ticketedSeat: { '@type': 'Seat', seatingType: 'Coach', seatNumber: '6D' } }, + }) as TransportBooking[]; + expect(card.reference).toBe('3165152579'); + expect(card.legs[0]).toMatchObject({ carrier: 'Flix', number: 'FLX30', seats: [{ passenger: 'Max Mustermann', seat: '6D' }] }); + expect(card.legs[0].departure.place).toEqual({ name: 'Hannover Hauptbahnhof', address: { locality: 'Hannover', country: 'DE' } }); + expect(JSON.stringify(card)).not.toContain('busName'); + }); + + it('reads hotels from Hotel subtypes with legacy names', () => { + const [card] = norm({ + '@context': ctx, '@type': 'LodgingReservation', reservationId: 'h-1', + reservationFor: { '@type': 'Hotel', name: 'Hotel Adlon', address: 'Unter den Linden 77, 10117 Berlin', telephone: '+49 30 1' }, + checkinTime: '2026-12-04T15:00:00+01:00', checkoutTime: '2026-12-06T12:00:00+01:00', Url: 'https://hotel.example.com/b/h-1', + }) as LodgingBooking[]; + expect(card).toMatchObject({ + kind: 'lodging', reference: 'h-1', name: 'Hotel Adlon', phone: '+49 30 1', address: { street: 'Unter den Linden 77, 10117 Berlin' }, + checkin: { date: '2026-12-04', time: '15:00:00' }, checkout: { date: '2026-12-06', time: '12:00:00' }, + links: { manage: { host: 'hotel.example.com' } }, + }); + }); + + it('reads restaurants, events with a date-only end, and rental cars', () => { + const [restaurant, event, car] = norm( + { '@context': ctx, '@type': 'FoodEstablishmentReservation', reservationNumber: 'R9', partySize: '2', startTime: '2026-12-05T19:30:00+01:00', reservationFor: { '@type': 'Restaurant', name: 'Tim Raue' } }, + { '@context': ctx, '@type': 'EventReservation', reservationNumber: 'E9', reservationFor: { '@type': 'MusicEvent', name: 'Konzert', startDate: '2026-12-05T20:00:00+01:00', endDate: '2026-12-05', location: { '@type': 'Place', name: 'Halle', address: { addressCountry: 'DE' } } } }, + { '@context': ctx, '@type': 'RentalCarReservation', reservationNumber: 'C9', reservationFor: { '@type': 'Car', brand: { name: 'VW' }, model: 'Golf', rentalCompany: { name: 'Sixt' } }, pickupTime: '2026-12-04T09:00:00+01:00', dropoffTime: '2026-12-07T18:00:00+01:00', pickupLocation: { name: 'BER' }, dropoffLocation: 'BER' }, + ); + expect(restaurant).toMatchObject({ kind: 'restaurant', name: 'Tim Raue', partySize: 2, start: { time: '19:30:00' } }); + expect(event).toMatchObject({ kind: 'event', name: 'Konzert', end: { date: '2026-12-05', time: '23:59:59' }, venue: { name: 'Halle' } }); + expect(car).toMatchObject({ kind: 'rentalcar', company: 'Sixt', car: 'VW Golf', pickup: { place: { name: 'BER' } }, dropoff: { place: { name: 'BER' } } }); + }); + + it('reads orders with German prices and parcels with their order', () => { + const [orderCard, parcelCard] = norm( + { + '@context': ctx, '@type': 'Order', merchant: { '@type': 'Organization', name: 'Kaufland.de' }, orderNumber: 'M1234AB', + price: '399,00 €', priceCurrency: 'EUR', orderStatus: ['', 'http://schema.org/OrderProcessing'], + acceptedOffer: { '@type': 'Offer', itemOffered: { '@type': 'Product', name: '3D-Drucker', image: 'https://media.example.com/p.jpg' }, eligibleQuantity: { value: '1' } }, + url: 'https://www.kaufland.de/account/orders', + }, + { + '@context': ctx, '@type': 'ParcelDelivery', carrier: 'DHL', trackingNumber: '0034', trackingUrl: 'https://www.dhl.de/t?id=0034', + expectedArrivalUntil: '2024-10-13CEST03:10:23', itemShipped: [{ '@type': 'Product', name: 'A' }, 'B'], + partOfOrder: { '@type': 'Order', orderNumber: 'M1234AB', merchant: 'Kaufland.de', orderStatus: 'http://schema.org/OrderInTransit' }, + }, + ) as Shipment[]; + expect(orderCard).toEqual({ + kind: 'shipment', key: 'shipment|order:m1234ab', merchant: 'Kaufland.de', orderNumber: 'M1234AB', status: 'processing', + items: [{ name: '3D-Drucker', quantity: 1, image: { url: 'https://media.example.com/p.jpg', host: 'media.example.com' } }], + total: { amount: '399.00', currency: 'EUR' }, orderUrl: { url: 'https://www.kaufland.de/account/orders', host: 'kaufland.de' }, + parcels: [], source: 'json-ld', + }); + expect(parcelCard).toMatchObject({ + key: 'shipment|order:m1234ab', merchant: 'Kaufland.de', status: 'in_transit', + parcels: [{ carrier: 'DHL', trackingNumber: '0034', trackingUrl: { host: 'dhl.de' }, expectedBy: { date: '2024-10-13' }, items: [{ name: 'A' }, { name: 'B' }] }], + }); + }); + + it('drops types it does not read', () => { + expect(norm( + { '@context': ctx, '@type': 'PromotionCard', image: 'https://x.example.com/a.png' }, + { '@context': ctx, '@type': 'EmailMessage', potentialAction: { '@type': 'ViewAction' } }, + { '@context': ctx, '@type': 'Invoice', totalPaymentDue: 3 }, + )).toEqual([]); + }); +}); diff --git a/lib/structured-mail/__tests__/validate-resolve.test.ts b/lib/structured-mail/__tests__/validate-resolve.test.ts new file mode 100644 index 000000000..188aa5a92 --- /dev/null +++ b/lib/structured-mail/__tests__/validate-resolve.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from 'vitest'; +import { normalizeItems } from '../normalize'; +import { needsAirportZones, resolveCards } from '../resolve'; +import { loadAirportZones } from '../time-zones'; +import type { StructuredCard, TransportBooking } from '../types'; +import { validateCard } from '../validate'; + +const ctx = 'http://schema.org'; +const MAIL = new Date('2026-10-30T10:00:00Z'); +const one = (item: unknown) => normalizeItems([{ item, source: 'json-ld' }])[0]; +const problems = (item: unknown, mailDate = MAIL) => validateCard(one(item), mailDate).problems; + +function flight(dep: [string, string, string], arr: [string, string, string], extra: Record = {}) { + return { + '@context': ctx, '@type': 'FlightReservation', reservationNumber: 'T1', underName: 'Erika Mustermann', + reservationFor: { + '@type': 'Flight', flightNumber: '100', airline: { name: 'Test Air', iataCode: 'TA' }, + departureAirport: { name: dep[0], iataCode: dep[1] }, departureTime: dep[2], + arrivalAirport: { name: arr[0], iataCode: arr[1] }, arrivalTime: arr[2], + }, + ...extra, + }; +} + +const sfoJfk = flight(['San Francisco', 'SFO', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']); + +describe('validateCard', () => { + it('accepts a complete flight near the mail date', () => { + expect(problems(sfoJfk)).toEqual([]); + }); + + it('rejects flights with missing times or places', () => { + const noArrival = flight(['San Francisco', 'SFO', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '']); + const bareDate = flight(['San Francisco', 'SFO', '2026-12-04'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']); + const noPlace = flight(['', '', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']); + expect(problems(noArrival)).toContain('arrival time missing'); + expect(problems(bareDate)).toContain('departure time missing'); + expect(problems(noPlace)).toContain('departure place missing'); + }); + + it('rejects bookings years away from the mail', () => { + expect(problems(sfoJfk, new Date('2024-11-12T00:00:00Z'))).toContain('date too far from the mail'); + expect(problems(sfoJfk, new Date('2028-01-01T00:00:00Z'))).toContain('date too far from the mail'); + }); + + it('rejects copies of Google documentation examples', () => { + expect(problems({ ...sfoJfk, reservationNumber: 'RXJ34P' })).toContain('documentation example'); + expect(problems({ '@context': ctx, '@type': 'ParcelDelivery', trackingNumber: '3453291231', carrier: 'FedEx' })).toContain('documentation example'); + }); + + it('rejects stays that end before they start', () => { + const stay = (checkin: string, checkout: string) => ({ + '@context': ctx, '@type': 'LodgingReservation', reservationNumber: 'H', reservationFor: { '@type': 'Hotel', name: 'Hotel' }, + checkinDate: checkin, checkoutDate: checkout, + }); + expect(problems(stay('2026-12-04', '2026-12-06'))).toEqual([]); + expect(problems(stay('2026-12-06', '2026-12-04'))).toContain('check-out not after check-in'); + expect(problems(stay('2026-12-04', '2026-12-04'))).toContain('check-out not after check-in'); + }); + + it('rejects orders with nothing to identify them, like Uber\'s empty Order', () => { + expect(problems({ '@context': ctx, '@type': 'Order', merchant: { name: 'UberEATS' } })).toContain('neither order number nor tracking'); + expect(problems({ '@context': ctx, '@type': 'Order', merchant: 'K4G', orderNumber: 'A1' })).toEqual([]); + }); +}); + +describe('resolveCards', () => { + const resolved = async (item: unknown): Promise => { + const cards = [one(item)]; + return needsAirportZones(cards) ? resolveCards(cards, await loadAirportZones()) : resolveCards(cards); + }; + + it('gives flight ends their airport zones and keeps the instants', async () => { + const [card] = (await resolved(sfoJfk)) as TransportBooking[]; + expect(card.legs[0].departure.at).toMatchObject({ zone: 'America/Los_Angeles', utc: '2026-12-05T04:15:00Z' }); + expect(card.legs[0].arrival?.at).toMatchObject({ zone: 'America/New_York', utc: '2026-12-05T11:30:00Z' }); + }); + + it('accepts eastbound date-line flights that land earlier on the wall clock', async () => { + const nrtHnl = flight(['Tokyo Narita', 'NRT', '2026-12-04T21:00:00+09:00'], ['Honolulu', 'HNL', '2026-12-04T09:15:00-10:00']); + const sydLax = flight(['Sydney', 'SYD', '2026-12-04T09:50:00+11:00'], ['Los Angeles', 'LAX', '2026-12-04T06:20:00-08:00']); + expect(await resolved(nrtHnl)).toHaveLength(1); + expect(await resolved(sydLax)).toHaveLength(1); + }); + + it('rejects arrival before departure and flights over 20 hours', async () => { + const backwards = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00+01:00'], ['New York', 'JFK', '2026-12-01T03:05:00-05:00']); + const tooLong = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00+01:00'], ['Sydney', 'SYD', '2026-12-02T23:00:00+11:00']); + expect(await resolved(backwards)).toEqual([]); + expect(await resolved(tooLong)).toEqual([]); + }); + + it('drops a flight whose offset contradicts the airport zone', async () => { + const wrongOffset = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00+00:00'], ['New York', 'JFK', '2026-12-01T13:05:00-05:00']); + expect(await resolved(wrongOffset)).toEqual([]); + }); + + it('reads a flight time without offset in the airport zone, and drops it when the airport is unknown', async () => { + const floating = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00'], ['New York', 'JFK', '2026-12-01T13:05:00']); + const [card] = (await resolved(floating)) as TransportBooking[]; + expect(card.legs[0].departure.at).toMatchObject({ zone: 'Europe/Berlin', utc: '2026-12-01T09:15:00Z' }); + const unknown = flight(['Nowhere', 'QQQ', '2026-12-01T10:15:00'], ['New York', 'JFK', '2026-12-01T13:05:00']); + expect(await resolved(unknown)).toEqual([]); + }); + + it('never shows a flight before the airport table is there', () => { + expect(resolveCards([one(sfoJfk)])).toEqual([]); + }); + + it('uses a fixed-offset zone where no airport code is known', async () => { + const noCode = flight(['San Francisco', '', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']); + const [card] = (await resolved(noCode)) as TransportBooking[]; + expect(card.legs[0].departure.at.zone).toBe('Etc/GMT+8'); + }); + + it('treats hotel times without offset or known country as dates only', async () => { + const [card] = await resolved({ + '@context': ctx, '@type': 'LodgingReservation', reservationNumber: 'H', reservationFor: { '@type': 'Hotel', name: 'Hotel', address: { addressCountry: 'US' } }, + checkinDate: '2026-12-04T15:00:00', checkoutDate: '2026-12-06T11:00:00', + }); + expect(card).toMatchObject({ checkin: { date: '2026-12-04' }, checkout: { date: '2026-12-06' } }); + expect((card as { checkin: { time?: string } }).checkin.time).toBeUndefined(); + }); + + it('reads bus times in the single zone of the stop\'s country', async () => { + const [card] = (await resolved({ + '@context': ctx, '@type': 'BusReservation', reservationNumber: 'B', + reservationFor: { + '@type': 'BusTrip', busNumber: 'FLX30', + departureBusStop: { name: 'Hannover', address: { addressCountry: 'DE' } }, departureTime: '2026-06-04T11:44:00+0200', + arrivalBusStop: { name: 'Berlin', address: { addressCountry: 'DE' } }, arrivalTime: '2026-06-04T13:44:00+0200', + }, + })) as TransportBooking[]; + expect(card.legs[0].departure.at.zone).toBe('Europe/Berlin'); + }); +}); diff --git a/lib/structured-mail/extract.ts b/lib/structured-mail/extract.ts new file mode 100644 index 000000000..77b3a1477 --- /dev/null +++ b/lib/structured-mail/extract.ts @@ -0,0 +1,123 @@ +/** + * Pull schema.org data out of a mail's raw HTML: JSON-LD blocks and + * microdata. The HTML is parsed into a detached document (DOMParser runs no + * scripts and loads nothing) that is never attached to the page; the sanitizer + * that renders the mail is a separate path and isn't touched. + */ +import { parseLenientJson } from './json-repair'; + +export type RawItem = Record; +export interface ExtractedItem { + item: unknown; + source: 'json-ld' | 'microdata'; +} + +const MAX_JSONLD_BLOCKS = 50; +const MAX_MICRODATA_ITEMS = 500; +const MAX_MICRODATA_DEPTH = 12; +const MAX_HTML_LENGTH = 8_000_000; + +/** Cheap test that a body may hold markup this feature reads. */ +const TYPE_HINT = /Reservation|ParcelDelivery|Order/; +const SCHEMA_ITEMTYPE = /^https?:\/\/(?:www\.)?schema\.org\/([A-Za-z]+)\/?$/i; + +/** + * Mail clients mark what they quote or forward with these containers; + * markup inside them belongs to someone else's mail. + */ +const QUOTED_CONTAINERS = 'blockquote, .gmail_quote, .moz-forward-container, #appendonsend, #divRplyFwdMsg'; + +export function defaultParseHtml(html: string): Document { + return new DOMParser().parseFromString(html, 'text/html'); +} + +export function mayContainStructuredData(html: string): boolean { + return TYPE_HINT.test(html); +} + +/** + * Elements that sit inside a quoted or forwarded part: inside one of the + * containers, or after Outlook's `#divRplyFwdMsg` separator (Outlook puts + * the forwarded message after it, not inside it). + */ +function isQuoted(element: Element, doc: Document): boolean { + if (element.closest(QUOTED_CONTAINERS)) return true; + const separator = doc.getElementById('divRplyFwdMsg'); + return !!separator && !!(separator.compareDocumentPosition(element) & 4 /* FOLLOWING */); +} + +function itemTypes(element: Element): string[] { + return (element.getAttribute('itemtype') ?? '') + .split(/\s+/) + .map((value) => SCHEMA_ITEMTYPE.exec(value)?.[1]) + .filter((value): value is string => !!value); +} + +function propertyValue(element: Element, depth: number): unknown { + if (element.hasAttribute('itemscope')) return microdataItem(element, depth + 1); + const content = element.getAttribute('content'); + if (content !== null) return content.trim(); + const tag = element.tagName.toLowerCase(); + const attr = (name: string) => (element.getAttribute(name) ?? '').trim(); + switch (tag) { + case 'a': case 'link': case 'area': return attr('href'); + case 'img': case 'audio': case 'video': case 'source': case 'embed': case 'iframe': case 'track': return attr('src'); + case 'object': return attr('data'); + case 'time': return element.hasAttribute('datetime') ? attr('datetime') : (element.textContent ?? '').trim(); + case 'data': case 'meter': return attr('value'); + default: return (element.textContent ?? '').replace(/\s+/g, ' ').trim(); + } +} + +/** + * One microdata item: properties are descendants with `itemprop` whose + * nearest `itemscope` ancestor is this item. A space-separated itemprop gives + * the value to each name; a repeated name becomes an array. + */ +function microdataItem(scope: Element, depth: number): RawItem | undefined { + if (depth > MAX_MICRODATA_DEPTH) return undefined; + const item: RawItem = {}; + const types = itemTypes(scope); + if (types.length > 0) item['@type'] = types.length === 1 ? types[0] : types; + const add = (name: string, value: unknown) => { + if (value === undefined || value === '') return; + const existing = item[name]; + if (existing === undefined) item[name] = value; + else if (Array.isArray(existing)) existing.push(value); + else item[name] = [existing, value]; + }; + for (const element of Array.from(scope.querySelectorAll('[itemprop]'))) { + const owner = element.parentElement?.closest('[itemscope]'); + if (owner !== scope) continue; + const value = propertyValue(element, depth); + for (const name of (element.getAttribute('itemprop') ?? '').split(/\s+/).filter(Boolean)) add(name, value); + } + return item; +} + +export function extractStructuredData( + html: string, + parseHtml: (html: string) => Document = defaultParseHtml, +): ExtractedItem[] { + if (!html || html.length > MAX_HTML_LENGTH || !mayContainStructuredData(html)) return []; + const doc = parseHtml(html); + const found: ExtractedItem[] = []; + + const scripts = Array.from(doc.querySelectorAll('script')).filter( + (script) => (script.getAttribute('type') ?? '').trim().toLowerCase() === 'application/ld+json', + ); + for (const script of scripts.slice(0, MAX_JSONLD_BLOCKS)) { + if (isQuoted(script, doc)) continue; + const value = parseLenientJson(script.textContent ?? ''); + if (value !== undefined && value !== null) found.push({ item: value, source: 'json-ld' }); + } + + const tops = Array.from(doc.querySelectorAll('[itemscope]')).filter((element) => !element.hasAttribute('itemprop')); + for (const scope of tops.slice(0, MAX_MICRODATA_ITEMS)) { + // Only items typed on schema.org count; an untyped or foreign item is noise. + if (itemTypes(scope).length === 0 || isQuoted(scope, doc)) continue; + const item = microdataItem(scope, 0); + if (item) found.push({ item: { '@context': 'https://schema.org', ...item }, source: 'microdata' }); + } + return found; +} diff --git a/lib/structured-mail/index.ts b/lib/structured-mail/index.ts new file mode 100644 index 000000000..b4461555d --- /dev/null +++ b/lib/structured-mail/index.ts @@ -0,0 +1,37 @@ +/** + * Structured mail: read the schema.org markup that airlines, hotels, shops + * and carriers put into their HTML mail, and turn it into cards. Plain + * TypeScript; the only DOM dependency is the HTML parser, which a caller + * without DOMParser (the native app) passes in. + * + * const cards = readStructuredMail(html, { mailDate }); + * const shown = needsAirportZones(cards) + * ? resolveCards(cards, await loadAirportZones()) + * : resolveCards(cards); + */ +import { extractStructuredData } from './extract'; +import { normalizeItems } from './normalize'; +import type { StructuredCard } from './types'; +import { validateCard } from './validate'; + +export interface ReadOptions { + /** When the mail was sent (else received): bookings far from it are doubtful. */ + mailDate: Date; + parseHtml?: (html: string) => Document; +} + +/** Cards from one HTML body, validated but with zones not yet resolved. */ +export function readStructuredMail(html: string, options: ReadOptions): StructuredCard[] { + const cards = normalizeItems(extractStructuredData(html, options.parseHtml)); + return cards + .map((card) => validateCard(card, options.mailDate)) + .filter((candidate) => candidate.problems.length === 0) + .map((candidate) => candidate.card); +} + +export { extractStructuredData, mayContainStructuredData } from './extract'; +export { normalizeItems } from './normalize'; +export { validateCard } from './validate'; +export { needsAirportZones, resolveCards } from './resolve'; +export { loadAirportZones } from './time-zones'; +export * from './types'; diff --git a/lib/structured-mail/json-repair.ts b/lib/structured-mail/json-repair.ts new file mode 100644 index 000000000..7ea2f3b84 --- /dev/null +++ b/lib/structured-mail/json-repair.ts @@ -0,0 +1,129 @@ +/** + * Parse the JSON-LD a sender wrote, including the broken forms real booking + * mail carries: missing commas between objects (Eurowings), several objects + * without the array brackets (Volotea), trailing commas (Eventbrite) and + * XML-entity-encoded JSON (Airbnb). Returns undefined when nothing sensible + * comes out; never throws. + */ + +const MAX_BLOCK_LENGTH = 200_000; + +function tryParse(text: string): { ok: true; value: unknown } | { ok: false } { + try { + return { ok: true, value: JSON.parse(text) }; + } catch { + return { ok: false }; + } +} + +function stripWrappers(text: string): string { + return text + .trim() + .replace(/^$/, '') + .replace(/^$/, '') + .trim(); +} + +const NAMED_ENTITIES: Record = { quot: '"', amp: '&', lt: '<', gt: '>', apos: "'", nbsp: ' ' }; + +function decodeEntities(text: string): string { + return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, body: string) => { + if (body[0] === '#') { + const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10); + return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match; + } + return NAMED_ENTITIES[body.toLowerCase()] ?? match; + }); +} + +/** + * One string-aware pass: drop commas right before `}` or `]`, put a comma + * between a closing `}` or `]` and an opening `{` or `[` (only where a + * comma or nothing could go), and escape raw control characters inside + * strings. Returns the text and how many values sit at the top level. + */ +function repairStructure(text: string): { text: string; topLevel: number } { + let out = ''; + let inString = false; + let depth = 0; + let topLevel = 0; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (inString) { + if (c === '\\') { + out += c + (text[i + 1] ?? ''); + i++; + continue; + } + if (c === '"') inString = false; + if (c < ' ') { + out += c === '\n' ? '\\n' : c === '\r' ? '\\r' : c === '\t' ? '\\t' : ''; + continue; + } + out += c; + continue; + } + if (c === '"') { + if (depth === 0) topLevel++; + inString = true; + out += c; + continue; + } + if (c === ',') { + let j = i + 1; + while (j < text.length && /\s/.test(text[j])) j++; + if (text[j] === '}' || text[j] === ']' || j >= text.length) continue; + out += c; + continue; + } + if (c === '{' || c === '[') { + if (depth === 0) { + topLevel++; + } else { + // `}{` or `]{` inside an array: the comma is missing. + const prev = out.trimEnd().slice(-1); + if (prev === '}' || prev === ']') out = out.trimEnd() + ','; + } + depth++; + out += c; + continue; + } + if (c === '}' || c === ']') { + depth = Math.max(0, depth - 1); + out += c; + continue; + } + out += c; + } + return { text: out, topLevel }; +} + +export function parseLenientJson(input: string): unknown { + if (!input || input.length > MAX_BLOCK_LENGTH) return undefined; + const direct = tryParse(input); + if (direct.ok) return direct.value; + + let text = stripWrappers(input); + let attempt = tryParse(text); + if (attempt.ok) return attempt.value; + + if (/&(quot|#34|#x22);/i.test(text)) { + text = decodeEntities(text); + attempt = tryParse(text); + if (attempt.ok) return attempt.value; + } + + const repaired = repairStructure(text); + attempt = tryParse(repaired.text); + if (attempt.ok) return attempt.value; + + if (repaired.topLevel > 1) { + // Several top-level values: the brackets are missing. The commas between + // them may be missing too, which the pass inside brackets adds. + attempt = tryParse(repairStructure(`[${repaired.text}]`).text); + if (attempt.ok) return attempt.value; + } + return undefined; +} diff --git a/lib/structured-mail/normalize.ts b/lib/structured-mail/normalize.ts new file mode 100644 index 000000000..9a650c6bf --- /dev/null +++ b/lib/structured-mail/normalize.ts @@ -0,0 +1,406 @@ +/** + * Turn extracted schema.org objects into the card model. Unknown types and + * properties are dropped; validation (validate.ts) decides afterwards which + * cards are believable. + */ +import type { ExtractedItem } from './extract'; +import type { + BookingKind, + BookingLinks, + CardSource, + Item, + Moment, + Parcel, + Place, + Seat, + Shipment, + StructuredCard, + TransportBooking, + TransportKind, + TransportLeg, +} from './types'; +import { + address, + all, + amount, + currency, + hasType, + isObj, + isSchemaContext, + link, + moment, + lenientDate, + nameOf, + obj, + orderStatus, + people, + positiveInt, + reservationStatus, + str, + typesOf, + type Obj, +} from './values'; + +const RESERVATION_KINDS: Record = { + FlightReservation: 'flight', + TrainReservation: 'train', + BusReservation: 'bus', + LodgingReservation: 'lodging', + RentalCarReservation: 'rentalcar', + EventReservation: 'event', + FoodEstablishmentReservation: 'restaurant', +}; + +const MAX_OBJECTS = 200; + +interface Found { + object: Obj; + source: CardSource; +} + +/** + * Flatten what the extractor found into typed top-level objects: arrays, + * `@graph`, ReservationPackage `subReservation`, and `reservationFor` arrays + * (one reservation per element). Only objects under a schema.org context + * count; nested objects inherit it. + */ +function flatten(extracted: ExtractedItem[]): Found[] { + const out: Found[] = []; + const visit = (value: unknown, source: CardSource, inherited: boolean, depth: number) => { + if (out.length >= MAX_OBJECTS || depth > 6) return; + if (Array.isArray(value)) { + for (const v of value) visit(v, source, inherited, depth + 1); + return; + } + if (!isObj(value)) return; + const inContext = inherited || isSchemaContext(value['@context']); + if (Array.isArray(value['@graph'])) { + visit(value['@graph'], source, inContext, depth + 1); + return; + } + if (!inContext) return; + if (hasType(value, ['ReservationPackage'])) { + visit(value.subReservation, source, true, depth + 1); + return; + } + const reservationFor = value.reservationFor; + if (Array.isArray(reservationFor) && reservationFor.length > 1) { + for (const one of reservationFor) out.push({ object: { ...value, reservationFor: one }, source }); + return; + } + out.push({ object: value, source }); + }; + for (const { item, source } of extracted) visit(item, source, false, 0); + return out; +} + +const lower = (s: string | undefined) => (s ?? '').toLowerCase().replace(/\s+/g, ' ').trim(); + +function place(value: unknown): Place { + const v = obj(value); + if (!v) { + const name = str(value); + return name ? { name } : {}; + } + const result: Place = {}; + const name = str(v.name); + if (name) result.name = name; + const code = str(v.iataCode)?.toUpperCase(); + if (code && /^[A-Z]{3}$/.test(code)) result.code = code; + const addr = address(v.address); + if (addr) result.address = addr; + return result; +} + +function bookingLinks(res: Obj, extra: Obj | undefined): BookingLinks { + const links: BookingLinks = {}; + const checkin = link(res.checkinUrl); + const manage = link(res.modifyReservationUrl) ?? link(res.url) ?? link(res.Url); + const ticket = link(res.ticketDownloadUrl) ?? link(res.ticketPrintUrl) ?? link(extra?.ticketDownloadUrl); + if (checkin) links.checkin = checkin; + if (manage) links.manage = manage; + if (ticket) links.ticket = ticket; + return links; +} + +function reference(res: Obj): string | undefined { + return str(res.reservationNumber) ?? str(res.reservationId) ?? str(res.confirmationNumber); +} + +function tickets(res: Obj): Obj[] { + return all(res.reservedTicket).filter(isObj); +} + +function seatOf(res: Obj): string | undefined { + const direct = str(res.airplaneSeat); + if (direct) return direct; + for (const ticket of tickets(res)) { + const seat = obj(ticket.ticketedSeat); + const number = str(seat?.seatNumber) ?? str(ticket.ticketedSeat); + if (number) { + const section = str(seat?.seatSection); + return section ? `${section}/${number}` : number; + } + } + return undefined; +} + +function splitFlightNumber(raw: string | undefined, airlineCode: string | undefined): { code?: string; number?: string } { + if (!raw) return { code: airlineCode }; + const compact = raw.replace(/\s+/g, '').toUpperCase(); + if (airlineCode && compact.startsWith(airlineCode) && /^\d{1,4}[A-Z]?$/.test(compact.slice(airlineCode.length))) { + return { code: airlineCode, number: compact.slice(airlineCode.length) }; + } + const match = /^([A-Z0-9]{2})(\d{1,4}[A-Z]?)$/.exec(compact); + if (match && /[A-Z]/.test(match[1]) && (!airlineCode || airlineCode === match[1])) return { code: match[1], number: match[2] }; + return { code: airlineCode, number: compact }; +} + +function transport(kind: TransportKind, res: Obj, source: CardSource): TransportBooking { + const trip = obj(res.reservationFor) ?? {}; + const passengers = people(res.underName); + let carrier: string | undefined; + let carrierCode: string | undefined; + let number: string | undefined; + let departure: Place; + let arrival: Place; + if (kind === 'flight') { + const airline = obj(trip.airline); + const airlineCode = str(airline?.iataCode)?.toUpperCase(); + carrier = nameOf(trip.airline) ?? nameOf(trip.provider) ?? nameOf(res.provider); + const split = splitFlightNumber(str(trip.flightNumber), airlineCode && /^[A-Z0-9]{2}$/.test(airlineCode) ? airlineCode : undefined); + carrierCode = split.code; + number = split.number; + departure = place(trip.departureAirport); + arrival = place(trip.arrivalAirport); + } else if (kind === 'train') { + carrier = nameOf(trip.provider) ?? nameOf(trip.trainCompany) ?? nameOf(res.provider); + number = str(trip.trainNumber) ?? str(trip.trainName); + departure = place(trip.departureStation); + arrival = place(trip.arrivalStation); + } else { + carrier = nameOf(trip.busCompany) ?? nameOf(trip.provider) ?? nameOf(res.provider); + // busName is ignored: FlixBus puts the destination there. + number = str(trip.busNumber); + departure = place(trip.departureBusStop ?? trip.departureStation); + arrival = place(trip.arrivalBusStop ?? trip.arrivalStation); + } + const depAt = moment(trip.departureTime); + const arrAt = moment(trip.arrivalTime); + const seat = seatOf(res); + const status = reservationStatus(res.reservationStatus); + const seats: Seat[] = seat ? [{ ...(passengers[0] ? { passenger: passengers[0] } : {}), seat }] : []; + const leg: TransportLeg = { + ...(carrier ? { carrier } : {}), + ...(carrierCode ? { carrierCode } : {}), + ...(number ? { number } : {}), + departure: { place: departure, at: depAt ?? { date: '' } }, + arrival: { place: arrival, ...(arrAt ? { at: arrAt } : {}) }, + seats, + ...(status ? { status } : {}), + }; + const ref = reference(res); + return { + key: '', + kind, + ...(ref ? { reference: ref } : {}), + ...(status ? { status } : {}), + passengers, + ...(carrier ? { provider: carrier } : {}), + links: bookingLinks(res, tickets(res)[0]), + ...(str(res.modifiedTime) ? { modified: str(res.modifiedTime) } : {}), + source, + legs: [leg], + }; +} + +function base(kind: BookingKind, res: Obj, source: CardSource) { + const ref = reference(res); + const status = reservationStatus(res.reservationStatus); + return { + key: '', + kind, + ...(ref ? { reference: ref } : {}), + ...(status ? { status } : {}), + passengers: people(res.underName), + links: bookingLinks(res, tickets(res)[0]), + ...(str(res.modifiedTime) ? { modified: str(res.modifiedTime) } : {}), + source, + }; +} + +const MISSING: Moment = { date: '' }; + +function reservation(kind: BookingKind, res: Obj, source: CardSource): StructuredCard | undefined { + if (kind === 'flight' || kind === 'train' || kind === 'bus') return transport(kind, res, source); + const target = obj(res.reservationFor) ?? {}; + const name = str(target.name) ?? (typeof res.reservationFor === 'string' ? str(res.reservationFor) : undefined); + switch (kind) { + case 'lodging': { + const addr = address(target.address); + const phone = str(target.telephone); + return { + ...base(kind, res, source), + kind, + name: name ?? '', + ...(name ? { provider: name } : {}), + ...(addr ? { address: addr } : {}), + ...(phone ? { phone } : {}), + checkin: moment(res.checkinDate) ?? moment(res.checkinTime) ?? MISSING, + checkout: moment(res.checkoutDate) ?? moment(res.checkoutTime) ?? MISSING, + }; + } + case 'rentalcar': { + const company = nameOf(target.rentalCompany) ?? nameOf(res.provider) ?? nameOf(target.brand); + const car = [nameOf(target.brand), str(target.model) ?? str(target.name)].filter(Boolean).join(' ') || undefined; + return { + ...base(kind, res, source), + kind, + ...(company ? { company, provider: company } : {}), + ...(car ? { car } : {}), + pickup: { place: place(res.pickupLocation), at: moment(res.pickupTime) ?? MISSING }, + dropoff: { place: place(res.dropoffLocation), at: moment(res.dropoffTime) ?? MISSING }, + }; + } + case 'event': { + const venue = place(target.location); + const ticketNumbers = [...new Set([str(res.ticketNumber), ...tickets(res).map((t) => str(t.ticketNumber))].filter((t): t is string => !!t))]; + let end = moment(target.endDate); + // An end date without a time means the end of that day. + if (end && !end.time) end = { ...end, time: '23:59:59' }; + return { + ...base(kind, res, source), + kind, + name: name ?? '', + ...(venue.name || venue.address ? { venue } : {}), + start: moment(target.startDate) ?? MISSING, + ...(end ? { end } : {}), + tickets: ticketNumbers, + ...(nameOf(res.provider) ? { provider: nameOf(res.provider) } : {}), + }; + } + case 'restaurant': { + const addr = address(target.address); + const partySize = positiveInt(res.partySize); + return { + ...base(kind, res, source), + kind, + name: name ?? '', + ...(name ? { provider: name } : {}), + ...(addr ? { address: addr } : {}), + start: moment(res.startTime) ?? moment(res.startDate) ?? MISSING, + ...(partySize ? { partySize } : {}), + }; + } + } +} + +function items(products: unknown, quantityOf?: (o: Obj) => number | undefined): Item[] { + const result: Item[] = []; + for (const p of all(products)) { + const name = nameOf(p); + if (!name) continue; + const product = isObj(p) ? p : undefined; + const image = product ? link(isObj(product.image) ? product.image.url ?? product.image.contentUrl : product.image) : undefined; + const quantity = product && quantityOf ? quantityOf(product) : undefined; + result.push({ name, ...(quantity ? { quantity } : {}), ...(image ? { image } : {}) }); + } + return result; +} + +function orderItems(order: Obj): Item[] { + const result: Item[] = []; + for (const offer of all(order.acceptedOffer).filter(isObj)) { + const quantity = positiveInt(offer.eligibleQuantity); + for (const item of items(offer.itemOffered)) result.push(quantity ? { ...item, quantity } : item); + } + for (const entry of all(order.orderedItem).filter(isObj)) { + if (hasType(entry, ['OrderItem'])) { + const quantity = positiveInt(entry.orderQuantity); + for (const item of items(entry.orderedItem)) result.push(quantity ? { ...item, quantity } : item); + } else { + result.push(...items(entry)); + } + } + return result; +} + +function order(o: Obj, source: CardSource): Shipment { + const merchant = nameOf(o.merchant) ?? nameOf(o.seller) ?? nameOf(o.broker); + const orderNumber = str(o.orderNumber); + const status = orderStatus(o.orderStatus); + const total = amount(o.price) ?? amount(o.totalPrice); + const cur = currency(o.priceCurrency); + const action = all(o.potentialAction).filter(isObj).find((a) => hasType(a, ['ViewAction'])); + const orderUrl = link(o.url) ?? link(o.Url) ?? link(action?.target) ?? link(action?.url); + return { + kind: 'shipment', + key: '', + ...(merchant ? { merchant } : {}), + ...(orderNumber ? { orderNumber } : {}), + ...(status ? { status } : {}), + items: orderItems(o), + ...(total && cur ? { total: { amount: total, currency: cur } } : {}), + ...(orderUrl ? { orderUrl } : {}), + parcels: [], + source, + }; +} + +function parcel(p: Obj, source: CardSource): Shipment { + const partOf = obj(p.partOfOrder); + const base: Shipment = partOf ? order(partOf, source) : { kind: 'shipment', key: '', items: [], parcels: [], source }; + const carrier = nameOf(p.carrier) ?? nameOf(p.provider); + const trackingNumber = str(p.trackingNumber); + const trackingUrl = link(p.trackingUrl); + const expectedBy = lenientDate(p.expectedArrivalUntil) ?? lenientDate(p.expectedArrivalFrom); + const shipped: Parcel = { + ...(carrier ? { carrier } : {}), + ...(trackingNumber ? { trackingNumber } : {}), + ...(trackingUrl ? { trackingUrl } : {}), + ...(expectedBy ? { expectedBy } : {}), + items: items(p.itemShipped), + }; + return { ...base, parcels: [shipped] }; +} + +/** One card per top-level object, unmerged and unvalidated, with a merge key. */ +export function normalizeItems(extracted: ExtractedItem[]): StructuredCard[] { + const cards: StructuredCard[] = []; + for (const { object, source } of flatten(extracted)) { + const types = typesOf(object); + const kindType = types.find((t) => RESERVATION_KINDS[t]); + let card: StructuredCard | undefined; + if (kindType) card = reservation(RESERVATION_KINDS[kindType], object, source); + else if (types.includes('ParcelDelivery')) card = parcel(object, source); + else if (types.includes('Order')) card = order(object, source); + if (card) cards.push({ ...card, key: cardKey(card) }); + } + return cards; +} + +/** The merge key: equal keys describe the same booking or order. */ +export function cardKey(card: StructuredCard): string { + switch (card.kind) { + case 'flight': case 'train': case 'bus': { + const leg = card.legs[0]; + return card.reference ? `${card.kind}|ref:${lower(card.reference)}` : `${card.kind}|leg:${legKey(leg)}`; + } + case 'lodging': + return `lodging|${lower(card.reference)}|${lower(card.name)}|${card.checkin.date}`; + case 'rentalcar': + return `rentalcar|${lower(card.reference)}|${lower(card.company)}|${card.pickup.at.date}`; + case 'event': case 'restaurant': + return `${card.kind}|${lower(card.reference)}|${lower(card.name)}|${card.start.date}`; + case 'shipment': { + if (card.orderNumber) return `shipment|order:${lower(card.orderNumber)}`; + const tracking = card.parcels.find((p) => p.trackingNumber)?.trackingNumber; + return tracking ? `shipment|parcel:${lower(tracking)}` : `shipment|anon:${lower(card.merchant)}|${card.parcels.length}`; + } + } +} + +/** A leg's identity inside a booking: carrier, number, departure day and place. */ +export function legKey(leg: TransportLeg): string { + return [lower(leg.carrierCode ?? leg.carrier), lower(leg.number), leg.departure.at.date, lower(leg.departure.place.code ?? leg.departure.place.name)].join('|'); +} diff --git a/lib/structured-mail/resolve.ts b/lib/structured-mail/resolve.ts new file mode 100644 index 000000000..b9787a2f0 --- /dev/null +++ b/lib/structured-mail/resolve.ts @@ -0,0 +1,139 @@ +/** + * Second validation pass, once zones can be looked up: give every time its + * zone and instant, then check order and durations on UTC instants (an + * eastbound flight over the date line lands "earlier" on the wall clock). + * A booking whose times don't hold together is dropped. + */ +import { countryZone, resolveZone, zonedTimeToUtc, type ResolvedZone } from './time-zones'; +import type { Moment, Place, StructuredCard, TransportBooking, TransportLeg } from './types'; + +type ZoneOf = (code: string | undefined) => string | undefined; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const LIMITS = { + flight: { min: 10 * MINUTE, max: 20 * HOUR }, + train: { min: MINUTE, max: 48 * HOUR }, + bus: { min: MINUTE, max: 48 * HOUR }, +} as const; + +/** Whether resolving these cards needs the airport table. */ +export function needsAirportZones(cards: StructuredCard[]): boolean { + return cards.some((card) => card.kind === 'flight'); +} + +interface Named { zone: string; source: 'airport' | 'country' } + +function namedZone(place: Place | undefined, kind: StructuredCard['kind'], zoneOf: ZoneOf | undefined, floating: boolean): Named | undefined { + if (kind === 'flight') { + const zone = zoneOf?.(place?.code); + return zone ? { zone, source: 'airport' } : undefined; + } + const zone = countryZone(place?.address?.country, { floating }); + return zone ? { zone, source: 'country' } : undefined; +} + +/** + * Attach zone and instant to a timed moment. 'drop' when table and mail + * disagree, or when a floating local time falls into a DST gap or overlap. + */ +function resolveMoment(m: Moment, named: Named | undefined): Moment | 'drop' { + if (!m.time) return m; + const utcMs = m.utc ? Date.parse(m.utc) : undefined; + const resolved: ResolvedZone | 'mismatch' = resolveZone({ named, offsetMinutes: m.offsetMinutes, utcMs }); + if (resolved === 'mismatch') return 'drop'; + if (m.utc || resolved.zone === null) return { ...m, zone: resolved.zone }; + const utc = zonedTimeToUtc(m.date, m.time, resolved.zone); + if (utc === null) return 'drop'; + return { ...m, zone: resolved.zone, utc: new Date(utc).toISOString().replace('.000Z', 'Z') }; +} + +function resolveLeg(kind: TransportBooking['kind'], leg: TransportLeg, zoneOf: ZoneOf | undefined): TransportLeg | null { + const dep = resolveMoment(leg.departure.at, namedZone(leg.departure.place, kind, zoneOf, leg.departure.at.offsetMinutes === undefined)); + if (dep === 'drop' || !dep.utc) return null; + let arrivalAt: Moment | undefined; + if (leg.arrival?.at) { + const arr = resolveMoment(leg.arrival.at, namedZone(leg.arrival.place, kind, zoneOf, leg.arrival.at.offsetMinutes === undefined)); + if (arr === 'drop' || !arr.utc) return null; + const duration = Date.parse(arr.utc) - Date.parse(dep.utc); + if (duration < LIMITS[kind].min || duration > LIMITS[kind].max) return null; + arrivalAt = arr; + } + return { + ...leg, + departure: { ...leg.departure, at: dep }, + ...(leg.arrival ? { arrival: { ...leg.arrival, ...(arrivalAt ? { at: arrivalAt } : {}) } } : {}), + }; +} + +/** Order two resolved moments: by instant when both have one, else by wall clock. */ +function notBefore(later: Moment, earlier: Moment): boolean { + if (later.utc && earlier.utc) return Date.parse(later.utc) >= Date.parse(earlier.utc); + return `${later.date}T${later.time ?? '00:00:00'}` >= `${earlier.date}T${earlier.time ?? '00:00:00'}`; +} + +function resolveCard(card: StructuredCard, zoneOf: ZoneOf | undefined): StructuredCard | null { + switch (card.kind) { + case 'flight': case 'train': case 'bus': { + const legs: TransportLeg[] = []; + for (const leg of card.legs) { + const resolved = resolveLeg(card.kind, leg, zoneOf); + if (!resolved) return null; + legs.push(resolved); + } + return { ...card, legs }; + } + case 'lodging': { + const place: Place = { address: card.address }; + const resolveEnd = (m: Moment): Moment | 'drop' => { + const named = namedZone(place, card.kind, zoneOf, m.offsetMinutes === undefined); + // A time with neither offset nor a known zone says nothing reliable: + // keep the date only. + if (m.time && m.offsetMinutes === undefined && !named) return { date: m.date }; + return resolveMoment(m, named); + }; + const checkin = resolveEnd(card.checkin); + const checkout = resolveEnd(card.checkout); + if (checkin === 'drop' || checkout === 'drop') return null; + if (checkin.time && checkout.time && !notBefore(checkout, checkin)) return null; + return { ...card, checkin, checkout }; + } + case 'rentalcar': { + const pickupAt = resolveMoment(card.pickup.at, namedZone(card.pickup.place, card.kind, zoneOf, card.pickup.at.offsetMinutes === undefined)); + const dropoffAt = resolveMoment(card.dropoff.at, namedZone(card.dropoff.place, card.kind, zoneOf, card.dropoff.at.offsetMinutes === undefined)); + if (pickupAt === 'drop' || dropoffAt === 'drop') return null; + if (!notBefore(dropoffAt, pickupAt) || notBefore(pickupAt, dropoffAt)) return null; + return { ...card, pickup: { ...card.pickup, at: pickupAt }, dropoff: { ...card.dropoff, at: dropoffAt } }; + } + case 'event': { + const named = (m: Moment) => namedZone(card.venue, card.kind, zoneOf, m.offsetMinutes === undefined); + const start = resolveMoment(card.start, named(card.start)); + const end = card.end ? resolveMoment(card.end, named(card.end)) : undefined; + if (start === 'drop' || end === 'drop') return null; + if (end && !notBefore(end, start)) return null; + return { ...card, start, ...(end ? { end } : {}) }; + } + case 'restaurant': { + const start = resolveMoment(card.start, namedZone({ address: card.address }, card.kind, zoneOf, card.start.offsetMinutes === undefined)); + if (start === 'drop') return null; + return { ...card, start }; + } + case 'shipment': + return card; + } +} + +/** + * Resolve zones and drop cards whose times don't hold together. `zoneOf` is + * the airport lookup from loadAirportZones(); without it, flight cards are + * dropped (the card must not show before its zones are known). + */ +export function resolveCards(cards: StructuredCard[], zoneOf?: ZoneOf): StructuredCard[] { + const out: StructuredCard[] = []; + for (const card of cards) { + if (card.kind === 'flight' && !zoneOf) continue; + const resolved = resolveCard(card, zoneOf); + if (resolved) out.push(resolved); + } + return out; +} diff --git a/lib/structured-mail/types.ts b/lib/structured-mail/types.ts new file mode 100644 index 000000000..80097266a --- /dev/null +++ b/lib/structured-mail/types.ts @@ -0,0 +1,176 @@ +/** + * The model booking, shipping and order mail is read into. Plain data, no + * React and no app imports, so the native app can reuse the parser. + */ + +export type BookingKind = 'flight' | 'train' | 'bus' | 'lodging' | 'rentalcar' | 'event' | 'restaurant'; +export type TransportKind = 'flight' | 'train' | 'bus'; +export type ReservationStatus = 'confirmed' | 'pending' | 'hold' | 'cancelled'; +export type OrderStatus = + | 'payment_due' | 'processing' | 'problem' | 'in_transit' + | 'pickup_available' | 'returned' | 'delivered' | 'cancelled'; + +/** A time as the mail wrote it, local to the place it belongs to. */ +export interface Moment { + /** YYYY-MM-DD, local. */ + date: string; + /** HH:mm:ss, local; absent for a date-only value. */ + time?: string; + /** Minutes east of UTC, from the value; absent when the mail gave none. */ + offsetMinutes?: number; + /** ISO instant; set when the time and its zone or offset are known. */ + utc?: string; + /** + * IANA zone after resolution (resolveCards): airport, country or a + * fixed-offset zone; null for a time that stays floating. Absent before. + */ + zone?: string | null; +} + +export interface Address { + street?: string; + locality?: string; + region?: string; + postalCode?: string; + /** ISO 3166-1 alpha-2, when the mail gave one. */ + country?: string; + /** Country as the mail named it, when that wasn't a code. */ + countryName?: string; +} + +export interface Place { + name?: string; + /** IATA airport code for airports, upper case. */ + code?: string; + address?: Address; +} + +/** A link a card may show: http(s) only, host shown next to it. */ +export interface Link { + url: string; + host: string; +} + +export interface Seat { + passenger?: string; + seat: string; +} + +export interface TransportLeg { + carrier?: string; + /** IATA airline code for flights. */ + carrierCode?: string; + number?: string; + departure: { place: Place; at: Moment }; + arrival?: { place: Place; at?: Moment }; + seats: Seat[]; + status?: ReservationStatus; +} + +export interface BookingLinks { + checkin?: Link; + manage?: Link; + ticket?: Link; +} + +export type CardSource = 'json-ld' | 'microdata' | 'pkpass'; + +interface BookingBase { + /** Merge key: equal keys are the same booking. */ + key: string; + kind: BookingKind; + reference?: string; + status?: ReservationStatus; + passengers: string[]; + provider?: string; + links: BookingLinks; + modified?: string; + source: CardSource; +} + +export interface TransportBooking extends BookingBase { + kind: TransportKind; + legs: TransportLeg[]; +} + +export interface LodgingBooking extends BookingBase { + kind: 'lodging'; + name: string; + address?: Address; + phone?: string; + checkin: Moment; + checkout: Moment; +} + +export interface RentalCarBooking extends BookingBase { + kind: 'rentalcar'; + company?: string; + car?: string; + pickup: { place: Place; at: Moment }; + dropoff: { place: Place; at: Moment }; +} + +export interface EventBooking extends BookingBase { + kind: 'event'; + name: string; + venue?: Place; + start: Moment; + end?: Moment; + tickets: string[]; +} + +export interface RestaurantBooking extends BookingBase { + kind: 'restaurant'; + name: string; + address?: Address; + start: Moment; + partySize?: number; +} + +export type Booking = TransportBooking | LodgingBooking | RentalCarBooking | EventBooking | RestaurantBooking; + +export interface Item { + name: string; + quantity?: number; + /** Remote image; a card shows it only when remote content may load. */ + image?: Link; +} + +export interface Parcel { + carrier?: string; + trackingNumber?: string; + trackingUrl?: Link; + expectedBy?: Moment; + items: Item[]; +} + +/** An Order and/or the ParcelDeliveries for one order number. */ +export interface Shipment { + kind: 'shipment'; + key: string; + merchant?: string; + orderNumber?: string; + status?: OrderStatus; + items: Item[]; + /** Amount normalised to a dot decimal ("399.00"). */ + total?: { amount: string; currency: string }; + orderUrl?: Link; + parcels: Parcel[]; + source: CardSource; +} + +export type StructuredCard = Booking | Shipment; + +/** A card plus the reasons validation found to doubt it (empty = fine). */ +export interface Candidate { + card: StructuredCard; + problems: string[]; +} + +export function isBooking(card: StructuredCard): card is Booking { + return card.kind !== 'shipment'; +} + +export function isTransport(card: StructuredCard): card is TransportBooking { + return card.kind === 'flight' || card.kind === 'train' || card.kind === 'bus'; +} diff --git a/lib/structured-mail/validate.ts b/lib/structured-mail/validate.ts new file mode 100644 index 000000000..f128cf3f3 --- /dev/null +++ b/lib/structured-mail/validate.ts @@ -0,0 +1,133 @@ +/** + * Plausibility checks that don't need time zones. A card with any problem is + * dropped, together with every other part of the same booking (merge.ts): + * a wrong card is worse than no card. Time order and durations are checked + * once the zones are known (resolve.ts). + */ +import type { Candidate, Moment, StructuredCard, TransportLeg } from './types'; + +const DAY = 86_400_000; +const PAST_WINDOW_DAYS = 366; +const TRANSPORT_FUTURE_DAYS = 400; +const OTHER_FUTURE_DAYS = 760; + +/** + * Values from Google's markup documentation examples. Senders paste those + * examples into newsletters (zave.it sent the "Eva Green" flight in a + * cashback mail, with a passing DKIM signature); they are never real. + */ +const DOCUMENTATION_VALUES = new Set([ + 'rxj34p', 'abc456', 'e123456789', 'ot12345', '546323', '3453291231', '176057', '123-4567890-1234567', +]); + +function dayNumber(date: string): number | undefined { + const ms = Date.parse(`${date}T00:00:00Z`); + return Number.isFinite(ms) ? Math.floor(ms / DAY) : undefined; +} + +function hasDate(m: Moment | undefined): m is Moment { + return !!m && /^\d{4}-\d{2}-\d{2}$/.test(m.date); +} + +function hasTime(m: Moment | undefined): boolean { + return hasDate(m) && !!m.time; +} + +function firstDate(card: StructuredCard): string | undefined { + switch (card.kind) { + case 'flight': case 'train': case 'bus': return card.legs[0]?.departure.at.date; + case 'lodging': return card.checkin.date; + case 'rentalcar': return card.pickup.at.date; + case 'event': case 'restaurant': return card.start.date; + case 'shipment': return undefined; + } +} + +function legProblems(kind: 'flight' | 'train' | 'bus', leg: TransportLeg, fromPass: boolean): string[] { + const problems: string[] = []; + if (!hasTime(leg.departure.at)) problems.push('departure time missing'); + if (!fromPass && !hasTime(leg.arrival?.at)) problems.push('arrival time missing'); + const named = (p: { code?: string; name?: string } | undefined) => !!(p && (kind === 'flight' ? p.code || p.name : p.name)); + if (!named(leg.departure.place)) problems.push('departure place missing'); + if (!named(leg.arrival?.place)) problems.push('arrival place missing'); + if (kind === 'flight' && !(leg.carrierCode || leg.carrier)) problems.push('airline missing'); + if (kind === 'flight' && !leg.number) problems.push('flight number missing'); + return problems; +} + +function cardProblems(card: StructuredCard): string[] { + switch (card.kind) { + case 'flight': case 'train': case 'bus': + if (card.legs.length === 0) return ['no legs']; + return card.legs.flatMap((leg) => legProblems(card.kind as 'flight' | 'train' | 'bus', leg, card.source === 'pkpass')); + case 'lodging': { + const problems: string[] = []; + if (!card.name) problems.push('hotel name missing'); + if (!hasDate(card.checkin) || !hasDate(card.checkout)) problems.push('stay dates missing'); + else { + const nights = dayNumber(card.checkout.date)! - dayNumber(card.checkin.date)!; + if (nights < 1) problems.push('check-out not after check-in'); + if (nights > 365) problems.push('stay longer than a year'); + } + return problems; + } + case 'rentalcar': { + const problems: string[] = []; + if (!hasTime(card.pickup.at) || !hasTime(card.dropoff.at)) problems.push('rental times missing'); + else { + const days = dayNumber(card.dropoff.at.date)! - dayNumber(card.pickup.at.date)!; + if (days < 0) problems.push('dropoff before pickup'); + if (days > 365) problems.push('rental longer than a year'); + } + return problems; + } + case 'event': { + const problems: string[] = []; + if (!card.name) problems.push('event name missing'); + if (!hasDate(card.start)) problems.push('event start missing'); + else if (card.end && hasDate(card.end)) { + const days = dayNumber(card.end.date)! - dayNumber(card.start.date)!; + if (days < 0) problems.push('event ends before it starts'); + if (days > 31) problems.push('event longer than a month'); + } + return problems; + } + case 'restaurant': { + const problems: string[] = []; + if (!card.name) problems.push('restaurant name missing'); + if (!hasTime(card.start)) problems.push('reservation time missing'); + return problems; + } + case 'shipment': { + const problems: string[] = []; + const tracked = card.parcels.some((p) => p.trackingNumber || p.trackingUrl); + if (!card.orderNumber && !tracked) problems.push('neither order number nor tracking'); + const showable = card.merchant || card.items.length > 0 || card.parcels.some((p) => p.carrier || p.items.length > 0 || p.expectedBy); + if (!showable) problems.push('nothing to show'); + return problems; + } + } +} + +function identifiers(card: StructuredCard): string[] { + if (card.kind === 'shipment') { + return [card.orderNumber, ...card.parcels.map((p) => p.trackingNumber)].filter((v): v is string => !!v); + } + return card.reference ? [card.reference] : []; +} + +/** Check one card against the mail's date. */ +export function validateCard(card: StructuredCard, mailDate: Date): Candidate { + const problems = cardProblems(card); + if (identifiers(card).some((id) => DOCUMENTATION_VALUES.has(id.toLowerCase()))) { + problems.push('documentation example'); + } + const date = firstDate(card); + const mailDay = Math.floor(mailDate.getTime() / DAY); + const day = date ? dayNumber(date) : undefined; + if (date !== undefined && day !== undefined && Number.isFinite(mailDay)) { + const future = card.kind === 'flight' || card.kind === 'train' || card.kind === 'bus' ? TRANSPORT_FUTURE_DAYS : OTHER_FUTURE_DAYS; + if (day < mailDay - PAST_WINDOW_DAYS || day > mailDay + future) problems.push('date too far from the mail'); + } + return { card, problems }; +} diff --git a/lib/structured-mail/values.ts b/lib/structured-mail/values.ts new file mode 100644 index 000000000..7ad53f3c8 --- /dev/null +++ b/lib/structured-mail/values.ts @@ -0,0 +1,254 @@ +/** + * Readers for single schema.org values, tolerant of the shapes real senders + * use: a string where an object belongs, an array where one value belongs, + * numbers for identifiers, German prices, offsets without a colon. + */ +import type { Address, Link, Moment, OrderStatus, ReservationStatus } from './types'; + +export type Obj = Record; + +export function isObj(value: unknown): value is Obj { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** The values of a property: an array as is, anything else as one element. */ +export function all(value: unknown): unknown[] { + if (value === undefined || value === null) return []; + return Array.isArray(value) ? value : [value]; +} + +/** The first object among the values. */ +export function obj(value: unknown): Obj | undefined { + return all(value).find(isObj); +} + +/** The first non-empty string (or number, as a string). */ +export function str(value: unknown): string | undefined { + for (const v of all(value)) { + if (typeof v === 'number' && Number.isFinite(v)) return String(v); + if (typeof v === 'string') { + const trimmed = v.replace(/\s+/g, ' ').trim(); + if (trimmed) return trimmed; + } + } + return undefined; +} + +/** A name: a plain string, or an object's `name` (Organization, Person, Place). */ +export function nameOf(value: unknown): string | undefined { + for (const v of all(value)) { + if (isObj(v)) { + const name = str(v.name) ?? [str(v.givenName), str(v.familyName)].filter(Boolean).join(' '); + if (name) return name; + } else { + const s = str(v); + if (s) return s; + } + } + return undefined; +} + +/** Types of an object, as their last path segment ("FlightReservation"). */ +export function typesOf(value: unknown): string[] { + if (!isObj(value)) return []; + return all(value['@type']) + .map((t) => (typeof t === 'string' ? t.trim().replace(/^schema:/i, '').split(/[/#]/).pop() ?? '' : '')) + .filter(Boolean); +} + +export function hasType(value: unknown, names: readonly string[]): boolean { + return typesOf(value).some((t) => names.includes(t)); +} + +const SCHEMA_CONTEXT = /^https?:\/\/(?:www\.)?schema\.org\/?$/i; + +/** Whether an `@context` names schema.org (string, array or `@vocab`). */ +export function isSchemaContext(context: unknown): boolean { + return all(context).some((c) => + (typeof c === 'string' && SCHEMA_CONTEXT.test(c.trim())) + || (isObj(c) && typeof c['@vocab'] === 'string' && SCHEMA_CONTEXT.test(c['@vocab'].trim()))); +} + +/** The last segment of an enum value: a URL or a bare token. */ +function enumToken(value: unknown): string | undefined { + const s = str(value); + if (!s) return undefined; + return s.split(/[/#:]/).filter(Boolean).pop(); +} + +const RESERVATION_STATUS: Record = { + confirmed: 'confirmed', cancelled: 'cancelled', canceled: 'cancelled', pending: 'pending', hold: 'hold', +}; + +/** ReservationConfirmed / Confirmed, full URL (http/https) or token. The first value that maps wins. */ +export function reservationStatus(value: unknown): ReservationStatus | undefined { + for (const v of all(value)) { + const token = enumToken(isObj(v) ? v['@id'] ?? v.name : v)?.toLowerCase().replace(/^reservation/, ''); + if (token && RESERVATION_STATUS[token]) return RESERVATION_STATUS[token]; + } + return undefined; +} + +const ORDER_STATUS: Record = { + paymentdue: 'payment_due', processing: 'processing', problem: 'problem', intransit: 'in_transit', + pickupavailable: 'pickup_available', returned: 'returned', delivered: 'delivered', + cancelled: 'cancelled', canceled: 'cancelled', +}; + +export function orderStatus(value: unknown): OrderStatus | undefined { + for (const v of all(value)) { + const token = enumToken(isObj(v) ? v['@id'] ?? v.name : v)?.toLowerCase().replace(/^order(status)?/, ''); + if (token && ORDER_STATUS[token]) return ORDER_STATUS[token]; + } + return undefined; +} + +const ISO_MOMENT = /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2})(?:[.,]\d+)?)?\s*(Z|[+-]\d{2}(?::?\d{2})?)?)?$/i; + +function validDate(y: number, m: number, d: number): boolean { + const date = new Date(Date.UTC(y, m - 1, d)); + return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d; +} + +function parseOffset(text: string): number | undefined { + if (/^z$/i.test(text)) return 0; + const match = /^([+-])(\d{2}):?(\d{2})?$/.exec(text); + if (!match) return undefined; + const hours = Number(match[2]); + const minutes = Number(match[3] ?? 0); + if (hours > 14 || minutes > 59) return undefined; + return (match[1] === '-' ? -1 : 1) * (hours * 60 + minutes); +} + +/** + * An ISO 8601 date or date-time. Offsets `Z`, `+01:00`, `+0100` and `+01`; + * fractional seconds dropped. Anything else is undefined, so a doubtful value + * counts as missing. + */ +export function moment(value: unknown): Moment | undefined { + const s = str(value); + if (!s) return undefined; + const match = ISO_MOMENT.exec(s); + if (!match) return undefined; + const [, y, mo, d, h, mi, se, off] = match; + if (!validDate(+y, +mo, +d)) return undefined; + const result: Moment = { date: `${y}-${mo}-${d}` }; + if (h === undefined) return result; + if (+h > 23 || +mi > 59 || +(se ?? 0) > 59) return undefined; + result.time = `${h}:${mi}:${se ?? '00'}`; + if (off !== undefined) { + const offset = parseOffset(off); + if (offset === undefined) return undefined; + result.offsetMinutes = offset; + const local = Date.UTC(+y, +mo - 1, +d, +h, +mi, +(se ?? 0)); + result.utc = new Date(local - offset * 60_000).toISOString().replace('.000Z', 'Z'); + } + return result; +} + +/** + * Only the date of a value whose time part is broken + * (`2024-10-13CEST03:10:23`, Kaufland). Used where a date is all a card + * shows (expected parcel arrival); nowhere else. + */ +export function lenientDate(value: unknown): Moment | undefined { + const parsed = moment(value); + if (parsed) return { date: parsed.date }; + const s = str(value); + const match = s ? /^(\d{4})-(\d{2})-(\d{2})(?![\d])/.exec(s) : null; + if (!match || !validDate(+match[1], +match[2], +match[3])) return undefined; + return { date: `${match[1]}-${match[2]}-${match[3]}` }; +} + +/** A PostalAddress, or a plain string kept as the street line. */ +export function address(value: unknown): Address | undefined { + const v = all(value)[0]; + if (typeof v === 'string') { + const street = str(v); + return street ? { street } : undefined; + } + if (!isObj(v)) return undefined; + const result: Address = {}; + const set = (key: keyof Address, raw: unknown) => { + const s = str(raw); + if (s) result[key] = s; + }; + set('street', v.streetAddress); + set('locality', v.addressLocality); + set('region', v.addressRegion); + set('postalCode', v.postalCode); + const country = isObj(v.addressCountry) ? str(v.addressCountry.identifier) ?? nameOf(v.addressCountry) : str(v.addressCountry); + if (country) { + if (/^[A-Za-z]{2}$/.test(country)) result.country = country.toUpperCase(); + else result.countryName = country; + } + return Object.keys(result).length > 0 ? result : undefined; +} + +const IPV4 = /^\d{1,3}(?:\.\d{1,3}){3}$/; + +/** + * A link a card may show: absolute http(s), a real host name (no single + * label like Ahoi's `http://default/`, no IP literal, no localhost), no + * credentials. Anything else is dropped. + */ +export function link(value: unknown): Link | undefined { + for (const v of all(value)) { + const raw = isObj(v) ? str(v.url) ?? str(v.target) ?? str(v['@id']) : str(v); + if (!raw) continue; + let url: URL; + try { + url = new URL(raw); + } catch { + continue; + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') continue; + if (url.username || url.password) continue; + const host = url.hostname.toLowerCase(); + if (!host.includes('.') || host.startsWith('[') || IPV4.test(host) || host === 'localhost' || host.endsWith('.localhost')) continue; + if (!/^[a-z0-9.-]+$/.test(host) || host.endsWith('.')) continue; + return { url: url.href, host: host.replace(/^www\./, '') }; + } + return undefined; +} + +/** + * A price as a dot-decimal string: numbers, "29.99", "14.9700", and German + * "399,00 €" / "1.299,00 EUR". Ambiguous values ("1,299") give undefined. + */ +export function amount(value: unknown): string | undefined { + for (const v of all(value)) { + if (typeof v === 'number' && Number.isFinite(v) && v >= 0) return v.toFixed(2); + if (typeof v !== 'string') continue; + const s = v.replace(/[^\d.,]/g, ''); + if (!s) continue; + let normalised: string | undefined; + if (/^\d+(\.\d+)?$/.test(s)) normalised = s; + else if (/^\d{1,3}(\.\d{3})+,\d{1,2}$/.test(s) || /^\d+,\d{1,2}$/.test(s)) normalised = s.replace(/\./g, '').replace(',', '.'); + else if (/^\d{1,3}(,\d{3})+\.\d{1,2}$/.test(s)) normalised = s.replace(/,/g, ''); + if (normalised === undefined) continue; + const n = Number(normalised); + if (Number.isFinite(n)) return n.toFixed(2); + } + return undefined; +} + +export function currency(value: unknown): string | undefined { + const s = str(value); + return s && /^[A-Za-z]{3}$/.test(s) ? s.toUpperCase() : undefined; +} + +export function positiveInt(value: unknown): number | undefined { + for (const v of all(value)) { + const raw = isObj(v) ? v.value : v; + const n = typeof raw === 'number' ? raw : typeof raw === 'string' && /^\s*\d+\s*$/.test(raw) ? Number(raw) : NaN; + if (Number.isInteger(n) && n > 0 && n < 10_000) return n; + } + return undefined; +} + +/** Names of people: Person objects, strings, arrays; duplicates dropped. */ +export function people(value: unknown): string[] { + const names = all(value).map((v) => nameOf(v)).filter((n): n is string => !!n); + return [...new Set(names)]; +} diff --git a/scripts/lite/scan-hosts.mjs b/scripts/lite/scan-hosts.mjs index c1822dfc4..9eda49ec8 100644 --- a/scripts/lite/scan-hosts.mjs +++ b/scripts/lite/scan-hosts.mjs @@ -39,6 +39,7 @@ export const LITE_BUNDLE_NON_REQUEST = [ { files: CHUNKS, why: "example addresses the translations localise", hosts: ["eksempel.dk", "mail.eksempel.dk", "piemers.lv", "pasts.piemers.lv", "ornek.com", "mail.ornek.com"] }, { files: CHUNKS, why: "links inside the sample mail of the demo mode (lib/demo/fixtures)", hosts: ["zoom.us", "notion.so", "www.figma.com", "vercel.com", "i.scdn.co", "cf.bstatic.com", "open.spotify.com", "linear.app", "sentry.io", "app.docusign.com", "github.com", "bulwark-webmail.vercel.app"] }, { files: CHUNKS, why: "JMAP capability URI and XML namespaces", hosts: ["bulwarkmail.com", "www.w3.org", "creativecommons.org"] }, + { files: CHUNKS, why: "schema.org vocabulary names in booking markup (lib/structured-mail), matched and never requested", hosts: ["schema.org"] }, { files: /./, why: "docs links in the translations and the README", hosts: ["bulwarkmail.org", "github.com"] }, { files: PAGES, why: "XML namespaces of inline SVG", hosts: ["www.w3.org"] }, { files: /^_next\/static\/chunks\/[^/]+\.css$/, why: "XML namespace of the SVG pointers in app/accessibility.css (large mouse pointer)", hosts: ["www.w3.org"] },