mirror of
https://github.com/bulwarkmail/webmail.git
synced 2026-10-30 16:16:19 +00:00
feat: read schema.org booking markup from mail bodies
This commit is contained in:
Linus Rath
@@ -170,6 +170,7 @@ export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[
|
||||
{ host: 'www.youtube-nocookie.com', why: 'example in a comment', files: ['lib/plugin-types.ts'] },
|
||||
{ host: 'bulwark-webmail.vercel.app', why: 'sample message text', files: ['stores/email-store.ts'] },
|
||||
{ host: 'randomuser.me', why: 'docs links in a comment of the mock server', files: ['app/api/dev-jmap/[...path]/route.ts'] },
|
||||
{ host: 'schema.org', why: 'vocabulary names in booking markup, matched and never requested', files: ['lib/structured-mail/extract.ts'] },
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { extractStructuredData, mayContainStructuredData } from '../extract';
|
||||
|
||||
const page = (head: string, body = '') => `<!DOCTYPE html><html><head>${head}</head><body>${body}</body></html>`;
|
||||
const ld = (value: unknown, type = 'application/ld+json') => `<script type="${type}">${JSON.stringify(value)}</script>`;
|
||||
const order = { '@context': 'http://schema.org', '@type': 'Order', orderNumber: '42' };
|
||||
|
||||
describe('extractStructuredData: JSON-LD', () => {
|
||||
it('reads blocks in head and body, type compared loosely', () => {
|
||||
const html = page(ld(order), ld({ ...order, orderNumber: '43' }, ' Application/LD+JSON '));
|
||||
expect(extractStructuredData(html).map((f) => f.item)).toEqual([order, { ...order, orderNumber: '43' }]);
|
||||
});
|
||||
|
||||
it('reads a block nested inside a microdata tree', () => {
|
||||
const html = page('', `<div itemscope itemtype="http://schema.org/EmailMessage">${ld(order)}</div>`);
|
||||
expect(extractStructuredData(html).filter((f) => f.source === 'json-ld').map((f) => f.item)).toEqual([order]);
|
||||
});
|
||||
|
||||
it('skips mail without any type it reads, without parsing', () => {
|
||||
const html = page(ld({ '@context': 'http://schema.org', '@type': 'PromotionCard' }));
|
||||
expect(mayContainStructuredData(html)).toBe(false);
|
||||
expect(extractStructuredData(html)).toEqual([]);
|
||||
});
|
||||
|
||||
it('ignores markup inside quoted or forwarded parts', () => {
|
||||
const quoted = `<blockquote>${ld(order)}</blockquote><div class="gmail_quote">${ld(order)}</div>`;
|
||||
const outlook = `<p>FYI</p><div id="divRplyFwdMsg">From: someone</div><div>${ld(order)}</div>`;
|
||||
expect(extractStructuredData(page('', quoted))).toEqual([]);
|
||||
expect(extractStructuredData(page('', outlook))).toEqual([]);
|
||||
});
|
||||
|
||||
it('caps the number of blocks it reads', () => {
|
||||
const html = page(Array.from({ length: 60 }, () => ld(order)).join(''));
|
||||
expect(extractStructuredData(html)).toHaveLength(50);
|
||||
});
|
||||
|
||||
it('runs no script and loads nothing while parsing', () => {
|
||||
const html = page('<script>window.__structuredMailRan = true</script>', `<img src="https://tracker.invalid/x.gif">${ld(order)}`);
|
||||
extractStructuredData(html);
|
||||
expect((window as unknown as Record<string, unknown>).__structuredMailRan).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractStructuredData: microdata', () => {
|
||||
it('reads values from content, href, src, datetime and text', () => {
|
||||
const html = page('', `
|
||||
<div itemscope itemtype="http://schema.org/ParcelDelivery">
|
||||
<meta itemprop="trackingNumber" content=" 123 ">
|
||||
<a itemprop="trackingUrl" href="https://track.example.com/123">track</a>
|
||||
<img itemprop="image" src="https://img.example.com/a.png">
|
||||
<time itemprop="expectedArrivalUntil" datetime="2026-11-02">2 Nov</time>
|
||||
<span itemprop="carrier"> DHL
|
||||
Express </span>
|
||||
</div>`);
|
||||
expect(extractStructuredData(html)).toEqual([{
|
||||
source: 'microdata',
|
||||
item: {
|
||||
'@context': 'https://schema.org', '@type': 'ParcelDelivery', trackingNumber: '123',
|
||||
trackingUrl: 'https://track.example.com/123', image: 'https://img.example.com/a.png',
|
||||
expectedArrivalUntil: '2026-11-02', carrier: 'DHL Express',
|
||||
},
|
||||
}]);
|
||||
});
|
||||
|
||||
it('nests items, splits itemprop lists and turns repeats into arrays', () => {
|
||||
const html = page('', `
|
||||
<div itemscope itemtype="https://schema.org/Order">
|
||||
<meta itemprop="orderNumber" content="7">
|
||||
<div itemprop="merchant seller" itemscope itemtype="http://schema.org/Organization"><meta itemprop="name" content="Shop"></div>
|
||||
<meta itemprop="orderStatus" content="">
|
||||
<link itemprop="orderStatus" href="http://schema.org/OrderProcessing">
|
||||
<div itemprop="acceptedOffer" itemscope itemtype="http://schema.org/Offer">
|
||||
<div itemprop="itemOffered" itemscope itemtype="http://schema.org/Product"><meta itemprop="name" content="One"></div>
|
||||
</div>
|
||||
<div itemprop="acceptedOffer" itemscope itemtype="http://schema.org/Offer">
|
||||
<div itemprop="itemOffered" itemscope itemtype="http://schema.org/Product"><meta itemprop="name" content="Two"></div>
|
||||
</div>
|
||||
</div>`);
|
||||
const [{ item }] = extractStructuredData(html);
|
||||
const shop = { '@type': 'Organization', name: 'Shop' };
|
||||
expect(item).toEqual({
|
||||
'@context': 'https://schema.org', '@type': 'Order', orderNumber: '7', merchant: shop, seller: shop,
|
||||
orderStatus: 'http://schema.org/OrderProcessing',
|
||||
acceptedOffer: [
|
||||
{ '@type': 'Offer', itemOffered: { '@type': 'Product', name: 'One' } },
|
||||
{ '@type': 'Offer', itemOffered: { '@type': 'Product', name: 'Two' } },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('ignores items not typed on schema.org', () => {
|
||||
const html = page('', `
|
||||
<div itemscope itemtype="http://example.com/Order"><meta itemprop="orderNumber" content="1"></div>
|
||||
<div itemscope><meta itemprop="orderNumber" content="2"></div>`);
|
||||
expect(extractStructuredData(html)).toEqual([]);
|
||||
});
|
||||
|
||||
it('accepts several itemtypes and keeps the schema.org ones', () => {
|
||||
const html = page('', '<div itemscope itemtype="http://example.com/X http://schema.org/Order"><meta itemprop="orderNumber" content="1"></div>');
|
||||
expect(extractStructuredData(html)[0].item).toMatchObject({ '@type': 'Order', orderNumber: '1' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseLenientJson } from '../json-repair';
|
||||
|
||||
const A = { '@type': 'FlightReservation', reservationNumber: 'A1' };
|
||||
const B = { '@type': 'FlightReservation', reservationNumber: 'B2' };
|
||||
|
||||
describe('parseLenientJson', () => {
|
||||
it('parses valid JSON unchanged', () => {
|
||||
expect(parseLenientJson(JSON.stringify([A, B]))).toEqual([A, B]);
|
||||
});
|
||||
|
||||
it('adds the commas Eurowings leaves out between array objects', () => {
|
||||
expect(parseLenientJson(`[${JSON.stringify(A)}\n${JSON.stringify(B)}]`)).toEqual([A, B]);
|
||||
expect(parseLenientJson(`[${JSON.stringify(A)}${JSON.stringify(B)}]`)).toEqual([A, B]);
|
||||
});
|
||||
|
||||
it('adds the brackets Volotea leaves out, with or without commas', () => {
|
||||
expect(parseLenientJson(`${JSON.stringify(A)},\n${JSON.stringify(B)}`)).toEqual([A, B]);
|
||||
expect(parseLenientJson(`${JSON.stringify(A)}\n${JSON.stringify(B)}`)).toEqual([A, B]);
|
||||
});
|
||||
|
||||
it('drops the trailing commas Eventbrite writes', () => {
|
||||
expect(parseLenientJson('{"@type": "EventReservation", "reservationNumber": "E1", }')).toEqual({ '@type': 'EventReservation', reservationNumber: 'E1' });
|
||||
expect(parseLenientJson('[{"a": [1, 2, ], }, ]')).toEqual([{ a: [1, 2] }]);
|
||||
});
|
||||
|
||||
it('decodes the XML entities Airbnb encodes its JSON with', () => {
|
||||
const encoded = JSON.stringify(A).replace(/"/g, '"');
|
||||
expect(parseLenientJson(encoded)).toEqual(A);
|
||||
expect(parseLenientJson('{"name": "Café & Bar"}')).toEqual({ name: 'Café & Bar' });
|
||||
});
|
||||
|
||||
it('strips comment and CDATA wrappers and escapes raw newlines in strings', () => {
|
||||
expect(parseLenientJson(`<!--\n${JSON.stringify(A)}\n-->`)).toEqual(A);
|
||||
expect(parseLenientJson(`<![CDATA[${JSON.stringify(A)}]]>`)).toEqual(A);
|
||||
expect(parseLenientJson('{"name": "two\nlines"}')).toEqual({ name: 'two\nlines' });
|
||||
});
|
||||
|
||||
it('leaves commas and braces inside strings alone', () => {
|
||||
expect(parseLenientJson('{"name": "a, } { b", }')).toEqual({ name: 'a, } { b' });
|
||||
});
|
||||
|
||||
it('gives up on input it cannot repair', () => {
|
||||
expect(parseLenientJson('{"a": ')).toBeUndefined();
|
||||
expect(parseLenientJson('not json at all')).toBeUndefined();
|
||||
expect(parseLenientJson('')).toBeUndefined();
|
||||
expect(parseLenientJson(`[${'x'.repeat(200_001)}]`)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { normalizeItems } from '../normalize';
|
||||
import type { LodgingBooking, Shipment, TransportBooking } from '../types';
|
||||
import { address, amount, lenientDate, link, moment, orderStatus, reservationStatus } from '../values';
|
||||
|
||||
const ctx = 'http://schema.org';
|
||||
const norm = (...items: unknown[]) => normalizeItems(items.map((item) => ({ item, source: 'json-ld' as const })));
|
||||
|
||||
describe('value readers', () => {
|
||||
it('reads ISO times with every offset form and computes the instant', () => {
|
||||
expect(moment('2027-03-04T20:15:00-08:00')).toEqual({ date: '2027-03-04', time: '20:15:00', offsetMinutes: -480, utc: '2027-03-05T04:15:00Z' });
|
||||
expect(moment('2024-06-04T11:44:00+0200')).toEqual({ date: '2024-06-04', time: '11:44:00', offsetMinutes: 120, utc: '2024-06-04T09:44:00Z' });
|
||||
expect(moment('2026-01-01T10:00+05')).toMatchObject({ offsetMinutes: 300, utc: '2026-01-01T05:00:00Z' });
|
||||
expect(moment('2026-01-01T10:00:00.123Z')).toMatchObject({ time: '10:00:00', offsetMinutes: 0, utc: '2026-01-01T10:00:00Z' });
|
||||
expect(moment('2026-01-01T10:00:00')).toEqual({ date: '2026-01-01', time: '10:00:00' });
|
||||
expect(moment('2026-01-01')).toEqual({ date: '2026-01-01' });
|
||||
});
|
||||
|
||||
it('refuses broken times instead of guessing', () => {
|
||||
expect(moment('2024-10-13CEST03:10:23')).toBeUndefined();
|
||||
expect(moment('2026-02-30T10:00:00Z')).toBeUndefined();
|
||||
expect(moment('2026-01-01T25:00:00Z')).toBeUndefined();
|
||||
expect(moment('2026-01-01T10:00:00+15:00')).toBeUndefined();
|
||||
expect(moment('next Friday')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps only the date of a broken value where a date is enough', () => {
|
||||
expect(lenientDate('2024-10-13CEST03:10:23')).toEqual({ date: '2024-10-13' });
|
||||
expect(lenientDate('2026-03-12T12:00:00-08:00')).toEqual({ date: '2026-03-12' });
|
||||
expect(lenientDate('20241013')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('reads both forms of the status enums over http and https', () => {
|
||||
expect(reservationStatus('http://schema.org/ReservationConfirmed')).toBe('confirmed');
|
||||
expect(reservationStatus('https://schema.org/Confirmed')).toBe('confirmed');
|
||||
expect(reservationStatus('ReservationCancelled')).toBe('cancelled');
|
||||
expect(reservationStatus(['', 'http://schema.org/Cancelled'])).toBe('cancelled');
|
||||
expect(reservationStatus('http://schema.org/Bogus')).toBeUndefined();
|
||||
expect(orderStatus(['', 'http://schema.org/OrderProcessing'])).toBe('processing');
|
||||
expect(orderStatus('http://schema.org/OrderStatus/InTransit')).toBe('in_transit');
|
||||
expect(orderStatus('OrderDelivered')).toBe('delivered');
|
||||
expect(orderStatus('Processing')).toBe('processing');
|
||||
});
|
||||
|
||||
it('reads prices in dot and German notation and refuses ambiguous ones', () => {
|
||||
expect(amount(25.99)).toBe('25.99');
|
||||
expect(amount('14.9700')).toBe('14.97');
|
||||
expect(amount('399,00 €')).toBe('399.00');
|
||||
expect(amount('1.299,00 EUR')).toBe('1299.00');
|
||||
expect(amount('$1,299.50')).toBe('1299.50');
|
||||
expect(amount('1,299')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps only http(s) links with a real host', () => {
|
||||
expect(link('https://www.fedex.com/track/1')).toEqual({ url: 'https://www.fedex.com/track/1', host: 'fedex.com' });
|
||||
expect(link({ '@type': 'ViewAction', target: 'http://shop.example.com/o/1' })?.host).toBe('shop.example.com');
|
||||
expect(link('http://default/buchung/VK4XKD')).toBeUndefined();
|
||||
expect(link('javascript:alert(1)')).toBeUndefined();
|
||||
expect(link('https://user:pw@example.com/')).toBeUndefined();
|
||||
expect(link('http://192.168.1.1/x')).toBeUndefined();
|
||||
expect(link('http://localhost:3000/x')).toBeUndefined();
|
||||
expect(link('/relative')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('reads addresses given as objects, strings and country objects', () => {
|
||||
expect(address('Unter den Linden 77, Berlin')).toEqual({ street: 'Unter den Linden 77, Berlin' });
|
||||
expect(address({ streetAddress: 'Koppenstr. 3', addressLocality: 'Berlin', postalCode: '10243', addressCountry: 'de', addressRegion: '' }))
|
||||
.toEqual({ street: 'Koppenstr. 3', locality: 'Berlin', postalCode: '10243', country: 'DE' });
|
||||
expect(address({ addressLocality: 'Paris', addressCountry: { '@type': 'Country', name: 'France' } }))
|
||||
.toEqual({ locality: 'Paris', countryName: 'France' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeItems', () => {
|
||||
const flight = {
|
||||
'@context': ctx, '@type': 'FlightReservation', reservationNumber: 'QX7K2M',
|
||||
reservationStatus: 'http://schema.org/ReservationConfirmed',
|
||||
underName: { '@type': 'Person', name: 'Erika Mustermann' }, airplaneSeat: '12C',
|
||||
checkinUrl: 'https://www.lufthansa.example.com/checkin',
|
||||
reservationFor: {
|
||||
'@type': 'Flight', flightNumber: 'LH400',
|
||||
airline: { '@type': 'Airline', name: 'Lufthansa', iataCode: 'LH' },
|
||||
departureAirport: { '@type': 'Airport', name: 'Frankfurt', iataCode: 'fra' }, departureTime: '2026-12-01T10:15:00+01:00',
|
||||
arrivalAirport: { '@type': 'Airport', name: 'New York JFK', iataCode: 'JFK' }, arrivalTime: '2026-12-01T13:05:00-05:00',
|
||||
},
|
||||
};
|
||||
|
||||
it('reads a flight reservation, splitting the airline code off the flight number', () => {
|
||||
const [card] = norm(flight) as TransportBooking[];
|
||||
expect(card).toEqual({
|
||||
key: 'flight|ref:qx7k2m', kind: 'flight', reference: 'QX7K2M', status: 'confirmed', passengers: ['Erika Mustermann'],
|
||||
provider: 'Lufthansa', links: { checkin: { url: 'https://www.lufthansa.example.com/checkin', host: 'lufthansa.example.com' } },
|
||||
source: 'json-ld',
|
||||
legs: [{
|
||||
carrier: 'Lufthansa', carrierCode: 'LH', number: '400',
|
||||
departure: { place: { name: 'Frankfurt', code: 'FRA' }, at: { date: '2026-12-01', time: '10:15:00', offsetMinutes: 60, utc: '2026-12-01T09:15:00Z' } },
|
||||
arrival: { place: { name: 'New York JFK', code: 'JFK' }, at: { date: '2026-12-01', time: '13:05:00', offsetMinutes: -300, utc: '2026-12-01T18:05:00Z' } },
|
||||
seats: [{ passenger: 'Erika Mustermann', seat: '12C' }], status: 'confirmed',
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
it('flattens arrays, @graph and reservationFor arrays', () => {
|
||||
const two = { ...flight, reservationFor: [flight.reservationFor, { ...flight.reservationFor, flightNumber: '401' }] };
|
||||
expect(norm([flight, flight]).length).toBe(2);
|
||||
expect(norm({ '@context': ctx, '@graph': [flight, flight] }).length).toBe(2);
|
||||
const legs = norm(two) as TransportBooking[];
|
||||
expect(legs.map((c) => c.legs[0].number)).toEqual(['400', '401']);
|
||||
});
|
||||
|
||||
it('accepts https contexts, @vocab and several @types, and drops foreign or missing contexts', () => {
|
||||
expect(norm({ ...flight, '@context': 'https://schema.org/' })).toHaveLength(1);
|
||||
expect(norm({ ...flight, '@context': { '@vocab': 'http://schema.org/' } })).toHaveLength(1);
|
||||
expect(norm({ ...flight, '@type': ['Thing', 'FlightReservation'] })).toHaveLength(1);
|
||||
expect(norm({ ...flight, '@context': 'http://example.com' })).toHaveLength(0);
|
||||
const { '@context': _dropped, ...noContext } = flight;
|
||||
expect(norm(noContext)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('reads bus trips the way FlixBus writes them', () => {
|
||||
const [card] = norm({
|
||||
'@context': ctx, '@type': 'BusReservation', reservationNumber: 3165152579,
|
||||
underName: { '@type': 'Person', name: 'Max Mustermann' },
|
||||
reservationFor: {
|
||||
'@type': 'BusTrip', busNumber: 'FLX30', busName: 'Berlin Ostbahnhof', busCompany: { '@type': 'Organization', name: 'Flix' },
|
||||
departureBusStop: { '@type': 'BusStop', name: 'Hannover Hauptbahnhof', address: { '@type': 'PostalAddress', addressLocality: 'Hannover', addressCountry: 'DE' } },
|
||||
departureTime: '2024-06-04T11:44:00+0200',
|
||||
arrivalBusStop: { '@type': 'BusStop', name: 'Berlin Ostbahnhof' }, arrivalTime: '2024-06-04T13:44:00+0200',
|
||||
},
|
||||
reservedTicket: { '@type': 'Ticket', ticketedSeat: { '@type': 'Seat', seatingType: 'Coach', seatNumber: '6D' } },
|
||||
}) as TransportBooking[];
|
||||
expect(card.reference).toBe('3165152579');
|
||||
expect(card.legs[0]).toMatchObject({ carrier: 'Flix', number: 'FLX30', seats: [{ passenger: 'Max Mustermann', seat: '6D' }] });
|
||||
expect(card.legs[0].departure.place).toEqual({ name: 'Hannover Hauptbahnhof', address: { locality: 'Hannover', country: 'DE' } });
|
||||
expect(JSON.stringify(card)).not.toContain('busName');
|
||||
});
|
||||
|
||||
it('reads hotels from Hotel subtypes with legacy names', () => {
|
||||
const [card] = norm({
|
||||
'@context': ctx, '@type': 'LodgingReservation', reservationId: 'h-1',
|
||||
reservationFor: { '@type': 'Hotel', name: 'Hotel Adlon', address: 'Unter den Linden 77, 10117 Berlin', telephone: '+49 30 1' },
|
||||
checkinTime: '2026-12-04T15:00:00+01:00', checkoutTime: '2026-12-06T12:00:00+01:00', Url: 'https://hotel.example.com/b/h-1',
|
||||
}) as LodgingBooking[];
|
||||
expect(card).toMatchObject({
|
||||
kind: 'lodging', reference: 'h-1', name: 'Hotel Adlon', phone: '+49 30 1', address: { street: 'Unter den Linden 77, 10117 Berlin' },
|
||||
checkin: { date: '2026-12-04', time: '15:00:00' }, checkout: { date: '2026-12-06', time: '12:00:00' },
|
||||
links: { manage: { host: 'hotel.example.com' } },
|
||||
});
|
||||
});
|
||||
|
||||
it('reads restaurants, events with a date-only end, and rental cars', () => {
|
||||
const [restaurant, event, car] = norm(
|
||||
{ '@context': ctx, '@type': 'FoodEstablishmentReservation', reservationNumber: 'R9', partySize: '2', startTime: '2026-12-05T19:30:00+01:00', reservationFor: { '@type': 'Restaurant', name: 'Tim Raue' } },
|
||||
{ '@context': ctx, '@type': 'EventReservation', reservationNumber: 'E9', reservationFor: { '@type': 'MusicEvent', name: 'Konzert', startDate: '2026-12-05T20:00:00+01:00', endDate: '2026-12-05', location: { '@type': 'Place', name: 'Halle', address: { addressCountry: 'DE' } } } },
|
||||
{ '@context': ctx, '@type': 'RentalCarReservation', reservationNumber: 'C9', reservationFor: { '@type': 'Car', brand: { name: 'VW' }, model: 'Golf', rentalCompany: { name: 'Sixt' } }, pickupTime: '2026-12-04T09:00:00+01:00', dropoffTime: '2026-12-07T18:00:00+01:00', pickupLocation: { name: 'BER' }, dropoffLocation: 'BER' },
|
||||
);
|
||||
expect(restaurant).toMatchObject({ kind: 'restaurant', name: 'Tim Raue', partySize: 2, start: { time: '19:30:00' } });
|
||||
expect(event).toMatchObject({ kind: 'event', name: 'Konzert', end: { date: '2026-12-05', time: '23:59:59' }, venue: { name: 'Halle' } });
|
||||
expect(car).toMatchObject({ kind: 'rentalcar', company: 'Sixt', car: 'VW Golf', pickup: { place: { name: 'BER' } }, dropoff: { place: { name: 'BER' } } });
|
||||
});
|
||||
|
||||
it('reads orders with German prices and parcels with their order', () => {
|
||||
const [orderCard, parcelCard] = norm(
|
||||
{
|
||||
'@context': ctx, '@type': 'Order', merchant: { '@type': 'Organization', name: 'Kaufland.de' }, orderNumber: 'M1234AB',
|
||||
price: '399,00 €', priceCurrency: 'EUR', orderStatus: ['', 'http://schema.org/OrderProcessing'],
|
||||
acceptedOffer: { '@type': 'Offer', itemOffered: { '@type': 'Product', name: '3D-Drucker', image: 'https://media.example.com/p.jpg' }, eligibleQuantity: { value: '1' } },
|
||||
url: 'https://www.kaufland.de/account/orders',
|
||||
},
|
||||
{
|
||||
'@context': ctx, '@type': 'ParcelDelivery', carrier: 'DHL', trackingNumber: '0034', trackingUrl: 'https://www.dhl.de/t?id=0034',
|
||||
expectedArrivalUntil: '2024-10-13CEST03:10:23', itemShipped: [{ '@type': 'Product', name: 'A' }, 'B'],
|
||||
partOfOrder: { '@type': 'Order', orderNumber: 'M1234AB', merchant: 'Kaufland.de', orderStatus: 'http://schema.org/OrderInTransit' },
|
||||
},
|
||||
) as Shipment[];
|
||||
expect(orderCard).toEqual({
|
||||
kind: 'shipment', key: 'shipment|order:m1234ab', merchant: 'Kaufland.de', orderNumber: 'M1234AB', status: 'processing',
|
||||
items: [{ name: '3D-Drucker', quantity: 1, image: { url: 'https://media.example.com/p.jpg', host: 'media.example.com' } }],
|
||||
total: { amount: '399.00', currency: 'EUR' }, orderUrl: { url: 'https://www.kaufland.de/account/orders', host: 'kaufland.de' },
|
||||
parcels: [], source: 'json-ld',
|
||||
});
|
||||
expect(parcelCard).toMatchObject({
|
||||
key: 'shipment|order:m1234ab', merchant: 'Kaufland.de', status: 'in_transit',
|
||||
parcels: [{ carrier: 'DHL', trackingNumber: '0034', trackingUrl: { host: 'dhl.de' }, expectedBy: { date: '2024-10-13' }, items: [{ name: 'A' }, { name: 'B' }] }],
|
||||
});
|
||||
});
|
||||
|
||||
it('drops types it does not read', () => {
|
||||
expect(norm(
|
||||
{ '@context': ctx, '@type': 'PromotionCard', image: 'https://x.example.com/a.png' },
|
||||
{ '@context': ctx, '@type': 'EmailMessage', potentialAction: { '@type': 'ViewAction' } },
|
||||
{ '@context': ctx, '@type': 'Invoice', totalPaymentDue: 3 },
|
||||
)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { normalizeItems } from '../normalize';
|
||||
import { needsAirportZones, resolveCards } from '../resolve';
|
||||
import { loadAirportZones } from '../time-zones';
|
||||
import type { StructuredCard, TransportBooking } from '../types';
|
||||
import { validateCard } from '../validate';
|
||||
|
||||
const ctx = 'http://schema.org';
|
||||
const MAIL = new Date('2026-10-30T10:00:00Z');
|
||||
const one = (item: unknown) => normalizeItems([{ item, source: 'json-ld' }])[0];
|
||||
const problems = (item: unknown, mailDate = MAIL) => validateCard(one(item), mailDate).problems;
|
||||
|
||||
function flight(dep: [string, string, string], arr: [string, string, string], extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
'@context': ctx, '@type': 'FlightReservation', reservationNumber: 'T1', underName: 'Erika Mustermann',
|
||||
reservationFor: {
|
||||
'@type': 'Flight', flightNumber: '100', airline: { name: 'Test Air', iataCode: 'TA' },
|
||||
departureAirport: { name: dep[0], iataCode: dep[1] }, departureTime: dep[2],
|
||||
arrivalAirport: { name: arr[0], iataCode: arr[1] }, arrivalTime: arr[2],
|
||||
},
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
const sfoJfk = flight(['San Francisco', 'SFO', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']);
|
||||
|
||||
describe('validateCard', () => {
|
||||
it('accepts a complete flight near the mail date', () => {
|
||||
expect(problems(sfoJfk)).toEqual([]);
|
||||
});
|
||||
|
||||
it('rejects flights with missing times or places', () => {
|
||||
const noArrival = flight(['San Francisco', 'SFO', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '']);
|
||||
const bareDate = flight(['San Francisco', 'SFO', '2026-12-04'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']);
|
||||
const noPlace = flight(['', '', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']);
|
||||
expect(problems(noArrival)).toContain('arrival time missing');
|
||||
expect(problems(bareDate)).toContain('departure time missing');
|
||||
expect(problems(noPlace)).toContain('departure place missing');
|
||||
});
|
||||
|
||||
it('rejects bookings years away from the mail', () => {
|
||||
expect(problems(sfoJfk, new Date('2024-11-12T00:00:00Z'))).toContain('date too far from the mail');
|
||||
expect(problems(sfoJfk, new Date('2028-01-01T00:00:00Z'))).toContain('date too far from the mail');
|
||||
});
|
||||
|
||||
it('rejects copies of Google documentation examples', () => {
|
||||
expect(problems({ ...sfoJfk, reservationNumber: 'RXJ34P' })).toContain('documentation example');
|
||||
expect(problems({ '@context': ctx, '@type': 'ParcelDelivery', trackingNumber: '3453291231', carrier: 'FedEx' })).toContain('documentation example');
|
||||
});
|
||||
|
||||
it('rejects stays that end before they start', () => {
|
||||
const stay = (checkin: string, checkout: string) => ({
|
||||
'@context': ctx, '@type': 'LodgingReservation', reservationNumber: 'H', reservationFor: { '@type': 'Hotel', name: 'Hotel' },
|
||||
checkinDate: checkin, checkoutDate: checkout,
|
||||
});
|
||||
expect(problems(stay('2026-12-04', '2026-12-06'))).toEqual([]);
|
||||
expect(problems(stay('2026-12-06', '2026-12-04'))).toContain('check-out not after check-in');
|
||||
expect(problems(stay('2026-12-04', '2026-12-04'))).toContain('check-out not after check-in');
|
||||
});
|
||||
|
||||
it('rejects orders with nothing to identify them, like Uber\'s empty Order', () => {
|
||||
expect(problems({ '@context': ctx, '@type': 'Order', merchant: { name: 'UberEATS' } })).toContain('neither order number nor tracking');
|
||||
expect(problems({ '@context': ctx, '@type': 'Order', merchant: 'K4G', orderNumber: 'A1' })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveCards', () => {
|
||||
const resolved = async (item: unknown): Promise<StructuredCard[]> => {
|
||||
const cards = [one(item)];
|
||||
return needsAirportZones(cards) ? resolveCards(cards, await loadAirportZones()) : resolveCards(cards);
|
||||
};
|
||||
|
||||
it('gives flight ends their airport zones and keeps the instants', async () => {
|
||||
const [card] = (await resolved(sfoJfk)) as TransportBooking[];
|
||||
expect(card.legs[0].departure.at).toMatchObject({ zone: 'America/Los_Angeles', utc: '2026-12-05T04:15:00Z' });
|
||||
expect(card.legs[0].arrival?.at).toMatchObject({ zone: 'America/New_York', utc: '2026-12-05T11:30:00Z' });
|
||||
});
|
||||
|
||||
it('accepts eastbound date-line flights that land earlier on the wall clock', async () => {
|
||||
const nrtHnl = flight(['Tokyo Narita', 'NRT', '2026-12-04T21:00:00+09:00'], ['Honolulu', 'HNL', '2026-12-04T09:15:00-10:00']);
|
||||
const sydLax = flight(['Sydney', 'SYD', '2026-12-04T09:50:00+11:00'], ['Los Angeles', 'LAX', '2026-12-04T06:20:00-08:00']);
|
||||
expect(await resolved(nrtHnl)).toHaveLength(1);
|
||||
expect(await resolved(sydLax)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('rejects arrival before departure and flights over 20 hours', async () => {
|
||||
const backwards = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00+01:00'], ['New York', 'JFK', '2026-12-01T03:05:00-05:00']);
|
||||
const tooLong = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00+01:00'], ['Sydney', 'SYD', '2026-12-02T23:00:00+11:00']);
|
||||
expect(await resolved(backwards)).toEqual([]);
|
||||
expect(await resolved(tooLong)).toEqual([]);
|
||||
});
|
||||
|
||||
it('drops a flight whose offset contradicts the airport zone', async () => {
|
||||
const wrongOffset = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00+00:00'], ['New York', 'JFK', '2026-12-01T13:05:00-05:00']);
|
||||
expect(await resolved(wrongOffset)).toEqual([]);
|
||||
});
|
||||
|
||||
it('reads a flight time without offset in the airport zone, and drops it when the airport is unknown', async () => {
|
||||
const floating = flight(['Frankfurt', 'FRA', '2026-12-01T10:15:00'], ['New York', 'JFK', '2026-12-01T13:05:00']);
|
||||
const [card] = (await resolved(floating)) as TransportBooking[];
|
||||
expect(card.legs[0].departure.at).toMatchObject({ zone: 'Europe/Berlin', utc: '2026-12-01T09:15:00Z' });
|
||||
const unknown = flight(['Nowhere', 'QQQ', '2026-12-01T10:15:00'], ['New York', 'JFK', '2026-12-01T13:05:00']);
|
||||
expect(await resolved(unknown)).toEqual([]);
|
||||
});
|
||||
|
||||
it('never shows a flight before the airport table is there', () => {
|
||||
expect(resolveCards([one(sfoJfk)])).toEqual([]);
|
||||
});
|
||||
|
||||
it('uses a fixed-offset zone where no airport code is known', async () => {
|
||||
const noCode = flight(['San Francisco', '', '2026-12-04T20:15:00-08:00'], ['New York', 'JFK', '2026-12-05T06:30:00-05:00']);
|
||||
const [card] = (await resolved(noCode)) as TransportBooking[];
|
||||
expect(card.legs[0].departure.at.zone).toBe('Etc/GMT+8');
|
||||
});
|
||||
|
||||
it('treats hotel times without offset or known country as dates only', async () => {
|
||||
const [card] = await resolved({
|
||||
'@context': ctx, '@type': 'LodgingReservation', reservationNumber: 'H', reservationFor: { '@type': 'Hotel', name: 'Hotel', address: { addressCountry: 'US' } },
|
||||
checkinDate: '2026-12-04T15:00:00', checkoutDate: '2026-12-06T11:00:00',
|
||||
});
|
||||
expect(card).toMatchObject({ checkin: { date: '2026-12-04' }, checkout: { date: '2026-12-06' } });
|
||||
expect((card as { checkin: { time?: string } }).checkin.time).toBeUndefined();
|
||||
});
|
||||
|
||||
it('reads bus times in the single zone of the stop\'s country', async () => {
|
||||
const [card] = (await resolved({
|
||||
'@context': ctx, '@type': 'BusReservation', reservationNumber: 'B',
|
||||
reservationFor: {
|
||||
'@type': 'BusTrip', busNumber: 'FLX30',
|
||||
departureBusStop: { name: 'Hannover', address: { addressCountry: 'DE' } }, departureTime: '2026-06-04T11:44:00+0200',
|
||||
arrivalBusStop: { name: 'Berlin', address: { addressCountry: 'DE' } }, arrivalTime: '2026-06-04T13:44:00+0200',
|
||||
},
|
||||
})) as TransportBooking[];
|
||||
expect(card.legs[0].departure.at.zone).toBe('Europe/Berlin');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Pull schema.org data out of a mail's raw HTML: JSON-LD blocks and
|
||||
* microdata. The HTML is parsed into a detached document (DOMParser runs no
|
||||
* scripts and loads nothing) that is never attached to the page; the sanitizer
|
||||
* that renders the mail is a separate path and isn't touched.
|
||||
*/
|
||||
import { parseLenientJson } from './json-repair';
|
||||
|
||||
export type RawItem = Record<string, unknown>;
|
||||
export interface ExtractedItem {
|
||||
item: unknown;
|
||||
source: 'json-ld' | 'microdata';
|
||||
}
|
||||
|
||||
const MAX_JSONLD_BLOCKS = 50;
|
||||
const MAX_MICRODATA_ITEMS = 500;
|
||||
const MAX_MICRODATA_DEPTH = 12;
|
||||
const MAX_HTML_LENGTH = 8_000_000;
|
||||
|
||||
/** Cheap test that a body may hold markup this feature reads. */
|
||||
const TYPE_HINT = /Reservation|ParcelDelivery|Order/;
|
||||
const SCHEMA_ITEMTYPE = /^https?:\/\/(?:www\.)?schema\.org\/([A-Za-z]+)\/?$/i;
|
||||
|
||||
/**
|
||||
* Mail clients mark what they quote or forward with these containers;
|
||||
* markup inside them belongs to someone else's mail.
|
||||
*/
|
||||
const QUOTED_CONTAINERS = 'blockquote, .gmail_quote, .moz-forward-container, #appendonsend, #divRplyFwdMsg';
|
||||
|
||||
export function defaultParseHtml(html: string): Document {
|
||||
return new DOMParser().parseFromString(html, 'text/html');
|
||||
}
|
||||
|
||||
export function mayContainStructuredData(html: string): boolean {
|
||||
return TYPE_HINT.test(html);
|
||||
}
|
||||
|
||||
/**
|
||||
* Elements that sit inside a quoted or forwarded part: inside one of the
|
||||
* containers, or after Outlook's `#divRplyFwdMsg` separator (Outlook puts
|
||||
* the forwarded message after it, not inside it).
|
||||
*/
|
||||
function isQuoted(element: Element, doc: Document): boolean {
|
||||
if (element.closest(QUOTED_CONTAINERS)) return true;
|
||||
const separator = doc.getElementById('divRplyFwdMsg');
|
||||
return !!separator && !!(separator.compareDocumentPosition(element) & 4 /* FOLLOWING */);
|
||||
}
|
||||
|
||||
function itemTypes(element: Element): string[] {
|
||||
return (element.getAttribute('itemtype') ?? '')
|
||||
.split(/\s+/)
|
||||
.map((value) => SCHEMA_ITEMTYPE.exec(value)?.[1])
|
||||
.filter((value): value is string => !!value);
|
||||
}
|
||||
|
||||
function propertyValue(element: Element, depth: number): unknown {
|
||||
if (element.hasAttribute('itemscope')) return microdataItem(element, depth + 1);
|
||||
const content = element.getAttribute('content');
|
||||
if (content !== null) return content.trim();
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const attr = (name: string) => (element.getAttribute(name) ?? '').trim();
|
||||
switch (tag) {
|
||||
case 'a': case 'link': case 'area': return attr('href');
|
||||
case 'img': case 'audio': case 'video': case 'source': case 'embed': case 'iframe': case 'track': return attr('src');
|
||||
case 'object': return attr('data');
|
||||
case 'time': return element.hasAttribute('datetime') ? attr('datetime') : (element.textContent ?? '').trim();
|
||||
case 'data': case 'meter': return attr('value');
|
||||
default: return (element.textContent ?? '').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One microdata item: properties are descendants with `itemprop` whose
|
||||
* nearest `itemscope` ancestor is this item. A space-separated itemprop gives
|
||||
* the value to each name; a repeated name becomes an array.
|
||||
*/
|
||||
function microdataItem(scope: Element, depth: number): RawItem | undefined {
|
||||
if (depth > MAX_MICRODATA_DEPTH) return undefined;
|
||||
const item: RawItem = {};
|
||||
const types = itemTypes(scope);
|
||||
if (types.length > 0) item['@type'] = types.length === 1 ? types[0] : types;
|
||||
const add = (name: string, value: unknown) => {
|
||||
if (value === undefined || value === '') return;
|
||||
const existing = item[name];
|
||||
if (existing === undefined) item[name] = value;
|
||||
else if (Array.isArray(existing)) existing.push(value);
|
||||
else item[name] = [existing, value];
|
||||
};
|
||||
for (const element of Array.from(scope.querySelectorAll('[itemprop]'))) {
|
||||
const owner = element.parentElement?.closest('[itemscope]');
|
||||
if (owner !== scope) continue;
|
||||
const value = propertyValue(element, depth);
|
||||
for (const name of (element.getAttribute('itemprop') ?? '').split(/\s+/).filter(Boolean)) add(name, value);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
export function extractStructuredData(
|
||||
html: string,
|
||||
parseHtml: (html: string) => Document = defaultParseHtml,
|
||||
): ExtractedItem[] {
|
||||
if (!html || html.length > MAX_HTML_LENGTH || !mayContainStructuredData(html)) return [];
|
||||
const doc = parseHtml(html);
|
||||
const found: ExtractedItem[] = [];
|
||||
|
||||
const scripts = Array.from(doc.querySelectorAll('script')).filter(
|
||||
(script) => (script.getAttribute('type') ?? '').trim().toLowerCase() === 'application/ld+json',
|
||||
);
|
||||
for (const script of scripts.slice(0, MAX_JSONLD_BLOCKS)) {
|
||||
if (isQuoted(script, doc)) continue;
|
||||
const value = parseLenientJson(script.textContent ?? '');
|
||||
if (value !== undefined && value !== null) found.push({ item: value, source: 'json-ld' });
|
||||
}
|
||||
|
||||
const tops = Array.from(doc.querySelectorAll('[itemscope]')).filter((element) => !element.hasAttribute('itemprop'));
|
||||
for (const scope of tops.slice(0, MAX_MICRODATA_ITEMS)) {
|
||||
// Only items typed on schema.org count; an untyped or foreign item is noise.
|
||||
if (itemTypes(scope).length === 0 || isQuoted(scope, doc)) continue;
|
||||
const item = microdataItem(scope, 0);
|
||||
if (item) found.push({ item: { '@context': 'https://schema.org', ...item }, source: 'microdata' });
|
||||
}
|
||||
return found;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Structured mail: read the schema.org markup that airlines, hotels, shops
|
||||
* and carriers put into their HTML mail, and turn it into cards. Plain
|
||||
* TypeScript; the only DOM dependency is the HTML parser, which a caller
|
||||
* without DOMParser (the native app) passes in.
|
||||
*
|
||||
* const cards = readStructuredMail(html, { mailDate });
|
||||
* const shown = needsAirportZones(cards)
|
||||
* ? resolveCards(cards, await loadAirportZones())
|
||||
* : resolveCards(cards);
|
||||
*/
|
||||
import { extractStructuredData } from './extract';
|
||||
import { normalizeItems } from './normalize';
|
||||
import type { StructuredCard } from './types';
|
||||
import { validateCard } from './validate';
|
||||
|
||||
export interface ReadOptions {
|
||||
/** When the mail was sent (else received): bookings far from it are doubtful. */
|
||||
mailDate: Date;
|
||||
parseHtml?: (html: string) => Document;
|
||||
}
|
||||
|
||||
/** Cards from one HTML body, validated but with zones not yet resolved. */
|
||||
export function readStructuredMail(html: string, options: ReadOptions): StructuredCard[] {
|
||||
const cards = normalizeItems(extractStructuredData(html, options.parseHtml));
|
||||
return cards
|
||||
.map((card) => validateCard(card, options.mailDate))
|
||||
.filter((candidate) => candidate.problems.length === 0)
|
||||
.map((candidate) => candidate.card);
|
||||
}
|
||||
|
||||
export { extractStructuredData, mayContainStructuredData } from './extract';
|
||||
export { normalizeItems } from './normalize';
|
||||
export { validateCard } from './validate';
|
||||
export { needsAirportZones, resolveCards } from './resolve';
|
||||
export { loadAirportZones } from './time-zones';
|
||||
export * from './types';
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* Parse the JSON-LD a sender wrote, including the broken forms real booking
|
||||
* mail carries: missing commas between objects (Eurowings), several objects
|
||||
* without the array brackets (Volotea), trailing commas (Eventbrite) and
|
||||
* XML-entity-encoded JSON (Airbnb). Returns undefined when nothing sensible
|
||||
* comes out; never throws.
|
||||
*/
|
||||
|
||||
const MAX_BLOCK_LENGTH = 200_000;
|
||||
|
||||
function tryParse(text: string): { ok: true; value: unknown } | { ok: false } {
|
||||
try {
|
||||
return { ok: true, value: JSON.parse(text) };
|
||||
} catch {
|
||||
return { ok: false };
|
||||
}
|
||||
}
|
||||
|
||||
function stripWrappers(text: string): string {
|
||||
return text
|
||||
.trim()
|
||||
.replace(/^<!--/, '')
|
||||
.replace(/-->$/, '')
|
||||
.replace(/^<!\[CDATA\[/, '')
|
||||
.replace(/\]\]>$/, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = { quot: '"', amp: '&', lt: '<', gt: '>', apos: "'", nbsp: ' ' };
|
||||
|
||||
function decodeEntities(text: string): string {
|
||||
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, body: string) => {
|
||||
if (body[0] === '#') {
|
||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;
|
||||
}
|
||||
return NAMED_ENTITIES[body.toLowerCase()] ?? match;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One string-aware pass: drop commas right before `}` or `]`, put a comma
|
||||
* between a closing `}` or `]` and an opening `{` or `[` (only where a
|
||||
* comma or nothing could go), and escape raw control characters inside
|
||||
* strings. Returns the text and how many values sit at the top level.
|
||||
*/
|
||||
function repairStructure(text: string): { text: string; topLevel: number } {
|
||||
let out = '';
|
||||
let inString = false;
|
||||
let depth = 0;
|
||||
let topLevel = 0;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (inString) {
|
||||
if (c === '\\') {
|
||||
out += c + (text[i + 1] ?? '');
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '"') inString = false;
|
||||
if (c < ' ') {
|
||||
out += c === '\n' ? '\\n' : c === '\r' ? '\\r' : c === '\t' ? '\\t' : '';
|
||||
continue;
|
||||
}
|
||||
out += c;
|
||||
continue;
|
||||
}
|
||||
if (c === '"') {
|
||||
if (depth === 0) topLevel++;
|
||||
inString = true;
|
||||
out += c;
|
||||
continue;
|
||||
}
|
||||
if (c === ',') {
|
||||
let j = i + 1;
|
||||
while (j < text.length && /\s/.test(text[j])) j++;
|
||||
if (text[j] === '}' || text[j] === ']' || j >= text.length) continue;
|
||||
out += c;
|
||||
continue;
|
||||
}
|
||||
if (c === '{' || c === '[') {
|
||||
if (depth === 0) {
|
||||
topLevel++;
|
||||
} else {
|
||||
// `}{` or `]{` inside an array: the comma is missing.
|
||||
const prev = out.trimEnd().slice(-1);
|
||||
if (prev === '}' || prev === ']') out = out.trimEnd() + ',';
|
||||
}
|
||||
depth++;
|
||||
out += c;
|
||||
continue;
|
||||
}
|
||||
if (c === '}' || c === ']') {
|
||||
depth = Math.max(0, depth - 1);
|
||||
out += c;
|
||||
continue;
|
||||
}
|
||||
out += c;
|
||||
}
|
||||
return { text: out, topLevel };
|
||||
}
|
||||
|
||||
export function parseLenientJson(input: string): unknown {
|
||||
if (!input || input.length > MAX_BLOCK_LENGTH) return undefined;
|
||||
const direct = tryParse(input);
|
||||
if (direct.ok) return direct.value;
|
||||
|
||||
let text = stripWrappers(input);
|
||||
let attempt = tryParse(text);
|
||||
if (attempt.ok) return attempt.value;
|
||||
|
||||
if (/&(quot|#34|#x22);/i.test(text)) {
|
||||
text = decodeEntities(text);
|
||||
attempt = tryParse(text);
|
||||
if (attempt.ok) return attempt.value;
|
||||
}
|
||||
|
||||
const repaired = repairStructure(text);
|
||||
attempt = tryParse(repaired.text);
|
||||
if (attempt.ok) return attempt.value;
|
||||
|
||||
if (repaired.topLevel > 1) {
|
||||
// Several top-level values: the brackets are missing. The commas between
|
||||
// them may be missing too, which the pass inside brackets adds.
|
||||
attempt = tryParse(repairStructure(`[${repaired.text}]`).text);
|
||||
if (attempt.ok) return attempt.value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
/**
|
||||
* Turn extracted schema.org objects into the card model. Unknown types and
|
||||
* properties are dropped; validation (validate.ts) decides afterwards which
|
||||
* cards are believable.
|
||||
*/
|
||||
import type { ExtractedItem } from './extract';
|
||||
import type {
|
||||
BookingKind,
|
||||
BookingLinks,
|
||||
CardSource,
|
||||
Item,
|
||||
Moment,
|
||||
Parcel,
|
||||
Place,
|
||||
Seat,
|
||||
Shipment,
|
||||
StructuredCard,
|
||||
TransportBooking,
|
||||
TransportKind,
|
||||
TransportLeg,
|
||||
} from './types';
|
||||
import {
|
||||
address,
|
||||
all,
|
||||
amount,
|
||||
currency,
|
||||
hasType,
|
||||
isObj,
|
||||
isSchemaContext,
|
||||
link,
|
||||
moment,
|
||||
lenientDate,
|
||||
nameOf,
|
||||
obj,
|
||||
orderStatus,
|
||||
people,
|
||||
positiveInt,
|
||||
reservationStatus,
|
||||
str,
|
||||
typesOf,
|
||||
type Obj,
|
||||
} from './values';
|
||||
|
||||
const RESERVATION_KINDS: Record<string, BookingKind> = {
|
||||
FlightReservation: 'flight',
|
||||
TrainReservation: 'train',
|
||||
BusReservation: 'bus',
|
||||
LodgingReservation: 'lodging',
|
||||
RentalCarReservation: 'rentalcar',
|
||||
EventReservation: 'event',
|
||||
FoodEstablishmentReservation: 'restaurant',
|
||||
};
|
||||
|
||||
const MAX_OBJECTS = 200;
|
||||
|
||||
interface Found {
|
||||
object: Obj;
|
||||
source: CardSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flatten what the extractor found into typed top-level objects: arrays,
|
||||
* `@graph`, ReservationPackage `subReservation`, and `reservationFor` arrays
|
||||
* (one reservation per element). Only objects under a schema.org context
|
||||
* count; nested objects inherit it.
|
||||
*/
|
||||
function flatten(extracted: ExtractedItem[]): Found[] {
|
||||
const out: Found[] = [];
|
||||
const visit = (value: unknown, source: CardSource, inherited: boolean, depth: number) => {
|
||||
if (out.length >= MAX_OBJECTS || depth > 6) return;
|
||||
if (Array.isArray(value)) {
|
||||
for (const v of value) visit(v, source, inherited, depth + 1);
|
||||
return;
|
||||
}
|
||||
if (!isObj(value)) return;
|
||||
const inContext = inherited || isSchemaContext(value['@context']);
|
||||
if (Array.isArray(value['@graph'])) {
|
||||
visit(value['@graph'], source, inContext, depth + 1);
|
||||
return;
|
||||
}
|
||||
if (!inContext) return;
|
||||
if (hasType(value, ['ReservationPackage'])) {
|
||||
visit(value.subReservation, source, true, depth + 1);
|
||||
return;
|
||||
}
|
||||
const reservationFor = value.reservationFor;
|
||||
if (Array.isArray(reservationFor) && reservationFor.length > 1) {
|
||||
for (const one of reservationFor) out.push({ object: { ...value, reservationFor: one }, source });
|
||||
return;
|
||||
}
|
||||
out.push({ object: value, source });
|
||||
};
|
||||
for (const { item, source } of extracted) visit(item, source, false, 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
const lower = (s: string | undefined) => (s ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
|
||||
|
||||
function place(value: unknown): Place {
|
||||
const v = obj(value);
|
||||
if (!v) {
|
||||
const name = str(value);
|
||||
return name ? { name } : {};
|
||||
}
|
||||
const result: Place = {};
|
||||
const name = str(v.name);
|
||||
if (name) result.name = name;
|
||||
const code = str(v.iataCode)?.toUpperCase();
|
||||
if (code && /^[A-Z]{3}$/.test(code)) result.code = code;
|
||||
const addr = address(v.address);
|
||||
if (addr) result.address = addr;
|
||||
return result;
|
||||
}
|
||||
|
||||
function bookingLinks(res: Obj, extra: Obj | undefined): BookingLinks {
|
||||
const links: BookingLinks = {};
|
||||
const checkin = link(res.checkinUrl);
|
||||
const manage = link(res.modifyReservationUrl) ?? link(res.url) ?? link(res.Url);
|
||||
const ticket = link(res.ticketDownloadUrl) ?? link(res.ticketPrintUrl) ?? link(extra?.ticketDownloadUrl);
|
||||
if (checkin) links.checkin = checkin;
|
||||
if (manage) links.manage = manage;
|
||||
if (ticket) links.ticket = ticket;
|
||||
return links;
|
||||
}
|
||||
|
||||
function reference(res: Obj): string | undefined {
|
||||
return str(res.reservationNumber) ?? str(res.reservationId) ?? str(res.confirmationNumber);
|
||||
}
|
||||
|
||||
function tickets(res: Obj): Obj[] {
|
||||
return all(res.reservedTicket).filter(isObj);
|
||||
}
|
||||
|
||||
function seatOf(res: Obj): string | undefined {
|
||||
const direct = str(res.airplaneSeat);
|
||||
if (direct) return direct;
|
||||
for (const ticket of tickets(res)) {
|
||||
const seat = obj(ticket.ticketedSeat);
|
||||
const number = str(seat?.seatNumber) ?? str(ticket.ticketedSeat);
|
||||
if (number) {
|
||||
const section = str(seat?.seatSection);
|
||||
return section ? `${section}/${number}` : number;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function splitFlightNumber(raw: string | undefined, airlineCode: string | undefined): { code?: string; number?: string } {
|
||||
if (!raw) return { code: airlineCode };
|
||||
const compact = raw.replace(/\s+/g, '').toUpperCase();
|
||||
if (airlineCode && compact.startsWith(airlineCode) && /^\d{1,4}[A-Z]?$/.test(compact.slice(airlineCode.length))) {
|
||||
return { code: airlineCode, number: compact.slice(airlineCode.length) };
|
||||
}
|
||||
const match = /^([A-Z0-9]{2})(\d{1,4}[A-Z]?)$/.exec(compact);
|
||||
if (match && /[A-Z]/.test(match[1]) && (!airlineCode || airlineCode === match[1])) return { code: match[1], number: match[2] };
|
||||
return { code: airlineCode, number: compact };
|
||||
}
|
||||
|
||||
function transport(kind: TransportKind, res: Obj, source: CardSource): TransportBooking {
|
||||
const trip = obj(res.reservationFor) ?? {};
|
||||
const passengers = people(res.underName);
|
||||
let carrier: string | undefined;
|
||||
let carrierCode: string | undefined;
|
||||
let number: string | undefined;
|
||||
let departure: Place;
|
||||
let arrival: Place;
|
||||
if (kind === 'flight') {
|
||||
const airline = obj(trip.airline);
|
||||
const airlineCode = str(airline?.iataCode)?.toUpperCase();
|
||||
carrier = nameOf(trip.airline) ?? nameOf(trip.provider) ?? nameOf(res.provider);
|
||||
const split = splitFlightNumber(str(trip.flightNumber), airlineCode && /^[A-Z0-9]{2}$/.test(airlineCode) ? airlineCode : undefined);
|
||||
carrierCode = split.code;
|
||||
number = split.number;
|
||||
departure = place(trip.departureAirport);
|
||||
arrival = place(trip.arrivalAirport);
|
||||
} else if (kind === 'train') {
|
||||
carrier = nameOf(trip.provider) ?? nameOf(trip.trainCompany) ?? nameOf(res.provider);
|
||||
number = str(trip.trainNumber) ?? str(trip.trainName);
|
||||
departure = place(trip.departureStation);
|
||||
arrival = place(trip.arrivalStation);
|
||||
} else {
|
||||
carrier = nameOf(trip.busCompany) ?? nameOf(trip.provider) ?? nameOf(res.provider);
|
||||
// busName is ignored: FlixBus puts the destination there.
|
||||
number = str(trip.busNumber);
|
||||
departure = place(trip.departureBusStop ?? trip.departureStation);
|
||||
arrival = place(trip.arrivalBusStop ?? trip.arrivalStation);
|
||||
}
|
||||
const depAt = moment(trip.departureTime);
|
||||
const arrAt = moment(trip.arrivalTime);
|
||||
const seat = seatOf(res);
|
||||
const status = reservationStatus(res.reservationStatus);
|
||||
const seats: Seat[] = seat ? [{ ...(passengers[0] ? { passenger: passengers[0] } : {}), seat }] : [];
|
||||
const leg: TransportLeg = {
|
||||
...(carrier ? { carrier } : {}),
|
||||
...(carrierCode ? { carrierCode } : {}),
|
||||
...(number ? { number } : {}),
|
||||
departure: { place: departure, at: depAt ?? { date: '' } },
|
||||
arrival: { place: arrival, ...(arrAt ? { at: arrAt } : {}) },
|
||||
seats,
|
||||
...(status ? { status } : {}),
|
||||
};
|
||||
const ref = reference(res);
|
||||
return {
|
||||
key: '',
|
||||
kind,
|
||||
...(ref ? { reference: ref } : {}),
|
||||
...(status ? { status } : {}),
|
||||
passengers,
|
||||
...(carrier ? { provider: carrier } : {}),
|
||||
links: bookingLinks(res, tickets(res)[0]),
|
||||
...(str(res.modifiedTime) ? { modified: str(res.modifiedTime) } : {}),
|
||||
source,
|
||||
legs: [leg],
|
||||
};
|
||||
}
|
||||
|
||||
function base(kind: BookingKind, res: Obj, source: CardSource) {
|
||||
const ref = reference(res);
|
||||
const status = reservationStatus(res.reservationStatus);
|
||||
return {
|
||||
key: '',
|
||||
kind,
|
||||
...(ref ? { reference: ref } : {}),
|
||||
...(status ? { status } : {}),
|
||||
passengers: people(res.underName),
|
||||
links: bookingLinks(res, tickets(res)[0]),
|
||||
...(str(res.modifiedTime) ? { modified: str(res.modifiedTime) } : {}),
|
||||
source,
|
||||
};
|
||||
}
|
||||
|
||||
const MISSING: Moment = { date: '' };
|
||||
|
||||
function reservation(kind: BookingKind, res: Obj, source: CardSource): StructuredCard | undefined {
|
||||
if (kind === 'flight' || kind === 'train' || kind === 'bus') return transport(kind, res, source);
|
||||
const target = obj(res.reservationFor) ?? {};
|
||||
const name = str(target.name) ?? (typeof res.reservationFor === 'string' ? str(res.reservationFor) : undefined);
|
||||
switch (kind) {
|
||||
case 'lodging': {
|
||||
const addr = address(target.address);
|
||||
const phone = str(target.telephone);
|
||||
return {
|
||||
...base(kind, res, source),
|
||||
kind,
|
||||
name: name ?? '',
|
||||
...(name ? { provider: name } : {}),
|
||||
...(addr ? { address: addr } : {}),
|
||||
...(phone ? { phone } : {}),
|
||||
checkin: moment(res.checkinDate) ?? moment(res.checkinTime) ?? MISSING,
|
||||
checkout: moment(res.checkoutDate) ?? moment(res.checkoutTime) ?? MISSING,
|
||||
};
|
||||
}
|
||||
case 'rentalcar': {
|
||||
const company = nameOf(target.rentalCompany) ?? nameOf(res.provider) ?? nameOf(target.brand);
|
||||
const car = [nameOf(target.brand), str(target.model) ?? str(target.name)].filter(Boolean).join(' ') || undefined;
|
||||
return {
|
||||
...base(kind, res, source),
|
||||
kind,
|
||||
...(company ? { company, provider: company } : {}),
|
||||
...(car ? { car } : {}),
|
||||
pickup: { place: place(res.pickupLocation), at: moment(res.pickupTime) ?? MISSING },
|
||||
dropoff: { place: place(res.dropoffLocation), at: moment(res.dropoffTime) ?? MISSING },
|
||||
};
|
||||
}
|
||||
case 'event': {
|
||||
const venue = place(target.location);
|
||||
const ticketNumbers = [...new Set([str(res.ticketNumber), ...tickets(res).map((t) => str(t.ticketNumber))].filter((t): t is string => !!t))];
|
||||
let end = moment(target.endDate);
|
||||
// An end date without a time means the end of that day.
|
||||
if (end && !end.time) end = { ...end, time: '23:59:59' };
|
||||
return {
|
||||
...base(kind, res, source),
|
||||
kind,
|
||||
name: name ?? '',
|
||||
...(venue.name || venue.address ? { venue } : {}),
|
||||
start: moment(target.startDate) ?? MISSING,
|
||||
...(end ? { end } : {}),
|
||||
tickets: ticketNumbers,
|
||||
...(nameOf(res.provider) ? { provider: nameOf(res.provider) } : {}),
|
||||
};
|
||||
}
|
||||
case 'restaurant': {
|
||||
const addr = address(target.address);
|
||||
const partySize = positiveInt(res.partySize);
|
||||
return {
|
||||
...base(kind, res, source),
|
||||
kind,
|
||||
name: name ?? '',
|
||||
...(name ? { provider: name } : {}),
|
||||
...(addr ? { address: addr } : {}),
|
||||
start: moment(res.startTime) ?? moment(res.startDate) ?? MISSING,
|
||||
...(partySize ? { partySize } : {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function items(products: unknown, quantityOf?: (o: Obj) => number | undefined): Item[] {
|
||||
const result: Item[] = [];
|
||||
for (const p of all(products)) {
|
||||
const name = nameOf(p);
|
||||
if (!name) continue;
|
||||
const product = isObj(p) ? p : undefined;
|
||||
const image = product ? link(isObj(product.image) ? product.image.url ?? product.image.contentUrl : product.image) : undefined;
|
||||
const quantity = product && quantityOf ? quantityOf(product) : undefined;
|
||||
result.push({ name, ...(quantity ? { quantity } : {}), ...(image ? { image } : {}) });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function orderItems(order: Obj): Item[] {
|
||||
const result: Item[] = [];
|
||||
for (const offer of all(order.acceptedOffer).filter(isObj)) {
|
||||
const quantity = positiveInt(offer.eligibleQuantity);
|
||||
for (const item of items(offer.itemOffered)) result.push(quantity ? { ...item, quantity } : item);
|
||||
}
|
||||
for (const entry of all(order.orderedItem).filter(isObj)) {
|
||||
if (hasType(entry, ['OrderItem'])) {
|
||||
const quantity = positiveInt(entry.orderQuantity);
|
||||
for (const item of items(entry.orderedItem)) result.push(quantity ? { ...item, quantity } : item);
|
||||
} else {
|
||||
result.push(...items(entry));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function order(o: Obj, source: CardSource): Shipment {
|
||||
const merchant = nameOf(o.merchant) ?? nameOf(o.seller) ?? nameOf(o.broker);
|
||||
const orderNumber = str(o.orderNumber);
|
||||
const status = orderStatus(o.orderStatus);
|
||||
const total = amount(o.price) ?? amount(o.totalPrice);
|
||||
const cur = currency(o.priceCurrency);
|
||||
const action = all(o.potentialAction).filter(isObj).find((a) => hasType(a, ['ViewAction']));
|
||||
const orderUrl = link(o.url) ?? link(o.Url) ?? link(action?.target) ?? link(action?.url);
|
||||
return {
|
||||
kind: 'shipment',
|
||||
key: '',
|
||||
...(merchant ? { merchant } : {}),
|
||||
...(orderNumber ? { orderNumber } : {}),
|
||||
...(status ? { status } : {}),
|
||||
items: orderItems(o),
|
||||
...(total && cur ? { total: { amount: total, currency: cur } } : {}),
|
||||
...(orderUrl ? { orderUrl } : {}),
|
||||
parcels: [],
|
||||
source,
|
||||
};
|
||||
}
|
||||
|
||||
function parcel(p: Obj, source: CardSource): Shipment {
|
||||
const partOf = obj(p.partOfOrder);
|
||||
const base: Shipment = partOf ? order(partOf, source) : { kind: 'shipment', key: '', items: [], parcels: [], source };
|
||||
const carrier = nameOf(p.carrier) ?? nameOf(p.provider);
|
||||
const trackingNumber = str(p.trackingNumber);
|
||||
const trackingUrl = link(p.trackingUrl);
|
||||
const expectedBy = lenientDate(p.expectedArrivalUntil) ?? lenientDate(p.expectedArrivalFrom);
|
||||
const shipped: Parcel = {
|
||||
...(carrier ? { carrier } : {}),
|
||||
...(trackingNumber ? { trackingNumber } : {}),
|
||||
...(trackingUrl ? { trackingUrl } : {}),
|
||||
...(expectedBy ? { expectedBy } : {}),
|
||||
items: items(p.itemShipped),
|
||||
};
|
||||
return { ...base, parcels: [shipped] };
|
||||
}
|
||||
|
||||
/** One card per top-level object, unmerged and unvalidated, with a merge key. */
|
||||
export function normalizeItems(extracted: ExtractedItem[]): StructuredCard[] {
|
||||
const cards: StructuredCard[] = [];
|
||||
for (const { object, source } of flatten(extracted)) {
|
||||
const types = typesOf(object);
|
||||
const kindType = types.find((t) => RESERVATION_KINDS[t]);
|
||||
let card: StructuredCard | undefined;
|
||||
if (kindType) card = reservation(RESERVATION_KINDS[kindType], object, source);
|
||||
else if (types.includes('ParcelDelivery')) card = parcel(object, source);
|
||||
else if (types.includes('Order')) card = order(object, source);
|
||||
if (card) cards.push({ ...card, key: cardKey(card) });
|
||||
}
|
||||
return cards;
|
||||
}
|
||||
|
||||
/** The merge key: equal keys describe the same booking or order. */
|
||||
export function cardKey(card: StructuredCard): string {
|
||||
switch (card.kind) {
|
||||
case 'flight': case 'train': case 'bus': {
|
||||
const leg = card.legs[0];
|
||||
return card.reference ? `${card.kind}|ref:${lower(card.reference)}` : `${card.kind}|leg:${legKey(leg)}`;
|
||||
}
|
||||
case 'lodging':
|
||||
return `lodging|${lower(card.reference)}|${lower(card.name)}|${card.checkin.date}`;
|
||||
case 'rentalcar':
|
||||
return `rentalcar|${lower(card.reference)}|${lower(card.company)}|${card.pickup.at.date}`;
|
||||
case 'event': case 'restaurant':
|
||||
return `${card.kind}|${lower(card.reference)}|${lower(card.name)}|${card.start.date}`;
|
||||
case 'shipment': {
|
||||
if (card.orderNumber) return `shipment|order:${lower(card.orderNumber)}`;
|
||||
const tracking = card.parcels.find((p) => p.trackingNumber)?.trackingNumber;
|
||||
return tracking ? `shipment|parcel:${lower(tracking)}` : `shipment|anon:${lower(card.merchant)}|${card.parcels.length}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A leg's identity inside a booking: carrier, number, departure day and place. */
|
||||
export function legKey(leg: TransportLeg): string {
|
||||
return [lower(leg.carrierCode ?? leg.carrier), lower(leg.number), leg.departure.at.date, lower(leg.departure.place.code ?? leg.departure.place.name)].join('|');
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Second validation pass, once zones can be looked up: give every time its
|
||||
* zone and instant, then check order and durations on UTC instants (an
|
||||
* eastbound flight over the date line lands "earlier" on the wall clock).
|
||||
* A booking whose times don't hold together is dropped.
|
||||
*/
|
||||
import { countryZone, resolveZone, zonedTimeToUtc, type ResolvedZone } from './time-zones';
|
||||
import type { Moment, Place, StructuredCard, TransportBooking, TransportLeg } from './types';
|
||||
|
||||
type ZoneOf = (code: string | undefined) => string | undefined;
|
||||
|
||||
const MINUTE = 60_000;
|
||||
const HOUR = 60 * MINUTE;
|
||||
const LIMITS = {
|
||||
flight: { min: 10 * MINUTE, max: 20 * HOUR },
|
||||
train: { min: MINUTE, max: 48 * HOUR },
|
||||
bus: { min: MINUTE, max: 48 * HOUR },
|
||||
} as const;
|
||||
|
||||
/** Whether resolving these cards needs the airport table. */
|
||||
export function needsAirportZones(cards: StructuredCard[]): boolean {
|
||||
return cards.some((card) => card.kind === 'flight');
|
||||
}
|
||||
|
||||
interface Named { zone: string; source: 'airport' | 'country' }
|
||||
|
||||
function namedZone(place: Place | undefined, kind: StructuredCard['kind'], zoneOf: ZoneOf | undefined, floating: boolean): Named | undefined {
|
||||
if (kind === 'flight') {
|
||||
const zone = zoneOf?.(place?.code);
|
||||
return zone ? { zone, source: 'airport' } : undefined;
|
||||
}
|
||||
const zone = countryZone(place?.address?.country, { floating });
|
||||
return zone ? { zone, source: 'country' } : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach zone and instant to a timed moment. 'drop' when table and mail
|
||||
* disagree, or when a floating local time falls into a DST gap or overlap.
|
||||
*/
|
||||
function resolveMoment(m: Moment, named: Named | undefined): Moment | 'drop' {
|
||||
if (!m.time) return m;
|
||||
const utcMs = m.utc ? Date.parse(m.utc) : undefined;
|
||||
const resolved: ResolvedZone | 'mismatch' = resolveZone({ named, offsetMinutes: m.offsetMinutes, utcMs });
|
||||
if (resolved === 'mismatch') return 'drop';
|
||||
if (m.utc || resolved.zone === null) return { ...m, zone: resolved.zone };
|
||||
const utc = zonedTimeToUtc(m.date, m.time, resolved.zone);
|
||||
if (utc === null) return 'drop';
|
||||
return { ...m, zone: resolved.zone, utc: new Date(utc).toISOString().replace('.000Z', 'Z') };
|
||||
}
|
||||
|
||||
function resolveLeg(kind: TransportBooking['kind'], leg: TransportLeg, zoneOf: ZoneOf | undefined): TransportLeg | null {
|
||||
const dep = resolveMoment(leg.departure.at, namedZone(leg.departure.place, kind, zoneOf, leg.departure.at.offsetMinutes === undefined));
|
||||
if (dep === 'drop' || !dep.utc) return null;
|
||||
let arrivalAt: Moment | undefined;
|
||||
if (leg.arrival?.at) {
|
||||
const arr = resolveMoment(leg.arrival.at, namedZone(leg.arrival.place, kind, zoneOf, leg.arrival.at.offsetMinutes === undefined));
|
||||
if (arr === 'drop' || !arr.utc) return null;
|
||||
const duration = Date.parse(arr.utc) - Date.parse(dep.utc);
|
||||
if (duration < LIMITS[kind].min || duration > LIMITS[kind].max) return null;
|
||||
arrivalAt = arr;
|
||||
}
|
||||
return {
|
||||
...leg,
|
||||
departure: { ...leg.departure, at: dep },
|
||||
...(leg.arrival ? { arrival: { ...leg.arrival, ...(arrivalAt ? { at: arrivalAt } : {}) } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Order two resolved moments: by instant when both have one, else by wall clock. */
|
||||
function notBefore(later: Moment, earlier: Moment): boolean {
|
||||
if (later.utc && earlier.utc) return Date.parse(later.utc) >= Date.parse(earlier.utc);
|
||||
return `${later.date}T${later.time ?? '00:00:00'}` >= `${earlier.date}T${earlier.time ?? '00:00:00'}`;
|
||||
}
|
||||
|
||||
function resolveCard(card: StructuredCard, zoneOf: ZoneOf | undefined): StructuredCard | null {
|
||||
switch (card.kind) {
|
||||
case 'flight': case 'train': case 'bus': {
|
||||
const legs: TransportLeg[] = [];
|
||||
for (const leg of card.legs) {
|
||||
const resolved = resolveLeg(card.kind, leg, zoneOf);
|
||||
if (!resolved) return null;
|
||||
legs.push(resolved);
|
||||
}
|
||||
return { ...card, legs };
|
||||
}
|
||||
case 'lodging': {
|
||||
const place: Place = { address: card.address };
|
||||
const resolveEnd = (m: Moment): Moment | 'drop' => {
|
||||
const named = namedZone(place, card.kind, zoneOf, m.offsetMinutes === undefined);
|
||||
// A time with neither offset nor a known zone says nothing reliable:
|
||||
// keep the date only.
|
||||
if (m.time && m.offsetMinutes === undefined && !named) return { date: m.date };
|
||||
return resolveMoment(m, named);
|
||||
};
|
||||
const checkin = resolveEnd(card.checkin);
|
||||
const checkout = resolveEnd(card.checkout);
|
||||
if (checkin === 'drop' || checkout === 'drop') return null;
|
||||
if (checkin.time && checkout.time && !notBefore(checkout, checkin)) return null;
|
||||
return { ...card, checkin, checkout };
|
||||
}
|
||||
case 'rentalcar': {
|
||||
const pickupAt = resolveMoment(card.pickup.at, namedZone(card.pickup.place, card.kind, zoneOf, card.pickup.at.offsetMinutes === undefined));
|
||||
const dropoffAt = resolveMoment(card.dropoff.at, namedZone(card.dropoff.place, card.kind, zoneOf, card.dropoff.at.offsetMinutes === undefined));
|
||||
if (pickupAt === 'drop' || dropoffAt === 'drop') return null;
|
||||
if (!notBefore(dropoffAt, pickupAt) || notBefore(pickupAt, dropoffAt)) return null;
|
||||
return { ...card, pickup: { ...card.pickup, at: pickupAt }, dropoff: { ...card.dropoff, at: dropoffAt } };
|
||||
}
|
||||
case 'event': {
|
||||
const named = (m: Moment) => namedZone(card.venue, card.kind, zoneOf, m.offsetMinutes === undefined);
|
||||
const start = resolveMoment(card.start, named(card.start));
|
||||
const end = card.end ? resolveMoment(card.end, named(card.end)) : undefined;
|
||||
if (start === 'drop' || end === 'drop') return null;
|
||||
if (end && !notBefore(end, start)) return null;
|
||||
return { ...card, start, ...(end ? { end } : {}) };
|
||||
}
|
||||
case 'restaurant': {
|
||||
const start = resolveMoment(card.start, namedZone({ address: card.address }, card.kind, zoneOf, card.start.offsetMinutes === undefined));
|
||||
if (start === 'drop') return null;
|
||||
return { ...card, start };
|
||||
}
|
||||
case 'shipment':
|
||||
return card;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve zones and drop cards whose times don't hold together. `zoneOf` is
|
||||
* the airport lookup from loadAirportZones(); without it, flight cards are
|
||||
* dropped (the card must not show before its zones are known).
|
||||
*/
|
||||
export function resolveCards(cards: StructuredCard[], zoneOf?: ZoneOf): StructuredCard[] {
|
||||
const out: StructuredCard[] = [];
|
||||
for (const card of cards) {
|
||||
if (card.kind === 'flight' && !zoneOf) continue;
|
||||
const resolved = resolveCard(card, zoneOf);
|
||||
if (resolved) out.push(resolved);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* The model booking, shipping and order mail is read into. Plain data, no
|
||||
* React and no app imports, so the native app can reuse the parser.
|
||||
*/
|
||||
|
||||
export type BookingKind = 'flight' | 'train' | 'bus' | 'lodging' | 'rentalcar' | 'event' | 'restaurant';
|
||||
export type TransportKind = 'flight' | 'train' | 'bus';
|
||||
export type ReservationStatus = 'confirmed' | 'pending' | 'hold' | 'cancelled';
|
||||
export type OrderStatus =
|
||||
| 'payment_due' | 'processing' | 'problem' | 'in_transit'
|
||||
| 'pickup_available' | 'returned' | 'delivered' | 'cancelled';
|
||||
|
||||
/** A time as the mail wrote it, local to the place it belongs to. */
|
||||
export interface Moment {
|
||||
/** YYYY-MM-DD, local. */
|
||||
date: string;
|
||||
/** HH:mm:ss, local; absent for a date-only value. */
|
||||
time?: string;
|
||||
/** Minutes east of UTC, from the value; absent when the mail gave none. */
|
||||
offsetMinutes?: number;
|
||||
/** ISO instant; set when the time and its zone or offset are known. */
|
||||
utc?: string;
|
||||
/**
|
||||
* IANA zone after resolution (resolveCards): airport, country or a
|
||||
* fixed-offset zone; null for a time that stays floating. Absent before.
|
||||
*/
|
||||
zone?: string | null;
|
||||
}
|
||||
|
||||
export interface Address {
|
||||
street?: string;
|
||||
locality?: string;
|
||||
region?: string;
|
||||
postalCode?: string;
|
||||
/** ISO 3166-1 alpha-2, when the mail gave one. */
|
||||
country?: string;
|
||||
/** Country as the mail named it, when that wasn't a code. */
|
||||
countryName?: string;
|
||||
}
|
||||
|
||||
export interface Place {
|
||||
name?: string;
|
||||
/** IATA airport code for airports, upper case. */
|
||||
code?: string;
|
||||
address?: Address;
|
||||
}
|
||||
|
||||
/** A link a card may show: http(s) only, host shown next to it. */
|
||||
export interface Link {
|
||||
url: string;
|
||||
host: string;
|
||||
}
|
||||
|
||||
export interface Seat {
|
||||
passenger?: string;
|
||||
seat: string;
|
||||
}
|
||||
|
||||
export interface TransportLeg {
|
||||
carrier?: string;
|
||||
/** IATA airline code for flights. */
|
||||
carrierCode?: string;
|
||||
number?: string;
|
||||
departure: { place: Place; at: Moment };
|
||||
arrival?: { place: Place; at?: Moment };
|
||||
seats: Seat[];
|
||||
status?: ReservationStatus;
|
||||
}
|
||||
|
||||
export interface BookingLinks {
|
||||
checkin?: Link;
|
||||
manage?: Link;
|
||||
ticket?: Link;
|
||||
}
|
||||
|
||||
export type CardSource = 'json-ld' | 'microdata' | 'pkpass';
|
||||
|
||||
interface BookingBase {
|
||||
/** Merge key: equal keys are the same booking. */
|
||||
key: string;
|
||||
kind: BookingKind;
|
||||
reference?: string;
|
||||
status?: ReservationStatus;
|
||||
passengers: string[];
|
||||
provider?: string;
|
||||
links: BookingLinks;
|
||||
modified?: string;
|
||||
source: CardSource;
|
||||
}
|
||||
|
||||
export interface TransportBooking extends BookingBase {
|
||||
kind: TransportKind;
|
||||
legs: TransportLeg[];
|
||||
}
|
||||
|
||||
export interface LodgingBooking extends BookingBase {
|
||||
kind: 'lodging';
|
||||
name: string;
|
||||
address?: Address;
|
||||
phone?: string;
|
||||
checkin: Moment;
|
||||
checkout: Moment;
|
||||
}
|
||||
|
||||
export interface RentalCarBooking extends BookingBase {
|
||||
kind: 'rentalcar';
|
||||
company?: string;
|
||||
car?: string;
|
||||
pickup: { place: Place; at: Moment };
|
||||
dropoff: { place: Place; at: Moment };
|
||||
}
|
||||
|
||||
export interface EventBooking extends BookingBase {
|
||||
kind: 'event';
|
||||
name: string;
|
||||
venue?: Place;
|
||||
start: Moment;
|
||||
end?: Moment;
|
||||
tickets: string[];
|
||||
}
|
||||
|
||||
export interface RestaurantBooking extends BookingBase {
|
||||
kind: 'restaurant';
|
||||
name: string;
|
||||
address?: Address;
|
||||
start: Moment;
|
||||
partySize?: number;
|
||||
}
|
||||
|
||||
export type Booking = TransportBooking | LodgingBooking | RentalCarBooking | EventBooking | RestaurantBooking;
|
||||
|
||||
export interface Item {
|
||||
name: string;
|
||||
quantity?: number;
|
||||
/** Remote image; a card shows it only when remote content may load. */
|
||||
image?: Link;
|
||||
}
|
||||
|
||||
export interface Parcel {
|
||||
carrier?: string;
|
||||
trackingNumber?: string;
|
||||
trackingUrl?: Link;
|
||||
expectedBy?: Moment;
|
||||
items: Item[];
|
||||
}
|
||||
|
||||
/** An Order and/or the ParcelDeliveries for one order number. */
|
||||
export interface Shipment {
|
||||
kind: 'shipment';
|
||||
key: string;
|
||||
merchant?: string;
|
||||
orderNumber?: string;
|
||||
status?: OrderStatus;
|
||||
items: Item[];
|
||||
/** Amount normalised to a dot decimal ("399.00"). */
|
||||
total?: { amount: string; currency: string };
|
||||
orderUrl?: Link;
|
||||
parcels: Parcel[];
|
||||
source: CardSource;
|
||||
}
|
||||
|
||||
export type StructuredCard = Booking | Shipment;
|
||||
|
||||
/** A card plus the reasons validation found to doubt it (empty = fine). */
|
||||
export interface Candidate {
|
||||
card: StructuredCard;
|
||||
problems: string[];
|
||||
}
|
||||
|
||||
export function isBooking(card: StructuredCard): card is Booking {
|
||||
return card.kind !== 'shipment';
|
||||
}
|
||||
|
||||
export function isTransport(card: StructuredCard): card is TransportBooking {
|
||||
return card.kind === 'flight' || card.kind === 'train' || card.kind === 'bus';
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Plausibility checks that don't need time zones. A card with any problem is
|
||||
* dropped, together with every other part of the same booking (merge.ts):
|
||||
* a wrong card is worse than no card. Time order and durations are checked
|
||||
* once the zones are known (resolve.ts).
|
||||
*/
|
||||
import type { Candidate, Moment, StructuredCard, TransportLeg } from './types';
|
||||
|
||||
const DAY = 86_400_000;
|
||||
const PAST_WINDOW_DAYS = 366;
|
||||
const TRANSPORT_FUTURE_DAYS = 400;
|
||||
const OTHER_FUTURE_DAYS = 760;
|
||||
|
||||
/**
|
||||
* Values from Google's markup documentation examples. Senders paste those
|
||||
* examples into newsletters (zave.it sent the "Eva Green" flight in a
|
||||
* cashback mail, with a passing DKIM signature); they are never real.
|
||||
*/
|
||||
const DOCUMENTATION_VALUES = new Set([
|
||||
'rxj34p', 'abc456', 'e123456789', 'ot12345', '546323', '3453291231', '176057', '123-4567890-1234567',
|
||||
]);
|
||||
|
||||
function dayNumber(date: string): number | undefined {
|
||||
const ms = Date.parse(`${date}T00:00:00Z`);
|
||||
return Number.isFinite(ms) ? Math.floor(ms / DAY) : undefined;
|
||||
}
|
||||
|
||||
function hasDate(m: Moment | undefined): m is Moment {
|
||||
return !!m && /^\d{4}-\d{2}-\d{2}$/.test(m.date);
|
||||
}
|
||||
|
||||
function hasTime(m: Moment | undefined): boolean {
|
||||
return hasDate(m) && !!m.time;
|
||||
}
|
||||
|
||||
function firstDate(card: StructuredCard): string | undefined {
|
||||
switch (card.kind) {
|
||||
case 'flight': case 'train': case 'bus': return card.legs[0]?.departure.at.date;
|
||||
case 'lodging': return card.checkin.date;
|
||||
case 'rentalcar': return card.pickup.at.date;
|
||||
case 'event': case 'restaurant': return card.start.date;
|
||||
case 'shipment': return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function legProblems(kind: 'flight' | 'train' | 'bus', leg: TransportLeg, fromPass: boolean): string[] {
|
||||
const problems: string[] = [];
|
||||
if (!hasTime(leg.departure.at)) problems.push('departure time missing');
|
||||
if (!fromPass && !hasTime(leg.arrival?.at)) problems.push('arrival time missing');
|
||||
const named = (p: { code?: string; name?: string } | undefined) => !!(p && (kind === 'flight' ? p.code || p.name : p.name));
|
||||
if (!named(leg.departure.place)) problems.push('departure place missing');
|
||||
if (!named(leg.arrival?.place)) problems.push('arrival place missing');
|
||||
if (kind === 'flight' && !(leg.carrierCode || leg.carrier)) problems.push('airline missing');
|
||||
if (kind === 'flight' && !leg.number) problems.push('flight number missing');
|
||||
return problems;
|
||||
}
|
||||
|
||||
function cardProblems(card: StructuredCard): string[] {
|
||||
switch (card.kind) {
|
||||
case 'flight': case 'train': case 'bus':
|
||||
if (card.legs.length === 0) return ['no legs'];
|
||||
return card.legs.flatMap((leg) => legProblems(card.kind as 'flight' | 'train' | 'bus', leg, card.source === 'pkpass'));
|
||||
case 'lodging': {
|
||||
const problems: string[] = [];
|
||||
if (!card.name) problems.push('hotel name missing');
|
||||
if (!hasDate(card.checkin) || !hasDate(card.checkout)) problems.push('stay dates missing');
|
||||
else {
|
||||
const nights = dayNumber(card.checkout.date)! - dayNumber(card.checkin.date)!;
|
||||
if (nights < 1) problems.push('check-out not after check-in');
|
||||
if (nights > 365) problems.push('stay longer than a year');
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
case 'rentalcar': {
|
||||
const problems: string[] = [];
|
||||
if (!hasTime(card.pickup.at) || !hasTime(card.dropoff.at)) problems.push('rental times missing');
|
||||
else {
|
||||
const days = dayNumber(card.dropoff.at.date)! - dayNumber(card.pickup.at.date)!;
|
||||
if (days < 0) problems.push('dropoff before pickup');
|
||||
if (days > 365) problems.push('rental longer than a year');
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
case 'event': {
|
||||
const problems: string[] = [];
|
||||
if (!card.name) problems.push('event name missing');
|
||||
if (!hasDate(card.start)) problems.push('event start missing');
|
||||
else if (card.end && hasDate(card.end)) {
|
||||
const days = dayNumber(card.end.date)! - dayNumber(card.start.date)!;
|
||||
if (days < 0) problems.push('event ends before it starts');
|
||||
if (days > 31) problems.push('event longer than a month');
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
case 'restaurant': {
|
||||
const problems: string[] = [];
|
||||
if (!card.name) problems.push('restaurant name missing');
|
||||
if (!hasTime(card.start)) problems.push('reservation time missing');
|
||||
return problems;
|
||||
}
|
||||
case 'shipment': {
|
||||
const problems: string[] = [];
|
||||
const tracked = card.parcels.some((p) => p.trackingNumber || p.trackingUrl);
|
||||
if (!card.orderNumber && !tracked) problems.push('neither order number nor tracking');
|
||||
const showable = card.merchant || card.items.length > 0 || card.parcels.some((p) => p.carrier || p.items.length > 0 || p.expectedBy);
|
||||
if (!showable) problems.push('nothing to show');
|
||||
return problems;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function identifiers(card: StructuredCard): string[] {
|
||||
if (card.kind === 'shipment') {
|
||||
return [card.orderNumber, ...card.parcels.map((p) => p.trackingNumber)].filter((v): v is string => !!v);
|
||||
}
|
||||
return card.reference ? [card.reference] : [];
|
||||
}
|
||||
|
||||
/** Check one card against the mail's date. */
|
||||
export function validateCard(card: StructuredCard, mailDate: Date): Candidate {
|
||||
const problems = cardProblems(card);
|
||||
if (identifiers(card).some((id) => DOCUMENTATION_VALUES.has(id.toLowerCase()))) {
|
||||
problems.push('documentation example');
|
||||
}
|
||||
const date = firstDate(card);
|
||||
const mailDay = Math.floor(mailDate.getTime() / DAY);
|
||||
const day = date ? dayNumber(date) : undefined;
|
||||
if (date !== undefined && day !== undefined && Number.isFinite(mailDay)) {
|
||||
const future = card.kind === 'flight' || card.kind === 'train' || card.kind === 'bus' ? TRANSPORT_FUTURE_DAYS : OTHER_FUTURE_DAYS;
|
||||
if (day < mailDay - PAST_WINDOW_DAYS || day > mailDay + future) problems.push('date too far from the mail');
|
||||
}
|
||||
return { card, problems };
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* Readers for single schema.org values, tolerant of the shapes real senders
|
||||
* use: a string where an object belongs, an array where one value belongs,
|
||||
* numbers for identifiers, German prices, offsets without a colon.
|
||||
*/
|
||||
import type { Address, Link, Moment, OrderStatus, ReservationStatus } from './types';
|
||||
|
||||
export type Obj = Record<string, unknown>;
|
||||
|
||||
export function isObj(value: unknown): value is Obj {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** The values of a property: an array as is, anything else as one element. */
|
||||
export function all(value: unknown): unknown[] {
|
||||
if (value === undefined || value === null) return [];
|
||||
return Array.isArray(value) ? value : [value];
|
||||
}
|
||||
|
||||
/** The first object among the values. */
|
||||
export function obj(value: unknown): Obj | undefined {
|
||||
return all(value).find(isObj);
|
||||
}
|
||||
|
||||
/** The first non-empty string (or number, as a string). */
|
||||
export function str(value: unknown): string | undefined {
|
||||
for (const v of all(value)) {
|
||||
if (typeof v === 'number' && Number.isFinite(v)) return String(v);
|
||||
if (typeof v === 'string') {
|
||||
const trimmed = v.replace(/\s+/g, ' ').trim();
|
||||
if (trimmed) return trimmed;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** A name: a plain string, or an object's `name` (Organization, Person, Place). */
|
||||
export function nameOf(value: unknown): string | undefined {
|
||||
for (const v of all(value)) {
|
||||
if (isObj(v)) {
|
||||
const name = str(v.name) ?? [str(v.givenName), str(v.familyName)].filter(Boolean).join(' ');
|
||||
if (name) return name;
|
||||
} else {
|
||||
const s = str(v);
|
||||
if (s) return s;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Types of an object, as their last path segment ("FlightReservation"). */
|
||||
export function typesOf(value: unknown): string[] {
|
||||
if (!isObj(value)) return [];
|
||||
return all(value['@type'])
|
||||
.map((t) => (typeof t === 'string' ? t.trim().replace(/^schema:/i, '').split(/[/#]/).pop() ?? '' : ''))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export function hasType(value: unknown, names: readonly string[]): boolean {
|
||||
return typesOf(value).some((t) => names.includes(t));
|
||||
}
|
||||
|
||||
const SCHEMA_CONTEXT = /^https?:\/\/(?:www\.)?schema\.org\/?$/i;
|
||||
|
||||
/** Whether an `@context` names schema.org (string, array or `@vocab`). */
|
||||
export function isSchemaContext(context: unknown): boolean {
|
||||
return all(context).some((c) =>
|
||||
(typeof c === 'string' && SCHEMA_CONTEXT.test(c.trim()))
|
||||
|| (isObj(c) && typeof c['@vocab'] === 'string' && SCHEMA_CONTEXT.test(c['@vocab'].trim())));
|
||||
}
|
||||
|
||||
/** The last segment of an enum value: a URL or a bare token. */
|
||||
function enumToken(value: unknown): string | undefined {
|
||||
const s = str(value);
|
||||
if (!s) return undefined;
|
||||
return s.split(/[/#:]/).filter(Boolean).pop();
|
||||
}
|
||||
|
||||
const RESERVATION_STATUS: Record<string, ReservationStatus> = {
|
||||
confirmed: 'confirmed', cancelled: 'cancelled', canceled: 'cancelled', pending: 'pending', hold: 'hold',
|
||||
};
|
||||
|
||||
/** ReservationConfirmed / Confirmed, full URL (http/https) or token. The first value that maps wins. */
|
||||
export function reservationStatus(value: unknown): ReservationStatus | undefined {
|
||||
for (const v of all(value)) {
|
||||
const token = enumToken(isObj(v) ? v['@id'] ?? v.name : v)?.toLowerCase().replace(/^reservation/, '');
|
||||
if (token && RESERVATION_STATUS[token]) return RESERVATION_STATUS[token];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const ORDER_STATUS: Record<string, OrderStatus> = {
|
||||
paymentdue: 'payment_due', processing: 'processing', problem: 'problem', intransit: 'in_transit',
|
||||
pickupavailable: 'pickup_available', returned: 'returned', delivered: 'delivered',
|
||||
cancelled: 'cancelled', canceled: 'cancelled',
|
||||
};
|
||||
|
||||
export function orderStatus(value: unknown): OrderStatus | undefined {
|
||||
for (const v of all(value)) {
|
||||
const token = enumToken(isObj(v) ? v['@id'] ?? v.name : v)?.toLowerCase().replace(/^order(status)?/, '');
|
||||
if (token && ORDER_STATUS[token]) return ORDER_STATUS[token];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const ISO_MOMENT = /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2})(?:[.,]\d+)?)?\s*(Z|[+-]\d{2}(?::?\d{2})?)?)?$/i;
|
||||
|
||||
function validDate(y: number, m: number, d: number): boolean {
|
||||
const date = new Date(Date.UTC(y, m - 1, d));
|
||||
return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
|
||||
}
|
||||
|
||||
function parseOffset(text: string): number | undefined {
|
||||
if (/^z$/i.test(text)) return 0;
|
||||
const match = /^([+-])(\d{2}):?(\d{2})?$/.exec(text);
|
||||
if (!match) return undefined;
|
||||
const hours = Number(match[2]);
|
||||
const minutes = Number(match[3] ?? 0);
|
||||
if (hours > 14 || minutes > 59) return undefined;
|
||||
return (match[1] === '-' ? -1 : 1) * (hours * 60 + minutes);
|
||||
}
|
||||
|
||||
/**
|
||||
* An ISO 8601 date or date-time. Offsets `Z`, `+01:00`, `+0100` and `+01`;
|
||||
* fractional seconds dropped. Anything else is undefined, so a doubtful value
|
||||
* counts as missing.
|
||||
*/
|
||||
export function moment(value: unknown): Moment | undefined {
|
||||
const s = str(value);
|
||||
if (!s) return undefined;
|
||||
const match = ISO_MOMENT.exec(s);
|
||||
if (!match) return undefined;
|
||||
const [, y, mo, d, h, mi, se, off] = match;
|
||||
if (!validDate(+y, +mo, +d)) return undefined;
|
||||
const result: Moment = { date: `${y}-${mo}-${d}` };
|
||||
if (h === undefined) return result;
|
||||
if (+h > 23 || +mi > 59 || +(se ?? 0) > 59) return undefined;
|
||||
result.time = `${h}:${mi}:${se ?? '00'}`;
|
||||
if (off !== undefined) {
|
||||
const offset = parseOffset(off);
|
||||
if (offset === undefined) return undefined;
|
||||
result.offsetMinutes = offset;
|
||||
const local = Date.UTC(+y, +mo - 1, +d, +h, +mi, +(se ?? 0));
|
||||
result.utc = new Date(local - offset * 60_000).toISOString().replace('.000Z', 'Z');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Only the date of a value whose time part is broken
|
||||
* (`2024-10-13CEST03:10:23`, Kaufland). Used where a date is all a card
|
||||
* shows (expected parcel arrival); nowhere else.
|
||||
*/
|
||||
export function lenientDate(value: unknown): Moment | undefined {
|
||||
const parsed = moment(value);
|
||||
if (parsed) return { date: parsed.date };
|
||||
const s = str(value);
|
||||
const match = s ? /^(\d{4})-(\d{2})-(\d{2})(?![\d])/.exec(s) : null;
|
||||
if (!match || !validDate(+match[1], +match[2], +match[3])) return undefined;
|
||||
return { date: `${match[1]}-${match[2]}-${match[3]}` };
|
||||
}
|
||||
|
||||
/** A PostalAddress, or a plain string kept as the street line. */
|
||||
export function address(value: unknown): Address | undefined {
|
||||
const v = all(value)[0];
|
||||
if (typeof v === 'string') {
|
||||
const street = str(v);
|
||||
return street ? { street } : undefined;
|
||||
}
|
||||
if (!isObj(v)) return undefined;
|
||||
const result: Address = {};
|
||||
const set = (key: keyof Address, raw: unknown) => {
|
||||
const s = str(raw);
|
||||
if (s) result[key] = s;
|
||||
};
|
||||
set('street', v.streetAddress);
|
||||
set('locality', v.addressLocality);
|
||||
set('region', v.addressRegion);
|
||||
set('postalCode', v.postalCode);
|
||||
const country = isObj(v.addressCountry) ? str(v.addressCountry.identifier) ?? nameOf(v.addressCountry) : str(v.addressCountry);
|
||||
if (country) {
|
||||
if (/^[A-Za-z]{2}$/.test(country)) result.country = country.toUpperCase();
|
||||
else result.countryName = country;
|
||||
}
|
||||
return Object.keys(result).length > 0 ? result : undefined;
|
||||
}
|
||||
|
||||
const IPV4 = /^\d{1,3}(?:\.\d{1,3}){3}$/;
|
||||
|
||||
/**
|
||||
* A link a card may show: absolute http(s), a real host name (no single
|
||||
* label like Ahoi's `http://default/`, no IP literal, no localhost), no
|
||||
* credentials. Anything else is dropped.
|
||||
*/
|
||||
export function link(value: unknown): Link | undefined {
|
||||
for (const v of all(value)) {
|
||||
const raw = isObj(v) ? str(v.url) ?? str(v.target) ?? str(v['@id']) : str(v);
|
||||
if (!raw) continue;
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(raw);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (url.protocol !== 'https:' && url.protocol !== 'http:') continue;
|
||||
if (url.username || url.password) continue;
|
||||
const host = url.hostname.toLowerCase();
|
||||
if (!host.includes('.') || host.startsWith('[') || IPV4.test(host) || host === 'localhost' || host.endsWith('.localhost')) continue;
|
||||
if (!/^[a-z0-9.-]+$/.test(host) || host.endsWith('.')) continue;
|
||||
return { url: url.href, host: host.replace(/^www\./, '') };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A price as a dot-decimal string: numbers, "29.99", "14.9700", and German
|
||||
* "399,00 €" / "1.299,00 EUR". Ambiguous values ("1,299") give undefined.
|
||||
*/
|
||||
export function amount(value: unknown): string | undefined {
|
||||
for (const v of all(value)) {
|
||||
if (typeof v === 'number' && Number.isFinite(v) && v >= 0) return v.toFixed(2);
|
||||
if (typeof v !== 'string') continue;
|
||||
const s = v.replace(/[^\d.,]/g, '');
|
||||
if (!s) continue;
|
||||
let normalised: string | undefined;
|
||||
if (/^\d+(\.\d+)?$/.test(s)) normalised = s;
|
||||
else if (/^\d{1,3}(\.\d{3})+,\d{1,2}$/.test(s) || /^\d+,\d{1,2}$/.test(s)) normalised = s.replace(/\./g, '').replace(',', '.');
|
||||
else if (/^\d{1,3}(,\d{3})+\.\d{1,2}$/.test(s)) normalised = s.replace(/,/g, '');
|
||||
if (normalised === undefined) continue;
|
||||
const n = Number(normalised);
|
||||
if (Number.isFinite(n)) return n.toFixed(2);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function currency(value: unknown): string | undefined {
|
||||
const s = str(value);
|
||||
return s && /^[A-Za-z]{3}$/.test(s) ? s.toUpperCase() : undefined;
|
||||
}
|
||||
|
||||
export function positiveInt(value: unknown): number | undefined {
|
||||
for (const v of all(value)) {
|
||||
const raw = isObj(v) ? v.value : v;
|
||||
const n = typeof raw === 'number' ? raw : typeof raw === 'string' && /^\s*\d+\s*$/.test(raw) ? Number(raw) : NaN;
|
||||
if (Number.isInteger(n) && n > 0 && n < 10_000) return n;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Names of people: Person objects, strings, arrays; duplicates dropped. */
|
||||
export function people(value: unknown): string[] {
|
||||
const names = all(value).map((v) => nameOf(v)).filter((n): n is string => !!n);
|
||||
return [...new Set(names)];
|
||||
}
|
||||
@@ -39,6 +39,7 @@ export const LITE_BUNDLE_NON_REQUEST = [
|
||||
{ files: CHUNKS, why: "example addresses the translations localise", hosts: ["eksempel.dk", "mail.eksempel.dk", "piemers.lv", "pasts.piemers.lv", "ornek.com", "mail.ornek.com"] },
|
||||
{ files: CHUNKS, why: "links inside the sample mail of the demo mode (lib/demo/fixtures)", hosts: ["zoom.us", "notion.so", "www.figma.com", "vercel.com", "i.scdn.co", "cf.bstatic.com", "open.spotify.com", "linear.app", "sentry.io", "app.docusign.com", "github.com", "bulwark-webmail.vercel.app"] },
|
||||
{ files: CHUNKS, why: "JMAP capability URI and XML namespaces", hosts: ["bulwarkmail.com", "www.w3.org", "creativecommons.org"] },
|
||||
{ files: CHUNKS, why: "schema.org vocabulary names in booking markup (lib/structured-mail), matched and never requested", hosts: ["schema.org"] },
|
||||
{ files: /./, why: "docs links in the translations and the README", hosts: ["bulwarkmail.org", "github.com"] },
|
||||
{ files: PAGES, why: "XML namespaces of inline SVG", hosts: ["www.w3.org"] },
|
||||
{ files: /^_next\/static\/chunks\/[^/]+\.css$/, why: "XML namespace of the SVG pointers in app/accessibility.css (large mouse pointer)", hosts: ["www.w3.org"] },
|
||||
|
||||
Reference in New Issue
Block a user