From 79ae1d51f8460079b104038b86923aa15925ee9e Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:46:59 +0100 Subject: [PATCH 1/7] feat: keep contacts on the device, search them offline and queue edits --- lib/store/__tests__/contacts.test.ts | 660 +++++++++++++++++++ lib/store/__tests__/helpers/fake-contacts.ts | 229 +++++++ lib/store/__tests__/helpers/fake-jmap.ts | 6 +- lib/store/area.ts | 2 + lib/store/areas.ts | 3 +- lib/store/contacts/index.ts | 27 + lib/store/contacts/methods.ts | 209 ++++++ lib/store/contacts/patch.ts | 88 +++ lib/store/contacts/pending.ts | 459 +++++++++++++ lib/store/contacts/records.ts | 116 ++++ lib/store/contacts/schema.ts | 75 +++ lib/store/contacts/sync.ts | 179 +++++ lib/store/engine/account-engine.ts | 2 + 13 files changed, 2051 insertions(+), 4 deletions(-) create mode 100644 lib/store/__tests__/contacts.test.ts create mode 100644 lib/store/__tests__/helpers/fake-contacts.ts create mode 100644 lib/store/contacts/index.ts create mode 100644 lib/store/contacts/methods.ts create mode 100644 lib/store/contacts/patch.ts create mode 100644 lib/store/contacts/pending.ts create mode 100644 lib/store/contacts/records.ts create mode 100644 lib/store/contacts/schema.ts create mode 100644 lib/store/contacts/sync.ts diff --git a/lib/store/__tests__/contacts.test.ts b/lib/store/__tests__/contacts.test.ts new file mode 100644 index 000000000..526595a85 --- /dev/null +++ b/lib/store/__tests__/contacts.test.ts @@ -0,0 +1,660 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { textFilter } from '@/lib/jmap/search-utils'; +import { DEFAULT_SCOPE, type StoreEvent } from '../area'; +import { contactsArea } from '../contacts'; +import { applyPatch, mergePatches } from '../contacts/patch'; +import { AccountEngine } from '../engine/account-engine'; +import type { MethodCall } from '../engine/transport'; +import type { StorageAdapter } from '../storage'; +import { JMAP_ACCOUNT } from './helpers/fake-jmap'; +import { CONTACTS_SESSION, FakeContacts, MAIL_ONLY_ACCOUNT, SHARED_ACCOUNT } from './helpers/fake-contacts'; +import { openFileStore, openMemoryStore } from './helpers/node-adapter'; + +function engineFor(server: FakeContacts, db: StorageAdapter = openMemoryStore(), emit?: (e: StoreEvent) => void) { + return new AccountEngine({ + account: 'a@fake.test', + db, + session: CONTACTS_SESSION, + transport: server, + areas: [contactsArea], + scope: DEFAULT_SCOPE, + emit, + now: () => Date.UTC(2026, 10, 3), + sleep: async () => {}, + }); +} + +async function synced(server: FakeContacts) { + const db = openMemoryStore(); + const engine = engineFor(server, db); + await engine.requestSync(); + return { db, engine }; +} + +function count(db: StorageAdapter, table: string): number { + return Number(db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM ${table}`)?.n); +} + +/* eslint-disable @typescript-eslint/no-explicit-any -- answers are shaped like the server's */ +async function call(engine: AccountEngine, name: string, args: Record): Promise { + const response = await engine.answer([[name, { accountId: JMAP_ACCOUNT, ...args }, '0']]); + if (!response) throw new Error('nothing answered'); + const [answered, result] = response.methodResponses[0]; + if (answered === 'error') throw new Error(result.type); + return result; +} + +/** The client's searchContacts: its filter, limit 50, then a get on the query's ids. */ +async function search(engine: AccountEngine, query: string, accountId = JMAP_ACCOUNT): Promise { + const calls: MethodCall[] = [ + ['ContactCard/query', { accountId, filter: textFilter(query), limit: 50 }, '0'], + ['ContactCard/get', { accountId, '#ids': { resultOf: '0', name: 'ContactCard/query', path: '/ids' } }, '1'], + ]; + const response = await engine.answer(calls); + return (response?.methodResponses[1][1].list as Array<{ name: { full: string } }>).map((c) => c.name.full); +} + +function setCalls(server: FakeContacts): Array> { + return server.calls.filter(([name]) => name === 'ContactCard/set').map(([, args]) => args as Record); +} + +describe('contacts sync', () => { + it('pages every card of every account with the capability, and only those', async () => { + const server = new FakeContacts(); + server.addBook({ id: 'teambook', name: 'Team' }, SHARED_ACCOUNT); + for (let i = 0; i < 2345; i += 1) server.addCard(); + server.addCard({ id: 'c1', name: { full: 'Same id in the shared account' } }, SHARED_ACCOUNT); + server.addCard({}, MAIL_ONLY_ACCOUNT); + const { db } = await synced(server); + + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM contact_card WHERE jmap = ?', [JMAP_ACCOUNT])?.n)).toBe(2345); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM contact_card WHERE jmap = ?', [SHARED_ACCOUNT])?.n)).toBe(1); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM contact_card WHERE jmap = ?', [MAIL_ONLY_ACCOUNT])?.n)).toBe(0); + // Same id in two accounts stays two cards. + const names = db.all<{ json: string }>("SELECT json FROM contact_card WHERE id = 'c1' ORDER BY jmap").map((r) => (JSON.parse(r.json) as { name: { full: string } }).name.full); + expect(names).toHaveLength(2); + for (const [name, args] of server.calls) { + if (name === 'ContactCard/query') expect(args.limit).toBeLessThanOrEqual(500); + } + }); + + it('continues a stopped full sync from its cursor after a reload', async () => { + const dir = mkdtempSync(path.join(os.tmpdir(), 'store-contacts-')); + try { + const file = path.join(dir, 's.db'); + const server = new FakeContacts(); + for (let i = 0; i < 2600; i += 1) server.addCard(); + const first = openFileStore(file); + const engine = engineFor(server, first); + let queries = 0; + server.beforeRequest = () => { + queries += 1; + // The address book call and the state capture, then one page of 2000: the network goes. + if (queries === 4) server.offline = true; + }; + await engine.requestSync(); + await engine.close(); + expect(count(first, 'contact_card')).toBe(2000); + first.close(); + + server.offline = false; + server.beforeRequest = null; + server.calls.length = 0; + const second = openFileStore(file); + await engineFor(server, second).requestSync(); + expect(count(second, 'contact_card')).toBe(2600); + const positions = server.calls.filter(([name]) => name === 'ContactCard/query').map(([, args]) => args.position); + expect(positions[0]).toBe(2000); + second.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('applies created, updated and destroyed cards as deltas, across pages', async () => { + const server = new FakeContacts(); + const a = server.addCard({ name: { full: 'Anna' } }); + const b = server.addCard({ name: { full: 'Bernd' } }); + const { db, engine } = await synced(server); + server.calls.length = 0; + + for (let i = 0; i < 1200; i += 1) server.addCard(); + server.editCard(a.id, { name: { full: 'Anna Neu' } }); + server.removeCard(b.id); + await engine.requestSync(); + + expect(server.calls.filter(([name]) => name === 'ContactCard/changes').length).toBeGreaterThan(1); + expect(server.calls.some(([name]) => name === 'ContactCard/query')).toBe(false); + expect(count(db, 'contact_card')).toBe(1201); + expect(await search(engine, 'anna')).toEqual(['Anna Neu']); + expect(await search(engine, 'bernd')).toEqual([]); + }); + + it('follows address book changes', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + server.addBook({ id: 'book2', name: 'Work' }); + await engine.requestSync(); + expect(server.calls.some(([name]) => name === 'AddressBook/changes')).toBe(true); + const books = await call(engine, 'AddressBook/get', {}); + expect(books.list.map((b: { id: string }) => b.id).sort()).toEqual(['book1', 'book2']); + }); + + it('starts over when the server can no longer give changes, and drops what it no longer has', async () => { + const server = new FakeContacts(); + const gone = server.addCard({ name: { full: 'Gone' } }); + const kept = server.addCard({ name: { full: 'Kept' } }); + const { db, engine } = await synced(server); + server.removeCard(gone.id); + server.account().coll.ContactCard.truncatedBelow = server.account().coll.ContactCard.log.length; + await engine.requestSync(); + expect(db.all('SELECT id FROM contact_card').map((r) => r.id)).toEqual([kept.id]); + }); + + it('keeps a queued create through a resync', async () => { + const server = new FakeContacts(); + server.addCard(); + const { db, engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Offline' }, addressBookIds: { book1: true } } } }); + // The create can't go out yet, and the sweep of a full sync must not take + // it for something the server deleted. + server.refuse.set(created.n.uid, { type: 'serverFail' }); + db.run("UPDATE sync_state SET state = 'bad state!' WHERE type = 'ContactCard'"); + await engine.requestSync(); + expect(await search(engine, 'offline')).toEqual(['Offline']); + server.refuse.clear(); + await engine.requestSync(); + expect(await search(engine, 'offline')).toEqual(['Offline']); + expect([...server.account().cards.values()].filter((c) => (c.name as { full: string }).full === 'Offline')).toHaveLength(1); + }); +}); + +describe('contact answers', () => { + it('says so instead of answering empty when nothing was synced yet', async () => { + const engine = engineFor(new FakeContacts()); + const response = await engine.answer([['ContactCard/query', { accountId: JMAP_ACCOUNT, limit: 50 }, '0'], ['AddressBook/get', { accountId: JMAP_ACCOUNT }, '1']]); + expect(response).toBeNull(); + }); + + it('gives the last server state, never a made-up one, and asked-for properties only', async () => { + const server = new FakeContacts(); + const card = server.addCard(); + const { engine } = await synced(server); + const got = await call(engine, 'ContactCard/get', { ids: [card.id, 'nope'], properties: ['uid'] }); + expect(got.state).toBe(server.account().coll.ContactCard.state()); + expect(got.list).toEqual([{ id: card.id, uid: card.uid }]); + expect(got.notFound).toEqual(['nope']); + const books = await call(engine, 'AddressBook/get', { properties: ['name'] }); + expect(books.state).toBe(server.account().coll.AddressBook.state()); + expect(books.list).toEqual([{ id: 'book1', name: 'book1' }]); + }); + + it('pages a query with total and filters by address book', async () => { + const server = new FakeContacts(); + server.addBook({ id: 'book2', name: 'Work' }); + for (let i = 0; i < 5; i += 1) server.addCard({ addressBookIds: { book1: true } }); + server.addCard({ addressBookIds: { book2: true } }); + const { engine } = await synced(server); + const page = await call(engine, 'ContactCard/query', { position: 2, limit: 2, calculateTotal: true }); + expect(page.ids).toHaveLength(2); + expect(page.position).toBe(2); + expect(page.total).toBe(6); + const work = await call(engine, 'ContactCard/query', { filter: { inAddressBook: 'book2' } }); + expect(work.ids).toHaveLength(1); + }); + + it('refuses a filter it cannot read, for that call only', async () => { + const server = new FakeContacts(); + server.addCard(); + const { engine } = await synced(server); + const response = await engine.answer([ + ['ContactCard/query', { accountId: JMAP_ACCOUNT, filter: { phone: '123' } }, '0'], + ['AddressBook/get', { accountId: JMAP_ACCOUNT }, '1'], + ]); + expect(response?.methodResponses[0][0]).toBe('error'); + expect(response?.methodResponses[0][1].type).toBe('unsupportedFilter'); + expect(response?.methodResponses[1][0]).toBe('AddressBook/get'); + }); +}); + +describe('autocomplete', () => { + async function people() { + const server = new FakeContacts(); + server.addCard({ + name: { full: 'Jörg Müller-Lüdenscheidt', components: [{ kind: 'given', value: 'Jörg' }, { kind: 'surname', value: 'Müller-Lüdenscheidt' }] }, + emails: { e: { address: 'joerg.mueller@example.org' } }, + organizations: { o: { name: 'Bäckerei Straße', units: [{ name: 'Backstube' }] } }, + }); + server.addCard({ name: { full: 'Johannes Weiß' }, emails: { e: { address: 'jw@firma.example' } } }); + server.addCard({ name: { full: 'José Ñandú' }, nicknames: { n: { name: 'Pepe' } } }); + server.addCard({ name: { components: [{ kind: 'given', value: 'Hartmann' }] }, emails: { e: { address: 'h@x.example' } } }); + server.addCard({ name: { full: 'Käthe Schön' } }); + return synced(server); + } + + it.each([ + ['Müller', 'Jörg Müller-Lüdenscheidt'], + ['müll', 'Jörg Müller-Lüdenscheidt'], + ['MÜLLER', 'Jörg Müller-Lüdenscheidt'], + ['muller', 'Jörg Müller-Lüdenscheidt'], + ['mueller', 'Jörg Müller-Lüdenscheidt'], + ['joerg', 'Jörg Müller-Lüdenscheidt'], + ['lüden', 'Jörg Müller-Lüdenscheidt'], + ['bäck', 'Jörg Müller-Lüdenscheidt'], + ['strasse', 'Jörg Müller-Lüdenscheidt'], + ['Straße', 'Jörg Müller-Lüdenscheidt'], + ['backstube', 'Jörg Müller-Lüdenscheidt'], + ['schoen', 'Käthe Schön'], + ['kaethe', 'Käthe Schön'], + ['schon', 'Käthe Schön'], + ['SCHÖN', 'Käthe Schön'], + ['hannes', 'Johannes Weiß'], + ['weiss', 'Johannes Weiß'], + ['firma', 'Johannes Weiß'], + ['jw@firma', 'Johannes Weiß'], + ['jose', 'José Ñandú'], + ['nandu', 'José Ñandú'], + ['pepe', 'José Ñandú'], + ['jö mül', 'Jörg Müller-Lüdenscheidt'], + ['"Jörg Müller"', 'Jörg Müller-Lüdenscheidt'], + ['jö', null], + ['jo*', null], + ])('finds %s', async (query, expected) => { + const { engine } = await people(); + const found = await search(engine, query); + if (expected) expect(found).toEqual([expected]); + else expect(found.length).toBeGreaterThan(1); + }); + + it('does not match words that only share a start with another card', async () => { + const { engine } = await people(); + expect(await search(engine, 'mül weiß')).toEqual([]); + expect(await search(engine, 'xyz')).toEqual([]); + expect(await search(engine, '*')).toEqual([]); + }); + + it('finds a card by a component when it has no full name', async () => { + const { db, engine } = await people(); + expect(await call(engine, 'ContactCard/query', { filter: { text: 'hart' } })).toMatchObject({ ids: [expect.any(String)] }); + expect(db.get<{ sort_key: string }>("SELECT sort_key FROM contact_card WHERE json LIKE '%Hartmann%'")?.sort_key).toBe('hartmann'); + }); + + it('lists names starting with the text before names containing it, and stops at the limit', async () => { + const server = new FakeContacts(); + for (let i = 0; i < 60; i += 1) server.addCard({ name: { full: `Aanna ${String(i).padStart(2, '0')}` } }); + server.addCard({ name: { full: 'Zoe Anna' } }); + const { engine } = await synced(server); + const found = await search(engine, 'anna'); + expect(found).toHaveLength(50); + expect(found[0]).toBe('Zoe Anna'); + }); + + it('keeps accounts apart', async () => { + const server = new FakeContacts(); + server.addCard({ name: { full: 'Own Person' } }); + server.addBook({ id: 'teambook', name: 'Team' }, SHARED_ACCOUNT); + server.addCard({ name: { full: 'Team Person' } }, SHARED_ACCOUNT); + const { engine } = await synced(server); + expect(await search(engine, 'person')).toEqual(['Own Person']); + expect(await search(engine, 'person', SHARED_ACCOUNT)).toEqual(['Team Person']); + }); + + it('finds the new name and not the old after an edit is queued', async () => { + const server = new FakeContacts(); + const card = server.addCard({ name: { full: 'Oldname' } }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { update: { [card.id]: { name: { full: 'Newname' } } } }); + expect(await search(engine, 'newname')).toEqual(['Newname']); + expect(await search(engine, 'oldname')).toEqual([]); + }); +}); + +describe('patches', () => { + it('applies pointers and removes on null, leaving the original alone', () => { + const card = { name: { full: 'A', given: 'B' }, note: 'x' }; + const out = applyPatch(card, { 'name/full': 'C', note: null, 'a~1b': 1 }); + expect(out).toEqual({ name: { full: 'C', given: 'B' }, 'a/b': 1 }); + expect(card.name.full).toBe('A'); + expect(() => applyPatch(card, { 'missing/x': 1 })).toThrow(); + }); + + it('merges: newer wins, a whole property replaces what is below it, a path goes into an older whole value', () => { + expect(mergePatches({ a: 1, b: 2 }, { b: 3 })).toEqual({ a: 1, b: 3 }); + expect(mergePatches({ 'name/full': 'A', 'name/given': 'B' }, { name: { full: 'C' } })).toEqual({ name: { full: 'C' } }); + expect(mergePatches({ name: { full: 'A', given: 'B' } }, { 'name/full': 'C' })).toEqual({ name: { full: 'C', given: 'B' } }); + expect(mergePatches({ note: 'x' }, { note: null })).toEqual({ note: null }); + }); +}); + +describe('queued edits', () => { + it('merges edits per card, and sends one patch after coming back', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old', name: { full: 'Anna' } }); + const { db, engine } = await synced(server); + server.offline = true; + engine.setOnline(false); + + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'first' } } }); + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'second', 'name/full': 'Anna B' } } }); + await call(engine, 'ContactCard/set', { update: { [card.id]: { 'name/full': 'Anna C' } } }); + expect(count(db, 'contact_pending')).toBe(1); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(1); + const seen = await call(engine, 'ContactCard/get', { ids: [card.id] }); + expect(seen.list[0]).toMatchObject({ note: 'second', name: { full: 'Anna C' } }); + + server.offline = false; + engine.setOnline(true); + await engine.areaCall('contacts.flush', undefined); + const sets = setCalls(server); + expect(sets).toHaveLength(1); + expect(sets[0].update[card.id]).toEqual({ note: 'second', 'name/full': 'Anna C' }); + expect(server.account().cards.get(card.id)).toMatchObject({ note: 'second', name: { full: 'Anna C' } }); + expect(count(db, 'contact_pending')).toBe(0); + }); + + it('lets stand-in ids work in later edits and sends one create', async () => { + const server = new FakeContacts(); + const { db, engine } = await synced(server); + engine.setOnline(false); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Neu' }, addressBookIds: { book1: true } } } }); + const standIn = created.n.id; + expect(created.n.uid).toMatch(/^urn:uuid:/); + + const fetched = await call(engine, 'ContactCard/get', { ids: [standIn] }); + expect(fetched.list[0]).toMatchObject({ id: standIn, name: { full: 'Neu' }, uid: created.n.uid }); + const updated = await call(engine, 'ContactCard/set', { update: { [standIn]: { note: 'added' } } }); + expect(updated.updated).toEqual({ [standIn]: null }); + + engine.setOnline(true); + await engine.requestSync(); + const sets = setCalls(server); + expect(sets).toHaveLength(1); + const sent = Object.values(sets[0].create)[0] as Record; + expect(sent).toMatchObject({ name: { full: 'Neu' }, note: 'added', uid: created.n.uid }); + expect(sent).not.toHaveProperty('id'); + const onServer = [...server.account().cards.values()]; + expect(onServer).toHaveLength(1); + + // The page may still hold the stand-in id: it finds the card, and can edit it. + const late = await call(engine, 'ContactCard/get', { ids: [standIn] }); + expect(late.list[0].id).toBe(onServer[0].id); + expect(db.get('SELECT 1 AS x FROM contact_card WHERE id = ?', [standIn])).toBeUndefined(); + expect(count(db, 'contact_card')).toBe(1); + engine.setOnline(false); + await call(engine, 'ContactCard/set', { update: { [standIn]: { note: 'later' } } }); + engine.setOnline(true); + await engine.requestSync(); + expect(server.account().cards.get(onServer[0].id)).toMatchObject({ note: 'later' }); + }); + + it('never sends a card created and deleted while offline', async () => { + const server = new FakeContacts(); + const { db, engine } = await synced(server); + engine.setOnline(false); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Short' }, addressBookIds: { book1: true } } } }); + const gone = await call(engine, 'ContactCard/set', { destroy: [created.n.id] }); + expect(gone.destroyed).toEqual([created.n.id]); + engine.setOnline(true); + await engine.requestSync(); + expect(setCalls(server)).toHaveLength(0); + expect(count(db, 'contact_card')).toBe(0); + }); + + it('hides a queued delete, also from sync, then deletes on the server', async () => { + const server = new FakeContacts(); + const card = server.addCard(); + const { db, engine } = await synced(server); + engine.setOnline(false); + await call(engine, 'ContactCard/set', { destroy: [card.id] }); + expect((await call(engine, 'ContactCard/get', { ids: [card.id] })).notFound).toEqual([card.id]); + const again = await call(engine, 'ContactCard/set', { destroy: [card.id], update: { [card.id]: { note: 'x' } } }); + expect(again.notDestroyed[card.id].type).toBe('notFound'); + expect(again.notUpdated[card.id].type).toBe('notFound'); + engine.setOnline(true); + await engine.requestSync(); + expect(server.account().cards.has(card.id)).toBe(false); + expect(count(db, 'contact_card')).toBe(0); + }); + + it('keeps a queued delete hidden when a full sync brings the card again', async () => { + const server = new FakeContacts(); + const card = server.addCard({ name: { full: 'Hidden' } }); + const { db, engine } = await synced(server); + await call(engine, 'ContactCard/set', { destroy: [card.id] }); + server.refuse.set(card.id, { type: 'serverFail' }); + db.run("UPDATE sync_state SET state = 'bad state!' WHERE type = 'ContactCard'"); + await engine.requestSync(); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(1); + expect(await search(engine, 'hidden')).toEqual([]); + server.refuse.clear(); + await engine.requestSync(); + expect(server.account().cards.has(card.id)).toBe(false); + }); + + it('keeps the queued edit on top when sync brings a newer card', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old', name: { full: 'Anna' } }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'mine' } } }); + // A pass with the flush step still to come: the row was already there, so the merge shows on the delta. + server.refuse.set(card.id, { type: 'serverFail' }); + server.editCard(card.id, { name: { full: 'Anna Server' } }); + await engine.requestSync(); + const seen = await call(engine, 'ContactCard/get', { ids: [card.id] }); + expect(seen.list[0]).toMatchObject({ note: 'mine', name: { full: 'Anna Server' } }); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(1); + server.refuse.clear(); + await engine.requestSync(); + expect(server.account().cards.get(card.id)).toMatchObject({ note: 'mine', name: { full: 'Anna Server' } }); + expect(await engine.areaCall('contacts.conflicts', undefined)).toEqual([]); + }); + + it('refuses a held upload as a photo, the way the client takes as a blob refusal', async () => { + const server = new FakeContacts(); + const card = server.addCard(); + const { engine } = await synced(server); + const result = await call(engine, 'ContactCard/set', { update: { [card.id]: { media: { p: { kind: 'photo', blobId: 'bulwark-local-123' } } } } }); + expect(result.notUpdated[card.id]).toMatchObject({ type: 'invalidProperties', properties: ['media'] }); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); + + it('refuses a card with no address book, since it could never be saved', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + const result = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Nowhere' } } } }); + expect(result.notCreated.n).toMatchObject({ type: 'invalidProperties', properties: ['addressBookIds'] }); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); + + it('answers a patch that leads nowhere per record', async () => { + const server = new FakeContacts(); + const card = server.addCard(); + const { engine } = await synced(server); + const result = await call(engine, 'ContactCard/set', { update: { [card.id]: { 'nope/x': 1 } } }); + expect(result.notUpdated[card.id].type).toBe('invalidPatch'); + }); +}); + +describe('lost responses', () => { + it('finds the card by uid instead of creating it twice', async () => { + const server = new FakeContacts(); + // Without the refusal Stalwart gives, only the uid lookup can stop a second card. + server.allowDuplicateUids = true; + const { db, engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Once' }, addressBookIds: { book1: true } } } }); + + server.loseNextSet = true; + await engine.requestSync(); + expect(server.account().cards.size).toBe(1); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(1); + + await engine.requestSync(); + expect(server.account().cards.size).toBe(1); + expect(setCalls(server)).toHaveLength(1); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + const real = [...server.account().cards.keys()][0]; + expect(db.all('SELECT id FROM contact_card').map((r) => r.id)).toEqual([real]); + expect((await call(engine, 'ContactCard/get', { ids: [created.n.id] })).list[0].id).toBe(real); + }); + + it('takes the server refusing the uid as the card being there', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Once' }, addressBookIds: { book1: true } } } }); + // The card reached the server by another way than this flush saw. + server.addCard({ uid: created.n.uid, name: { full: 'Once' } }); + await engine.requestSync(); + expect(server.account().cards.size).toBe(1); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + expect(await engine.areaCall('contacts.conflicts', undefined)).toEqual([]); + expect((await call(engine, 'ContactCard/query', {})).ids).toHaveLength(1); + }); + + it('does not take its own lost update for a change on the server', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old' }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'mine' } } }); + server.loseNextSet = true; + await engine.requestSync(); + expect(server.account().cards.get(card.id)).toMatchObject({ note: 'mine' }); + await engine.requestSync(); + expect(setCalls(server)).toHaveLength(1); + expect(await engine.areaCall('contacts.conflicts', undefined)).toEqual([]); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); + + it('keeps everything queued when the network fails, and a delete after a lost create still deletes', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Twice' }, addressBookIds: { book1: true } } } }); + server.loseNextSet = true; + await engine.requestSync(); + expect(server.account().cards.size).toBe(1); + await call(engine, 'ContactCard/set', { destroy: [created.n.id] }); + await engine.requestSync(); + expect(server.account().cards.size).toBe(0); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); +}); + +describe('conflicts', () => { + it('records a refused create and takes it out of the answers', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Refused' }, addressBookIds: { book1: true } } } }); + const uid = created.n.uid; + server.refuse.set(uid, { type: 'forbidden', description: 'No rights here' }); + await engine.requestSync(); + const conflicts = await engine.areaCall('contacts.conflicts', undefined) as Array>; + expect(conflicts).toHaveLength(1); + expect(conflicts[0]).toMatchObject({ kind: 'refused', summary: 'Refused', error: 'forbidden: No rights here', jmap: JMAP_ACCOUNT, local: { op: 'create' } }); + expect(await search(engine, 'refused')).toEqual([]); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); + + it('records a refused update and goes back to the server copy', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'server' }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'mine' } } }); + server.refuse.set(card.id, { type: 'forbidden' }); + await engine.requestSync(); + const conflicts = await engine.areaCall('contacts.conflicts', undefined) as Array>; + expect(conflicts[0]).toMatchObject({ kind: 'refused', recordId: card.id, local: { op: 'update', payload: { note: 'mine' } } }); + expect((await call(engine, 'ContactCard/get', { ids: [card.id] })).list[0].note).toBe('server'); + await engine.areaCall('contacts.dismissConflict', { id: conflicts[0].id }); + expect(await engine.areaCall('contacts.conflicts', undefined)).toEqual([]); + }); + + it('records an edit to a property the server changed too, and does not overwrite it', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old' }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'mine' } } }); + server.editCard(card.id, { note: 'theirs' }); + await engine.requestSync(); + const conflicts = await engine.areaCall('contacts.conflicts', undefined) as Array>; + expect(conflicts).toHaveLength(1); + expect(conflicts[0]).toMatchObject({ kind: 'changed', local: { payload: { note: 'mine' } }, server: { note: 'theirs' } }); + expect(setCalls(server)).toHaveLength(0); + expect(server.account().cards.get(card.id)?.note).toBe('theirs'); + expect((await call(engine, 'ContactCard/get', { ids: [card.id] })).list[0].note).toBe('theirs'); + }); + + it('sends an edit when the server only changed other properties', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old', name: { full: 'A' } }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'mine' } } }); + server.editCard(card.id, { name: { full: 'B' } }); + await engine.requestSync(); + expect(await engine.areaCall('contacts.conflicts', undefined)).toEqual([]); + expect(server.account().cards.get(card.id)).toMatchObject({ note: 'mine', name: { full: 'B' } }); + }); + + it('records an edit to a card deleted on the server', async () => { + const server = new FakeContacts(); + const card = server.addCard({ name: { full: 'Vanished' } }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { update: { [card.id]: { note: 'mine' } } }); + server.removeCard(card.id); + await engine.requestSync(); + const conflicts = await engine.areaCall('contacts.conflicts', undefined) as Array>; + expect(conflicts).toHaveLength(1); + expect(conflicts[0]).toMatchObject({ kind: 'gone', summary: 'Vanished', recordId: card.id }); + expect(await search(engine, 'vanished')).toEqual([]); + }); + + it('does not delete a card that changed on the server since the delete was queued', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old', name: { full: 'Keep me' } }); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { destroy: [card.id] }); + server.editCard(card.id, { note: 'someone cares' }); + await engine.requestSync(); + expect(server.account().cards.has(card.id)).toBe(true); + expect(await search(engine, 'keep')).toEqual(['Keep me']); + const conflicts = await engine.areaCall('contacts.conflicts', undefined) as Array>; + expect(conflicts[0]).toMatchObject({ kind: 'changed', local: { op: 'destroy' } }); + }); + + it('does not count a delete of a card already gone as a conflict', async () => { + const server = new FakeContacts(); + const card = server.addCard(); + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { destroy: [card.id] }); + server.removeCard(card.id); + await engine.requestSync(); + expect(await engine.areaCall('contacts.conflicts', undefined)).toEqual([]); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); +}); + +describe('edits while sending', () => { + it('keeps an edit made while the create was on its way', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Racing' }, note: 'one', addressBookIds: { book1: true } } } }); + let edited = false; + const original = server.request.bind(server); + server.request = async (calls: MethodCall[]) => { + if (!edited && calls.some(([name]) => name === 'ContactCard/set')) { + edited = true; + await call(engine, 'ContactCard/set', { update: { [created.n.id]: { note: 'two' } } }); + } + return original(calls); + }; + await engine.requestSync(); + await engine.requestSync(); + const cards = [...server.account().cards.values()]; + expect(cards).toHaveLength(1); + expect(cards[0].note).toBe('two'); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); +}); +/* eslint-enable @typescript-eslint/no-explicit-any */ diff --git a/lib/store/__tests__/helpers/fake-contacts.ts b/lib/store/__tests__/helpers/fake-contacts.ts new file mode 100644 index 000000000..e6b408755 --- /dev/null +++ b/lib/store/__tests__/helpers/fake-contacts.ts @@ -0,0 +1,229 @@ +/** + * FakeJmap with contacts, as measured on both Stalwarts (2026-11-03): uid is a + * query filter, a second card with a uid is refused with invalidProperties, + * unknown ids give per-record notFound, AddressBook state doesn't move when + * cards change, `ContactCard/get` with ids null stops at 500. + */ + +import { TransportError, type JmapResponse, type MethodCall, type StoreSession } from '../../engine/transport'; +import { Collection, FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './fake-jmap'; + +export const CONTACTS = 'urn:ietf:params:jmap:contacts'; +export const SHARED_ACCOUNT = 'shared1'; +export const MAIL_ONLY_ACCOUNT = 'mailonly'; + +export const CONTACTS_SESSION: StoreSession = { + ...FAKE_SESSION, + primaryAccounts: { ...FAKE_SESSION.primaryAccounts, [CONTACTS]: JMAP_ACCOUNT }, + accounts: { + [JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [CONTACTS]: {} } }, + [SHARED_ACCOUNT]: { name: 'team@example.org', isPersonal: false, accountCapabilities: { [CONTACTS]: {} } }, + [MAIL_ONLY_ACCOUNT]: { name: 'list@example.org', isPersonal: false, accountCapabilities: { 'urn:ietf:params:jmap:mail': {} } }, + }, + capabilities: { [CONTACTS]: {} }, +}; + +type Obj = Record & { id: string }; + +/** The /changes rules of the base fake, for another collection. */ +export function changesFor(coll: Collection, args: Record): Record { + const since = String(args.sinceState); + if (!/^s\d+$/.test(since)) throw Object.assign(new Error('bad state'), { jmapError: 'invalidArguments' }); + const from = Number(since.slice(1)); + if (from < coll.truncatedBelow || from > coll.log.length) { + throw Object.assign(new Error('log truncated'), { jmapError: 'cannotCalculateChanges' }); + } + const maxChanges = Math.min(Number(args.maxChanges ?? 5000) || 5000, 5000); + return { accountId: args.accountId, oldState: since, ...coll.changes(since, maxChanges) }; +} + +function applyServerPatch(card: Obj, patch: Record): void { + for (const [path, value] of Object.entries(patch)) { + const steps = path.split('/'); + let node = card as Record; + for (const step of steps.slice(0, -1)) node = node[step] as Record; + const last = steps[steps.length - 1]; + if (value === null) delete node[last]; + else node[last] = value; + } +} + +class ContactAccount { + readonly books = new Map(); + readonly cards = new Map(); + readonly coll = { AddressBook: new Collection(), ContactCard: new Collection() }; +} + +export class FakeContacts extends FakeJmap { + private readonly contactAccounts = new Map(); + private nextCard = 1; + /** Set to apply the next ContactCard/set and then lose its response. */ + loseNextSet = false; + /** Card ids (or creation ids) whose next set is refused with this error. */ + readonly refuse = new Map(); + /** A server that lets two cards have one uid, as nothing in the RFC forbids it. */ + allowDuplicateUids = false; + + constructor(books: string[] = ['book1']) { + super(); + for (const id of books) this.addBook({ id, name: id, isDefault: id === books[0] }); + } + + account(id: string = JMAP_ACCOUNT): ContactAccount { + let found = this.contactAccounts.get(id); + if (!found) { + found = new ContactAccount(); + this.contactAccounts.set(id, found); + } + return found; + } + + addBook(book: Obj, accountId: string = JMAP_ACCOUNT): void { + const account = this.account(accountId); + account.books.set(book.id, book); + account.coll.AddressBook.push('created', book.id); + } + + addCard(partial: Partial = {}, accountId: string = JMAP_ACCOUNT): Obj { + const account = this.account(accountId); + const n = this.nextCard++; + const card: Obj = { + '@type': 'Card', + version: '1.0', + id: `c${n}`, + uid: `urn:uuid:card-${n}`, + addressBookIds: { [[...account.books.keys()][0] ?? 'book1']: true }, + kind: 'individual', + name: { full: `Person ${n}` }, + ...partial, + }; + account.cards.set(card.id, card); + account.coll.ContactCard.push('created', card.id); + return card; + } + + editCard(id: string, patch: Record, accountId: string = JMAP_ACCOUNT): void { + const account = this.account(accountId); + applyServerPatch(account.cards.get(id) as Obj, patch); + account.coll.ContactCard.push('updated', id); + } + + removeCard(id: string, accountId: string = JMAP_ACCOUNT): void { + const account = this.account(accountId); + account.cards.delete(id); + account.coll.ContactCard.push('destroyed', id); + } + + private contactSet(account: ContactAccount, args: Record): Record { + const result: Record = { accountId: args.accountId, oldState: account.coll.ContactCard.state() }; + const created: Record = {}; + const notCreated: Record = {}; + const updated: Record = {}; + const notUpdated: Record = {}; + const destroyed: string[] = []; + const notDestroyed: Record = {}; + for (const [cid, raw] of Object.entries((args.create ?? {}) as Record>)) { + const refused = this.refuse.get(cid) ?? this.refuse.get(String(raw.uid)); + const clash = [...account.cards.values()].find((card) => card.uid === raw.uid); + if (refused) { + notCreated[cid] = refused; + } else if (clash && !this.allowDuplicateUids) { + notCreated[cid] = { type: 'invalidProperties', description: `Contact with UID ${String(raw.uid)} already exists with id ${clash.id}.`, properties: ['uid'] }; + } else { + const id = `c${this.nextCard++}`; + account.cards.set(id, { '@type': 'Card', version: '1.0', ...raw, id }); + account.coll.ContactCard.push('created', id); + created[cid] = { id }; + } + } + for (const [id, patch] of Object.entries((args.update ?? {}) as Record>)) { + const card = account.cards.get(id); + const refused = this.refuse.get(id); + if (refused) notUpdated[id] = refused; + else if (!card) notUpdated[id] = { type: 'notFound' }; + else { + applyServerPatch(card, patch); + account.coll.ContactCard.push('updated', id); + updated[id] = null; + } + } + for (const id of (args.destroy ?? []) as string[]) { + const refused = this.refuse.get(id); + if (refused) notDestroyed[id] = refused; + else if (!account.cards.has(id)) notDestroyed[id] = { type: 'notFound' }; + else { + account.cards.delete(id); + account.coll.ContactCard.push('destroyed', id); + destroyed.push(id); + } + } + for (const [key, value] of Object.entries({ created, notCreated, updated, notUpdated, notDestroyed })) { + if (Object.keys(value).length > 0) result[key] = value; + } + if (destroyed.length > 0) result.destroyed = destroyed; + result.newState = account.coll.ContactCard.state(); + return result; + } + + protected override handle(name: string, args: Record): Record { + const account = this.account(String(args.accountId)); + switch (name) { + case 'AddressBook/get': { + const ids = args.ids as string[] | null | undefined; + const all = [...account.books.values()]; + return { + accountId: args.accountId, + state: account.coll.AddressBook.state(), + list: ids ? all.filter((b) => ids.includes(b.id)) : all, + notFound: ids ? ids.filter((id) => !account.books.has(id)) : [], + }; + } + case 'AddressBook/changes': + return changesFor(account.coll.AddressBook, args); + case 'ContactCard/changes': + return changesFor(account.coll.ContactCard, args); + case 'ContactCard/query': { + const filter = (args.filter ?? {}) as { uid?: string; inAddressBook?: string }; + let list = [...account.cards.values()]; + if (filter.uid) list = list.filter((c) => c.uid === filter.uid); + if (filter.inAddressBook) list = list.filter((c) => (c.addressBookIds as Record)[filter.inAddressBook as string]); + const position = Number(args.position ?? 0); + const limit = Number(args.limit ?? 5000) || 5000; + return { + accountId: args.accountId, + queryState: account.coll.ContactCard.state(), + canCalculateChanges: true, + position, + ids: list.slice(position, position + Math.min(limit, 5000)).map((c) => c.id), + ...(args.calculateTotal ? { total: list.length } : {}), + }; + } + case 'ContactCard/get': { + const ids = args.ids as string[] | null | undefined; + if (ids && ids.length > 500) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' }); + const chosen = ids ?? [...account.cards.keys()].slice(0, 500); + return { + accountId: args.accountId, + state: account.coll.ContactCard.state(), + list: chosen.filter((id) => account.cards.has(id)).map((id) => structuredClone(account.cards.get(id))), + notFound: chosen.filter((id) => !account.cards.has(id)), + }; + } + case 'ContactCard/set': + return this.contactSet(account, args); + default: + return super.handle(name, args); + } + } + + override async request(calls: MethodCall[], using?: string[]): Promise { + // The real server fails the whole request for a capability it doesn't list. + void using; + const response = await super.request(calls); + if (this.loseNextSet && calls.some(([name]) => name === 'ContactCard/set')) { + this.loseNextSet = false; + throw new TransportError('network', 'Connection dropped after the server took the request'); + } + return response; + } +} diff --git a/lib/store/__tests__/helpers/fake-jmap.ts b/lib/store/__tests__/helpers/fake-jmap.ts index 9aa917be5..1fd274c04 100644 --- a/lib/store/__tests__/helpers/fake-jmap.ts +++ b/lib/store/__tests__/helpers/fake-jmap.ts @@ -25,7 +25,7 @@ type ChangeKind = 'created' | 'updated' | 'destroyed'; interface Log { kind: ChangeKind; id: string } -class Collection { +export class Collection { log: Log[] = []; /** Below this log position, states are refused as truncated. */ truncatedBelow = 0; @@ -169,7 +169,7 @@ export class FakeJmap implements JmapTransport { return list.map((e) => e.id); } - private handle(name: string, args: Record): Record { + protected handle(name: string, args: Record): Record { switch (name) { case 'Mailbox/get': { const ids = args.ids as string[] | null; @@ -275,7 +275,7 @@ export class FakeJmap implements JmapTransport { return { methodResponses: done }; } - async download(): Promise { + async download(_jmap: string, _blobId: string, _type: string, _name: string): Promise { return new Uint8Array(); } diff --git a/lib/store/area.ts b/lib/store/area.ts index fceed110e..9bd11fd35 100644 --- a/lib/store/area.ts +++ b/lib/store/area.ts @@ -52,6 +52,8 @@ export interface StoreContext { scope: StoreScope; now(): number; emit(event: StoreEvent): void; + /** Starts a sync pass, or one more after the running pass. Resolves when they're done. */ + requestSync(): Promise; } export interface SyncContext extends StoreContext { diff --git a/lib/store/areas.ts b/lib/store/areas.ts index 6bff665ca..c8882f3ee 100644 --- a/lib/store/areas.ts +++ b/lib/store/areas.ts @@ -1,6 +1,7 @@ /** Every part of the local store. New areas go here. */ import type { StoreArea } from './area'; +import { contactsArea } from './contacts'; import { mailArea } from './mail'; -export const STORE_AREAS: readonly StoreArea[] = [mailArea]; +export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea]; diff --git a/lib/store/contacts/index.ts b/lib/store/contacts/index.ts new file mode 100644 index 000000000..01b129a8a --- /dev/null +++ b/lib/store/contacts/index.ts @@ -0,0 +1,27 @@ +import type { StoreArea } from '../area'; +import { CONTACT_METHODS } from './methods'; +import { listConflicts, pendingCount } from './pending'; +import { CONTACT_MIGRATIONS } from './schema'; +import { contactsSyncer } from './sync'; + +export const contactsArea: StoreArea = { + name: 'contacts', + migrations: CONTACT_MIGRATIONS, + methods: CONTACT_METHODS, + syncers: [contactsSyncer], + calls: { + /** Sends what's queued, in the next sync pass, and says what's left. */ + 'contacts.flush': async (ctx) => { + await ctx.requestSync(); + return { pending: pendingCount(ctx.db), conflicts: listConflicts(ctx.db).length }; + }, + /** How many cards have edits waiting to go out. */ + 'contacts.pending': (ctx) => pendingCount(ctx.db), + /** Edits the server refused or that met a change there. The server's copy is the one kept. */ + 'contacts.conflicts': (ctx) => listConflicts(ctx.db), + /** The user has seen this one. */ + 'contacts.dismissConflict': (ctx, params: { id: number }) => { + ctx.db.run('DELETE FROM contact_conflict WHERE id = ?', [params.id]); + }, + }, +}; diff --git a/lib/store/contacts/methods.ts b/lib/store/contacts/methods.ts new file mode 100644 index 000000000..f3c885909 --- /dev/null +++ b/lib/store/contacts/methods.ts @@ -0,0 +1,209 @@ +/** Contact reads and queued edits, answered from the store. */ + +import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area'; +import { listObjects } from '../engine/core-schema'; +import type { SqlValue } from '../storage'; +import { queueCreate, queueDestroy, queueUpdate, resolveCardId, type SetError } from './pending'; +import { CONTACTS_CAPABILITY, foldText } from './records'; +import type { JsonObject, Patch } from './patch'; + +type Filter = Record; + +interface Sql { sql: string; params: SqlValue[] } + +function jmapOf(ctx: MethodContext, args: Record): string { + return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[CONTACTS_CAPABILITY] ?? ctx.defaultJmap; +} + +/** The last state from the server. Throws when this type was never synced, so an empty answer always means empty. */ +function syncedState(ctx: MethodContext, jmap: string, type: string): string | null { + const row = ctx.db.get<{ state: string | null; phase: string | null }>('SELECT state, phase FROM sync_state WHERE jmap = ? AND type = ?', [jmap, type]); + if (!row || (!row.state && !row.phase)) throw new MethodError('serverUnavailable', 'Contacts not kept yet'); + return row.state; +} + +function pick(object: Record, properties: unknown): Record { + if (!Array.isArray(properties)) return object; + const out: Record = { id: object.id }; + for (const property of properties as string[]) if (property in object) out[property] = object[property]; + return out; +} + +function quote(term: string): string { + return `"${term.replace(/"/g, '""')}"`; +} + +function terms(text: string): string[] { + return foldText(text).split(/[^\p{L}\p{N}]+/u).filter(Boolean); +} + +/** + * A `text` condition, one part of the client's textFilter (search-utils.ts): + * words, `word*` or a "quoted phrase". Every word has to match somewhere. The + * trigram index finds it anywhere in a name, which takes three characters. + * Shorter ones match the start of a word. + */ +function textCondition(value: string, rank: string[]): Sql { + const phrase = /^"(.+)"$/.exec(value.trim()); + const words = terms(phrase ? phrase[1] : value); + if (words.length === 0) return { sql: '0', params: [] }; + if (phrase) { + rank.push(quote(words.join(' '))); + return { sql: 'c.rowid IN (SELECT rowid FROM contact_fts_word WHERE contact_fts_word MATCH ?)', params: [quote(words.join(' '))] }; + } + const parts: string[] = []; + const params: SqlValue[] = []; + for (const word of words) { + rank.push(`${quote(word)}*`); + if ([...word].length >= 3) { + parts.push('c.rowid IN (SELECT rowid FROM contact_fts_tri WHERE contact_fts_tri MATCH ?)'); + params.push(quote(word)); + } else { + parts.push('c.rowid IN (SELECT rowid FROM contact_fts_word WHERE contact_fts_word MATCH ?)'); + params.push(`${quote(word)}*`); + } + } + return { sql: parts.join(' AND '), params }; +} + +function filterSql(filter: Filter, rank: string[]): Sql { + if (typeof filter.operator === 'string') { + const parts = (Array.isArray(filter.conditions) ? filter.conditions as Filter[] : []).map((c) => filterSql(c, rank)); + const params = parts.flatMap((p) => p.params); + const joined = (joiner: string) => parts.map((p) => `(${p.sql})`).join(joiner); + if (filter.operator === 'AND') return { sql: parts.length ? joined(' AND ') : '1', params }; + if (filter.operator === 'OR') return { sql: parts.length ? joined(' OR ') : '0', params }; + if (filter.operator === 'NOT') return { sql: parts.length ? `NOT (${joined(' OR ')})` : '1', params }; + throw new MethodError('unsupportedFilter', `Unknown operator ${String(filter.operator)}`); + } + const conditions: string[] = []; + const params: SqlValue[] = []; + for (const [key, value] of Object.entries(filter)) { + if (key === 'text') { + const text = textCondition(String(value), rank); + conditions.push(text.sql); + params.push(...text.params); + } else if (key === 'inAddressBook') { + conditions.push('EXISTS (SELECT 1 FROM contact_card_book b WHERE b.card_rowid = c.rowid AND b.book_id = ?)'); + params.push(String(value)); + } else { + throw new MethodError('unsupportedFilter', `${key} isn't available offline`); + } + } + return { sql: conditions.length ? conditions.join(' AND ') : '1', params }; +} + +const addressBookGet: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + const state = syncedState(ctx, jmap, 'AddressBook'); + const books = listObjects>(ctx.db, jmap, 'AddressBook'); + const ids = Array.isArray(args.ids) ? args.ids as string[] : null; + const chosen = ids ? books.filter((book) => ids.includes(book.id as string)) : books; + return { + accountId: jmap, + ...(state ? { state } : {}), + list: chosen.map((book) => pick(book, args.properties)), + notFound: ids ? ids.filter((id) => !books.some((book) => book.id === id)) : [], + }; +}; + +const contactQuery: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + syncedState(ctx, jmap, 'ContactCard'); + if (args.anchor !== undefined) throw new MethodError('unsupportedFilter', 'anchor needs the server'); + if (Array.isArray(args.sort) && args.sort.length > 0) throw new MethodError('unsupportedSort', 'Sorted lists need the server'); + const rank: string[] = []; + const where = args.filter ? filterSql(args.filter as Filter, rank) : { sql: '1', params: [] }; + const position = Math.max(0, Number(args.position ?? 0)); + // limit 0 means no limit on Stalwart, and the client can't tell a local answer apart. + const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1; + // Names starting with what was typed come before ones merely containing it. + const first = rank.length > 0 ? 'c.rowid IN (SELECT rowid FROM contact_fts_word WHERE contact_fts_word MATCH ?) DESC, ' : ''; + const ids = ctx.db.all<{ id: string }>( + `SELECT c.id FROM contact_card c WHERE c.jmap = ? AND (${where.sql}) ORDER BY ${first}c.sort_key, c.id LIMIT ? OFFSET ?`, + [jmap, ...where.params, ...(rank.length > 0 ? [rank.join(' AND ')] : []), limit, position], + ).map((row) => String(row.id)); + const result: MethodResult = { + accountId: jmap, + // Local answers aren't a position in the server's log, so this is a token only. + queryState: 'local', + canCalculateChanges: false, + position, + ids, + }; + if (args.calculateTotal) { + result.total = Number(ctx.db.get<{ n: number }>( + `SELECT COUNT(*) AS n FROM contact_card c WHERE c.jmap = ? AND (${where.sql})`, + [jmap, ...where.params], + )?.n ?? 0); + } + return result; +}; + +const contactGet: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + const state = syncedState(ctx, jmap, 'ContactCard'); + const list: Array> = []; + const notFound: string[] = []; + const rows = Array.isArray(args.ids) + ? (args.ids as string[]).map((id) => ({ id, row: ctx.db.get<{ json: string }>('SELECT json FROM contact_card WHERE jmap = ? AND id = ?', [jmap, resolveCardId(ctx.db, jmap, id)]) })) + : ctx.db.all<{ json: string; id: string }>('SELECT json, id FROM contact_card WHERE jmap = ? ORDER BY sort_key, id', [jmap]).map((row) => ({ id: row.id, row })); + for (const { id, row } of rows) { + if (row) list.push(pick(JSON.parse(row.json) as Record, args.properties)); + else notFound.push(id); + } + return { accountId: jmap, ...(state ? { state } : {}), list, notFound }; +}; + +/** + * Queues the edits. The answer leaves out oldState and newState: nothing + * reached the server, so there's no real state to name, and the client reads + * neither from ContactCard/set. + */ +const contactSet: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + syncedState(ctx, jmap, 'ContactCard'); + const created: Record = {}; + const notCreated: Record = {}; + const updated: Record = {}; + const notUpdated: Record = {}; + const destroyed: string[] = []; + const notDestroyed: Record = {}; + ctx.db.transaction(() => { + for (const [cid, card] of Object.entries((args.create ?? {}) as Record)) { + const outcome = queueCreate(ctx, jmap, card); + if ('id' in outcome) created[cid] = outcome; + else notCreated[cid] = outcome; + } + for (const [id, patch] of Object.entries((args.update ?? {}) as Record)) { + const refused = queueUpdate(ctx, jmap, id, patch); + if (refused) notUpdated[id] = refused; + else updated[id] = null; + } + for (const id of (args.destroy ?? []) as string[]) { + const refused = queueDestroy(ctx, jmap, id); + if (refused) notDestroyed[id] = refused; + else destroyed.push(id); + } + }); + if (Object.keys(created).length + Object.keys(updated).length + destroyed.length > 0) { + ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] }); + } + const some = (name: string, value: object) => (Object.keys(value).length > 0 ? { [name]: value } : {}); + return { + accountId: jmap, + ...some('created', created), + ...some('notCreated', notCreated), + ...some('updated', updated), + ...some('notUpdated', notUpdated), + ...some('destroyed', destroyed), + ...some('notDestroyed', notDestroyed), + }; +}; + +export const CONTACT_METHODS: Readonly> = { + 'AddressBook/get': addressBookGet, + 'ContactCard/query': contactQuery, + 'ContactCard/get': contactGet, + 'ContactCard/set': contactSet, +}; diff --git a/lib/store/contacts/patch.ts b/lib/store/contacts/patch.ts new file mode 100644 index 000000000..07247b3d9 --- /dev/null +++ b/lib/store/contacts/patch.ts @@ -0,0 +1,88 @@ +/** + * RFC 8620 §5.3 patches: keys are JSON pointers without the leading slash, a + * null value removes the property. Written so the common outbox can use the + * same rules for its per-record merge. + */ + +export type JsonObject = Record; +export type Patch = Record; + +export class InvalidPatchError extends Error { + constructor(path: string) { + super(`Can't apply a patch at ${path}`); + this.name = 'InvalidPatchError'; + } +} + +function steps(path: string): string[] { + return path.split('/').map((step) => step.replace(/~1/g, '/').replace(/~0/g, '~')); +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** A copy of the object with the patch applied. Throws when a path leads nowhere. */ +export function applyPatch(object: T, patch: Patch): T { + const root: JsonObject = { ...object }; + for (const [path, value] of Object.entries(patch)) { + const parts = steps(path); + let node = root; + for (const step of parts.slice(0, -1)) { + const next = node[step]; + if (!isObject(next)) throw new InvalidPatchError(path); + const copy = { ...next }; + node[step] = copy; + node = copy; + } + const last = parts[parts.length - 1]; + if (value === null) delete node[last]; + else node[last] = value; + } + return root as T; +} + +/** One patch that does what the older one and then the newer one would do. */ +export function mergePatches(older: Patch, newer: Patch): Patch { + const merged: Patch = { ...older }; + for (const [path, value] of Object.entries(newer)) { + // A newer write to a whole property replaces older writes below it. + for (const key of Object.keys(merged)) { + if (key.startsWith(`${path}/`)) delete merged[key]; + } + // A newer write below an older whole-property write goes into that value. + const parent = Object.keys(merged).find((key) => path.startsWith(`${key}/`)); + if (parent && isObject(merged[parent])) { + const holder = applyPatch({ value: merged[parent] }, { [`value/${path.slice(parent.length + 1)}`]: value }); + merged[parent] = holder.value; + } else { + merged[path] = value; + } + } + return merged; +} + +/** The top-level properties a patch writes to. */ +export function touchedProperties(patch: Patch): string[] { + return [...new Set(Object.keys(patch).map((path) => steps(path)[0]))]; +} + +/** Deep equality that ignores key order. Absent and null count as the same. */ +export function sameJson(a: unknown, b: unknown): boolean { + if (a === b || (a == null && b == null)) return true; + if (Array.isArray(a) && Array.isArray(b)) { + return a.length === b.length && a.every((item, i) => sameJson(item, b[i])); + } + if (isObject(a) && isObject(b)) { + return [...new Set([...Object.keys(a), ...Object.keys(b)])].every((key) => sameJson(a[key], b[key])); + } + return false; +} + +/** Whether `actual` holds everything in `wanted`. A server may add properties of its own. */ +export function coversJson(wanted: unknown, actual: unknown): boolean { + if (isObject(wanted) && isObject(actual)) { + return Object.keys(wanted).every((key) => coversJson(wanted[key], actual[key])); + } + return sameJson(wanted, actual); +} diff --git a/lib/store/contacts/pending.ts b/lib/store/contacts/pending.ts new file mode 100644 index 000000000..aaadc64b7 --- /dev/null +++ b/lib/store/contacts/pending.ts @@ -0,0 +1,459 @@ +/** + * Contact edits made while the server can't be reached, and sending them. + * + * One row per card, whatever number of edits it had. The visible card in + * contact_card already has the edits on top, and sync keeps it so. Sending + * runs at the start of each sync pass, so it never overlaps with itself. + * + * What a row becomes on its way out: + * - Update: the server's card is read first. Nothing to do if it already has + * the values (a lost response). A conflict if the server changed one of the + * properties the edit touches. Otherwise the patch goes out. + * - Destroy: done if the card is gone, a conflict if it changed since. + * - Create: a request that may have reached the server looks for the card's + * uid first, so a lost response doesn't make a second card. Stalwart also + * refuses a second card with the same uid, which ends up the same way. + */ + +import type { StoreContext, SyncContext } from '../area'; +import { responseFor, type MethodCall } from '../engine/transport'; +import type { StorageAdapter } from '../storage'; +import { applyPatch, coversJson, InvalidPatchError, mergePatches, sameJson, touchedProperties, type JsonObject, type Patch } from './patch'; +import { cardDisplayName, CONTACTS_USING, readCard, removeCard, writeCard, type Card } from './records'; + +export const STAND_IN_PREFIX = 'bulwark-local-card-'; +/** What a held upload's stand-in blob id starts with (lib/offline/queue.ts). */ +const LOCAL_BLOB_MARK = '"blobId":"bulwark-local-'; + +const SET_BATCH = 100; +const SET_BATCH_BYTES = 5 * 1024 * 1024; +const GET_BATCH = 500; +const CALLS_PER_REQUEST = 16; +/** Set errors that say nothing about the edit itself: it goes out again next time. */ +const TRANSIENT_ERRORS = new Set(['serverFail', 'serverUnavailable', 'rateLimit']); +/** Server-kept timestamps: their changing doesn't mean someone edited the card. */ +const SERVER_STAMPS = ['updated', 'created']; + +export interface PendingRow { + seq: number; + jmap: string; + id: string; + op: 'create' | 'update' | 'destroy'; + uid: string | null; + payload: string | null; + base: string | null; + attempted: number; + queued_at: number; +} + +export interface SetError { type: string; description?: string; properties?: string[] } + +export interface ContactConflict { + id: number; + jmap: string; + recordId: string; + kind: 'refused' | 'changed' | 'gone'; + summary: string; + local: { op: string; payload: unknown }; + server: unknown; + error: string | null; + createdAt: number; +} + +export function pendingFor(db: StorageAdapter, jmap: string, id: string): PendingRow | undefined { + return db.get('SELECT * FROM contact_pending WHERE jmap = ? AND id = ?', [jmap, id]); +} + +function dropPending(db: StorageAdapter, jmap: string, id: string): void { + db.run('DELETE FROM contact_pending WHERE jmap = ? AND id = ?', [jmap, id]); +} + +function savePending(db: StorageAdapter, ctx: { now(): number }, row: { jmap: string; id: string; op: PendingRow['op']; uid?: string | null; payload: string | null; base: string | null }): void { + db.run( + `INSERT INTO contact_pending (jmap, id, op, uid, payload, base, queued_at) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (jmap, id) DO UPDATE SET op = excluded.op, uid = excluded.uid, payload = excluded.payload, base = excluded.base`, + [row.jmap, row.id, row.op, row.uid ?? null, row.payload, row.base, ctx.now()], + ); +} + +/** A stand-in id that already went through becomes the real one. */ +export function resolveCardId(db: StorageAdapter, jmap: string, id: string): string { + return db.get<{ real_id: string }>('SELECT real_id FROM contact_id_map WHERE jmap = ? AND local_id = ?', [jmap, id])?.real_id ?? id; +} + +/** A held upload can't be named in a card that goes out later. The client falls back to a data: URI. */ +function holdsLocalBlob(value: unknown): boolean { + return JSON.stringify(value).includes(LOCAL_BLOB_MARK); +} + +const LOCAL_BLOB_ERROR: SetError = { + type: 'invalidProperties', + description: 'A photo that is not on the server yet cannot be used here', + properties: ['media'], +}; + +export function queueCreate(ctx: StoreContext, jmap: string, card: JsonObject): { id: string; uid: string } | SetError { + if (holdsLocalBlob(card)) return LOCAL_BLOB_ERROR; + if (Object.keys((card.addressBookIds ?? {}) as object).length === 0) { + // The client picks the default book. None known means there's nowhere to put it. + return { type: 'invalidProperties', description: 'A contact needs an address book', properties: ['addressBookIds'] }; + } + const id = `${STAND_IN_PREFIX}${crypto.randomUUID()}`; + const uid = typeof card.uid === 'string' && card.uid ? card.uid : `urn:uuid:${crypto.randomUUID()}`; + const payload = { ...card, uid }; + savePending(ctx.db, ctx, { jmap, id, op: 'create', uid, payload: JSON.stringify(payload), base: null }); + writeCard(ctx.db, jmap, { '@type': 'Card', version: '1.0', ...payload, id }); + return { id, uid }; +} + +export function queueUpdate(ctx: StoreContext, jmap: string, rawId: string, patch: Patch): SetError | null { + const id = resolveCardId(ctx.db, jmap, rawId); + const card = readCard(ctx.db, jmap, id); + if (!card) return { type: 'notFound' }; + if ('id' in patch) return { type: 'invalidProperties', description: 'The id property is immutable.', properties: ['id'] }; + if (holdsLocalBlob(patch)) return LOCAL_BLOB_ERROR; + const pending = pendingFor(ctx.db, jmap, id); + let visible: Card; + try { + visible = applyPatch(card, patch); + } catch (error) { + if (error instanceof InvalidPatchError) return { type: 'invalidPatch', description: error.message }; + throw error; + } + if (pending?.op === 'create') { + const payload = applyPatch(JSON.parse(pending.payload as string) as JsonObject, patch); + savePending(ctx.db, ctx, { jmap, id, op: 'create', uid: pending.uid, payload: JSON.stringify(payload), base: null }); + } else { + const merged = pending ? mergePatches(JSON.parse(pending.payload as string) as Patch, patch) : patch; + savePending(ctx.db, ctx, { jmap, id, op: 'update', payload: JSON.stringify(merged), base: pending ? pending.base : JSON.stringify(card) }); + } + writeCard(ctx.db, jmap, visible); + return null; +} + +export function queueDestroy(ctx: StoreContext, jmap: string, rawId: string): SetError | null { + const id = resolveCardId(ctx.db, jmap, rawId); + const card = readCard(ctx.db, jmap, id); + if (!card) return { type: 'notFound' }; + const pending = pendingFor(ctx.db, jmap, id); + if (pending?.op === 'create' && !pending.attempted) { + // Never reached the server, so it never has to. + dropPending(ctx.db, jmap, id); + } else if (pending?.op === 'create') { + // It may have reached the server. Sending decides, then destroys it. + savePending(ctx.db, ctx, { jmap, id, op: 'destroy', uid: pending.uid, payload: pending.payload, base: null }); + } else { + savePending(ctx.db, ctx, { jmap, id, op: 'destroy', payload: null, base: pending ? pending.base : JSON.stringify(card) }); + } + removeCard(ctx.db, jmap, id); + return null; +} + +// ---- What sync does around queued edits ---- + +/** Stores a card the server sent, with whatever is queued for it on top. */ +export function storeServerCard(db: StorageAdapter, jmap: string, card: Card, gen?: number): void { + const pending = pendingFor(db, jmap, card.id); + if (pending?.op === 'destroy') return; + let visible = card; + if (pending?.op === 'update') { + try { + visible = applyPatch(card, JSON.parse(pending.payload as string) as Patch); + } catch (error) { + // The patch no longer fits the card. Sending turns this into a conflict. + if (!(error instanceof InvalidPatchError)) throw error; + } + } + writeCard(db, jmap, visible, gen); +} + +/** The server no longer has this card. */ +export function serverDestroyed(ctx: StoreContext, jmap: string, id: string): void { + const pending = pendingFor(ctx.db, jmap, id); + if (pending?.op === 'update') recordConflict(ctx, pending, 'gone', null, null); + if (pending && pending.op !== 'create') dropPending(ctx.db, jmap, id); + removeCard(ctx.db, jmap, id); +} + +function recordConflict(ctx: StoreContext, row: PendingRow, kind: ContactConflict['kind'], server: Card | null, error: SetError | null): void { + const local = row.payload ? JSON.parse(row.payload) as JsonObject : null; + const known = server ?? readCard(ctx.db, row.jmap, row.id) ?? (row.base ? JSON.parse(row.base) as JsonObject : local); + ctx.db.run( + 'INSERT INTO contact_conflict (jmap, record_id, kind, summary, local, server, error, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + [ + row.jmap, + row.id, + kind, + cardDisplayName(known ?? {}), + JSON.stringify({ op: row.op, payload: local }), + server ? JSON.stringify(server) : null, + error ? [error.type, error.description].filter(Boolean).join(': ') : null, + ctx.now(), + ], + ); +} + +export function listConflicts(db: StorageAdapter): ContactConflict[] { + return db.all<{ id: number; jmap: string; record_id: string; kind: ContactConflict['kind']; summary: string; local: string; server: string | null; error: string | null; created_at: number }>( + 'SELECT * FROM contact_conflict ORDER BY id', + ).map((row) => ({ + id: Number(row.id), + jmap: row.jmap, + recordId: row.record_id, + kind: row.kind, + summary: row.summary, + local: JSON.parse(row.local) as ContactConflict['local'], + server: row.server ? JSON.parse(row.server) : null, + error: row.error, + createdAt: Number(row.created_at), + })); +} + +export function pendingCount(db: StorageAdapter): number { + return Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM contact_pending')?.n ?? 0); +} + +// ---- Sending ---- + +function withoutStamps(card: JsonObject): JsonObject { + const copy = { ...card }; + for (const key of SERVER_STAMPS) delete copy[key]; + return copy; +} + +async function readServerCards(ctx: SyncContext, jmap: string, ids: string[]): Promise> { + const cards = new Map(); + for (let i = 0; i < ids.length; i += GET_BATCH) { + const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + GET_BATCH) }, 'g']], CONTACTS_USING); + for (const card of responseFor(response, 'g')[1].list as Card[]) cards.set(card.id, card); + } + return cards; +} + +/** The ids the server has for these uids. */ +async function lookUpUids(ctx: SyncContext, jmap: string, uids: string[]): Promise> { + const found = new Map(); + for (let i = 0; i < uids.length; i += CALLS_PER_REQUEST) { + const group = uids.slice(i, i + CALLS_PER_REQUEST); + const calls: MethodCall[] = group.map((uid, n) => ['ContactCard/query', { accountId: jmap, filter: { uid }, limit: 1 }, `q${n}`]); + const response = await ctx.transport.request(calls, CONTACTS_USING); + group.forEach((uid, n) => { + const ids = responseFor(response, `q${n}`)[1].ids as string[]; + if (ids.length > 0) found.set(uid, ids[0]); + }); + } + return found; +} + +/** The create went through, or had already. The stand-in row becomes the server's card. */ +function settleCreated(ctx: SyncContext, sent: PendingRow, server: Card): void { + const { db } = ctx; + const { jmap, id: standIn } = sent; + const now = pendingFor(db, jmap, standIn); + db.run('INSERT OR REPLACE INTO contact_id_map (jmap, local_id, real_id) VALUES (?, ?, ?)', [jmap, standIn, server.id]); + dropPending(db, jmap, standIn); + removeCard(db, jmap, standIn); + if (now?.op === 'destroy' || !now) { + // Deleted while the create was on its way: delete the real one next. + savePending(db, ctx, { jmap, id: server.id, op: 'destroy', payload: null, base: JSON.stringify(server) }); + return; + } + if (now.payload !== sent.payload) { + // Edited while the create was on its way: what differs goes out as an edit. + const before = JSON.parse(sent.payload as string) as JsonObject; + const after = JSON.parse(now.payload as string) as JsonObject; + const patch: Patch = {}; + for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) { + if (!sameJson(before[key], after[key])) patch[key.replace(/~/g, '~0').replace(/\//g, '~1')] = after[key] ?? null; + } + savePending(db, ctx, { jmap, id: server.id, op: 'update', payload: JSON.stringify(patch), base: JSON.stringify(server) }); + writeCard(db, jmap, applyPatch(server, patch)); + return; + } + writeCard(db, jmap, server); +} + +interface Decision { create: Map; update: Map; destroy: Map } + +/** What to do with one row now that the server's copy is known. Conflicts are settled here. */ +function decide(ctx: SyncContext, row: PendingRow, server: Card | undefined, serverByUid: Map, out: Decision): void { + const { db } = ctx; + const { jmap, id } = row; + const uidHit = row.uid ? serverByUid.get(row.uid) : undefined; + if (row.op === 'create') { + if (uidHit) { + settleCreated(ctx, row, { ...(JSON.parse(row.payload as string) as JsonObject), ...uidHit }); + return; + } + out.create.set(id, row); + return; + } + if (row.op === 'destroy' && id.startsWith(STAND_IN_PREFIX)) { + // A create that may have gone out, deleted since. Without a card on the server, nothing to do. + dropPending(db, jmap, id); + if (uidHit) savePending(db, ctx, { jmap, id: uidHit.id, op: 'destroy', payload: null, base: JSON.stringify(uidHit) }); + return; + } + if (row.op === 'destroy') { + if (!server) { + dropPending(db, jmap, id); + } else if (row.base && !sameJson(withoutStamps(JSON.parse(row.base) as JsonObject), withoutStamps(server))) { + recordConflict(ctx, row, 'changed', server, null); + dropPending(db, jmap, id); + writeCard(db, jmap, server); + } else { + out.destroy.set(id, row); + } + return; + } + if (!server) { + recordConflict(ctx, row, 'gone', null, null); + dropPending(db, jmap, id); + removeCard(db, jmap, id); + return; + } + const patch = JSON.parse(row.payload as string) as Patch; + const base = row.base ? JSON.parse(row.base) as JsonObject : null; + let applied: Card | null = null; + try { + applied = applyPatch(server, patch); + } catch (error) { + if (!(error instanceof InvalidPatchError)) throw error; + } + const touched = touchedProperties(patch); + if (applied && touched.every((key) => coversJson((applied as Card)[key], server[key]))) { + dropPending(db, jmap, id); + writeCard(db, jmap, server); + return; + } + if (!applied || (base && touched.some((key) => !sameJson(base[key], server[key])))) { + recordConflict(ctx, row, 'changed', server, null); + dropPending(db, jmap, id); + writeCard(db, jmap, server); + return; + } + out.update.set(id, row); +} + +function settleUpdated(ctx: SyncContext, sent: PendingRow, server: Card): void { + const now = pendingFor(ctx.db, sent.jmap, sent.id); + if (!now) return; + if (now.op === 'update' && now.payload === sent.payload) { + dropPending(ctx.db, sent.jmap, sent.id); + return; + } + // Edited again while this was on its way: that edit now builds on what was sent. + const base = applyPatch(server, JSON.parse(sent.payload as string) as Patch); + ctx.db.run('UPDATE contact_pending SET base = ? WHERE seq = ?', [JSON.stringify(base), now.seq]); +} + +function settleRefused(ctx: SyncContext, row: PendingRow, error: SetError, server: Card | undefined): void { + const { db } = ctx; + if (TRANSIENT_ERRORS.has(error.type)) return; + if (row.op === 'destroy' && error.type === 'notFound') { + dropPending(db, row.jmap, row.id); + return; + } + const kind = error.type === 'notFound' ? 'gone' : 'refused'; + recordConflict(ctx, row, kind, server ?? null, error); + dropPending(db, row.jmap, row.id); + if (server && kind === 'refused') writeCard(db, row.jmap, server); + else removeCard(db, row.jmap, row.id); +} + +/** Takes rows up to about the size of one request. */ +function nextBatch(db: StorageAdapter, jmap: string, afterSeq: number): PendingRow[] { + const rows = db.all('SELECT * FROM contact_pending WHERE jmap = ? AND seq > ? ORDER BY seq LIMIT ?', [jmap, afterSeq, SET_BATCH]); + let bytes = 0; + const batch: PendingRow[] = []; + for (const row of rows) { + bytes += row.payload?.length ?? 0; + if (batch.length > 0 && bytes > SET_BATCH_BYTES) break; + batch.push(row); + } + return batch; +} + +/** Sends what's queued for one JMAP account. Network failures reach the engine and the rows stay. */ +export async function flushPending(ctx: SyncContext, jmap: string): Promise { + const { db } = ctx; + let afterSeq = 0; + let settled = false; + for (;;) { + if (ctx.stopped()) break; + const batch = nextBatch(db, jmap, afterSeq); + if (batch.length === 0) break; + afterSeq = batch[batch.length - 1].seq; + settled = true; + + const current = await readServerCards(ctx, jmap, batch.filter((row) => row.op !== 'create' && !row.id.startsWith(STAND_IN_PREFIX)).map((row) => row.id)); + const attempted = batch.filter((row) => row.uid && row.attempted).map((row) => row.uid as string); + const uidIds = await lookUpUids(ctx, jmap, attempted); + const uidCards = await readServerCards(ctx, jmap, [...uidIds.values()]); + const serverByUid = new Map(); + for (const [uid, id] of uidIds) { + const card = uidCards.get(id); + if (card) serverByUid.set(uid, card); + } + + const out: Decision = { create: new Map(), update: new Map(), destroy: new Map() }; + db.transaction(() => { + for (const { seq, id } of batch) { + // The page may have edited it while the reads were on their way. + const row = pendingFor(db, jmap, id); + if (row && row.seq === seq) decide(ctx, row, current.get(id), serverByUid, out); + } + for (const row of out.create.values()) db.run('UPDATE contact_pending SET attempted = 1 WHERE seq = ?', [row.seq]); + }); + if (out.create.size + out.update.size + out.destroy.size === 0) continue; + + const creationIds = new Map([...out.create.values()].map((row) => [`c${row.seq}`, row])); + const response = await ctx.transport.request([['ContactCard/set', { + accountId: jmap, + ...(creationIds.size > 0 ? { create: Object.fromEntries([...creationIds].map(([cid, row]) => [cid, JSON.parse(row.payload as string)])) } : {}), + ...(out.update.size > 0 ? { update: Object.fromEntries([...out.update.values()].map((row) => [row.id, JSON.parse(row.payload as string)])) } : {}), + ...(out.destroy.size > 0 ? { destroy: [...out.destroy.keys()] } : {}), + }, 's']], CONTACTS_USING); + const [, result] = responseFor(response, 's'); + + const refusedUid: Array<[PendingRow, SetError]> = []; + db.transaction(() => { + for (const [cid, row] of creationIds) { + const created = result.created?.[cid] as JsonObject | undefined; + const refused = result.notCreated?.[cid] as SetError | undefined; + if (created) { + settleCreated(ctx, row, { ...(JSON.parse(row.payload as string) as JsonObject), ...created, id: String(created.id) }); + } else if (refused?.properties?.includes('uid')) { + refusedUid.push([row, refused]); + } else if (refused) { + settleRefused(ctx, row, refused, undefined); + } + } + for (const [id, row] of out.update) { + const refused = result.notUpdated?.[id] as SetError | undefined; + if (refused) settleRefused(ctx, row, refused, current.get(id)); + else if (result.updated && id in result.updated) settleUpdated(ctx, row, current.get(id) as Card); + } + for (const [id, row] of out.destroy) { + const refused = result.notDestroyed?.[id] as SetError | undefined; + if (refused) settleRefused(ctx, row, refused, current.get(id)); + else if ((result.destroyed as string[] | undefined)?.includes(id)) dropPending(db, jmap, id); + } + }); + + // The server has a card with this uid already: this one, sent before. + if (refusedUid.length > 0) { + const ids = await lookUpUids(ctx, jmap, refusedUid.map(([row]) => row.uid as string)); + const cards = await readServerCards(ctx, jmap, [...ids.values()]); + db.transaction(() => { + for (const [row, refused] of refusedUid) { + const id = ids.get(row.uid as string); + const card = id ? cards.get(id) : undefined; + if (card) settleCreated(ctx, row, { ...(JSON.parse(row.payload as string) as JsonObject), ...card }); + else settleRefused(ctx, row, refused, undefined); + } + }); + } + } + if (settled) ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] }); +} diff --git a/lib/store/contacts/records.ts b/lib/store/contacts/records.ts new file mode 100644 index 000000000..07dc2f73b --- /dev/null +++ b/lib/store/contacts/records.ts @@ -0,0 +1,116 @@ +/** How a ContactCard becomes rows, and how the search index reads it. */ + +import type { SqlValue, StorageAdapter } from '../storage'; + +export const CONTACTS_CAPABILITY = 'urn:ietf:params:jmap:contacts'; +export const CONTACTS_USING = ['urn:ietf:params:jmap:core', CONTACTS_CAPABILITY]; + +/** As the client asks for them. Without them Stalwart may leave out shareWith and myRights. */ +export const ADDRESS_BOOK_PROPERTIES = ['id', 'name', 'description', 'sortOrder', 'isDefault', 'isSubscribed', 'shareWith', 'myRights']; + +export type Card = Record & { id: string }; + +function values(map: unknown): Array> { + if (typeof map !== 'object' || map === null) return []; + return Object.values(map).filter((v): v is Record => typeof v === 'object' && v !== null); +} + +function strings(items: unknown[]): string[] { + return items.filter((item): item is string => typeof item === 'string' && item.length > 0); +} + +/** Lower case, no accents: what both the index and the search terms go through. */ +export function foldText(text: string): string { + return text + .normalize('NFKD') + .replace(/\p{M}+/gu, '') + .toLowerCase() + .replace(/ß/g, 'ss') + .replace(/æ/g, 'ae') + .replace(/œ/g, 'oe') + .replace(/[øđł]/g, (c) => ({ ø: 'o', đ: 'd', ł: 'l' })[c] as string); +} + +/** + * The text to index. A German name is indexed both ways, since people type + * "Mueller" for "Müller" as often as "Muller". + */ +function indexText(text: string): string { + const folded = foldText(text); + const german = foldText( + text.normalize('NFC').replace(/[äÄ]/g, 'ae').replace(/[öÖ]/g, 'oe').replace(/[üÜ]/g, 'ue'), + ); + return german === folded ? folded : `${folded} ${german}`; +} + +export function cardDisplayName(card: Record): string { + const name = card.name as { full?: unknown; components?: unknown } | undefined; + if (typeof name?.full === 'string' && name.full) return name.full; + const parts = Array.isArray(name?.components) ? strings((name.components as Array<{ value?: unknown }>).map((c) => c?.value)) : []; + if (parts.length > 0) return parts.join(' '); + return strings(values(card.emails).map((e) => e.address))[0] ?? ''; +} + +function cardIndex(card: Record): { name: string; email: string; org: string } { + const name = card.name as { full?: unknown; components?: unknown } | undefined; + const components = Array.isArray(name?.components) ? (name.components as Array<{ value?: unknown }>).map((c) => c?.value) : []; + const nicknames = values(card.nicknames).map((n) => n.name); + const emails = values(card.emails).map((e) => e.address); + const organizations = values(card.organizations).flatMap((o) => [ + o.name, + ...(Array.isArray(o.units) ? (o.units as Array<{ name?: unknown }>).map((u) => u?.name) : []), + ]); + return { + name: indexText(strings([name?.full, ...components, ...nicknames]).join(' ')), + email: indexText(strings(emails).join(' ')), + org: indexText(strings(organizations).join(' ')), + }; +} + +function books(card: Record): string[] { + const ids = card.addressBookIds; + if (typeof ids !== 'object' || ids === null) return []; + return Object.entries(ids).filter(([, on]) => on).map(([id]) => id); +} + +/** Writes one card and its index rows. Without a gen, the row keeps its own. */ +export function writeCard(db: StorageAdapter, jmap: string, card: Card, gen?: number): void { + const existing = db.get<{ rowid: number; gen: number }>('SELECT rowid, gen FROM contact_card WHERE jmap = ? AND id = ?', [jmap, card.id]); + const row: SqlValue[] = [ + typeof card.uid === 'string' ? card.uid : null, + foldText(cardDisplayName(card)), + JSON.stringify(card), + gen ?? Number(existing?.gen ?? 0), + ]; + let rowid: number; + if (existing) { + rowid = Number(existing.rowid); + db.run('UPDATE contact_card SET uid = ?, sort_key = ?, json = ?, gen = ? WHERE rowid = ?', [...row, rowid]); + dropIndex(db, rowid); + } else { + rowid = db.run('INSERT INTO contact_card (uid, sort_key, json, gen, jmap, id) VALUES (?, ?, ?, ?, ?, ?)', [...row, jmap, card.id]).lastInsertRowid; + } + const text = cardIndex(card); + for (const table of ['contact_fts_word', 'contact_fts_tri']) { + db.run(`INSERT INTO ${table} (rowid, name, email, org) VALUES (?, ?, ?, ?)`, [rowid, text.name, text.email, text.org]); + } + for (const book of books(card)) db.run('INSERT OR IGNORE INTO contact_card_book (card_rowid, book_id) VALUES (?, ?)', [rowid, book]); +} + +function dropIndex(db: StorageAdapter, rowid: number): void { + db.run('DELETE FROM contact_fts_word WHERE rowid = ?', [rowid]); + db.run('DELETE FROM contact_fts_tri WHERE rowid = ?', [rowid]); + db.run('DELETE FROM contact_card_book WHERE card_rowid = ?', [rowid]); +} + +export function removeCard(db: StorageAdapter, jmap: string, id: string): void { + const existing = db.get<{ rowid: number }>('SELECT rowid FROM contact_card WHERE jmap = ? AND id = ?', [jmap, id]); + if (!existing) return; + dropIndex(db, Number(existing.rowid)); + db.run('DELETE FROM contact_card WHERE rowid = ?', [existing.rowid]); +} + +export function readCard(db: StorageAdapter, jmap: string, id: string): Card | null { + const row = db.get<{ json: string }>('SELECT json FROM contact_card WHERE jmap = ? AND id = ?', [jmap, id]); + return row ? JSON.parse(row.json) as Card : null; +} diff --git a/lib/store/contacts/schema.ts b/lib/store/contacts/schema.ts new file mode 100644 index 000000000..66b30832c --- /dev/null +++ b/lib/store/contacts/schema.ts @@ -0,0 +1,75 @@ +import type { Migration } from '../storage'; + +export const CONTACT_MIGRATIONS: readonly Migration[] = [ + { + id: 600, + name: 'contacts', + sql: ` + CREATE TABLE contact_card ( + rowid INTEGER PRIMARY KEY, + jmap TEXT NOT NULL, + id TEXT NOT NULL, + uid TEXT, + sort_key TEXT NOT NULL DEFAULT '', + -- What the UI sees: the server's card with queued edits on top. + json TEXT NOT NULL, + gen INTEGER NOT NULL DEFAULT 0, + UNIQUE (jmap, id) + ); + CREATE INDEX contact_card_sort ON contact_card (jmap, sort_key, id); + CREATE TABLE contact_card_book ( + card_rowid INTEGER NOT NULL, + book_id TEXT NOT NULL, + PRIMARY KEY (card_rowid, book_id) + ) WITHOUT ROWID; + CREATE INDEX contact_card_book_lookup ON contact_card_book (book_id, card_rowid); + -- Text goes in through foldText (records.ts), which also takes care of accents. + CREATE VIRTUAL TABLE contact_fts_word USING fts5( + name, email, org, + content = '', contentless_delete = 1, + tokenize = 'unicode61 remove_diacritics 0', + prefix = '2 3' + ); + CREATE VIRTUAL TABLE contact_fts_tri USING fts5( + name, email, org, + content = '', contentless_delete = 1, + tokenize = 'trigram' + ); + -- One row per card with something to send. Edits to a card merge here. + CREATE TABLE contact_pending ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + jmap TEXT NOT NULL, + id TEXT NOT NULL, + op TEXT NOT NULL, + uid TEXT, + -- create: the whole card. update: the merged patch. destroy: nothing. + payload TEXT, + -- The server's card when the first edit was queued, to see what changed there since. + base TEXT, + -- A request may have reached the server, so a create looks for its uid first. + attempted INTEGER NOT NULL DEFAULT 0, + queued_at INTEGER NOT NULL, + UNIQUE (jmap, id) + ); + -- Stand-in ids the page may still hold after a create went through. + CREATE TABLE contact_id_map ( + jmap TEXT NOT NULL, + local_id TEXT NOT NULL, + real_id TEXT NOT NULL, + PRIMARY KEY (jmap, local_id) + ) WITHOUT ROWID; + CREATE TABLE contact_conflict ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + jmap TEXT NOT NULL, + record_id TEXT NOT NULL, + -- refused: the server said no. changed: edited there too. gone: deleted there. + kind TEXT NOT NULL, + summary TEXT NOT NULL, + local TEXT NOT NULL, + server TEXT, + error TEXT, + created_at INTEGER NOT NULL + ); + `, + }, +]; diff --git a/lib/store/contacts/sync.ts b/lib/store/contacts/sync.ts new file mode 100644 index 000000000..f734263d6 --- /dev/null +++ b/lib/store/contacts/sync.ts @@ -0,0 +1,179 @@ +/** + * Contact sync for one JMAP account: first what's queued goes out, then the + * address books, then the cards. A full pass over the cards captures the + * state first and pages, with its cursor in sync_state, so a reload continues + * where it stopped. Cards the pass didn't see are gone from the server. + */ + +import type { SyncContext, Syncer } from '../area'; +import { runChangesLoop } from '../engine/changes'; +import { deleteObject, putObject } from '../engine/core-schema'; +import { responseFor, type MethodCall } from '../engine/transport'; +import { flushPending, serverDestroyed, storeServerCard } from './pending'; +import { ADDRESS_BOOK_PROPERTIES, CONTACTS_CAPABILITY, CONTACTS_USING, type Card } from './records'; + +const PAGE = 500; +const PAIRS_PER_REQUEST = 4; + +// ---- Address books ---- + +async function fullBooks(ctx: SyncContext, jmap: string): Promise { + const response = await ctx.transport.request([['AddressBook/get', { accountId: jmap, properties: ADDRESS_BOOK_PROPERTIES }, 'b']], CONTACTS_USING); + const [, result] = responseFor(response, 'b'); + ctx.db.transaction(() => { + ctx.db.run("DELETE FROM object WHERE jmap = ? AND type = 'AddressBook'", [jmap]); + for (const book of result.list as Array<{ id: string }>) putObject(ctx.db, jmap, 'AddressBook', book.id, book); + ctx.setSyncState(jmap, 'AddressBook', { state: String(result.state), phase: null, cursor: null }); + }); + ctx.emit({ type: 'changed', account: ctx.account, collections: ['AddressBook'] }); +} + +async function syncBooks(ctx: SyncContext, jmap: string): Promise { + const { state, target } = ctx.syncState(jmap, 'AddressBook'); + if (!state) return fullBooks(ctx, jmap); + const outcome = await runChangesLoop({ + transport: ctx.transport, + type: 'AddressBook', + jmap, + using: CONTACTS_USING, + since: state, + target, + stopped: ctx.stopped, + apply: async (page) => { + const ids = [...page.created, ...page.updated]; + const list: Array<{ id: string }> = []; + for (let i = 0; i < ids.length; i += PAGE) { + const response = await ctx.transport.request([['AddressBook/get', { accountId: jmap, ids: ids.slice(i, i + PAGE), properties: ADDRESS_BOOK_PROPERTIES }, 'b']], CONTACTS_USING); + list.push(...responseFor(response, 'b')[1].list); + } + ctx.db.transaction(() => { + for (const book of list) putObject(ctx.db, jmap, 'AddressBook', book.id, book); + for (const id of page.destroyed) deleteObject(ctx.db, jmap, 'AddressBook', id); + ctx.setSyncState(jmap, 'AddressBook', { state: page.newState }); + }); + if (ids.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['AddressBook'] }); + }, + }); + if (outcome.outcome === 'resync') await fullBooks(ctx, jmap); +} + +// ---- Cards ---- + +interface ListCursor { position: number; gen: number; captured: string } + +async function startFullList(ctx: SyncContext, jmap: string): Promise { + const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: [] }, 's']], CONTACTS_USING); + const captured = String(responseFor(response, 's')[1].state); + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM contact_card WHERE jmap = ?', [jmap])?.gen ?? 0) + 1; + const cursor: ListCursor = { position: 0, gen, captured }; + ctx.db.transaction(() => ctx.setSyncState(jmap, 'ContactCard', { state: null, phase: 'list', cursor: { ...cursor } })); + return cursor; +} + +/** Cards from before this pass that it didn't find, leaving alone creates still on this device. */ +function sweep(ctx: SyncContext, jmap: string, gen: number): void { + const stale = ctx.db.all<{ id: string }>( + `SELECT c.id FROM contact_card c WHERE c.jmap = ? AND c.gen < ? + AND NOT EXISTS (SELECT 1 FROM contact_pending p WHERE p.jmap = c.jmap AND p.id = c.id AND p.op = 'create')`, + [jmap, gen], + ); + for (const row of stale) serverDestroyed(ctx, jmap, String(row.id)); +} + +async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise { + const cursor = { ...start }; + for (;;) { + if (ctx.stopped()) return false; + const calls: MethodCall[] = []; + for (let i = 0; i < PAIRS_PER_REQUEST; i += 1) { + calls.push(['ContactCard/query', { accountId: jmap, position: cursor.position + i * PAGE, limit: PAGE, calculateTotal: i === 0 }, `q${i}`]); + calls.push(['ContactCard/get', { accountId: jmap, '#ids': { resultOf: `q${i}`, name: 'ContactCard/query', path: '/ids' } }, `g${i}`]); + } + const response = await ctx.transport.request(calls, CONTACTS_USING); + let total: number | null = null; + let got = 0; + let short = false; + const cards: Card[] = []; + for (let i = 0; i < PAIRS_PER_REQUEST && !short; i += 1) { + const [, query] = responseFor(response, `q${i}`); + if (i === 0 && typeof query.total === 'number') total = query.total; + cards.push(...responseFor(response, `g${i}`)[1].list as Card[]); + got += (query.ids as string[]).length; + short = (query.ids as string[]).length < PAGE; + } + cursor.position += got; + const done = got === 0 || (total === null ? short : cursor.position >= total); + ctx.db.transaction(() => { + for (const card of cards) storeServerCard(ctx.db, jmap, card, cursor.gen); + if (done) { + sweep(ctx, jmap, cursor.gen); + ctx.setSyncState(jmap, 'ContactCard', { state: cursor.captured, phase: null, cursor: null }); + } else { + ctx.setSyncState(jmap, 'ContactCard', { cursor: { ...cursor } }); + } + }); + ctx.emit({ type: 'progress', account: ctx.account, area: 'contacts', phase: 'list', done: cursor.position, total }); + if (done) { + ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] }); + return true; + } + } +} + +async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: string | null): Promise<'done' | 'resync' | 'stopped'> { + const outcome = await runChangesLoop({ + transport: ctx.transport, + type: 'ContactCard', + jmap, + using: CONTACTS_USING, + since, + target, + stopped: ctx.stopped, + apply: async (page) => { + const ids = [...page.created, ...page.updated]; + const fetched: Card[] = []; + for (let i = 0; i < ids.length; i += PAGE) { + const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + PAGE) }, 'g']], CONTACTS_USING); + fetched.push(...responseFor(response, 'g')[1].list as Card[]); + } + ctx.db.transaction(() => { + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM contact_card WHERE jmap = ?', [jmap])?.gen ?? 0); + for (const card of fetched) storeServerCard(ctx.db, jmap, card, gen); + for (const id of page.destroyed) serverDestroyed(ctx, jmap, id); + ctx.setSyncState(jmap, 'ContactCard', { state: page.newState }); + }); + if (ids.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] }); + }, + }); + if (outcome.outcome === 'resync') return 'resync'; + return outcome.outcome === 'stopped' ? 'stopped' : 'done'; +} + +async function syncCards(ctx: SyncContext, jmap: string): Promise { + const row = ctx.syncState(jmap, 'ContactCard'); + let cursor = row.phase === 'list' ? row.cursor as unknown as ListCursor : null; + if (!cursor && !row.state) cursor = await startFullList(ctx, jmap); + if (!cursor && row.state) { + const outcome = await syncDelta(ctx, jmap, row.state, row.target); + if (outcome === 'stopped') return; + if (outcome === 'resync') cursor = await startFullList(ctx, jmap); + } + if (cursor) { + if (!(await runFullList(ctx, jmap, cursor))) return; + // Catch what changed while paging. + const after = ctx.syncState(jmap, 'ContactCard'); + if (after.state) await syncDelta(ctx, jmap, after.state, after.target); + } +} + +export const contactsSyncer: Syncer = { + name: 'contacts', + capability: CONTACTS_CAPABILITY, + async run(ctx, jmap) { + await flushPending(ctx, jmap); + if (ctx.stopped()) return; + await syncBooks(ctx, jmap); + if (ctx.stopped()) return; + await syncCards(ctx, jmap); + }, +}; diff --git a/lib/store/engine/account-engine.ts b/lib/store/engine/account-engine.ts index 6281afc71..a3d56298e 100644 --- a/lib/store/engine/account-engine.ts +++ b/lib/store/engine/account-engine.ts @@ -196,6 +196,7 @@ export class AccountEngine { transport: this.transport, now: this.now, emit: this.emitter, + requestSync: () => this.requestSync(), stopped: () => this.closed || !this.online, syncState: (jmap, type) => this.syncState(jmap, type), setSyncState: (jmap, type, row) => this.setSyncState(jmap, type, row), @@ -210,6 +211,7 @@ export class AccountEngine { scope: this.scope, now: this.now, emit: this.emitter, + requestSync: () => this.requestSync(), }; } From 7bb8447743db33c21b59d6cda97806a2a2930bbb Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:47:00 +0100 Subject: [PATCH 2/7] fix: keep the contact list when the address book can't be read --- lib/jmap/client.ts | 27 ++++- lib/store/__tests__/client-contacts.test.ts | 113 ++++++++++++++++++++ stores/__tests__/contact-store.test.ts | 34 +++++- stores/contact-store.ts | 5 +- 4 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 lib/store/__tests__/client-contacts.test.ts diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index 6d35dcdfa..56258dd86 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -7327,7 +7327,10 @@ export class JMAPClient implements IJMAPClient { } } + /** Throws when no account could answer, so a failure doesn't read as having no address books. */ async getAllAddressBooks(): Promise { + let answered = 0; + let failure: unknown; try { const allBooks: AddressBook[] = []; const primaryId = this.getContactsAccountId(); @@ -7353,17 +7356,23 @@ export class JMAPClient implements IJMAPClient { isShared: !isPrimary, })); allBooks.push(...books); + answered += 1; + } else if (response.methodResponses?.[0]?.[0] === "error") { + const refused = response.methodResponses[0][1]; + throw new Error(refused?.description || refused?.type || "AddressBook/get failed"); } } catch (error) { console.error(`Failed to fetch address books for account ${accountId}:`, error); + failure ??= error; } } - return allBooks; + if (answered > 0 || !failure) return allBooks; } catch (error) { console.error('Failed to fetch all address books:', error); return this.getAddressBooks(); } + throw failure; } async createAddressBook(name: string): Promise { @@ -8230,6 +8239,8 @@ export class JMAPClient implements IJMAPClient { ], this.contactUsing()); const queryResult = response.methodResponses?.[0]; + // A refused call isn't an empty address book: the caller keeps what it has. + if (queryResult?.[0] === "error") throw new Error(queryResult[1]?.description || queryResult[1]?.type || "ContactCard/query failed"); if (queryResult?.[0] !== "ContactCard/query") break; const ids: string[] = queryResult[1].ids || []; @@ -8252,8 +8263,10 @@ export class JMAPClient implements IJMAPClient { ["ContactCard/get", { accountId, ids: chunk }, "g"], ], this.contactUsing()); - if (response.methodResponses?.[0]?.[0] === "ContactCard/get") { - const list = (response.methodResponses[0][1].list || []) as ContactCard[]; + const getResult = response.methodResponses?.[0]; + if (getResult?.[0] === "error") throw new Error(getResult[1]?.description || getResult[1]?.type || "ContactCard/get failed"); + if (getResult?.[0] === "ContactCard/get") { + const list = (getResult[1].list || []) as ContactCard[]; allContacts.push(...list.map(contactFromWire)); } } @@ -8273,7 +8286,10 @@ export class JMAPClient implements IJMAPClient { } } + /** Throws when no account could answer, so a failure doesn't read as an empty address book. */ async getAllContacts(): Promise { + let answered = 0; + let failure: unknown; try { const allContacts: ContactCard[] = []; const primaryId = this.getContactsAccountId(); @@ -8297,16 +8313,19 @@ export class JMAPClient implements IJMAPClient { isShared: !isPrimary, })); allContacts.push(...contacts); + answered += 1; } catch (error) { console.error(`Failed to fetch contacts for account ${accountId}:`, error); + failure ??= error; } } - return allContacts; + if (answered > 0 || !failure) return allContacts; } catch (error) { console.error('Failed to fetch all contacts:', error); return this.getContacts(); } + throw failure; } async getContact(contactId: string, accountId?: string): Promise { diff --git a/lib/store/__tests__/client-contacts.test.ts b/lib/store/__tests__/client-contacts.test.ts new file mode 100644 index 000000000..87d516a34 --- /dev/null +++ b/lib/store/__tests__/client-contacts.test.ts @@ -0,0 +1,113 @@ +// @vitest-environment node +/** + * The real JMAPClient reading the store's answers: what the client's own code + * makes of them is the contract, not the store's idea of the shape. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { JMAPClient } from '@/lib/jmap/client'; +import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks'; +import { DEFAULT_SCOPE } from '../area'; +import { contactsArea } from '../contacts'; +import { AccountEngine } from '../engine/account-engine'; +import { JMAP_ACCOUNT } from './helpers/fake-jmap'; +import { CONTACTS_SESSION, FakeContacts, SHARED_ACCOUNT } from './helpers/fake-contacts'; +import { openMemoryStore } from './helpers/node-adapter'; + +afterEach(() => setOfflineHooks(null)); + +async function offlineClient(server: FakeContacts): Promise<{ client: JMAPClient; engine: AccountEngine }> { + const engine = new AccountEngine({ + account: 'a@fake.test', + db: openMemoryStore(), + session: CONTACTS_SESSION, + transport: server, + areas: [contactsArea], + scope: DEFAULT_SCOPE, + sleep: async () => {}, + }); + await engine.requestSync(); + engine.setOnline(false); + const hooks: Partial = { + skipNetwork: () => true, + answer: (_client, calls) => engine.answer(calls), + record: () => {}, + noteNetwork: () => {}, + }; + setOfflineHooks(hooks as OfflineHooks); + const client = JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', { + ...CONTACTS_SESSION, + capabilities: { ...CONTACTS_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 2 } }, + }); + return { client, engine }; +} + +function seeded(): FakeContacts { + const server = new FakeContacts(); + for (const full of ['Anna Bauer', 'Bernd Graf', 'Carla Müller']) server.addCard({ name: { full } }); + server.addBook({ id: 'teambook', name: 'Team' }, SHARED_ACCOUNT); + server.addCard({ id: 'c1', name: { full: 'Dora Team' }, addressBookIds: { teambook: true } }, SHARED_ACCOUNT); + return server; +} + +describe('the client over the store', () => { + it('reports a store that has nothing yet as a failure, not as no contacts', async () => { + const engine = new AccountEngine({ + account: 'a@fake.test', db: openMemoryStore(), session: CONTACTS_SESSION, transport: new FakeContacts(), + areas: [contactsArea], scope: DEFAULT_SCOPE, sleep: async () => {}, + }); + // Answered per call, with one more call the store can answer: the error isn't hidden by a null answer. + const perCall: Partial = { + skipNetwork: () => true, + answer: async (_client, calls) => (await engine.answer(calls)) ?? { + methodResponses: calls.map(([, , id]) => ['error', { type: 'serverUnavailable' }, id]), + }, + record: () => {}, + noteNetwork: () => {}, + }; + setOfflineHooks(perCall as OfflineHooks); + const client = JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', CONTACTS_SESSION); + await expect(client.getAllContacts()).rejects.toThrow(); + await expect(client.getAllAddressBooks()).rejects.toThrow(); + }); + + it('lists all contacts and books across accounts, paging in its own batch size', async () => { + const { client } = await offlineClient(seeded()); + const contacts = await client.getAllContacts(); + expect(contacts.map((c) => c.name?.full).sort()).toEqual(['Anna Bauer', 'Bernd Graf', 'Carla Müller', 'Dora Team']); + const shared = contacts.find((c) => c.name?.full === 'Dora Team'); + expect(shared).toMatchObject({ id: `${SHARED_ACCOUNT}:c1`, originalId: 'c1', isShared: true }); + const books = await client.getAllAddressBooks(); + expect(books.map((b) => b.id).sort()).toEqual(['book1', `${SHARED_ACCOUNT}:teambook`]); + }); + + it('searches with its own filter', async () => { + const { client } = await offlineClient(seeded()); + expect((await client.searchContacts('müll')).map((c) => c.name?.full)).toEqual(['Carla Müller']); + expect((await client.searchContacts('team')).map((c) => c.originalId)).toEqual(['c1']); + }); + + it('creates, edits and deletes through its own methods with the stand-in id', async () => { + const { client, engine } = await offlineClient(seeded()); + const created = await client.createContact({ name: { full: 'Neue Person' }, emails: { e: { address: 'neu@example.org' } } }); + expect(created.id).toBeTruthy(); + expect(created.addressBookIds).toEqual({ book1: true }); + await client.updateContact(created.id, { name: { full: 'Neue Person B' } }); + expect((await client.getContact(created.id))?.name?.full).toBe('Neue Person B'); + expect((await client.searchContacts('person b')).map((c) => c.id)).toEqual([created.id]); + await client.deleteContact(created.id); + expect(await client.getContact(created.id)).toBeNull(); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + }); + + it('throws the server-style error for an edit to a card it does not have', async () => { + const { client } = await offlineClient(seeded()); + await expect(client.updateContact('nope', { name: { full: 'X' } })).rejects.toThrow(); + await expect(client.deleteContact('nope')).rejects.toThrow(); + }); + + it('reads an account the store has as the primary contacts account', async () => { + const { client } = await offlineClient(seeded()); + expect(client.getContactsAccountId()).toBe(JMAP_ACCOUNT); + expect((await client.getContacts('book1')).length).toBe(3); + }); +}); diff --git a/stores/__tests__/contact-store.test.ts b/stores/__tests__/contact-store.test.ts index 1a6fb072d..f9dfabcca 100644 --- a/stores/__tests__/contact-store.test.ts +++ b/stores/__tests__/contact-store.test.ts @@ -9,7 +9,7 @@ import { normalizeContactPhotoUri, TRUSTED_SENDERS_BOOK_NAME, } from '../contact-store'; -import type { ContactCard, ContactMedia } from '@/lib/jmap/types'; +import type { AddressBook, ContactCard, ContactMedia } from '@/lib/jmap/types'; import type { IJMAPClient } from '@/lib/jmap/client-interface'; import { clearContactPhotoCache } from '@/lib/contact-photo-cache'; import { PrincipalsRequestError } from '@/lib/jmap/principals-error'; @@ -1364,6 +1364,38 @@ describe('contact-store', () => { }); }); + describe('an account that cannot answer', () => { + it('keeps its contacts and books while the other accounts refresh', async () => { + const failing = { + getAllContacts: vi.fn().mockRejectedValue(new Error('offline')), + getAllAddressBooks: vi.fn().mockRejectedValue(new Error('offline')), + } as unknown as IJMAPClient; + const working = { + getAllContacts: vi.fn().mockResolvedValue([makeContact({ id: 'b1' })]), + getAllAddressBooks: vi.fn().mockResolvedValue([{ id: 'bookb', name: 'B' }]), + } as unknown as IJMAPClient; + const kept = makeContact({ id: 'acc-a::a1', localAccountId: 'acc-a' }); + const keptBook = { id: 'acc-a::booka', name: 'A', localAccountId: 'acc-a' } as AddressBook; + useContactStore.setState({ contacts: [kept], addressBooks: [keptBook] }); + const accounts = [{ client: failing, localAccountId: 'acc-a' }, { client: working, localAccountId: 'acc-b' }]; + + await useContactStore.getState().fetchAllAccountsContacts(accounts); + await useContactStore.getState().fetchAllAccountsAddressBooks(accounts); + + const state = useContactStore.getState(); + expect(state.contacts.map((c) => c.id)).toEqual(['acc-a::a1', 'acc-b::b1']); + expect(state.addressBooks.map((b) => b.id)).toEqual(['acc-a::booka', 'acc-b::bookb']); + }); + + it('keeps the list when the only account fails', async () => { + const failing = { getAllContacts: vi.fn().mockRejectedValue(new Error('offline')) } as unknown as IJMAPClient; + useContactStore.setState({ contacts: [makeContact()] }); + await useContactStore.getState().fetchContacts(failing); + expect(useContactStore.getState().contacts).toHaveLength(1); + expect(useContactStore.getState().error).toBe('Failed to fetch contacts'); + }); + }); + describe('persistence/partialize', () => { it('should persist contacts when supportsSync is false', () => { const { partialize } = (useContactStore as unknown as { persist: { getOptions: () => { partialize: (state: Record) => Record } } }).persist.getOptions(); diff --git a/stores/contact-store.ts b/stores/contact-store.ts index cd10a91f7..3403088b2 100644 --- a/stores/contact-store.ts +++ b/stores/contact-store.ts @@ -671,7 +671,8 @@ export const useContactStore = create()( return prefixContactsWithLocalAccount(list, localAccountId); } catch (error) { debug.error(`Failed to fetch contacts for account ${localAccountId}:`, error); - return [] as ContactCard[]; + // Keep what this account's cards were: it just couldn't answer. + return get().contacts.filter((contact) => contact.localAccountId === localAccountId); } }), ); @@ -694,7 +695,7 @@ export const useContactStore = create()( return prefixAddressBooksWithLocalAccount(list, localAccountId); } catch (error) { debug.error(`Failed to fetch address books for account ${localAccountId}:`, error); - return [] as AddressBook[]; + return get().addressBooks.filter((book) => book.localAccountId === localAccountId); } }), ); From 698510fa08c18e5c15b741a09db6f522c8c316dd Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:47:39 +0100 Subject: [PATCH 3/7] feat: list files offline and keep pinned and recently opened files on the device --- lib/store/__tests__/client-files.test.ts | 88 ++++ lib/store/__tests__/files.test.ts | 535 ++++++++++++++++++++++ lib/store/__tests__/helpers/fake-files.ts | 192 ++++++++ lib/store/areas.ts | 3 +- lib/store/files/index.ts | 27 ++ lib/store/files/keep.ts | 295 ++++++++++++ lib/store/files/methods.ts | 108 +++++ lib/store/files/records.ts | 50 ++ lib/store/files/schema.ts | 47 ++ lib/store/files/sync.ts | 160 +++++++ 10 files changed, 1504 insertions(+), 1 deletion(-) create mode 100644 lib/store/__tests__/client-files.test.ts create mode 100644 lib/store/__tests__/files.test.ts create mode 100644 lib/store/__tests__/helpers/fake-files.ts create mode 100644 lib/store/files/index.ts create mode 100644 lib/store/files/keep.ts create mode 100644 lib/store/files/methods.ts create mode 100644 lib/store/files/records.ts create mode 100644 lib/store/files/schema.ts create mode 100644 lib/store/files/sync.ts diff --git a/lib/store/__tests__/client-files.test.ts b/lib/store/__tests__/client-files.test.ts new file mode 100644 index 000000000..08efd48d4 --- /dev/null +++ b/lib/store/__tests__/client-files.test.ts @@ -0,0 +1,88 @@ +// @vitest-environment node +/** + * The real JMAPClient listing files from the store, in each way a server says + * what a node is, plus what it does with a write. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { JMAPClient } from '@/lib/jmap/client'; +import { fileNodeKind } from '@/lib/jmap/filenode-kind'; +import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks'; +import { DEFAULT_SCOPE } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { filesArea } from '../files'; +import { FakeFiles, FILES, FILES_SESSION, SHARED_FILES_ACCOUNT, type FakeFilesOptions } from './helpers/fake-files'; +import { openMemoryStore } from './helpers/node-adapter'; + +afterEach(() => setOfflineHooks(null)); + +async function offlineClient(server: FakeFiles): Promise { + const engine = new AccountEngine({ + account: 'a@fake.test', + db: openMemoryStore(), + session: FILES_SESSION, + transport: server, + areas: [filesArea], + scope: DEFAULT_SCOPE, + sleep: async () => {}, + }); + await engine.requestSync(); + engine.setOnline(false); + setOfflineHooks({ + skipNetwork: () => true, + answer: (_client, calls) => engine.answer(calls), + record: () => {}, + noteNetwork: () => {}, + } as Partial as OfflineHooks); + return JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', { + ...FILES_SESSION, + capabilities: { ...FILES_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500 } }, + }); +} + +describe('file listing offline', () => { + it.each<[string, FakeFilesOptions]>([ + ['nodeType (0.16.6 and later, 1.0 as tested)', { style: 'nodeType' }], + ['blobId null only (0.16.0-5)', { style: 'blob' }], + ['@type and mediaType (1.0.0)', { style: 'draft' }], + ])('tells folders from files: %s', async (_label, options) => { + const server = new FakeFiles(options); + const folder = server.addFolder('Docs'); + server.addFile('note.txt', folder.id); + const client = await offlineClient(server); + expect(client.supportsFiles()).toBe(true); + + const nodes = await client.listAllFileNodes(); + const byName = new Map(nodes.map((n) => [n.name, n])); + expect(fileNodeKind(byName.get('Docs') as never)).toBe('directory'); + expect(fileNodeKind(byName.get('note.txt') as never)).toBe('file'); + expect(byName.get('note.txt')).toMatchObject({ parentId: folder.id, type: 'text/plain', size: 10 }); + expect((await client.listFileNodes(folder.id)).map((n) => n.name)).toEqual(['note.txt']); + expect((await client.listFileNodes(null)).map((n) => n.name)).toEqual(['Docs']); + }); + + it('lists an account larger than one /get, ending its paging cleanly', async () => { + const server = new FakeFiles({ getAll: 'refuse' }); + const folder = server.addFolder('Docs'); + for (let i = 0; i < 700; i += 1) server.addFile(`f${i}`, folder.id); + const client = await offlineClient(server); + expect(await client.listAllFileNodes()).toHaveLength(701); + }); + + it('lists shared accounts with namespaced ids', async () => { + const server = new FakeFiles(); + server.addFolder('Mine'); + const shared = server.addFolder('Theirs', null, undefined, SHARED_FILES_ACCOUNT); + const client = await offlineClient(server); + const nodes = await client.listAllFileNodesAcrossAccounts(); + expect(nodes.find((n) => n.name === 'Theirs')).toMatchObject({ id: `${SHARED_FILES_ACCOUNT}:${shared.id}`, isShared: true }); + expect(nodes.find((n) => n.name === 'Mine')).toMatchObject({ isShared: false }); + }); + + it('fails a write with an error rather than losing it', async () => { + const server = new FakeFiles(); + const client = await offlineClient(server); + await expect(client.createFileDirectory('New', null)).rejects.toThrow(); + expect(FILES in FILES_SESSION.capabilities).toBe(true); + expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false); + }); +}); diff --git a/lib/store/__tests__/files.test.ts b/lib/store/__tests__/files.test.ts new file mode 100644 index 000000000..acf81b483 --- /dev/null +++ b/lib/store/__tests__/files.test.ts @@ -0,0 +1,535 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { DEFAULT_SCOPE, type StoreEvent, type StoreScope } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError } from '../engine/transport'; +import { fileBlob, FILE_CACHE_CAP_BYTES, filesArea } from '../files'; +import type { StorageAdapter } from '../storage'; +import { JMAP_ACCOUNT } from './helpers/fake-jmap'; +import { FakeFiles, FILES_SESSION, SHARED_FILES_ACCOUNT } from './helpers/fake-files'; +import { openMemoryStore } from './helpers/node-adapter'; + +const NOW = Date.UTC(2026, 10, 3); + +/* eslint-disable @typescript-eslint/no-explicit-any -- answers are shaped like the server's */ + +interface Rig { + db: StorageAdapter; + engine: AccountEngine; + clock: { at: number }; + events: StoreEvent[]; +} + +function rigFor(server: FakeFiles, scope: Partial = {}): Rig { + const db = openMemoryStore(); + const clock = { at: NOW }; + const events: StoreEvent[] = []; + const engine = new AccountEngine({ + account: 'a@fake.test', + db, + session: FILES_SESSION, + transport: server, + areas: [filesArea], + scope: { ...DEFAULT_SCOPE, ...scope }, + emit: (event) => events.push(event), + now: () => clock.at, + sleep: async () => {}, + }); + return { db, engine, clock, events }; +} + +async function synced(server: FakeFiles, scope: Partial = {}): Promise { + const rig = rigFor(server, scope); + await rig.engine.requestSync(); + return rig; +} + +async function call(engine: AccountEngine, name: string, args: Record = {}): Promise { + const response = await engine.answer([[name, { accountId: JMAP_ACCOUNT, ...args }, '0']]); + if (!response) throw new Error('nothing answered'); + const [answered, result] = response.methodResponses[0]; + if (answered === 'error') throw new Error(result.type); + return result; +} + +const ref = (id: string, jmap = JMAP_ACCOUNT) => ({ jmap, id }); + +function names(rows: Array<{ name?: unknown }>): unknown[] { + return rows.map((row) => row.name).sort(); +} + +describe('file metadata sync', () => { + it('stores small accounts from one FileNode/get', async () => { + const server = new FakeFiles(); + const folder = server.addFolder('Docs'); + server.addFile('a.txt', folder.id); + const { engine } = await synced(server); + expect(server.calls.filter(([n]) => n === 'FileNode/query')).toHaveLength(0); + expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['Docs', 'a.txt']); + }); + + it.each([ + ['0.16 cuts the list at 500', { getAll: 'cut' as const }], + ['1.0 refuses a list over 500', { getAll: 'refuse' as const }], + ])('pages past the /get limit: %s', async (_label, options) => { + const server = new FakeFiles(options); + const folder = server.addFolder('Docs'); + for (let i = 0; i < 1234; i += 1) server.addFile(`f${i}.txt`, folder.id); + const { db } = await synced(server); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(1235); + for (const [name, args] of server.calls) { + if (name === 'FileNode/get' && Array.isArray(args.ids)) expect(args.ids.length).toBeLessThanOrEqual(500); + } + }); + + it('finds folders the query leaves out on old servers, by the parents of what it did find', async () => { + const server = new FakeFiles({ queryOmitsFolders: true, style: 'blob' }); + for (let i = 0; i < 600; i += 1) server.addFile(`early${i}.txt`); + const top = server.addFolder('Top'); + const inner = server.addFolder('Inner', top.id); + server.addFile('deep.txt', inner.id); + const { db } = await synced(server); + const kinds = db.all<{ name: string; kind: string }>("SELECT name, kind FROM file_node WHERE name IN ('Top', 'Inner', 'deep.txt') ORDER BY name"); + expect(kinds).toEqual([{ name: 'Inner', kind: 'directory' }, { name: 'Top', kind: 'directory' }, { name: 'deep.txt', kind: 'file' }]); + }); + + it('continues a stopped full sync from its cursor', async () => { + const server = new FakeFiles({ getAll: 'refuse' }); + for (let i = 0; i < 6000; i += 1) server.addFile(`f${i}`); + const rig = rigFor(server); + let gets = 0; + server.beforeRequest = () => { + gets += 1; + // State capture, the refused list, a query page of 5000 and two requests of gets: then the network goes. + if (gets === 6) server.offline = true; + }; + await rig.engine.requestSync(); + expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(5000); + server.offline = false; + server.beforeRequest = null; + server.calls.length = 0; + await rig.engine.requestSync(); + expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(6000); + expect(server.calls.find(([n]) => n === 'FileNode/query')?.[1].position).toBe(5000); + expect(server.calls.filter(([n, a]) => n === 'FileNode/get' && a.ids === null && a.accountId === JMAP_ACCOUNT)).toHaveLength(0); + }); + + it('follows changes through FileNode/changes', async () => { + const server = new FakeFiles(); + const a = server.addFile('a.txt'); + const b = server.addFile('b.txt'); + const { engine } = await synced(server); + server.calls.length = 0; + server.renameNode(a.id, 'renamed.txt'); + server.removeNode(b.id); + server.addFolder('New'); + await engine.requestSync(); + expect(server.calls.some(([n]) => n === 'FileNode/changes')).toBe(true); + expect(server.calls.some(([n, args]) => n === 'FileNode/get' && args.ids === null)).toBe(false); + expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['New', 'renamed.txt']); + }); + + it('starts over when the server can no longer give changes, and keeps a pin on a file still there', async () => { + const server = new FakeFiles(); + const gone = server.addFile('gone.txt'); + const kept = server.addFile('kept.txt'); + const { db, engine } = await synced(server); + await engine.areaCall('files.pin', ref(kept.id)); + await engine.requestSync(); + server.removeNode(gone.id); + server.account().coll.truncatedBelow = server.account().coll.log.length; + await engine.requestSync(); + expect(db.all('SELECT id FROM file_node').map((r) => r.id)).toEqual([kept.id]); + expect(db.get<{ pinned: number }>('SELECT pinned FROM file_node WHERE id = ?', [kept.id])?.pinned).toBe(1); + }); + + it('keeps accounts apart, even with the same ids', async () => { + const server = new FakeFiles(); + server.addFile('own.txt', null, 5, { id: 'same' }); + server.addFile('team.txt', null, 5, { id: 'same', blobId: 'team-blob' }, SHARED_FILES_ACCOUNT); + const { engine } = await synced(server); + expect(names((await call(engine, 'FileNode/get', { ids: ['same'] })).list)).toEqual(['own.txt']); + expect(names((await call(engine, 'FileNode/get', { ids: ['same'], accountId: SHARED_FILES_ACCOUNT })).list)).toEqual(['team.txt']); + }); +}); + +describe('file answers', () => { + it('says so instead of an empty list when nothing was read yet', async () => { + const { engine } = rigFor(new FakeFiles()); + expect(await engine.answer([['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '0']])).toBeNull(); + }); + + it('gives the last server state and only asked-for properties', async () => { + const server = new FakeFiles(); + const file = server.addFile('a.txt'); + const { engine } = await synced(server); + const got = await call(engine, 'FileNode/get', { ids: [file.id, 'nope'], properties: ['name', 'parentId'] }); + expect(got.state).toBe(server.account().coll.state()); + expect(got.list).toEqual([{ id: file.id, name: 'a.txt', parentId: null }]); + expect(got.notFound).toEqual(['nope']); + }); + + it('answers a query by parent, also at the root, which the real server refuses to take', async () => { + const server = new FakeFiles(); + const folder = server.addFolder('Docs'); + const inside = server.addFile('in.txt', folder.id); + const root = server.addFile('root.txt'); + const { engine } = await synced(server); + expect((await call(engine, 'FileNode/query', { filter: { parentId: folder.id } })).ids).toEqual([inside.id]); + expect((await call(engine, 'FileNode/query', { filter: { parentId: null } })).ids).toEqual([folder.id, root.id]); + }); + + it('pages and sorts a query', async () => { + const server = new FakeFiles(); + server.addFile('b', null, 30); + server.addFile('a', null, 10); + server.addFile('c', null, 20); + const { engine } = await synced(server); + const page = await call(engine, 'FileNode/query', { filter: {}, position: 1, limit: 1, calculateTotal: true }); + expect(page).toMatchObject({ position: 1, total: 3, ids: [expect.any(String)] }); + const bySize = await call(engine, 'FileNode/query', { filter: {}, sort: [{ property: 'size', isAscending: false }] }); + const list = (await call(engine, 'FileNode/get', { ids: bySize.ids })).list; + expect(list.map((n: { name: string }) => n.name)).toEqual(['b', 'c', 'a']); + }); + + it('refuses what it cannot read, for that call only', async () => { + const server = new FakeFiles(); + server.addFile('a'); + const { engine } = await synced(server); + const response = await engine.answer([ + ['FileNode/query', { accountId: JMAP_ACCOUNT, filter: { hasFoo: true } }, '0'], + ['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'], + ]); + expect(response?.methodResponses[0][1].type).toBe('unsupportedFilter'); + expect(response?.methodResponses[1][0]).toBe('FileNode/get'); + }); + + it('answers FileNode/set with a clear per-call error, not silence', async () => { + const server = new FakeFiles(); + server.addFile('a'); + const { engine } = await synced(server); + const response = await engine.answer([ + ['FileNode/set', { accountId: JMAP_ACCOUNT, create: { n0: { name: 'x' } } }, '0'], + ['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'], + ]); + expect(response?.methodResponses[0]).toEqual(['error', { type: 'serverUnavailable', description: 'FileNode/set needs the server' }, '0']); + expect(response?.methodResponses[1][0]).toBe('FileNode/get'); + expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false); + }); +}); + +describe('pinned files', () => { + it('fetches a pinned file and gives it back byte for byte, past the chunk size', async () => { + const server = new FakeFiles(); + const content = new Uint8Array(3 * 1024 * 1024 + 17).map((_, i) => i % 251); + const file = server.addFile('big.bin', null, content); + const { db, engine } = await synced(server); + expect(server.downloads).toEqual([]); + + expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: false }); + await engine.requestSync(); + expect(server.downloads).toEqual([{ jmap: JMAP_ACCOUNT, blobId: file.blobId }]); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(4); + + const read = await engine.areaCall('files.read', ref(file.id)) as any; + expect(read.name).toBe('big.bin'); + expect(read.type).toBe('text/plain'); + expect(Buffer.compare(read.data, content)).toBe(0); + expect(Buffer.compare(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)?.data as Uint8Array, content)).toBe(0); + expect(await engine.areaCall('files.blob', { jmap: JMAP_ACCOUNT, blobId: file.blobId })).toMatchObject({ size: content.length }); + // Already kept: not fetched again. + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: true }); + }); + + it('names a file with a %-encoded name from 0.16 WebDAV the way the client shows it', async () => { + const server = new FakeFiles(); + const file = server.addFile('My%20Report.pdf'); + const { engine } = await synced(server); + await engine.areaCall('files.pin', ref(file.id)); + await engine.requestSync(); + expect((await engine.areaCall('files.read', ref(file.id)) as any).name).toBe('My Report.pdf'); + expect((await engine.areaCall('files.kept', undefined) as any).files[0].name).toBe('My Report.pdf'); + // The stored node stays as the server sent it. The client decodes it once. + expect((await call(engine, 'FileNode/get', { ids: [file.id] })).list[0].name).toBe('My%20Report.pdf'); + }); + + it('gives nothing for a file not kept', async () => { + const server = new FakeFiles(); + const file = server.addFile('a.txt'); + const { db, engine } = await synced(server); + expect(await engine.areaCall('files.read', ref(file.id))).toBeNull(); + expect(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)).toBeNull(); + }); + + it('refuses to pin a folder or an unknown id', async () => { + const server = new FakeFiles(); + const folder = server.addFolder('Docs'); + const { engine } = await synced(server); + await expect(engine.areaCall('files.pin', ref(folder.id))).rejects.toThrow('Only files'); + await expect(engine.areaCall('files.pin', ref('nope'))).rejects.toThrow(); + }); + + it('downloads one at a time', async () => { + const server = new FakeFiles(); + const files = [server.addFile('a'), server.addFile('b'), server.addFile('c')]; + const { engine } = await synced(server); + for (const file of files) await engine.areaCall('files.pin', ref(file.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(3); + expect(server.mostAtOnce).toBe(1); + }); + + it('fetches a pinned file over the one-file limit, and skips an opened one', async () => { + const server = new FakeFiles(); + const pinned = server.addFile('pinned.bin', null, 100); + const opened = server.addFile('opened.bin', null, 100); + const { engine } = await synced(server, { maxAttachmentBytes: 50 }); + await engine.areaCall('files.pin', ref(pinned.id)); + await engine.areaCall('files.noteOpened', ref(opened.id)); + await engine.requestSync(); + expect(server.downloads.map((d) => d.blobId)).toEqual([pinned.blobId]); + const status = await engine.areaCall('files.kept', undefined) as any; + const row = (name: string) => status.files.find((f: { name: string }) => f.name === name); + expect(row('pinned.bin')).toMatchObject({ pinned: true, kept: true, error: null }); + expect(row('opened.bin')).toMatchObject({ pinned: false, kept: false, error: 'tooLarge' }); + }); + + it('does not take pins past the total limit, and says so', async () => { + const server = new FakeFiles(); + const a = server.addFile('a', null, 60); + const b = server.addFile('b', null, 60); + const { engine, clock } = await synced(server, { maxTotalBytes: 100 }); + await engine.areaCall('files.pin', ref(a.id)); + clock.at += 1; + await engine.areaCall('files.pin', ref(b.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + const status = await engine.areaCall('files.kept', undefined) as any; + expect(status.files.filter((f: { kept: boolean }) => f.kept)).toHaveLength(1); + expect(status.files.filter((f: { error: string | null }) => f.error === 'overLimit')).toHaveLength(1); + }); + + it('keeps nothing when the scope forbids it, and drops what was kept', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(file.id)); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + + rig.engine.setScope({ ...DEFAULT_SCOPE, attachments: false }); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).toBeNull(); + expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0); + expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ pinned: true, kept: false, error: 'notAllowed' }); + + rig.engine.setScope(DEFAULT_SCOPE); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + }); + + it('goes by the real size when the metadata has it wrong', async () => { + const server = new FakeFiles(); + const opened = server.addFile('opened', null, new Uint8Array(100), { size: 1 }); + const { engine } = await synced(server, { maxAttachmentBytes: 50 }); + await engine.areaCall('files.noteOpened', ref(opened.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + expect(await engine.areaCall('files.read', ref(opened.id))).toBeNull(); + expect((await engine.areaCall('files.kept', undefined) as any).files[0].error).toBe('tooLarge'); + }); + + it('does not store a file unpinned while it was downloading', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const { engine } = await synced(server); + await engine.areaCall('files.pin', ref(file.id)); + const original = server.download.bind(server); + server.download = async (jmap, blobId) => { + await engine.areaCall('files.unpin', ref(file.id)); + return original(jmap, blobId); + }; + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + expect(await engine.areaCall('files.read', ref(file.id))).toBeNull(); + }); + + it('lets a never-opened file go on unpin, and keeps an opened one as recent', async () => { + const server = new FakeFiles(); + const never = server.addFile('never'); + const opened = server.addFile('opened'); + const { engine } = await synced(server); + await engine.areaCall('files.pin', ref(never.id)); + await engine.areaCall('files.pin', ref(opened.id)); + await engine.areaCall('files.noteOpened', ref(opened.id)); + await engine.requestSync(); + await engine.areaCall('files.unpin', ref(never.id)); + await engine.areaCall('files.unpin', ref(opened.id)); + expect(await engine.areaCall('files.read', ref(never.id))).toBeNull(); + expect(await engine.areaCall('files.read', ref(opened.id))).not.toBeNull(); + const status = await engine.areaCall('files.kept', undefined) as any; + expect(status.files.map((f: { name: string; pinned: boolean }) => [f.name, f.pinned])).toEqual([['opened', false]]); + }); + + it('drops the old content and fetches the new one when a pinned file changes', async () => { + const server = new FakeFiles(); + const file = server.addFile('a.txt', null, new Uint8Array([1, 2, 3])); + const { db, engine } = await synced(server); + await engine.areaCall('files.pin', ref(file.id)); + await engine.requestSync(); + const old = file.blobId as string; + const fresh = server.replaceContent(file.id, new Uint8Array([9, 9])); + await engine.requestSync(); + expect(fileBlob({ db }, JMAP_ACCOUNT, old)).toBeNull(); + expect([...(fileBlob({ db }, JMAP_ACCOUNT, fresh)?.data ?? [])]).toEqual([9, 9]); + expect(Array.from((await engine.areaCall('files.read', ref(file.id)) as any).data)).toEqual([9, 9]); + }); + + it('drops the content when the file is removed on the server, unless another node shares it', async () => { + const server = new FakeFiles(); + const a = server.addFile('a', null, 5, { blobId: 'shared-blob' }); + const b = server.addFile('b', null, 5, { blobId: 'shared-blob' }); + const { db, engine } = await synced(server); + await engine.areaCall('files.pin', ref(a.id)); + await engine.areaCall('files.pin', ref(b.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + server.removeNode(a.id); + await engine.requestSync(); + expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).not.toBeNull(); + server.removeNode(b.id); + await engine.requestSync(); + expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).toBeNull(); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0); + }); + + it('carries on past one bad file, and hands a network failure to the engine', async () => { + const server = new FakeFiles(); + const bad = server.addFile('bad'); + const good = server.addFile('good'); + server.blobs.delete(bad.blobId as string); + const { engine, clock } = await synced(server); + await engine.areaCall('files.pin', ref(bad.id)); + clock.at += 1; + await engine.areaCall('files.pin', ref(good.id)); + await engine.requestSync(); + expect(await engine.areaCall('files.read', ref(good.id))).not.toBeNull(); + const status = await engine.areaCall('files.kept', undefined) as any; + expect(status.files.find((f: { name: string }) => f.name === 'bad').error).toBe('download'); + + }); + + it('hands a network failure to the engine, which ends the pass, and goes on next time', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(file.id)); + const original = server.download.bind(server); + server.download = async () => { + throw new TransportError('network', 'Failed to fetch'); + }; + const syncedBefore = rig.events.filter((e) => e.type === 'synced').length; + await rig.engine.requestSync(); + expect(rig.events.some((e) => e.type === 'error')).toBe(false); + expect(rig.events.filter((e) => e.type === 'synced').length).toBe(syncedBefore); + expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ kept: false, error: null }); + + server.download = original; + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + }); +}); + +describe('recently opened files', () => { + it('keeps an opened file without a pin', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const { engine } = await synced(server); + expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: false }); + await engine.requestSync(); + expect(await engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: true }); + }); + + it('pushes out the oldest unpinned first, and never a pinned one', async () => { + const server = new FakeFiles(); + const files = Object.fromEntries(['old', 'pinnedOldest', 'mid', 'new', 'newest'].map((name) => [name, server.addFile(name, null, 10)])); + const rig = await synced(server, { maxTotalBytes: 30 }); + const open = async (name: string) => { + rig.clock.at += 1000; + await rig.engine.areaCall('files.noteOpened', ref(files[name].id)); + await rig.engine.requestSync(); + }; + await open('pinnedOldest'); + await rig.engine.areaCall('files.pin', ref(files.pinnedOldest.id)); + await open('old'); + await open('mid'); + const kept = async () => (await rig.engine.areaCall('files.kept', undefined) as any).files + .filter((f: { kept: boolean }) => f.kept).map((f: { name: string }) => f.name).sort(); + expect(await kept()).toEqual(['mid', 'old', 'pinnedOldest']); + + await open('new'); + expect(await kept()).toEqual(['mid', 'new', 'pinnedOldest']); + await open('mid'); + await open('newest'); + expect(await kept()).toEqual(['mid', 'newest', 'pinnedOldest']); + expect((await rig.engine.areaCall('files.kept', undefined) as any).bytes).toBe(30); + + // A pinned file was never pushed out and had to come back. + expect(server.downloads.filter((d) => d.blobId === files.pinnedOldest.blobId)).toHaveLength(1); + // Pushed out means forgotten: the next pass doesn't fetch it straight back. + expect(server.downloads.filter((d) => d.blobId === files.old.blobId)).toHaveLength(1); + const count = server.downloads.length; + await rig.engine.requestSync(); + expect(server.downloads).toHaveLength(count); + expect((await rig.engine.areaCall('files.kept', undefined) as any).files.map((f: { name: string }) => f.name)).not.toContain('old'); + }); + + it('pushes out recent files for a pin, and keeps a pin past the cap', async () => { + const server = new FakeFiles(); + const recent = server.addFile('recent', null, 20); + const pin = server.addFile('pin', null, 20); + const rig = await synced(server, { maxTotalBytes: 25 }); + await rig.engine.areaCall('files.noteOpened', ref(recent.id)); + await rig.engine.requestSync(); + rig.clock.at += 1000; + await rig.engine.areaCall('files.pin', ref(pin.id)); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(recent.id))).toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(pin.id))).not.toBeNull(); + }); + + it('keeps recently opened files under 500 MB whatever the total limit is', async () => { + const server = new FakeFiles(); + const file = server.addFile('new', null, 10); + const rig = await synced(server, { maxTotalBytes: 2 * FILE_CACHE_CAP_BYTES }); + // As if an older file already took all of it: only the row counts here. + rig.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [JMAP_ACCOUNT, 'huge', 'x', FILE_CACHE_CAP_BYTES, NOW - 1000]); + rig.db.run("INSERT INTO file_node (jmap, id, name, kind, blob_id, size, json, opened_at) VALUES (?, 'hugenode', 'huge', 'file', 'huge', ?, '{}', ?)", [JMAP_ACCOUNT, FILE_CACHE_CAP_BYTES, NOW - 1000]); + await rig.engine.areaCall('files.noteOpened', ref(file.id)); + await rig.engine.requestSync(); + expect(fileBlob({ db: rig.db }, JMAP_ACCOUNT, 'huge')).toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + expect((await rig.engine.areaCall('files.kept', undefined) as any).capBytes).toBe(FILE_CACHE_CAP_BYTES); + }); + + it('drops an old opened file that does not fit rather than a newer one', async () => { + const server = new FakeFiles(); + const a = server.addFile('a', null, 20); + const b = server.addFile('b', null, 20); + const rig = await synced(server, { maxTotalBytes: 25 }); + await rig.engine.areaCall('files.noteOpened', ref(a.id)); + rig.clock.at += 1000; + await rig.engine.areaCall('files.noteOpened', ref(b.id)); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(b.id))).not.toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(a.id))).toBeNull(); + expect(server.downloads.map((d) => d.blobId)).toEqual([b.blobId, a.blobId]); + const count = server.downloads.length; + await rig.engine.requestSync(); + expect(server.downloads).toHaveLength(count); + }); +}); +/* eslint-enable @typescript-eslint/no-explicit-any */ diff --git a/lib/store/__tests__/helpers/fake-files.ts b/lib/store/__tests__/helpers/fake-files.ts new file mode 100644 index 000000000..0ff22f4ed --- /dev/null +++ b/lib/store/__tests__/helpers/fake-files.ts @@ -0,0 +1,192 @@ +/** + * FakeJmap with files, as measured on both Stalwarts (2026-11-03) and as the + * client's comments record: `filter: {parentId: null}` is an HTTP 400 that + * fails the whole request, a folder has null blobId, type and size. What differs + * between versions is set per server: + * - `style` is how a node says what it is: `nodeType` (0.16.6 and later, 1.0 as + * tested), `blob` (0.16.0-5: nothing but blobId null), `draft` (1.0.0: `@type` + * and `mediaType`). + * - `getAll`: ids null past 500 is silently cut on 0.16 and refused on 1.0. + * - `queryOmitsFolders`: FileNode/query before 0.16.6. + */ + +import { TransportError, type StoreSession } from '../../engine/transport'; +import { changesFor } from './fake-contacts'; +import { Collection, FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './fake-jmap'; + +export const FILES = 'urn:ietf:params:jmap:filenode'; +export const SHARED_FILES_ACCOUNT = 'shared1'; + +export const FILES_SESSION: StoreSession = { + ...FAKE_SESSION, + primaryAccounts: { ...FAKE_SESSION.primaryAccounts, [FILES]: JMAP_ACCOUNT }, + accounts: { + [JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [FILES]: {} } }, + [SHARED_FILES_ACCOUNT]: { name: 'team@example.org', isPersonal: false, accountCapabilities: { [FILES]: {} } }, + }, + capabilities: { [FILES]: {} }, +}; + +export interface FakeNode { + id: string; + parentId: string | null; + name: string; + blobId: string | null; + type: string | null; + size: number | null; + modified?: string; +} + +export type FolderStyle = 'nodeType' | 'blob' | 'draft'; + +export interface FakeFilesOptions { + style?: FolderStyle; + getAll?: 'cut' | 'refuse'; + queryOmitsFolders?: boolean; +} + +class FilesAccount { + readonly nodes = new Map(); + readonly coll = new Collection(); +} + +export class FakeFiles extends FakeJmap { + private readonly fileAccounts = new Map(); + private readonly options: Required; + private nextNode = 1; + readonly blobs = new Map(); + /** In order, with the account. */ + readonly downloads: Array<{ jmap: string; blobId: string }> = []; + downloading = 0; + mostAtOnce = 0; + + constructor(options: FakeFilesOptions = {}) { + super(); + this.options = { style: 'nodeType', getAll: 'cut', queryOmitsFolders: false, ...options }; + } + + account(id: string = JMAP_ACCOUNT): FilesAccount { + let found = this.fileAccounts.get(id); + if (!found) { + found = new FilesAccount(); + this.fileAccounts.set(id, found); + } + return found; + } + + addFolder(name: string, parentId: string | null = null, id?: string, accountId: string = JMAP_ACCOUNT): FakeNode { + const account = this.account(accountId); + const node: FakeNode = { id: id ?? `n${this.nextNode++}`, parentId, name, blobId: null, type: null, size: null }; + account.nodes.set(node.id, node); + account.coll.push('created', node.id); + return node; + } + + addFile(name: string, parentId: string | null = null, content: Uint8Array | number = 10, extra: Partial = {}, accountId: string = JMAP_ACCOUNT): FakeNode { + const account = this.account(accountId); + const id = extra.id ?? `n${this.nextNode++}`; + const blobId = extra.blobId ?? `blob-${id}`; + const data = typeof content === 'number' ? new Uint8Array(content).fill(id.length) : content; + this.blobs.set(blobId, data); + const node: FakeNode = { id, parentId, name, blobId, type: 'text/plain', size: data.length, ...extra }; + account.nodes.set(id, node); + account.coll.push('created', id); + return node; + } + + /** A new version of the file: new blob, same node. */ + replaceContent(id: string, content: Uint8Array, accountId: string = JMAP_ACCOUNT): string { + const account = this.account(accountId); + const node = account.nodes.get(id) as FakeNode; + const blobId = `${node.blobId as string}-v`; + this.blobs.set(blobId, content); + node.blobId = blobId; + node.size = content.length; + account.coll.push('updated', id); + return blobId; + } + + renameNode(id: string, name: string, accountId: string = JMAP_ACCOUNT): void { + const account = this.account(accountId); + (account.nodes.get(id) as FakeNode).name = name; + account.coll.push('updated', id); + } + + removeNode(id: string, accountId: string = JMAP_ACCOUNT): void { + const account = this.account(accountId); + account.nodes.delete(id); + account.coll.push('destroyed', id); + } + + private wire(node: FakeNode, properties: string[] | undefined): Record { + const isFolder = node.blobId === null; + const full: Record = { id: node.id, parentId: node.parentId, name: node.name, blobId: node.blobId, size: node.size, created: '2026-10-01T00:00:00Z', modified: node.modified ?? '2026-10-01T00:00:00Z' }; + if (this.options.style === 'draft') { + full['@type'] = isFolder ? 'Directory' : 'File'; + full.mediaType = node.type; + } else { + full.type = node.type; + if (this.options.style === 'nodeType') full.nodeType = isFolder ? 'directory' : 'file'; + } + if (!properties) return full; + const out: Record = { id: node.id }; + for (const p of properties) if (p in full) out[p] = full[p]; + return out; + } + + protected override handle(name: string, args: Record): Record { + if (!name.startsWith('FileNode/')) return super.handle(name, args); + const account = this.account(String(args.accountId)); + switch (name) { + case 'FileNode/changes': + return changesFor(account.coll, args); + case 'FileNode/get': { + const ids = args.ids as string[] | null; + if (ids && ids.length > 500) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' }); + if (!ids && account.nodes.size > 500 && this.options.getAll === 'refuse') { + throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' }); + } + const chosen = ids ?? [...account.nodes.keys()].slice(0, 500); + const properties = args.properties as string[] | undefined; + return { + accountId: args.accountId, + state: account.coll.state(), + list: chosen.filter((id) => account.nodes.has(id)).map((id) => this.wire(account.nodes.get(id) as FakeNode, properties)), + notFound: chosen.filter((id) => !account.nodes.has(id)), + }; + } + case 'FileNode/query': { + const filter = (args.filter ?? {}) as { parentId?: string | null }; + // Measured on both servers: this fails the whole request. + if (filter.parentId === null) throw new TransportError('http', 'JMAP request failed: 400', 400, 'urn:ietf:params:jmap:error:notRequest'); + let list = [...account.nodes.values()]; + if (this.options.queryOmitsFolders) list = list.filter((n) => n.blobId !== null); + if (filter.parentId) list = list.filter((n) => n.parentId === filter.parentId); + const position = Number(args.position ?? 0); + const limit = Math.min(Number(args.limit ?? 5000) || 5000, 5000); + return { + accountId: args.accountId, + queryState: account.coll.state(), + canCalculateChanges: true, + position, + ids: list.slice(position, position + limit).map((n) => n.id), + ...(args.calculateTotal ? { total: list.length } : {}), + }; + } + default: + throw Object.assign(new Error(name), { jmapError: 'unknownMethod' }); + } + } + + override async download(jmap: string, blobId: string): Promise { + if (this.offline) throw new TransportError('network', 'Failed to fetch'); + this.downloads.push({ jmap, blobId }); + this.downloading += 1; + this.mostAtOnce = Math.max(this.mostAtOnce, this.downloading); + await Promise.resolve(); + this.downloading -= 1; + const data = this.blobs.get(blobId); + if (!data) throw new TransportError('http', 'Download failed: 404', 404); + return data; + } +} diff --git a/lib/store/areas.ts b/lib/store/areas.ts index c8882f3ee..7fb966931 100644 --- a/lib/store/areas.ts +++ b/lib/store/areas.ts @@ -2,6 +2,7 @@ import type { StoreArea } from './area'; import { contactsArea } from './contacts'; +import { filesArea } from './files'; import { mailArea } from './mail'; -export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea]; +export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea, filesArea]; diff --git a/lib/store/files/index.ts b/lib/store/files/index.ts new file mode 100644 index 000000000..0957ced89 --- /dev/null +++ b/lib/store/files/index.ts @@ -0,0 +1,27 @@ +import type { StoreArea } from '../area'; +import { fileBlob, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep'; +import { FILE_METHODS } from './methods'; +import { FILE_MIGRATIONS } from './schema'; +import { filesSyncer } from './sync'; + +export { fileBlob, FILE_CACHE_CAP_BYTES } from './keep'; + +export const filesArea: StoreArea = { + name: 'files', + migrations: FILE_MIGRATIONS, + methods: FILE_METHODS, + syncers: [filesSyncer], + calls: { + /** Keep this file's content on the device until it's unpinned. */ + 'files.pin': (ctx, params: NodeRef) => pinFile(ctx, params), + 'files.unpin': (ctx, params: NodeRef) => unpinFile(ctx, params), + /** The page opened this file: keep its content for a while. */ + 'files.noteOpened': (ctx, params: NodeRef) => noteOpened(ctx, params), + /** A kept file's content and name, or null. */ + 'files.read': (ctx, params: NodeRef) => readFile(ctx, params), + /** A kept content by blob id, or null. */ + 'files.blob': (ctx, params: { jmap: string; blobId: string }) => fileBlob(ctx, params.jmap, params.blobId), + /** Pinned and recently opened files, for a settings page. */ + 'files.kept': (ctx) => keptStatus(ctx), + }, +}; diff --git a/lib/store/files/keep.ts b/lib/store/files/keep.ts new file mode 100644 index 000000000..a11b93455 --- /dev/null +++ b/lib/store/files/keep.ts @@ -0,0 +1,295 @@ +/** + * File contents kept on the device: pinned files, and recently opened ones + * within a cap. Metadata for every file is in file_node regardless. + * + * What gets kept: + * - A pinned file stays until it's unpinned or removed on the server. The cap + * doesn't push it out and the size limit for one file doesn't apply. The + * account's total limit (scope.maxTotalBytes, which a policy can lower) does. + * - A recently opened file is kept if it's within scope.maxAttachmentBytes, and + * pushed out oldest first when the cap is reached. Its opened_at goes with + * it, so it isn't fetched straight back. + * - With scope.attachments off, nothing is kept. + */ + +import { TransportError } from '../engine/transport'; +import { MethodError, type StoreContext, type SyncContext } from '../area'; +import type { StorageAdapter } from '../storage'; +import { nodeMediaType, type WireNode } from './records'; + +export const FILE_CACHE_CAP_BYTES = 500 * 1024 * 1024; +const CHUNK_BYTES = 1024 * 1024; + +export interface KeptFile { data: Uint8Array; type: string; size: number } + +export interface NodeRef { jmap: string; id: string } + +interface NodeRow { + jmap: string; + id: string; + name: string; + kind: string; + blob_id: string | null; + size: number | null; + json: string; + pinned: number; + opened_at: number | null; + keep_error: string | null; +} + +interface BlobRow { + jmap: string; + blob_id: string; + size: number; + kept_at: number; + pinned: number; + opened_at: number | null; +} + +function capBytes(ctx: Pick): number { + return Math.min(FILE_CACHE_CAP_BYTES, ctx.scope.maxTotalBytes); +} + +/** A kept file's content, for the client's blob lookup. */ +export function fileBlob(ctx: Pick, jmap: string, blobId: string): KeptFile | null { + const row = ctx.db.get<{ type: string; size: number }>('SELECT type, size FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + if (!row) return null; + const data = new Uint8Array(Number(row.size)); + let at = 0; + for (const chunk of ctx.db.all<{ data: Uint8Array }>('SELECT data FROM file_blob_chunk WHERE jmap = ? AND blob_id = ? ORDER BY seq', [jmap, blobId])) { + data.set(chunk.data, at); + at += chunk.data.length; + } + return { data, type: row.type, size: Number(row.size) }; +} + +function hasBlob(db: StorageAdapter, jmap: string, blobId: string): boolean { + return Boolean(db.get('SELECT 1 AS x FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId])); +} + +function dropBlob(db: StorageAdapter, jmap: string, blobId: string): void { + db.run('DELETE FROM file_blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + db.run('DELETE FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); +} + +function storeBlob(ctx: StoreContext, jmap: string, blobId: string, type: string, data: Uint8Array): void { + dropBlob(ctx.db, jmap, blobId); + ctx.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [jmap, blobId, type, data.length, ctx.now()]); + for (let seq = 0, at = 0; at < data.length; seq += 1, at += CHUNK_BYTES) { + ctx.db.run('INSERT INTO file_blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, data.subarray(at, at + CHUNK_BYTES)]); + } +} + +/** All kept contents in the store, with what the nodes pointing at each say about it. */ +function keptBlobs(db: StorageAdapter): BlobRow[] { + return db.all( + `SELECT b.jmap, b.blob_id, b.size, b.kept_at, COALESCE(MAX(n.pinned), 0) AS pinned, MAX(n.opened_at) AS opened_at + FROM file_blob b LEFT JOIN file_node n ON n.jmap = b.jmap AND n.blob_id = b.blob_id + GROUP BY b.jmap, b.blob_id`, + ); +} + +function ageOf(blob: BlobRow): number { + return Number(blob.opened_at ?? blob.kept_at); +} + +function pushOut(db: StorageAdapter, blob: BlobRow): void { + dropBlob(db, blob.jmap, blob.blob_id); + db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]); +} + +/** + * Pushes out unpinned contents, oldest first, so `size` more fits under the + * cap. A recently opened file only pushes out older ones and says false when + * that isn't enough. A pinned one always gets in. + */ +function makeRoom(ctx: StoreContext, size: number, incoming: { pinned: boolean; openedAt: number }): boolean { + const cap = capBytes(ctx); + const blobs = keptBlobs(ctx.db); + let used = blobs.reduce((sum, blob) => sum + Number(blob.size), 0); + const candidates = blobs + .filter((blob) => !blob.pinned && (incoming.pinned || ageOf(blob) < incoming.openedAt)) + .sort((a, b) => ageOf(a) - ageOf(b)); + const chosen: BlobRow[] = []; + for (const blob of candidates) { + if (used + size <= cap) break; + used -= Number(blob.size); + chosen.push(blob); + } + if (used + size > cap && !incoming.pinned) return false; + for (const blob of chosen) pushOut(ctx.db, blob); + return true; +} + +function pinnedBytes(db: StorageAdapter): number { + return keptBlobs(db).filter((blob) => blob.pinned).reduce((sum, blob) => sum + Number(blob.size), 0); +} + +function setError(db: StorageAdapter, jmap: string, blobId: string, error: string | null): void { + db.run('UPDATE file_node SET keep_error = ? WHERE jmap = ? AND blob_id = ?', [error, jmap, blobId]); +} + +/** Contents no node points at any more: the file was removed or has a new version. */ +export function dropOrphanBlobs(db: StorageAdapter): void { + for (const blob of keptBlobs(db)) { + const used = db.get('SELECT 1 AS x FROM file_node WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]); + if (!used) dropBlob(db, blob.jmap, blob.blob_id); + } +} + +interface Wanted { blob_id: string; id: string; name: string; size: number | null; json: string; pinned: number; opened_at: number | null } + +function wantedRow(db: StorageAdapter, jmap: string, blobId: string): Wanted | undefined { + return db.get( + `SELECT blob_id, id, name, size, json, pinned, opened_at FROM file_node + WHERE jmap = ? AND blob_id = ? AND (pinned = 1 OR opened_at IS NOT NULL) + ORDER BY pinned DESC, opened_at DESC LIMIT 1`, + [jmap, blobId], + ); +} + +/** Fetches what should be kept and isn't yet, one file at a time, pinned files first. */ +export async function fetchKeptFiles(ctx: SyncContext, jmap: string): Promise { + const { db } = ctx; + if (!ctx.scope.attachments) { + db.transaction(() => { + for (const blob of keptBlobs(db)) dropBlob(db, blob.jmap, blob.blob_id); + db.run("UPDATE file_node SET keep_error = 'notAllowed' WHERE jmap = ? AND (pinned = 1 OR opened_at IS NOT NULL)", [jmap]); + }); + return; + } + db.transaction(() => { + db.run("UPDATE file_node SET keep_error = NULL WHERE jmap = ? AND keep_error = 'notAllowed'", [jmap]); + // The limit may have gone down since last time. + makeRoom(ctx, 0, { pinned: true, openedAt: 0 }); + }); + const todo = db.all<{ blob_id: string }>( + `SELECT n.blob_id FROM file_node n + WHERE n.jmap = ? AND n.blob_id IS NOT NULL AND (n.pinned = 1 OR n.opened_at IS NOT NULL) + AND NOT EXISTS (SELECT 1 FROM file_blob b WHERE b.jmap = n.jmap AND b.blob_id = n.blob_id) + GROUP BY n.blob_id ORDER BY MAX(n.pinned) DESC, MAX(n.opened_at) DESC`, + [jmap], + ).map((row) => String(row.blob_id)); + let done = 0; + for (const blobId of todo) { + if (ctx.stopped()) return; + // The page may have changed its mind during the last download. + const want = wantedRow(db, jmap, blobId); + if (!want) continue; + const known = Number(want.size ?? 0); + if (!want.pinned && known > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) { + setError(db, jmap, blobId, 'tooLarge'); + continue; + } + if (want.pinned && pinnedBytes(db) + known > ctx.scope.maxTotalBytes) { + setError(db, jmap, blobId, 'overLimit'); + continue; + } + let data: Uint8Array; + const type = nodeMediaType(JSON.parse(want.json) as WireNode); + try { + data = await ctx.transport.download(jmap, blobId, type, want.name); + } catch (error) { + // No network or no credentials reach the engine. One bad file doesn't stop the others. + if (error instanceof TransportError && error.kind === 'http') { + setError(db, jmap, blobId, 'download'); + continue; + } + throw error; + } + db.transaction(() => { + const now = wantedRow(db, jmap, blobId); + if (!now) return; + if (now.pinned ? pinnedBytes(db) + data.length > ctx.scope.maxTotalBytes : data.length > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) { + // The size in the metadata was wrong or missing. With the real one + // noted, the next pass sees this without downloading it again. + db.run('UPDATE file_node SET size = ? WHERE jmap = ? AND blob_id = ?', [data.length, jmap, blobId]); + setError(db, jmap, blobId, now.pinned ? 'overLimit' : 'tooLarge'); + return; + } + if (!makeRoom(ctx, data.length, { pinned: Boolean(now.pinned), openedAt: Number(now.opened_at ?? 0) })) { + db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ? AND pinned = 0', [jmap, blobId]); + return; + } + storeBlob(ctx, jmap, blobId, type, data); + setError(db, jmap, blobId, null); + }); + done += 1; + ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'contents', done, total: todo.length }); + } + if (done > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] }); +} + +// ---- Calls from the page ---- + +function fileRow(db: StorageAdapter, { jmap, id }: NodeRef): NodeRow { + const row = db.get('SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]); + if (!row) throw new MethodError('notFound', `No file ${id}`); + if (!row.blob_id || row.kind !== 'file') throw new MethodError('invalidArguments', 'Only files can be kept'); + return row; +} + +export function pinFile(ctx: StoreContext, ref: NodeRef): { kept: boolean } { + const row = fileRow(ctx.db, ref); + ctx.db.run('UPDATE file_node SET pinned = 1, keep_error = NULL WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]); + const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string); + if (!kept) void ctx.requestSync(); + return { kept }; +} + +/** Back to being a recently opened file, or let go of it if it never was one. */ +export function unpinFile(ctx: StoreContext, ref: NodeRef): void { + const row = fileRow(ctx.db, ref); + ctx.db.transaction(() => { + ctx.db.run('UPDATE file_node SET pinned = 0 WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]); + if (!wantedRow(ctx.db, ref.jmap, row.blob_id as string)) dropBlob(ctx.db, ref.jmap, row.blob_id as string); + }); +} + +export function noteOpened(ctx: StoreContext, ref: NodeRef): { kept: boolean } { + const row = fileRow(ctx.db, ref); + ctx.db.run('UPDATE file_node SET opened_at = ?, keep_error = NULL WHERE jmap = ? AND id = ?', [ctx.now(), ref.jmap, ref.id]); + const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string); + if (!kept) void ctx.requestSync(); + return { kept }; +} + +export function readFile(ctx: StoreContext, ref: NodeRef): (KeptFile & { name: string }) | null { + const row = fileRow(ctx.db, ref); + const kept = fileBlob(ctx, ref.jmap, row.blob_id as string); + return kept ? { ...kept, name: row.name } : null; +} + +export interface KeptFileStatus { + jmap: string; + id: string; + name: string; + size: number | null; + pinned: boolean; + openedAt: number | null; + kept: boolean; + /** Why it isn't kept: tooLarge, overLimit, notAllowed or download. */ + error: string | null; +} + +/** Pinned and recently opened files, and how much room their contents take. */ +export function keptStatus(ctx: StoreContext): { files: KeptFileStatus[]; bytes: number; capBytes: number } { + const rows = ctx.db.all( + `SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node + WHERE pinned = 1 OR opened_at IS NOT NULL ORDER BY pinned DESC, opened_at DESC, name`, + ); + return { + files: rows.map((row) => ({ + jmap: row.jmap, + id: row.id, + name: row.name, + size: row.size === null ? null : Number(row.size), + pinned: Boolean(row.pinned), + openedAt: row.opened_at === null ? null : Number(row.opened_at), + kept: row.blob_id !== null && hasBlob(ctx.db, row.jmap, row.blob_id), + error: row.keep_error, + })), + bytes: keptBlobs(ctx.db).reduce((sum, blob) => sum + Number(blob.size), 0), + capBytes: capBytes(ctx), + }; +} diff --git a/lib/store/files/methods.ts b/lib/store/files/methods.ts new file mode 100644 index 000000000..0945a33eb --- /dev/null +++ b/lib/store/files/methods.ts @@ -0,0 +1,108 @@ +/** + * FileNode reads answered from the store, so listings work offline. Nothing + * here answers FileNode/set: the responder gives it a per-call + * serverUnavailable, since the design has no offline file writes. + */ + +import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area'; +import type { SqlValue } from '../storage'; +import { FILES_CAPABILITY } from './records'; + +function jmapOf(ctx: MethodContext, args: Record): string { + return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[FILES_CAPABILITY] ?? ctx.defaultJmap; +} + +/** The last state from the server. Throws when the list was never read, so an empty answer always means empty. */ +function syncedState(ctx: MethodContext, jmap: string): string | null { + const row = ctx.db.get<{ state: string | null; phase: string | null }>("SELECT state, phase FROM sync_state WHERE jmap = ? AND type = 'FileNode'", [jmap]); + if (!row || (!row.state && !row.phase)) throw new MethodError('serverUnavailable', 'Files not kept yet'); + return row.state; +} + +function pick(node: Record, properties: unknown): Record { + if (!Array.isArray(properties)) return node; + const out: Record = { id: node.id }; + for (const property of properties as string[]) if (property in node) out[property] = node[property]; + return out; +} + +const SORTS: Record = { + name: 'name COLLATE NOCASE', + size: 'size', + created: "json_extract(json, '$.created')", + modified: "json_extract(json, '$.modified')", +}; + +function orderSql(sort: unknown): string { + const parts: string[] = []; + for (const level of (Array.isArray(sort) ? sort : []) as Array<{ property?: string; isAscending?: boolean }>) { + const column = SORTS[String(level.property)]; + if (!column) throw new MethodError('unsupportedSort', `${String(level.property)} isn't available offline`); + parts.push(`${column} ${level.isAscending === false ? 'DESC' : 'ASC'}`); + } + parts.push('name COLLATE NOCASE ASC', 'id ASC'); + return parts.join(', '); +} + +const fileNodeGet: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + const state = syncedState(ctx, jmap); + const list: Array> = []; + const notFound: string[] = []; + if (Array.isArray(args.ids)) { + for (const id of args.ids as string[]) { + const row = ctx.db.get<{ json: string }>('SELECT json FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]); + if (row) list.push(pick(JSON.parse(row.json) as Record, args.properties)); + else notFound.push(id); + } + } else { + // The whole tree at once. The client reads a long list as possibly cut, + // pages FileNode/query to check, and finds nothing missing. + for (const row of ctx.db.all<{ json: string }>('SELECT json FROM file_node WHERE jmap = ?', [jmap])) { + list.push(pick(JSON.parse(row.json) as Record, args.properties)); + } + } + return { accountId: jmap, ...(state ? { state } : {}), list, notFound }; +}; + +const fileNodeQuery: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + syncedState(ctx, jmap); + if (args.anchor !== undefined) throw new MethodError('unsupportedFilter', 'anchor needs the server'); + const filter = (args.filter ?? {}) as Record; + const conditions = ['jmap = ?']; + const params: SqlValue[] = [jmap]; + for (const [key, value] of Object.entries(filter)) { + if (key !== 'parentId') throw new MethodError('unsupportedFilter', `${key} isn't available offline`); + if (value === null) { + conditions.push('parent_id IS NULL'); + } else { + conditions.push('parent_id = ?'); + params.push(String(value)); + } + } + const where = conditions.join(' AND '); + const position = Math.max(0, Number(args.position ?? 0)); + const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1; + const ids = ctx.db.all<{ id: string }>( + `SELECT id FROM file_node WHERE ${where} ORDER BY ${orderSql(args.sort)} LIMIT ? OFFSET ?`, + [...params, limit, position], + ).map((row) => String(row.id)); + const result: MethodResult = { + accountId: jmap, + // Local answers aren't a position in the server's log, so this is a token only. + queryState: 'local', + canCalculateChanges: false, + position, + ids, + }; + if (args.calculateTotal) { + result.total = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM file_node WHERE ${where}`, params)?.n ?? 0); + } + return result; +}; + +export const FILE_METHODS: Readonly> = { + 'FileNode/get': fileNodeGet, + 'FileNode/query': fileNodeQuery, +}; diff --git a/lib/store/files/records.ts b/lib/store/files/records.ts new file mode 100644 index 000000000..fd5cbcb51 --- /dev/null +++ b/lib/store/files/records.ts @@ -0,0 +1,50 @@ +/** How a FileNode becomes a row. The server's own copy stays in `json`, in whichever draft it speaks. */ + +import { fileNodeKind, fromDraftFileNode } from '@/lib/jmap/filenode-kind'; +import { decodeFileNodeName } from '@/lib/jmap/filenode-name'; +import type { FileNode } from '@/lib/jmap/types'; +import type { StorageAdapter } from '../storage'; + +export const FILES_CAPABILITY = 'urn:ietf:params:jmap:filenode'; +export const FILES_USING = ['urn:ietf:params:jmap:core', FILES_CAPABILITY]; + +/** The client's list (lib/jmap/client.ts). Stalwart leaves out shareWith and myRights unless asked. */ +export const FILE_NODE_PROPERTIES = [ + 'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified', + 'nodeType', '@type', 'mediaType', 'shareWith', 'myRights', +]; + +export type WireNode = Record & { id: string }; + +/** The node's media type, whichever draft named it. */ +export function nodeMediaType(node: WireNode): string { + const { type } = fromDraftFileNode(node as unknown as Partial & { '@type'?: unknown; mediaType?: unknown }); + return typeof type === 'string' && type ? type : 'application/octet-stream'; +} + +export function writeNode(db: StorageAdapter, jmap: string, node: WireNode, gen: number): void { + const normal = fromDraftFileNode(node as unknown as FileNode & { '@type'?: unknown; mediaType?: unknown }); + db.run( + `INSERT INTO file_node (jmap, id, parent_id, name, kind, blob_id, size, json, gen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (jmap, id) DO UPDATE SET parent_id = excluded.parent_id, name = excluded.name, kind = excluded.kind, + size = excluded.size, json = excluded.json, gen = excluded.gen, + keep_error = CASE WHEN blob_id IS excluded.blob_id THEN keep_error END, + blob_id = excluded.blob_id`, + [ + jmap, + node.id, + typeof node.parentId === 'string' ? node.parentId : null, + // The client decodes names itself from `json`. This column is for the store's own lists. + typeof node.name === 'string' ? decodeFileNodeName(node.name) : '', + fileNodeKind(normal), + typeof node.blobId === 'string' ? node.blobId : null, + typeof node.size === 'number' ? node.size : null, + JSON.stringify(node), + gen, + ], + ); +} + +export function removeNode(db: StorageAdapter, jmap: string, id: string): void { + db.run('DELETE FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]); +} diff --git a/lib/store/files/schema.ts b/lib/store/files/schema.ts new file mode 100644 index 000000000..9a3f20e35 --- /dev/null +++ b/lib/store/files/schema.ts @@ -0,0 +1,47 @@ +import type { Migration } from '../storage'; + +export const FILE_MIGRATIONS: readonly Migration[] = [ + { + id: 700, + name: 'files', + sql: ` + CREATE TABLE file_node ( + jmap TEXT NOT NULL, + id TEXT NOT NULL, + parent_id TEXT, + name TEXT NOT NULL DEFAULT '', + -- file, directory or symlink, whichever way the server says so. + kind TEXT NOT NULL, + blob_id TEXT, + size INTEGER, + json TEXT NOT NULL, + gen INTEGER NOT NULL DEFAULT 0, + pinned INTEGER NOT NULL DEFAULT 0, + -- When the page last opened it. Cleared when its content is pushed out. + opened_at INTEGER, + -- Why the content isn't kept: tooLarge, overLimit, notAllowed or download. + keep_error TEXT, + PRIMARY KEY (jmap, id) + ) WITHOUT ROWID; + CREATE INDEX file_node_parent ON file_node (jmap, parent_id); + CREATE INDEX file_node_blob ON file_node (jmap, blob_id); + CREATE TABLE file_blob ( + jmap TEXT NOT NULL, + blob_id TEXT NOT NULL, + type TEXT NOT NULL, + size INTEGER NOT NULL, + kept_at INTEGER NOT NULL, + PRIMARY KEY (jmap, blob_id) + ) WITHOUT ROWID; + -- In pieces: a whole file bound at once would grow the WASM heap by its + -- size, and that memory never comes back. + CREATE TABLE file_blob_chunk ( + jmap TEXT NOT NULL, + blob_id TEXT NOT NULL, + seq INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (jmap, blob_id, seq) + ); + `, + }, +]; diff --git a/lib/store/files/sync.ts b/lib/store/files/sync.ts new file mode 100644 index 000000000..cc9d99918 --- /dev/null +++ b/lib/store/files/sync.ts @@ -0,0 +1,160 @@ +/** + * FileNode metadata for one JMAP account, then the contents worth keeping. + * + * A full pass reads the tree the way the client does (fetchAllFileNodes, + * lib/jmap/client.ts). `FileNode/get` with ids null returns everything, but + * 0.16 silently stops at maxObjectsInGet and 1.0 refuses a larger account with + * requestTooLarge. Past that, ids come from paging FileNode/query. Before + * 0.16.6 that query leaves out folders, so parents still unknown afterwards are + * read by id. The cursor is in sync_state, so a reload continues. + */ + +import type { SyncContext, Syncer } from '../area'; +import { runChangesLoop } from '../engine/changes'; +import { JmapMethodError, responseFor, type MethodCall } from '../engine/transport'; +import { dropOrphanBlobs, fetchKeptFiles } from './keep'; +import { FILE_NODE_PROPERTIES, FILES_CAPABILITY, FILES_USING, removeNode, writeNode, type WireNode } from './records'; + +const MAX_GET = 500; +const GETS_PER_REQUEST = 8; +const QUERY_PAGE = 5000; + +function getBatch(ctx: SyncContext): number { + const core = ctx.session.capabilities['urn:ietf:params:jmap:core'] as { maxObjectsInGet?: number } | undefined; + return Math.min(MAX_GET, core?.maxObjectsInGet || MAX_GET); +} + +async function fetchNodes(ctx: SyncContext, jmap: string, ids: string[]): Promise { + const size = getBatch(ctx); + const nodes: WireNode[] = []; + for (let at = 0; at < ids.length;) { + const calls: MethodCall[] = []; + for (let i = 0; i < GETS_PER_REQUEST && at < ids.length; i += 1, at += size) { + calls.push(['FileNode/get', { accountId: jmap, ids: ids.slice(at, at + size), properties: FILE_NODE_PROPERTIES }, `g${i}`]); + } + const response = await ctx.transport.request(calls, FILES_USING); + calls.forEach(([, , callId]) => nodes.push(...responseFor(response, callId)[1].list as WireNode[])); + } + return nodes; +} + +interface ListCursor { step: 'all' | 'query' | 'parents'; position: number; gen: number; captured: string } + +async function startFullList(ctx: SyncContext, jmap: string): Promise { + const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: [], properties: ['id'] }, 's']], FILES_USING); + const captured = String(responseFor(response, 's')[1].state); + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0) + 1; + const cursor: ListCursor = { step: 'all', position: 0, gen, captured }; + ctx.db.transaction(() => ctx.setSyncState(jmap, 'FileNode', { state: null, phase: 'list', cursor: { ...cursor } })); + return cursor; +} + +function saveStep(ctx: SyncContext, jmap: string, nodes: WireNode[], cursor: ListCursor): void { + ctx.db.transaction(() => { + for (const node of nodes) writeNode(ctx.db, jmap, node, cursor.gen); + ctx.setSyncState(jmap, 'FileNode', { cursor: { ...cursor } }); + }); +} + +async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise { + const cursor = { ...start }; + + if (cursor.step === 'all') { + let nodes: WireNode[] = []; + let complete = false; + try { + const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: null, properties: FILE_NODE_PROPERTIES }, 'a']], FILES_USING); + nodes = responseFor(response, 'a')[1].list as WireNode[]; + complete = nodes.length < getBatch(ctx); + } catch (error) { + if (!(error instanceof JmapMethodError && error.type === 'requestTooLarge')) throw error; + } + if (!complete) cursor.step = 'query'; + else cursor.step = 'parents'; + saveStep(ctx, jmap, nodes, cursor); + } + + while (cursor.step === 'query') { + if (ctx.stopped()) return false; + const response = await ctx.transport.request([['FileNode/query', { accountId: jmap, filter: {}, position: cursor.position, limit: QUERY_PAGE, calculateTotal: true }, 'q']], FILES_USING); + const [, query] = responseFor(response, 'q'); + const ids = query.ids as string[]; + const missing = ids.filter((id) => Number(ctx.db.get<{ gen: number }>('SELECT gen FROM file_node WHERE jmap = ? AND id = ?', [jmap, id])?.gen ?? -1) < cursor.gen); + const nodes = await fetchNodes(ctx, jmap, missing); + cursor.position += ids.length; + // The server may clamp the limit, so only an empty page or the total ends this. + if (ids.length === 0 || (typeof query.total === 'number' && cursor.position >= query.total)) cursor.step = 'parents'; + saveStep(ctx, jmap, nodes, cursor); + ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'list', done: cursor.position, total: typeof query.total === 'number' ? query.total : null }); + } + + const asked = new Set(); + for (;;) { + if (ctx.stopped()) return false; + const parents = ctx.db.all<{ id: string }>( + `SELECT DISTINCT n.parent_id AS id FROM file_node n + WHERE n.jmap = ? AND n.gen = ? AND n.parent_id IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM file_node p WHERE p.jmap = n.jmap AND p.id = n.parent_id AND p.gen = ?)`, + [jmap, cursor.gen, cursor.gen], + ).map((row) => String(row.id)).filter((id) => !asked.has(id)); + if (parents.length === 0) break; + for (const id of parents) asked.add(id); + saveStep(ctx, jmap, await fetchNodes(ctx, jmap, parents), cursor); + } + + ctx.db.transaction(() => { + // Whatever this pass didn't see is gone from the server. + ctx.db.run('DELETE FROM file_node WHERE jmap = ? AND gen < ?', [jmap, cursor.gen]); + ctx.setSyncState(jmap, 'FileNode', { state: cursor.captured, phase: null, cursor: null }); + }); + ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] }); + return true; +} + +async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: string | null): Promise<'done' | 'resync' | 'stopped'> { + const outcome = await runChangesLoop({ + transport: ctx.transport, + type: 'FileNode', + jmap, + using: FILES_USING, + since, + target, + stopped: ctx.stopped, + apply: async (page) => { + const nodes = await fetchNodes(ctx, jmap, [...page.created, ...page.updated]); + ctx.db.transaction(() => { + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0); + for (const node of nodes) writeNode(ctx.db, jmap, node, gen); + for (const id of page.destroyed) removeNode(ctx.db, jmap, id); + ctx.setSyncState(jmap, 'FileNode', { state: page.newState }); + }); + if (nodes.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] }); + }, + }); + if (outcome.outcome === 'resync') return 'resync'; + return outcome.outcome === 'stopped' ? 'stopped' : 'done'; +} + +export const filesSyncer: Syncer = { + name: 'files', + capability: FILES_CAPABILITY, + async run(ctx, jmap) { + const row = ctx.syncState(jmap, 'FileNode'); + let cursor = row.phase === 'list' ? row.cursor as unknown as ListCursor : null; + if (!cursor && !row.state) cursor = await startFullList(ctx, jmap); + if (!cursor && row.state) { + const outcome = await syncDelta(ctx, jmap, row.state, row.target); + if (outcome === 'stopped') return; + if (outcome === 'resync') cursor = await startFullList(ctx, jmap); + } + if (cursor) { + if (!(await runFullList(ctx, jmap, cursor))) return; + // Catch what changed while paging. + const after = ctx.syncState(jmap, 'FileNode'); + if (after.state && (await syncDelta(ctx, jmap, after.state, after.target)) === 'stopped') return; + } + if (ctx.stopped()) return; + ctx.db.transaction(() => dropOrphanBlobs(ctx.db)); + await fetchKeptFiles(ctx, jmap); + }, +}; From 64434bd5a6e7f3e22692ccf5a4c3443c1d016979 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:47:40 +0100 Subject: [PATCH 4/7] fix: keep the file list when refreshing a folder fails --- stores/__tests__/file-store.test.ts | 16 ++++++++++++++++ stores/file-store.ts | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/stores/__tests__/file-store.test.ts b/stores/__tests__/file-store.test.ts index 6cb46aab6..bf7080b02 100644 --- a/stores/__tests__/file-store.test.ts +++ b/stores/__tests__/file-store.test.ts @@ -120,6 +120,22 @@ describe('file-store hierarchy (issue #379)', () => { }); }); + it('keeps the listing when refreshing the same folder fails, and starts empty in a new one', async () => { + const client = makeMockClient([dir('stuff', 'Stuff', null), file('notes', 'Notes.md', null), file('inner', 'Inner.md', 'stuff')]); + useFileStore.getState().initClient(client); + await useFileStore.getState().navigate(null); + expect(useFileStore.getState().resources.map(r => r.name).sort()).toEqual(['Notes.md', 'Stuff']); + + client.listAllFileNodes = async () => { throw new Error('offline'); }; + await useFileStore.getState().navigate(null); + expect(useFileStore.getState().error).toBe('offline'); + expect(useFileStore.getState().resources.map(r => r.name).sort()).toEqual(['Notes.md', 'Stuff']); + + await useFileStore.getState().navigate('stuff', 'Stuff'); + expect(useFileStore.getState().error).toBe('offline'); + expect(useFileStore.getState().resources).toEqual([]); + }); + it('lists only root-level nodes at the root, not the whole tree flattened', async () => { // Stuff > Nonsense > Notes.md / Other.md (the exact shape from the issue) const client = makeMockClient([ diff --git a/stores/file-store.ts b/stores/file-store.ts index f9ec1a6b5..3feada71e 100644 --- a/stores/file-store.ts +++ b/stores/file-store.ts @@ -982,7 +982,9 @@ export const useFileStore = create((set, get) => ({ error: error instanceof Error ? error.message : 'Failed to list directory', hasListed: true, isLoading: false, - resources: [], + // A refresh that fails keeps what the folder showed. Another folder's + // rows would be wrong here, so one being opened starts empty. + resources: opensFolder ? [] : get().resources, }); } }, From 691deff583e4ddf7c39a15dcec516c4f04e3cb58 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:50:58 +0100 Subject: [PATCH 5/7] fix: refuse a queued contact too large to ever send --- lib/store/__tests__/contacts.test.ts | 13 +++++++++++++ lib/store/contacts/pending.ts | 21 ++++++++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/store/__tests__/contacts.test.ts b/lib/store/__tests__/contacts.test.ts index 526595a85..2331572e2 100644 --- a/lib/store/__tests__/contacts.test.ts +++ b/lib/store/__tests__/contacts.test.ts @@ -470,6 +470,19 @@ describe('queued edits', () => { expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); }); + it('refuses a card too large to ever send, so it cannot hold up the rest', async () => { + const server = new FakeContacts(); + const card = server.addCard(); + const { engine } = await synced(server); + const photo = { p: { kind: 'photo', uri: `data:image/jpeg;base64,${'A'.repeat(9 * 1024 * 1024)}` } }; + const made = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Big' }, addressBookIds: { book1: true }, media: photo } } }); + expect(made.notCreated.n.type).toBe('tooLarge'); + const edited = await call(engine, 'ContactCard/set', { update: { [card.id]: { media: photo } } }); + expect(edited.notUpdated[card.id].type).toBe('tooLarge'); + expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); + expect((await call(engine, 'ContactCard/get', { ids: [card.id] })).list[0]).not.toHaveProperty('media'); + }); + it('answers a patch that leads nowhere per record', async () => { const server = new FakeContacts(); const card = server.addCard(); diff --git a/lib/store/contacts/pending.ts b/lib/store/contacts/pending.ts index aaadc64b7..4cf2feba2 100644 --- a/lib/store/contacts/pending.ts +++ b/lib/store/contacts/pending.ts @@ -27,6 +27,8 @@ const LOCAL_BLOB_MARK = '"blobId":"bulwark-local-'; const SET_BATCH = 100; const SET_BATCH_BYTES = 5 * 1024 * 1024; +/** The server takes 10 MB a request. A bigger card would hold up everything queued behind it, for good. */ +const MAX_RECORD_BYTES = 8 * 1024 * 1024; const GET_BATCH = 500; const CALLS_PER_REQUEST = 16; /** Set errors that say nothing about the edit itself: it goes out again next time. */ @@ -86,6 +88,8 @@ function holdsLocalBlob(value: unknown): boolean { return JSON.stringify(value).includes(LOCAL_BLOB_MARK); } +const TOO_LARGE_ERROR: SetError = { type: 'tooLarge', description: 'This contact is too large to save' }; + const LOCAL_BLOB_ERROR: SetError = { type: 'invalidProperties', description: 'A photo that is not on the server yet cannot be used here', @@ -100,9 +104,10 @@ export function queueCreate(ctx: StoreContext, jmap: string, card: JsonObject): } const id = `${STAND_IN_PREFIX}${crypto.randomUUID()}`; const uid = typeof card.uid === 'string' && card.uid ? card.uid : `urn:uuid:${crypto.randomUUID()}`; - const payload = { ...card, uid }; - savePending(ctx.db, ctx, { jmap, id, op: 'create', uid, payload: JSON.stringify(payload), base: null }); - writeCard(ctx.db, jmap, { '@type': 'Card', version: '1.0', ...payload, id }); + const payload = JSON.stringify({ ...card, uid }); + if (payload.length > MAX_RECORD_BYTES) return TOO_LARGE_ERROR; + savePending(ctx.db, ctx, { jmap, id, op: 'create', uid, payload, base: null }); + writeCard(ctx.db, jmap, { '@type': 'Card', version: '1.0', ...card, uid, id }); return { id, uid }; } @@ -121,11 +126,13 @@ export function queueUpdate(ctx: StoreContext, jmap: string, rawId: string, patc throw error; } if (pending?.op === 'create') { - const payload = applyPatch(JSON.parse(pending.payload as string) as JsonObject, patch); - savePending(ctx.db, ctx, { jmap, id, op: 'create', uid: pending.uid, payload: JSON.stringify(payload), base: null }); + const payload = JSON.stringify(applyPatch(JSON.parse(pending.payload as string) as JsonObject, patch)); + if (payload.length > MAX_RECORD_BYTES) return TOO_LARGE_ERROR; + savePending(ctx.db, ctx, { jmap, id, op: 'create', uid: pending.uid, payload, base: null }); } else { - const merged = pending ? mergePatches(JSON.parse(pending.payload as string) as Patch, patch) : patch; - savePending(ctx.db, ctx, { jmap, id, op: 'update', payload: JSON.stringify(merged), base: pending ? pending.base : JSON.stringify(card) }); + const merged = JSON.stringify(pending ? mergePatches(JSON.parse(pending.payload as string) as Patch, patch) : patch); + if (merged.length > MAX_RECORD_BYTES) return TOO_LARGE_ERROR; + savePending(ctx.db, ctx, { jmap, id, op: 'update', payload: merged, base: pending ? pending.base : JSON.stringify(card) }); } writeCard(ctx.db, jmap, visible); return null; From 9fbe33004bf71f5f10dcf57e4ca989b6956a4efe Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:20:39 +0100 Subject: [PATCH 6/7] fix: send contact edits that remove entries and stop reading own edits as conflicts An edit that removed an entry was dropped at flush, because a server value with more keys counted as already applied. Other fixes in the same path: - the local copy after a flush is the server's read-back, never what was sent, since Stalwart 1.0 stores a data: URI as a blob - a delete or a newer edit after a lost response no longer reads as a change on the server, and an edit made during a refused send stays - a create only becomes a card with its uid if that card holds what was sent, otherwise it is a refused conflict - a refused or failed flush no longer stops contacts syncing - request limits come from the session - answers wait for the first full pass - paging goes by anchor, so a deleted card doesn't make the next one get skipped and swept contacts.test.ts: "continues a stopped full sync" now checks the anchor instead of position 2000, because the cursor no longer pages by position. --- lib/store/__tests__/contacts.test.ts | 392 ++++++++++++++++++- lib/store/__tests__/helpers/fake-contacts.ts | 51 ++- lib/store/contacts/methods.ts | 8 +- lib/store/contacts/patch.ts | 35 +- lib/store/contacts/pending.ts | 353 +++++++++++------ lib/store/contacts/schema.ts | 2 + lib/store/contacts/sync.ts | 83 ++-- lib/store/engine/limits.ts | 31 ++ 8 files changed, 788 insertions(+), 167 deletions(-) create mode 100644 lib/store/engine/limits.ts diff --git a/lib/store/__tests__/contacts.test.ts b/lib/store/__tests__/contacts.test.ts index 2331572e2..c05334fab 100644 --- a/lib/store/__tests__/contacts.test.ts +++ b/lib/store/__tests__/contacts.test.ts @@ -6,19 +6,19 @@ import path from 'node:path'; import { textFilter } from '@/lib/jmap/search-utils'; import { DEFAULT_SCOPE, type StoreEvent } from '../area'; import { contactsArea } from '../contacts'; -import { applyPatch, mergePatches } from '../contacts/patch'; +import { applyPatch, coversJson, mergePatches, sameJson } from '../contacts/patch'; import { AccountEngine } from '../engine/account-engine'; -import type { MethodCall } from '../engine/transport'; +import type { MethodCall, StoreSession } from '../engine/transport'; import type { StorageAdapter } from '../storage'; import { JMAP_ACCOUNT } from './helpers/fake-jmap'; import { CONTACTS_SESSION, FakeContacts, MAIL_ONLY_ACCOUNT, SHARED_ACCOUNT } from './helpers/fake-contacts'; import { openFileStore, openMemoryStore } from './helpers/node-adapter'; -function engineFor(server: FakeContacts, db: StorageAdapter = openMemoryStore(), emit?: (e: StoreEvent) => void) { +function engineFor(server: FakeContacts, db: StorageAdapter = openMemoryStore(), emit?: (e: StoreEvent) => void, session: StoreSession = CONTACTS_SESSION) { return new AccountEngine({ account: 'a@fake.test', db, - session: CONTACTS_SESSION, + session, transport: server, areas: [contactsArea], scope: DEFAULT_SCOPE, @@ -107,9 +107,11 @@ describe('contacts sync', () => { const second = openFileStore(file); await engineFor(server, second).requestSync(); expect(count(second, 'contact_card')).toBe(2600); - const positions = server.calls.filter(([name]) => name === 'ContactCard/query').map(([, args]) => args.position); - expect(positions[0]).toBe(2000); second.close(); + // It goes on from the last card read, not from the start. + const first_ = server.calls.find(([name]) => name === 'ContactCard/query')?.[1]; + expect(first_?.anchor).toBeTruthy(); + expect(first_?.anchorOffset).toBe(1); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -670,4 +672,382 @@ describe('edits while sending', () => { expect(await engine.areaCall('contacts.pending', undefined)).toBe(0); }); }); + +const mine = (engine: AccountEngine, patch: Record, id: string) => call(engine, 'ContactCard/set', { update: { [id]: patch } }); +const conflictsOf = async (engine: AccountEngine) => await engine.areaCall('contacts.conflicts', undefined) as Array>; +const pendingOf = (engine: AccountEngine) => engine.areaCall('contacts.pending', undefined); + +describe('edits that make a card smaller', () => { + it.each<[string, Record, Record, Record]>([ + ['the last email', { emails: { e1: { address: 'a@example.org' } } }, { emails: {} }, { emails: {} }], + ['one of two emails', { emails: { e1: { address: 'a@example.org' }, e2: { address: 'b@example.org' } } }, { emails: { e1: { address: 'a@example.org' } } }, { emails: { e1: { address: 'a@example.org' } } }], + ['an email by pointer', { emails: { e1: { address: 'a@example.org' }, e2: { address: 'b@example.org' } } }, { 'emails/e2': null }, { emails: { e1: { address: 'a@example.org' } } }], + ['a keyword', { keywords: { work: true, home: true } }, { keywords: { work: true } }, { keywords: { work: true } }], + ['an address book', { addressBookIds: { book1: true, book2: true } }, { addressBookIds: { book2: true } }, { addressBookIds: { book2: true } }], + ['a group member', { kind: 'group', members: { 'urn:uuid:a': true, 'urn:uuid:b': true } }, { members: { 'urn:uuid:a': true } }, { members: { 'urn:uuid:a': true } }], + ])('sends the removal of %s', async (_label, start, patch, expected) => { + const server = new FakeContacts(['book1', 'book2']); + const card = server.addCard(start); + const { engine } = await synced(server); + await mine(engine, patch, card.id); + await engine.requestSync(); + expect(server.account().cards.get(card.id)).toMatchObject(expected); + for (const [key, value] of Object.entries(expected)) expect(server.account().cards.get(card.id)?.[key]).toEqual(value); + expect(await conflictsOf(engine)).toEqual([]); + expect(await pendingOf(engine)).toBe(0); + }); + + it('reads a key only the server has as its own, and one it had before as not yet removed', () => { + const wanted = { e1: { address: 'a' } }; + expect(coversJson(wanted, { e1: { address: 'a' }, added: { address: 'x' } }, { e1: { address: 'a' } })).toBe(true); + expect(coversJson(wanted, { e1: { address: 'a' }, e2: { address: 'b' } }, { e1: { address: 'a' }, e2: { address: 'b' } })).toBe(false); + expect(coversJson({ e: { '@type': 'x' } }, { e: { '@type': 'x', extra: 1 } }, undefined)).toBe(true); + // Stalwart 1.0 keeps a data: URI as a blob. + expect(coversJson({ m: { kind: 'photo', uri: 'data:image/png;base64,AA' } }, { m: { kind: 'photo', blobId: 'b1' } }, undefined)).toBe(true); + expect(coversJson({ m: { kind: 'logo', uri: 'data:image/png;base64,AA' } }, { m: { kind: 'photo', blobId: 'b1' } }, undefined)).toBe(false); + expect(sameJson({}, undefined)).toBe(true); + }); +}); + +describe('flush failures', () => { + it('still syncs after the server refuses the set as a whole, and records the edits', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'server' }); + const { engine } = await synced(server); + await mine(engine, { note: 'mine' }, card.id); + await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'New' }, addressBookIds: { book1: true } } } }); + server.failSet = () => ({ jmapError: 'forbidden' }); + server.addCard({ name: { full: 'From the server' } }); + await engine.requestSync(); + expect(await search(engine, 'from the server')).toEqual(['From the server']); + expect((await conflictsOf(engine)).map((c) => c.kind)).toEqual(['refused', 'refused']); + expect(await pendingOf(engine)).toBe(0); + expect((await call(engine, 'ContactCard/get', { ids: [card.id] })).list[0].note).toBe('server'); + }); + + it('keeps edits and still syncs when the failure says nothing about them', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'server' }); + const { engine } = await synced(server); + await mine(engine, { note: 'mine' }, card.id); + server.failSet = () => ({ jmapError: 'serverFail' }); + server.addCard({ name: { full: 'From the server' } }); + await engine.requestSync(); + expect(await search(engine, 'from the server')).toEqual(['From the server']); + expect(await conflictsOf(engine)).toEqual([]); + expect(await pendingOf(engine)).toBe(1); + server.failSet = null; + await engine.requestSync(); + expect(server.account().cards.get(card.id)?.note).toBe('mine'); + }); + + it('takes the rest of a batch through when one card makes the request fail', async () => { + const server = new FakeContacts(); + const good = server.addCard({ note: 'a' }); + const bad = server.addCard({ note: 'b' }); + const { engine } = await synced(server); + await mine(engine, { note: 'good edit' }, good.id); + await mine(engine, { note: 'poison' }, bad.id); + server.failSet = (args) => (JSON.stringify(args).includes('poison') ? { http: 400 } : null); + await engine.requestSync(); + expect(server.account().cards.get(good.id)?.note).toBe('good edit'); + expect(server.account().cards.get(bad.id)?.note).toBe('b'); + const conflicts = await conflictsOf(engine); + expect(conflicts.map((c) => [c.recordId, c.kind])).toEqual([[bad.id, 'refused']]); + expect(await pendingOf(engine)).toBe(0); + }); + + it('takes the request limit from the session', async () => { + const server = new FakeContacts(); + const card = server.addCard(); + const session = { ...CONTACTS_SESSION, capabilities: { ...CONTACTS_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxSizeRequest: 1000 } } }; + const db = openMemoryStore(); + const engine = engineFor(server, db, undefined, session); + await engine.requestSync(); + const result = await mine(engine, { note: 'x'.repeat(2000) }, card.id); + expect(result.notUpdated[card.id].type).toBe('tooLarge'); + }); +}); + +describe('a uid another card has', () => { + it('does not turn an offline create into that card', async () => { + const server = new FakeContacts(); + const other = server.addCard({ uid: 'urn:uuid:taken', name: { full: 'Their card' } }); + const { db, engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { uid: 'urn:uuid:taken', name: { full: 'My card' }, addressBookIds: { book1: true } } } }); + await engine.requestSync(); + const conflicts = await conflictsOf(engine); + expect(conflicts).toHaveLength(1); + expect(conflicts[0]).toMatchObject({ kind: 'refused', recordId: created.n.id, summary: 'My card', local: { op: 'create', payload: { name: { full: 'My card' } } } }); + expect((await call(engine, 'ContactCard/get', { ids: [created.n.id] })).notFound).toEqual([created.n.id]); + expect(db.all('SELECT id FROM contact_card').map((r) => r.id)).toEqual([other.id]); + expect(server.account().cards.get(other.id)).toMatchObject({ name: { full: 'Their card' } }); + }); + + it('does not take someone else\'s card for its own after a failed attempt either', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { uid: 'urn:uuid:race', name: { full: 'Mine' }, addressBookIds: { book1: true } } } }); + server.offlineBeforeSet = true; + await engine.requestSync(); + const other = server.addCard({ uid: 'urn:uuid:race', name: { full: 'Theirs' } }); + await engine.requestSync(); + expect((await conflictsOf(engine)).map((c) => [c.recordId, c.kind])).toEqual([[created.n.id, 'refused']]); + expect(server.account().cards.size).toBe(1); + expect(server.account().cards.get(other.id)).toMatchObject({ name: { full: 'Theirs' } }); + }); + + it('does not delete someone else\'s card when the create was deleted again', async () => { + const server = new FakeContacts(); + const { engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { uid: 'urn:uuid:race', name: { full: 'Mine' }, addressBookIds: { book1: true } } } }); + server.offlineBeforeSet = true; + await engine.requestSync(); + await call(engine, 'ContactCard/set', { destroy: [created.n.id] }); + const other = server.addCard({ uid: 'urn:uuid:race', name: { full: 'Theirs' } }); + await engine.requestSync(); + expect(server.account().cards.has(other.id)).toBe(true); + expect(await pendingOf(engine)).toBe(0); + }); +}); + +describe('what the server really stored', () => { + const photo = { p: { kind: 'photo', uri: 'data:image/png;base64,iVBORw0KGgo=', mediaType: 'image/png' } }; + + it('does not read its own stored photo as a change after the network went, on 1.0', async () => { + const server = new FakeContacts(); + server.mediaAsBlobs = true; + const { engine } = await synced(server); + const { created } = await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Foto' }, addressBookIds: { book1: true }, media: photo } } }); + server.offlineAfterSet = true; + await engine.requestSync(); + expect(server.account().cards.size).toBe(1); + server.offline = false; + await call(engine, 'ContactCard/set', { destroy: [created.n.id] }); + await engine.requestSync(); + expect(await conflictsOf(engine)).toEqual([]); + expect(server.account().cards.size).toBe(0); + }); + + it('adopts its own card by uid though the server keeps the photo as a blob', async () => { + const server = new FakeContacts(); + server.mediaAsBlobs = true; + const { engine } = await synced(server); + await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Foto' }, addressBookIds: { book1: true }, media: photo } } }); + server.loseNextSet = true; + await engine.requestSync(); + await engine.requestSync(); + expect(await conflictsOf(engine)).toEqual([]); + expect(await pendingOf(engine)).toBe(0); + expect(server.account().cards.size).toBe(1); + }); + + it('keeps the card as the server read it back, not as it was sent, before any delta', async () => { + const server = new FakeContacts(); + server.mediaAsBlobs = true; + const { db, engine } = await synced(server); + await call(engine, 'ContactCard/set', { create: { n: { name: { full: 'Foto' }, addressBookIds: { book1: true }, media: photo } } }); + // The network goes right after the read-back, before the delta of this pass. + server.beforeRequest = () => { + const set = server.calls.findIndex(([name]) => name === 'ContactCard/set'); + if (set >= 0 && server.calls.slice(set).some(([name]) => name === 'ContactCard/get')) server.offline = true; + }; + await engine.requestSync(); + server.beforeRequest = null; + server.offline = false; + const row = db.get<{ json: string }>('SELECT json FROM contact_card'); + expect((JSON.parse(row?.json as string) as { media: Record }).media.p).toHaveProperty('blobId'); + const id = [...server.account().cards.keys()][0]; + await mine(engine, { note: 'after' }, id); + await engine.requestSync(); + expect(await conflictsOf(engine)).toEqual([]); + expect(server.account().cards.get(id)?.note).toBe('after'); + }); + + it('does not take a lost photo update for someone else changing it, on 1.0', async () => { + const server = new FakeContacts(); + server.mediaAsBlobs = true; + const card = server.addCard({ note: 'old' }); + const { engine } = await synced(server); + await mine(engine, { media: photo }, card.id); + server.loseNextSet = true; + await engine.requestSync(); + await engine.requestSync(); + expect(await conflictsOf(engine)).toEqual([]); + expect(await pendingOf(engine)).toBe(0); + }); +}); + +describe('after a lost update', () => { + it('still deletes the card', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old' }); + const { engine } = await synced(server); + await mine(engine, { note: 'mine' }, card.id); + server.loseNextSet = true; + await engine.requestSync(); + await call(engine, 'ContactCard/set', { destroy: [card.id] }); + await engine.requestSync(); + expect(await conflictsOf(engine)).toEqual([]); + expect(server.account().cards.has(card.id)).toBe(false); + }); + + it('does not delete when someone else changed another property too', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old', name: { full: 'A' } }); + const { engine } = await synced(server); + await mine(engine, { note: 'mine' }, card.id); + server.loseNextSet = true; + await engine.requestSync(); + await call(engine, 'ContactCard/set', { destroy: [card.id] }); + server.editCard(card.id, { name: { full: 'B' } }); + await engine.requestSync(); + expect(server.account().cards.has(card.id)).toBe(true); + expect((await conflictsOf(engine)).map((c) => c.kind)).toEqual(['changed']); + }); + + it('sends a newer edit of the same property', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old' }); + const { engine } = await synced(server); + await mine(engine, { note: 'first' }, card.id); + server.loseNextSet = true; + await engine.requestSync(); + await mine(engine, { note: 'second' }, card.id); + await engine.requestSync(); + expect(await conflictsOf(engine)).toEqual([]); + expect(server.account().cards.get(card.id)?.note).toBe('second'); + }); +}); + +describe('a refusal while the page kept editing', () => { + it('keeps the newer edit, and only records a conflict if it is refused too', async () => { + const server = new FakeContacts(); + const card = server.addCard({ note: 'old', name: { full: 'A' } }); + const { engine } = await synced(server); + await mine(engine, { note: 'one' }, card.id); + server.refuse.set(card.id, { type: 'forbidden' }); + const original = server.request.bind(server); + let edited = false; + server.request = async (calls: MethodCall[]) => { + if (!edited && calls.some(([name]) => name === 'ContactCard/set')) { + edited = true; + await mine(engine, { 'name/full': 'B' }, card.id); + } + return original(calls); + }; + await engine.requestSync(); + expect(await pendingOf(engine)).toBe(1); + expect(await conflictsOf(engine)).toEqual([]); + server.refuse.clear(); + await engine.requestSync(); + expect(server.account().cards.get(card.id)).toMatchObject({ note: 'one', name: { full: 'B' } }); + expect(await pendingOf(engine)).toBe(0); + }); +}); + +describe('session limits', () => { + it('takes maxObjectsInGet and maxCallsInRequest from the session', async () => { + const server = new FakeContacts(); + server.maxObjectsInGet = 2; + const cards = [server.addCard(), server.addCard(), server.addCard(), server.addCard(), server.addCard()]; + const session = { ...CONTACTS_SESSION, capabilities: { ...CONTACTS_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 2, maxCallsInRequest: 4 } } }; + const db = openMemoryStore(); + const events: StoreEvent[] = []; + const engine = engineFor(server, db, (e) => events.push(e), session); + await engine.requestSync(); + expect(events.filter((e) => e.type === 'error')).toEqual([]); + expect(count(db, 'contact_card')).toBe(5); + for (const [name, args] of server.calls) { + if (name === 'ContactCard/query') expect(args.limit).toBeLessThanOrEqual(2); + } + for (const card of cards.slice(0, 3)) await mine(engine, { note: 'x' }, card.id); + await engine.requestSync(); + expect(events.filter((e) => e.type === 'error')).toEqual([]); + expect(await pendingOf(engine)).toBe(0); + expect(cards.slice(0, 3).map((c) => server.account().cards.get(c.id)?.note)).toEqual(['x', 'x', 'x']); + }); +}); + +describe('an interrupted first sync', () => { + it('answers serverUnavailable until the first full pass is done', async () => { + const server = new FakeContacts(); + for (let i = 0; i < 2600; i += 1) server.addCard(); + const db = openMemoryStore(); + const engine = engineFor(server, db); + let requests = 0; + server.beforeRequest = () => { + requests += 1; + if (requests === 4) server.offline = true; + }; + await engine.requestSync(); + expect(count(db, 'contact_card')).toBe(2000); + expect(await engine.answer([['ContactCard/query', { accountId: JMAP_ACCOUNT, limit: 50 }, '0']])).toBeNull(); + expect(await engine.answer([['ContactCard/get', { accountId: JMAP_ACCOUNT, ids: null }, '0']])).toBeNull(); + server.offline = false; + server.beforeRequest = null; + await engine.requestSync(); + expect((await call(engine, 'ContactCard/query', { calculateTotal: true })).total).toBe(2600); + }); + + it('goes on answering from the old copy while a later resync runs', async () => { + const server = new FakeContacts(); + for (let i = 0; i < 2600; i += 1) server.addCard(); + const { db, engine } = await synced(server); + db.run("UPDATE sync_state SET state = 'bad state!' WHERE type = 'ContactCard'"); + let requests = 0; + server.beforeRequest = () => { + requests += 1; + if (requests === 3) server.offline = true; + }; + await engine.requestSync(); + expect((await call(engine, 'ContactCard/query', { calculateTotal: true })).total).toBe(2600); + }); +}); + +describe('paging while cards change', () => { + it('does not skip a card when an earlier one is deleted between requests', async () => { + const server = new FakeContacts(); + const cards = Array.from({ length: 2600 }, () => server.addCard()); + const db = openMemoryStore(); + const engine = engineFor(server, db); + let queries = 0; + const original = server.request.bind(server); + server.request = async (calls: MethodCall[]) => { + if (calls.some(([name]) => name === 'ContactCard/query')) { + queries += 1; + if (queries === 2) server.removeCard(cards[0].id); + } + return original(calls); + }; + await engine.requestSync(); + const kept = new Set(db.all<{ id: string }>('SELECT id FROM contact_card').map((r) => r.id)); + expect(cards.slice(1).filter((c) => !kept.has(c.id))).toEqual([]); + expect(kept.has(cards[0].id)).toBe(false); + const anchored = server.calls.filter(([name, args]) => name === 'ContactCard/query' && args.anchor !== undefined); + expect(anchored.length).toBeGreaterThan(0); + }); + + it('starts over when the card it paged from is gone', async () => { + const server = new FakeContacts(); + const cards = Array.from({ length: 2600 }, () => server.addCard()); + const db = openMemoryStore(); + const engine = engineFor(server, db); + let queries = 0; + const original = server.request.bind(server); + server.request = async (calls: MethodCall[]) => { + if (calls.some(([name]) => name === 'ContactCard/query')) { + queries += 1; + if (queries === 2) server.removeCard(cards[1999].id); + } + return original(calls); + }; + const events: StoreEvent[] = []; + await engine.requestSync(); + expect(events).toEqual([]); + const kept = new Set(db.all<{ id: string }>('SELECT id FROM contact_card').map((r) => r.id)); + expect(kept.size).toBe(2599); + expect(kept.has(cards[1999].id)).toBe(false); + }); +}); /* eslint-enable @typescript-eslint/no-explicit-any */ diff --git a/lib/store/__tests__/helpers/fake-contacts.ts b/lib/store/__tests__/helpers/fake-contacts.ts index e6b408755..6cec59cbf 100644 --- a/lib/store/__tests__/helpers/fake-contacts.ts +++ b/lib/store/__tests__/helpers/fake-contacts.ts @@ -37,6 +37,15 @@ export function changesFor(coll: Collection, args: Record): Rec return { accountId: args.accountId, oldState: since, ...coll.changes(since, maxChanges) }; } +function storedMedia(card: Obj): void { + const media = card.media as Record> | undefined; + for (const [key, entry] of Object.entries(media ?? {})) { + if (typeof entry.uri !== 'string' || !entry.uri.startsWith('data:')) continue; + const { uri, ...rest } = entry; + (media as Record)[key] = { ...rest, blobId: `stored-${uri.length}-${key}` }; + } +} + function applyServerPatch(card: Obj, patch: Record): void { for (const [path, value] of Object.entries(patch)) { const steps = path.split('/'); @@ -63,6 +72,16 @@ export class FakeContacts extends FakeJmap { readonly refuse = new Map(); /** A server that lets two cards have one uid, as nothing in the RFC forbids it. */ allowDuplicateUids = false; + /** Stalwart 1.0 stores a data: URI in `media` as a blob: `uri` goes, `blobId` comes (measured). */ + mediaAsBlobs = false; + /** Set to apply the next ContactCard/set, answer it, and then lose the network. */ + offlineAfterSet = false; + /** Set to lose the network before the next ContactCard/set reaches the server. */ + offlineBeforeSet = false; + /** The most ids ContactCard/get and calls a request take, as a session can say. */ + maxObjectsInGet = 500; + /** Answers a ContactCard/set call as a whole, instead of handling it. */ + failSet: ((args: Record) => { jmapError: string } | { http: number } | null) | null = null; constructor(books: string[] = ['book1']) { super(); @@ -131,7 +150,9 @@ export class FakeContacts extends FakeJmap { notCreated[cid] = { type: 'invalidProperties', description: `Contact with UID ${String(raw.uid)} already exists with id ${clash.id}.`, properties: ['uid'] }; } else { const id = `c${this.nextCard++}`; - account.cards.set(id, { '@type': 'Card', version: '1.0', ...raw, id }); + const stored: Obj = { '@type': 'Card', version: '1.0', ...structuredClone(raw), id }; + if (this.mediaAsBlobs) storedMedia(stored); + account.cards.set(id, stored); account.coll.ContactCard.push('created', id); created[cid] = { id }; } @@ -142,7 +163,8 @@ export class FakeContacts extends FakeJmap { if (refused) notUpdated[id] = refused; else if (!card) notUpdated[id] = { type: 'notFound' }; else { - applyServerPatch(card, patch); + applyServerPatch(card, structuredClone(patch)); + if (this.mediaAsBlobs) storedMedia(card); account.coll.ContactCard.push('updated', id); updated[id] = null; } @@ -187,7 +209,12 @@ export class FakeContacts extends FakeJmap { let list = [...account.cards.values()]; if (filter.uid) list = list.filter((c) => c.uid === filter.uid); if (filter.inAddressBook) list = list.filter((c) => (c.addressBookIds as Record)[filter.inAddressBook as string]); - const position = Number(args.position ?? 0); + let position = Number(args.position ?? 0); + if (args.anchor !== undefined) { + const at = list.findIndex((c) => c.id === args.anchor); + if (at < 0) throw Object.assign(new Error('anchorNotFound'), { jmapError: 'anchorNotFound' }); + position = at + Number(args.anchorOffset ?? 0); + } const limit = Number(args.limit ?? 5000) || 5000; return { accountId: args.accountId, @@ -200,8 +227,8 @@ export class FakeContacts extends FakeJmap { } case 'ContactCard/get': { const ids = args.ids as string[] | null | undefined; - if (ids && ids.length > 500) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' }); - const chosen = ids ?? [...account.cards.keys()].slice(0, 500); + if (ids && ids.length > this.maxObjectsInGet) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' }); + const chosen = ids ?? [...account.cards.keys()].slice(0, this.maxObjectsInGet); return { accountId: args.accountId, state: account.coll.ContactCard.state(), @@ -209,8 +236,12 @@ export class FakeContacts extends FakeJmap { notFound: chosen.filter((id) => !account.cards.has(id)), }; } - case 'ContactCard/set': + case 'ContactCard/set': { + const failure = this.failSet?.(args); + if (failure && 'http' in failure) throw new TransportError('http', `JMAP request failed: ${failure.http}`, failure.http, 'urn:ietf:params:jmap:error:limit'); + if (failure) throw Object.assign(new Error(failure.jmapError), failure); return this.contactSet(account, args); + } default: return super.handle(name, args); } @@ -219,7 +250,15 @@ export class FakeContacts extends FakeJmap { override async request(calls: MethodCall[], using?: string[]): Promise { // The real server fails the whole request for a capability it doesn't list. void using; + if (this.offlineBeforeSet && calls.some(([name]) => name === 'ContactCard/set')) { + this.offlineBeforeSet = false; + throw new TransportError('network', 'Failed to fetch'); + } const response = await super.request(calls); + if (this.offlineAfterSet && calls.some(([name]) => name === 'ContactCard/set')) { + this.offlineAfterSet = false; + this.offline = true; + } if (this.loseNextSet && calls.some(([name]) => name === 'ContactCard/set')) { this.loseNextSet = false; throw new TransportError('network', 'Connection dropped after the server took the request'); diff --git a/lib/store/contacts/methods.ts b/lib/store/contacts/methods.ts index f3c885909..9ef5a391f 100644 --- a/lib/store/contacts/methods.ts +++ b/lib/store/contacts/methods.ts @@ -15,10 +15,14 @@ function jmapOf(ctx: MethodContext, args: Record): string { return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[CONTACTS_CAPABILITY] ?? ctx.defaultJmap; } -/** The last state from the server. Throws when this type was never synced, so an empty answer always means empty. */ +/** + * The last state from the server. Throws until a first full pass finished, so + * an answer never passes off part of a list as the whole. A later resync + * (phase `list`) goes on answering from the copy it's refreshing. + */ function syncedState(ctx: MethodContext, jmap: string, type: string): string | null { const row = ctx.db.get<{ state: string | null; phase: string | null }>('SELECT state, phase FROM sync_state WHERE jmap = ? AND type = ?', [jmap, type]); - if (!row || (!row.state && !row.phase)) throw new MethodError('serverUnavailable', 'Contacts not kept yet'); + if (!row || (!row.state && row.phase !== 'list')) throw new MethodError('serverUnavailable', 'Contacts not kept yet'); return row.state; } diff --git a/lib/store/contacts/patch.ts b/lib/store/contacts/patch.ts index 07247b3d9..61a5c1050 100644 --- a/lib/store/contacts/patch.ts +++ b/lib/store/contacts/patch.ts @@ -67,9 +67,13 @@ export function touchedProperties(patch: Patch): string[] { return [...new Set(Object.keys(patch).map((path) => steps(path)[0]))]; } -/** Deep equality that ignores key order. Absent and null count as the same. */ +function isNothing(value: unknown): boolean { + return value == null || (isObject(value) && Object.keys(value).length === 0); +} + +/** Deep equality that ignores key order. Absent, null and an empty map count as the same. */ export function sameJson(a: unknown, b: unknown): boolean { - if (a === b || (a == null && b == null)) return true; + if (a === b || (isNothing(a) && isNothing(b))) return true; if (Array.isArray(a) && Array.isArray(b)) { return a.length === b.length && a.every((item, i) => sameJson(item, b[i])); } @@ -79,10 +83,27 @@ export function sameJson(a: unknown, b: unknown): boolean { return false; } -/** Whether `actual` holds everything in `wanted`. A server may add properties of its own. */ -export function coversJson(wanted: unknown, actual: unknown): boolean { - if (isObject(wanted) && isObject(actual)) { - return Object.keys(wanted).every((key) => coversJson(wanted[key], actual[key])); +/** Stalwart 1.0 keeps a data: URI in `media` as a blob: `uri` goes and `blobId` comes (measured). */ +function inlineKeptAsBlob(wanted: JsonObject, actual: JsonObject): boolean { + return typeof wanted.uri === 'string' && wanted.uri.startsWith('data:') && actual.uri == null && typeof actual.blobId === 'string'; +} + +/** + * Whether `actual` is `wanted` as far as a server can tell. A key only the + * server has is its own, unless `base` shows it was there before the edit: + * then the edit meant to remove it, and it hasn't happened yet. + */ +export function coversJson(wanted: unknown, actual: unknown, base?: unknown): boolean { + if (!isObject(wanted) || !isObject(actual)) return sameJson(wanted, actual); + const before = isObject(base) ? base : {}; + const inline = inlineKeptAsBlob(wanted, actual); + for (const key of Object.keys(wanted)) { + if (inline && key === 'uri') continue; + if (!coversJson(wanted[key], actual[key], before[key])) return false; } - return sameJson(wanted, actual); + for (const key of Object.keys(actual)) { + if (key in wanted || (inline && key === 'blobId')) continue; + if (before[key] != null) return false; + } + return true; } diff --git a/lib/store/contacts/pending.ts b/lib/store/contacts/pending.ts index 4cf2feba2..f6d2bdc78 100644 --- a/lib/store/contacts/pending.ts +++ b/lib/store/contacts/pending.ts @@ -12,11 +12,21 @@ * - Destroy: done if the card is gone, a conflict if it changed since. * - Create: a request that may have reached the server looks for the card's * uid first, so a lost response doesn't make a second card. Stalwart also - * refuses a second card with the same uid, which ends up the same way. + * refuses a second card with the same uid. Either way the card found is only + * ours if it holds what was sent, or an offline card with someone else's uid + * would silently turn into theirs. + * + * The local copy after sending is always the server's own read-back, never what + * was sent: Stalwart 1.0 stores a data: URI as a blob. A base built from what + * was sent would read that as someone else's change. If the network goes before + * the read-back, that counts as a lost response: the row stays, and the next + * pass finds its edit already there. `sent` is what the last request held, so + * the next pass can tell its own edit from someone else's. */ import type { StoreContext, SyncContext } from '../area'; -import { responseFor, type MethodCall } from '../engine/transport'; +import { requestLimits, type RequestLimits } from '../engine/limits'; +import { JmapMethodError, responseFor, TransportError, type MethodCall } from '../engine/transport'; import type { StorageAdapter } from '../storage'; import { applyPatch, coversJson, InvalidPatchError, mergePatches, sameJson, touchedProperties, type JsonObject, type Patch } from './patch'; import { cardDisplayName, CONTACTS_USING, readCard, removeCard, writeCard, type Card } from './records'; @@ -26,11 +36,12 @@ export const STAND_IN_PREFIX = 'bulwark-local-card-'; const LOCAL_BLOB_MARK = '"blobId":"bulwark-local-'; const SET_BATCH = 100; -const SET_BATCH_BYTES = 5 * 1024 * 1024; -/** The server takes 10 MB a request. A bigger card would hold up everything queued behind it, for good. */ -const MAX_RECORD_BYTES = 8 * 1024 * 1024; -const GET_BATCH = 500; -const CALLS_PER_REQUEST = 16; + +/** Of the session's request limit. A bigger card would never go out. */ +function maxRecordBytes(ctx: StoreContext): number { + return Math.floor(requestLimits(ctx.session).maxSizeRequest * 0.8); +} + /** Set errors that say nothing about the edit itself: it goes out again next time. */ const TRANSIENT_ERRORS = new Set(['serverFail', 'serverUnavailable', 'rateLimit']); /** Server-kept timestamps: their changing doesn't mean someone edited the card. */ @@ -45,6 +56,7 @@ export interface PendingRow { payload: string | null; base: string | null; attempted: number; + sent: string | null; queued_at: number; } @@ -105,7 +117,7 @@ export function queueCreate(ctx: StoreContext, jmap: string, card: JsonObject): const id = `${STAND_IN_PREFIX}${crypto.randomUUID()}`; const uid = typeof card.uid === 'string' && card.uid ? card.uid : `urn:uuid:${crypto.randomUUID()}`; const payload = JSON.stringify({ ...card, uid }); - if (payload.length > MAX_RECORD_BYTES) return TOO_LARGE_ERROR; + if (payload.length > maxRecordBytes(ctx)) return TOO_LARGE_ERROR; savePending(ctx.db, ctx, { jmap, id, op: 'create', uid, payload, base: null }); writeCard(ctx.db, jmap, { '@type': 'Card', version: '1.0', ...card, uid, id }); return { id, uid }; @@ -127,11 +139,11 @@ export function queueUpdate(ctx: StoreContext, jmap: string, rawId: string, patc } if (pending?.op === 'create') { const payload = JSON.stringify(applyPatch(JSON.parse(pending.payload as string) as JsonObject, patch)); - if (payload.length > MAX_RECORD_BYTES) return TOO_LARGE_ERROR; + if (payload.length > maxRecordBytes(ctx)) return TOO_LARGE_ERROR; savePending(ctx.db, ctx, { jmap, id, op: 'create', uid: pending.uid, payload, base: null }); } else { const merged = JSON.stringify(pending ? mergePatches(JSON.parse(pending.payload as string) as Patch, patch) : patch); - if (merged.length > MAX_RECORD_BYTES) return TOO_LARGE_ERROR; + if (merged.length > maxRecordBytes(ctx)) return TOO_LARGE_ERROR; savePending(ctx.db, ctx, { jmap, id, op: 'update', payload: merged, base: pending ? pending.base : JSON.stringify(card) }); } writeCard(ctx.db, jmap, visible); @@ -228,30 +240,44 @@ function withoutStamps(card: JsonObject): JsonObject { return copy; } -async function readServerCards(ctx: SyncContext, jmap: string, ids: string[]): Promise> { +async function readServerCards(ctx: SyncContext, jmap: string, ids: string[], limits: RequestLimits): Promise> { const cards = new Map(); - for (let i = 0; i < ids.length; i += GET_BATCH) { - const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + GET_BATCH) }, 'g']], CONTACTS_USING); + for (let i = 0; i < ids.length; i += limits.maxObjectsInGet) { + const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + limits.maxObjectsInGet) }, 'g']], CONTACTS_USING); for (const card of responseFor(response, 'g')[1].list as Card[]) cards.set(card.id, card); } return cards; } -/** The ids the server has for these uids. */ -async function lookUpUids(ctx: SyncContext, jmap: string, uids: string[]): Promise> { - const found = new Map(); - for (let i = 0; i < uids.length; i += CALLS_PER_REQUEST) { - const group = uids.slice(i, i + CALLS_PER_REQUEST); +/** The server's cards with these uids. */ +async function cardsByUid(ctx: SyncContext, jmap: string, uids: string[], limits: RequestLimits): Promise> { + const ids = new Map(); + for (let i = 0; i < uids.length; i += limits.maxCallsInRequest) { + const group = uids.slice(i, i + limits.maxCallsInRequest); const calls: MethodCall[] = group.map((uid, n) => ['ContactCard/query', { accountId: jmap, filter: { uid }, limit: 1 }, `q${n}`]); const response = await ctx.transport.request(calls, CONTACTS_USING); group.forEach((uid, n) => { - const ids = responseFor(response, `q${n}`)[1].ids as string[]; - if (ids.length > 0) found.set(uid, ids[0]); + const found = responseFor(response, `q${n}`)[1].ids as string[]; + if (found.length > 0) ids.set(uid, found[0]); }); } - return found; + const cards = await readServerCards(ctx, jmap, [...ids.values()], limits); + const byUid = new Map(); + for (const [uid, id] of ids) { + const card = cards.get(id); + if (card) byUid.set(uid, card); + } + return byUid; } +/** Whether a card the server has under our uid is the one we sent. */ +function holdsSent(sentPayload: string, card: Card): boolean { + const sent = JSON.parse(sentPayload) as JsonObject; + return Object.keys(sent).every((key) => coversJson(sent[key], card[key])); +} + +const UID_TAKEN: SetError = { type: 'invalidProperties', description: 'Another contact already has this uid', properties: ['uid'] }; + /** The create went through, or had already. The stand-in row becomes the server's card. */ function settleCreated(ctx: SyncContext, sent: PendingRow, server: Card): void { const { db } = ctx; @@ -282,29 +308,59 @@ function settleCreated(ctx: SyncContext, sent: PendingRow, server: Card): void { interface Decision { create: Map; update: Map; destroy: Map } +/** + * Whether someone else moved one of these properties since the edit was queued. + * The server holding what our last request sent is our own doing, not a change: + * its response may have been lost. + */ +function movedOnServer(row: PendingRow, server: Card, keys: readonly string[]): boolean { + if (!row.base) return false; + const base = JSON.parse(row.base) as JsonObject; + let own: JsonObject | null = null; + if (row.sent) { + try { + own = applyPatch(base, JSON.parse(row.sent) as Patch); + } catch (error) { + if (!(error instanceof InvalidPatchError)) throw error; + } + } + return keys.some((key) => !sameJson(base[key], server[key]) && !(own && coversJson(own[key], server[key], base[key]))); +} + /** What to do with one row now that the server's copy is known. Conflicts are settled here. */ function decide(ctx: SyncContext, row: PendingRow, server: Card | undefined, serverByUid: Map, out: Decision): void { const { db } = ctx; const { jmap, id } = row; const uidHit = row.uid ? serverByUid.get(row.uid) : undefined; if (row.op === 'create') { - if (uidHit) { - settleCreated(ctx, row, { ...(JSON.parse(row.payload as string) as JsonObject), ...uidHit }); - return; + if (!uidHit) { + out.create.set(id, row); + } else if (holdsSent(row.sent ?? (row.payload as string), uidHit)) { + settleCreated(ctx, { ...row, payload: row.sent ?? row.payload }, uidHit); + } else { + recordConflict(ctx, row, 'refused', null, UID_TAKEN); + dropPending(db, jmap, id); + removeCard(db, jmap, id); } - out.create.set(id, row); return; } if (row.op === 'destroy' && id.startsWith(STAND_IN_PREFIX)) { - // A create that may have gone out, deleted since. Without a card on the server, nothing to do. + // A create that may have gone out, deleted since. Only a card holding what was sent is ours to delete. dropPending(db, jmap, id); - if (uidHit) savePending(db, ctx, { jmap, id: uidHit.id, op: 'destroy', payload: null, base: JSON.stringify(uidHit) }); + const sent = row.sent ?? row.payload; + if (uidHit && sent && holdsSent(sent, uidHit)) { + savePending(db, ctx, { jmap, id: uidHit.id, op: 'destroy', payload: null, base: JSON.stringify(uidHit) }); + } return; } if (row.op === 'destroy') { if (!server) { dropPending(db, jmap, id); - } else if (row.base && !sameJson(withoutStamps(JSON.parse(row.base) as JsonObject), withoutStamps(server))) { + return; + } + const stable = withoutStamps(server); + const base = row.base ? withoutStamps(JSON.parse(row.base) as JsonObject) : {}; + if (movedOnServer(row, server, [...new Set([...Object.keys(stable), ...Object.keys(base)])])) { recordConflict(ctx, row, 'changed', server, null); dropPending(db, jmap, id); writeCard(db, jmap, server); @@ -320,7 +376,7 @@ function decide(ctx: SyncContext, row: PendingRow, server: Card | undefined, ser return; } const patch = JSON.parse(row.payload as string) as Patch; - const base = row.base ? JSON.parse(row.base) as JsonObject : null; + const base = row.base ? JSON.parse(row.base) as JsonObject : {}; let applied: Card | null = null; try { applied = applyPatch(server, patch); @@ -328,12 +384,12 @@ function decide(ctx: SyncContext, row: PendingRow, server: Card | undefined, ser if (!(error instanceof InvalidPatchError)) throw error; } const touched = touchedProperties(patch); - if (applied && touched.every((key) => coversJson((applied as Card)[key], server[key]))) { + if (applied && touched.every((key) => coversJson((applied as Card)[key], server[key], base[key]))) { dropPending(db, jmap, id); writeCard(db, jmap, server); return; } - if (!applied || (base && touched.some((key) => !sameJson(base[key], server[key])))) { + if (!applied || movedOnServer(row, server, touched)) { recordConflict(ctx, row, 'changed', server, null); dropPending(db, jmap, id); writeCard(db, jmap, server); @@ -342,125 +398,176 @@ function decide(ctx: SyncContext, row: PendingRow, server: Card | undefined, ser out.update.set(id, row); } -function settleUpdated(ctx: SyncContext, sent: PendingRow, server: Card): void { - const now = pendingFor(ctx.db, sent.jmap, sent.id); +function settleUpdated(ctx: SyncContext, sent: PendingRow, server: Card | undefined): void { + const { db } = ctx; + const now = pendingFor(db, sent.jmap, sent.id); if (!now) return; if (now.op === 'update' && now.payload === sent.payload) { - dropPending(ctx.db, sent.jmap, sent.id); + dropPending(db, sent.jmap, sent.id); + if (server) writeCard(db, sent.jmap, server); return; } - // Edited again while this was on its way: that edit now builds on what was sent. - const base = applyPatch(server, JSON.parse(sent.payload as string) as Patch); - ctx.db.run('UPDATE contact_pending SET base = ? WHERE seq = ?', [JSON.stringify(base), now.seq]); + // Edited again while this was on its way: that edit now builds on what the server holds. + if (!server) return; + db.run('UPDATE contact_pending SET base = ?, sent = NULL, attempted = 0 WHERE seq = ?', [JSON.stringify(server), now.seq]); + if (now.op === 'update') storeServerCard(db, sent.jmap, server); } -function settleRefused(ctx: SyncContext, row: PendingRow, error: SetError, server: Card | undefined): void { +function settleRefused(ctx: SyncContext, sent: PendingRow, error: SetError, server: Card | undefined): void { const { db } = ctx; if (TRANSIENT_ERRORS.has(error.type)) return; - if (row.op === 'destroy' && error.type === 'notFound') { - dropPending(db, row.jmap, row.id); + const now = pendingFor(db, sent.jmap, sent.id); + if (now && (now.op !== sent.op || now.payload !== sent.payload)) { + // Edited again while this was on its way, so the refused edit is still in + // the row. It goes out once more, and a second refusal is the conflict. + db.run('UPDATE contact_pending SET sent = NULL, attempted = 0 WHERE seq = ?', [now.seq]); + return; + } + if (sent.op === 'destroy' && error.type === 'notFound') { + dropPending(db, sent.jmap, sent.id); return; } const kind = error.type === 'notFound' ? 'gone' : 'refused'; - recordConflict(ctx, row, kind, server ?? null, error); - dropPending(db, row.jmap, row.id); - if (server && kind === 'refused') writeCard(db, row.jmap, server); - else removeCard(db, row.jmap, row.id); + recordConflict(ctx, sent, kind, server ?? null, error); + dropPending(db, sent.jmap, sent.id); + const restore = server ?? (sent.base ? JSON.parse(sent.base) as Card : null); + if (restore && kind === 'refused') writeCard(db, sent.jmap, restore); + else removeCard(db, sent.jmap, sent.id); } /** Takes rows up to about the size of one request. */ -function nextBatch(db: StorageAdapter, jmap: string, afterSeq: number): PendingRow[] { - const rows = db.all('SELECT * FROM contact_pending WHERE jmap = ? AND seq > ? ORDER BY seq LIMIT ?', [jmap, afterSeq, SET_BATCH]); +function nextBatch(db: StorageAdapter, jmap: string, afterSeq: number, limits: RequestLimits): PendingRow[] { + const rows = db.all('SELECT * FROM contact_pending WHERE jmap = ? AND seq > ? ORDER BY seq LIMIT ?', [jmap, afterSeq, Math.min(SET_BATCH, limits.maxObjectsInSet)]); + const room = limits.maxSizeRequest / 2; let bytes = 0; const batch: PendingRow[] = []; for (const row of rows) { bytes += row.payload?.length ?? 0; - if (batch.length > 0 && bytes > SET_BATCH_BYTES) break; + if (batch.length > 0 && bytes > room) break; batch.push(row); } return batch; } +/** One exchange with the server for these rows: read, decide, send, read back, settle. */ +async function exchange(ctx: SyncContext, jmap: string, rows: readonly PendingRow[], limits: RequestLimits): Promise { + const { db } = ctx; + const current = await readServerCards(ctx, jmap, rows.filter((row) => row.op !== 'create' && !row.id.startsWith(STAND_IN_PREFIX)).map((row) => row.id), limits); + const serverByUid = await cardsByUid(ctx, jmap, rows.filter((row) => row.uid && row.attempted).map((row) => row.uid as string), limits); + + const out: Decision = { create: new Map(), update: new Map(), destroy: new Map() }; + db.transaction(() => { + for (const { seq, id } of rows) { + // The page may have edited it while the reads were on their way. + const row = pendingFor(db, jmap, id); + if (row && row.seq === seq) decide(ctx, row, current.get(id), serverByUid, out); + } + for (const row of [...out.create.values(), ...out.update.values()]) { + db.run('UPDATE contact_pending SET attempted = 1, sent = payload WHERE seq = ?', [row.seq]); + } + }); + if (out.create.size + out.update.size + out.destroy.size === 0) return; + + const creationIds = new Map([...out.create.values()].map((row) => [`c${row.seq}`, row])); + const response = await ctx.transport.request([['ContactCard/set', { + accountId: jmap, + ...(creationIds.size > 0 ? { create: Object.fromEntries([...creationIds].map(([cid, row]) => [cid, JSON.parse(row.payload as string)])) } : {}), + ...(out.update.size > 0 ? { update: Object.fromEntries([...out.update.values()].map((row) => [row.id, JSON.parse(row.payload as string)])) } : {}), + ...(out.destroy.size > 0 ? { destroy: [...out.destroy.keys()] } : {}), + }, 's']], CONTACTS_USING); + const [, result] = responseFor(response, 's'); + + // What the server now holds. A network failure here is a lost response: the rows stay. + const createdIds = new Map(); + for (const cid of creationIds.keys()) { + const created = result.created?.[cid] as { id?: unknown } | undefined; + if (created?.id) createdIds.set(cid, String(created.id)); + } + const updatedIds = [...out.update.keys()].filter((id) => result.updated && id in result.updated); + const readBack = await readServerCards(ctx, jmap, [...createdIds.values(), ...updatedIds], limits); + // A card already has this uid: maybe this one, sent before. + const uidRefused = [...creationIds].filter(([cid]) => (result.notCreated?.[cid] as SetError | undefined)?.properties?.includes('uid')); + const taken = await cardsByUid(ctx, jmap, uidRefused.map(([, row]) => row.uid as string), limits); + + db.transaction(() => { + for (const [cid, row] of creationIds) { + const realId = createdIds.get(cid); + const refused = result.notCreated?.[cid] as SetError | undefined; + if (realId) { + const card = readBack.get(realId); + if (card) { + settleCreated(ctx, row, card); + } else { + // Created and deleted again already. The delta will say the same. + dropPending(db, jmap, row.id); + removeCard(db, jmap, row.id); + } + } else if (refused?.properties?.includes('uid')) { + const card = taken.get(row.uid as string); + if (card && holdsSent(row.payload as string, card)) settleCreated(ctx, row, card); + else settleRefused(ctx, row, refused, undefined); + } else if (refused) { + settleRefused(ctx, row, refused, undefined); + } + } + for (const [id, row] of out.update) { + const refused = result.notUpdated?.[id] as SetError | undefined; + if (refused) settleRefused(ctx, row, refused, current.get(id)); + else if (updatedIds.includes(id)) settleUpdated(ctx, row, readBack.get(id)); + } + for (const [id, row] of out.destroy) { + const refused = result.notDestroyed?.[id] as SetError | undefined; + if (refused) settleRefused(ctx, row, refused, current.get(id)); + else if ((result.destroyed as string[] | undefined)?.includes(id)) dropPending(db, jmap, id); + } + }); +} + +/** The server said no to the request itself, for a reason that won't go away by itself. */ +function requestRefusal(error: unknown): SetError | null { + if (error instanceof JmapMethodError) return TRANSIENT_ERRORS.has(error.type) ? null : { type: error.type, description: error.message }; + if (error instanceof TransportError && error.kind === 'http' && error.status !== undefined && error.status >= 400 && error.status < 500) { + return { type: 'requestRefused', description: error.message }; + } + return null; +} + +/** One card may be what makes a request fail: the others go one by one, and that one becomes a conflict. */ +async function sendRows(ctx: SyncContext, jmap: string, rows: readonly PendingRow[], limits: RequestLimits): Promise { + try { + await exchange(ctx, jmap, rows, limits); + } catch (error) { + const refusal = requestRefusal(error); + if (!refusal) throw error; + if (rows.length > 1) { + for (const row of rows) { + if (ctx.stopped()) return; + await sendRows(ctx, jmap, [row], limits); + } + return; + } + ctx.db.transaction(() => { + const row = pendingFor(ctx.db, jmap, rows[0].id); + if (row) settleRefused(ctx, row, refusal, undefined); + }); + } +} + /** Sends what's queued for one JMAP account. Network failures reach the engine and the rows stay. */ export async function flushPending(ctx: SyncContext, jmap: string): Promise { - const { db } = ctx; + const limits = requestLimits(ctx.session); let afterSeq = 0; let settled = false; - for (;;) { - if (ctx.stopped()) break; - const batch = nextBatch(db, jmap, afterSeq); - if (batch.length === 0) break; - afterSeq = batch[batch.length - 1].seq; - settled = true; - - const current = await readServerCards(ctx, jmap, batch.filter((row) => row.op !== 'create' && !row.id.startsWith(STAND_IN_PREFIX)).map((row) => row.id)); - const attempted = batch.filter((row) => row.uid && row.attempted).map((row) => row.uid as string); - const uidIds = await lookUpUids(ctx, jmap, attempted); - const uidCards = await readServerCards(ctx, jmap, [...uidIds.values()]); - const serverByUid = new Map(); - for (const [uid, id] of uidIds) { - const card = uidCards.get(id); - if (card) serverByUid.set(uid, card); - } - - const out: Decision = { create: new Map(), update: new Map(), destroy: new Map() }; - db.transaction(() => { - for (const { seq, id } of batch) { - // The page may have edited it while the reads were on their way. - const row = pendingFor(db, jmap, id); - if (row && row.seq === seq) decide(ctx, row, current.get(id), serverByUid, out); - } - for (const row of out.create.values()) db.run('UPDATE contact_pending SET attempted = 1 WHERE seq = ?', [row.seq]); - }); - if (out.create.size + out.update.size + out.destroy.size === 0) continue; - - const creationIds = new Map([...out.create.values()].map((row) => [`c${row.seq}`, row])); - const response = await ctx.transport.request([['ContactCard/set', { - accountId: jmap, - ...(creationIds.size > 0 ? { create: Object.fromEntries([...creationIds].map(([cid, row]) => [cid, JSON.parse(row.payload as string)])) } : {}), - ...(out.update.size > 0 ? { update: Object.fromEntries([...out.update.values()].map((row) => [row.id, JSON.parse(row.payload as string)])) } : {}), - ...(out.destroy.size > 0 ? { destroy: [...out.destroy.keys()] } : {}), - }, 's']], CONTACTS_USING); - const [, result] = responseFor(response, 's'); - - const refusedUid: Array<[PendingRow, SetError]> = []; - db.transaction(() => { - for (const [cid, row] of creationIds) { - const created = result.created?.[cid] as JsonObject | undefined; - const refused = result.notCreated?.[cid] as SetError | undefined; - if (created) { - settleCreated(ctx, row, { ...(JSON.parse(row.payload as string) as JsonObject), ...created, id: String(created.id) }); - } else if (refused?.properties?.includes('uid')) { - refusedUid.push([row, refused]); - } else if (refused) { - settleRefused(ctx, row, refused, undefined); - } - } - for (const [id, row] of out.update) { - const refused = result.notUpdated?.[id] as SetError | undefined; - if (refused) settleRefused(ctx, row, refused, current.get(id)); - else if (result.updated && id in result.updated) settleUpdated(ctx, row, current.get(id) as Card); - } - for (const [id, row] of out.destroy) { - const refused = result.notDestroyed?.[id] as SetError | undefined; - if (refused) settleRefused(ctx, row, refused, current.get(id)); - else if ((result.destroyed as string[] | undefined)?.includes(id)) dropPending(db, jmap, id); - } - }); - - // The server has a card with this uid already: this one, sent before. - if (refusedUid.length > 0) { - const ids = await lookUpUids(ctx, jmap, refusedUid.map(([row]) => row.uid as string)); - const cards = await readServerCards(ctx, jmap, [...ids.values()]); - db.transaction(() => { - for (const [row, refused] of refusedUid) { - const id = ids.get(row.uid as string); - const card = id ? cards.get(id) : undefined; - if (card) settleCreated(ctx, row, { ...(JSON.parse(row.payload as string) as JsonObject), ...card }); - else settleRefused(ctx, row, refused, undefined); - } - }); + try { + for (;;) { + if (ctx.stopped()) break; + const batch = nextBatch(ctx.db, jmap, afterSeq, limits); + if (batch.length === 0) break; + afterSeq = batch[batch.length - 1].seq; + settled = true; + await sendRows(ctx, jmap, batch, limits); } + } finally { + if (settled) ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] }); } - if (settled) ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] }); } diff --git a/lib/store/contacts/schema.ts b/lib/store/contacts/schema.ts index 66b30832c..9aceaf22b 100644 --- a/lib/store/contacts/schema.ts +++ b/lib/store/contacts/schema.ts @@ -48,6 +48,8 @@ export const CONTACT_MIGRATIONS: readonly Migration[] = [ base TEXT, -- A request may have reached the server, so a create looks for its uid first. attempted INTEGER NOT NULL DEFAULT 0, + -- The payload that request held: the server may have this, not the newer payload. + sent TEXT, queued_at INTEGER NOT NULL, UNIQUE (jmap, id) ); diff --git a/lib/store/contacts/sync.ts b/lib/store/contacts/sync.ts index f734263d6..b06ddc835 100644 --- a/lib/store/contacts/sync.ts +++ b/lib/store/contacts/sync.ts @@ -2,18 +2,24 @@ * Contact sync for one JMAP account: first what's queued goes out, then the * address books, then the cards. A full pass over the cards captures the * state first and pages, with its cursor in sync_state, so a reload continues - * where it stopped. Cards the pass didn't see are gone from the server. + * where it stopped. Cards the pass didn't see are gone from the server, so a + * page must not skip any: it goes from the last card read (`anchor`), since a + * position moves when an earlier card is deleted. Both Stalwarts take that. */ import type { SyncContext, Syncer } from '../area'; import { runChangesLoop } from '../engine/changes'; import { deleteObject, putObject } from '../engine/core-schema'; -import { responseFor, type MethodCall } from '../engine/transport'; +import { requestLimits } from '../engine/limits'; +import { JmapMethodError, responseFor, TransportError, type MethodCall } from '../engine/transport'; import { flushPending, serverDestroyed, storeServerCard } from './pending'; import { ADDRESS_BOOK_PROPERTIES, CONTACTS_CAPABILITY, CONTACTS_USING, type Card } from './records'; -const PAGE = 500; -const PAIRS_PER_REQUEST = 4; +const MOST_PAIRS_PER_REQUEST = 4; + +function pageSize(ctx: SyncContext): number { + return requestLimits(ctx.session).maxObjectsInGet; +} // ---- Address books ---- @@ -42,8 +48,9 @@ async function syncBooks(ctx: SyncContext, jmap: string): Promise { apply: async (page) => { const ids = [...page.created, ...page.updated]; const list: Array<{ id: string }> = []; - for (let i = 0; i < ids.length; i += PAGE) { - const response = await ctx.transport.request([['AddressBook/get', { accountId: jmap, ids: ids.slice(i, i + PAGE), properties: ADDRESS_BOOK_PROPERTIES }, 'b']], CONTACTS_USING); + const size = pageSize(ctx); + for (let i = 0; i < ids.length; i += size) { + const response = await ctx.transport.request([['AddressBook/get', { accountId: jmap, ids: ids.slice(i, i + size), properties: ADDRESS_BOOK_PROPERTIES }, 'b']], CONTACTS_USING); list.push(...responseFor(response, 'b')[1].list); } ctx.db.transaction(() => { @@ -59,14 +66,16 @@ async function syncBooks(ctx: SyncContext, jmap: string): Promise { // ---- Cards ---- -interface ListCursor { position: number; gen: number; captured: string } +interface ListCursor { position: number; anchor: string | null; gen: number; captured: string } async function startFullList(ctx: SyncContext, jmap: string): Promise { const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: [] }, 's']], CONTACTS_USING); const captured = String(responseFor(response, 's')[1].state); const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM contact_card WHERE jmap = ?', [jmap])?.gen ?? 0) + 1; - const cursor: ListCursor = { position: 0, gen, captured }; - ctx.db.transaction(() => ctx.setSyncState(jmap, 'ContactCard', { state: null, phase: 'list', cursor: { ...cursor } })); + const cursor: ListCursor = { position: 0, anchor: null, gen, captured }; + // Answers wait for the end of a first pass (phase `first`). A resync has an older copy to give. + const phase = ctx.syncState(jmap, 'ContactCard').state ? 'list' : 'first'; + ctx.db.transaction(() => ctx.setSyncState(jmap, 'ContactCard', { state: null, phase, cursor: { ...cursor } })); return cursor; } @@ -82,27 +91,47 @@ function sweep(ctx: SyncContext, jmap: string, gen: number): void { async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise { const cursor = { ...start }; + const limits = requestLimits(ctx.session); + const page = limits.maxObjectsInGet; + const pairs = Math.max(1, Math.min(MOST_PAIRS_PER_REQUEST, Math.floor(limits.maxCallsInRequest / 2))); for (;;) { if (ctx.stopped()) return false; const calls: MethodCall[] = []; - for (let i = 0; i < PAIRS_PER_REQUEST; i += 1) { - calls.push(['ContactCard/query', { accountId: jmap, position: cursor.position + i * PAGE, limit: PAGE, calculateTotal: i === 0 }, `q${i}`]); + for (let i = 0; i < pairs; i += 1) { + const from = cursor.anchor === null ? { position: i * page } : { anchor: cursor.anchor, anchorOffset: 1 + i * page }; + calls.push(['ContactCard/query', { accountId: jmap, ...from, limit: page, calculateTotal: i === 0 }, `q${i}`]); calls.push(['ContactCard/get', { accountId: jmap, '#ids': { resultOf: `q${i}`, name: 'ContactCard/query', path: '/ids' } }, `g${i}`]); } const response = await ctx.transport.request(calls, CONTACTS_USING); let total: number | null = null; let got = 0; + let reached = 0; let short = false; + let last: string | null = null; const cards: Card[] = []; - for (let i = 0; i < PAIRS_PER_REQUEST && !short; i += 1) { - const [, query] = responseFor(response, `q${i}`); - if (i === 0 && typeof query.total === 'number') total = query.total; - cards.push(...responseFor(response, `g${i}`)[1].list as Card[]); - got += (query.ids as string[]).length; - short = (query.ids as string[]).length < PAGE; + try { + for (let i = 0; i < pairs && !short; i += 1) { + const [, query] = responseFor(response, `q${i}`); + const ids = query.ids as string[]; + if (i === 0 && typeof query.total === 'number') total = query.total; + cards.push(...responseFor(response, `g${i}`)[1].list as Card[]); + got += ids.length; + if (ids.length > 0) last = ids[ids.length - 1]; + reached = Number(query.position ?? 0) + ids.length; + short = ids.length < page; + } + } catch (error) { + if (!(error instanceof JmapMethodError && error.type === 'anchorNotFound')) throw error; + // The card it went from is gone. Going through again costs requests, not cards. + cursor.anchor = null; + cursor.position = 0; + ctx.db.transaction(() => ctx.setSyncState(jmap, 'ContactCard', { cursor: { ...cursor } })); + continue; } - cursor.position += got; - const done = got === 0 || (total === null ? short : cursor.position >= total); + if (last !== null) cursor.anchor = last; + cursor.position = reached; + // The server may clamp a page, so a short one only ends this if there's no total to go by. + const done = got === 0 || (total === null ? short : reached >= total); ctx.db.transaction(() => { for (const card of cards) storeServerCard(ctx.db, jmap, card, cursor.gen); if (done) { @@ -132,8 +161,9 @@ async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: apply: async (page) => { const ids = [...page.created, ...page.updated]; const fetched: Card[] = []; - for (let i = 0; i < ids.length; i += PAGE) { - const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + PAGE) }, 'g']], CONTACTS_USING); + const size = pageSize(ctx); + for (let i = 0; i < ids.length; i += size) { + const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + size) }, 'g']], CONTACTS_USING); fetched.push(...responseFor(response, 'g')[1].list as Card[]); } ctx.db.transaction(() => { @@ -151,7 +181,7 @@ async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: async function syncCards(ctx: SyncContext, jmap: string): Promise { const row = ctx.syncState(jmap, 'ContactCard'); - let cursor = row.phase === 'list' ? row.cursor as unknown as ListCursor : null; + let cursor = row.phase === 'list' || row.phase === 'first' ? row.cursor as unknown as ListCursor : null; if (!cursor && !row.state) cursor = await startFullList(ctx, jmap); if (!cursor && row.state) { const outcome = await syncDelta(ctx, jmap, row.state, row.target); @@ -170,7 +200,14 @@ export const contactsSyncer: Syncer = { name: 'contacts', capability: CONTACTS_CAPABILITY, async run(ctx, jmap) { - await flushPending(ctx, jmap); + try { + await flushPending(ctx, jmap); + } catch (error) { + // No network, no credentials or a busy server is the engine's to handle. + // Anything else leaves the edits queued and mustn't stop contacts syncing. + if (error instanceof TransportError && error.kind !== 'http') throw error; + ctx.emit({ type: 'error', account: ctx.account, area: 'contacts', message: error instanceof Error ? error.message : String(error) }); + } if (ctx.stopped()) return; await syncBooks(ctx, jmap); if (ctx.stopped()) return; diff --git a/lib/store/engine/limits.ts b/lib/store/engine/limits.ts new file mode 100644 index 000000000..e79f8da8e --- /dev/null +++ b/lib/store/engine/limits.ts @@ -0,0 +1,31 @@ +/** What one request may hold: the session's limits, never past the ones the engine was built for. */ + +import type { StoreSession } from './transport'; + +export interface RequestLimits { + maxObjectsInGet: number; + maxObjectsInSet: number; + maxCallsInRequest: number; + maxSizeRequest: number; +} + +const DEFAULTS: RequestLimits = { + maxObjectsInGet: 500, + maxObjectsInSet: 500, + maxCallsInRequest: 16, + maxSizeRequest: 10 * 1024 * 1024, +}; + +export function requestLimits(session: StoreSession): RequestLimits { + const core = (session.capabilities['urn:ietf:params:jmap:core'] ?? {}) as Partial; + const limit = (key: keyof RequestLimits) => { + const value = core[key]; + return typeof value === 'number' && value > 0 ? Math.min(value, DEFAULTS[key]) : DEFAULTS[key]; + }; + return { + maxObjectsInGet: limit('maxObjectsInGet'), + maxObjectsInSet: limit('maxObjectsInSet'), + maxCallsInRequest: limit('maxCallsInRequest'), + maxSizeRequest: limit('maxSizeRequest'), + }; +} From 4c242e6a0cc530f40c1e9af4efc0045d514830e2 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:20:40 +0100 Subject: [PATCH 7/7] fix: keep an account's contacts when only another account could answer --- lib/jmap/client.ts | 36 +++++++++++------- lib/jmap/partial-accounts.ts | 27 ++++++++++++++ lib/store/__tests__/client-contacts.test.ts | 23 ++++++++++++ stores/__tests__/contact-store.test.ts | 41 +++++++++++++++++++++ stores/contact-store.ts | 30 ++++++++++++--- 5 files changed, 138 insertions(+), 19 deletions(-) create mode 100644 lib/jmap/partial-accounts.ts diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index 56258dd86..af3a7e095 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -19,6 +19,7 @@ import { isTaskLikeObject, type ScannedCalendarObject } from "@/lib/calendar-com import { DEFAULT_CALENDAR_COMPONENTS, mkCalendarCollection, newCalendarCollectionName } from "@/lib/webdav/calendar-collection"; import { sanitizeDisplayName, splitMailbox } from "@/lib/rfc5322-mailbox"; import { decodeFileNodeName } from "./filenode-name"; +import { PartialAccountsError } from "./partial-accounts"; import { fileNodeKind, fromDraftFileNode, toDraftFileNodeCreate } from "./filenode-kind"; import { fileNodeDepth } from "./filenode-depth"; import { contactFromWire, contactToWire } from "./contact-wire"; @@ -7327,13 +7328,16 @@ export class JMAPClient implements IJMAPClient { } } - /** Throws when no account could answer, so a failure doesn't read as having no address books. */ + /** + * Throws PartialAccountsError when an account couldn't answer, with what the + * others gave, so a failure doesn't read as that account having no books. + */ async getAllAddressBooks(): Promise { - let answered = 0; + const failed: string[] = []; let failure: unknown; + const allBooks: AddressBook[] = []; + const primaryId = this.getContactsAccountId(); try { - const allBooks: AddressBook[] = []; - const primaryId = this.getContactsAccountId(); const accountIds = this.getContactCapableAccountIds(); for (const accountId of accountIds) { @@ -7356,23 +7360,23 @@ export class JMAPClient implements IJMAPClient { isShared: !isPrimary, })); allBooks.push(...books); - answered += 1; } else if (response.methodResponses?.[0]?.[0] === "error") { const refused = response.methodResponses[0][1]; throw new Error(refused?.description || refused?.type || "AddressBook/get failed"); } } catch (error) { console.error(`Failed to fetch address books for account ${accountId}:`, error); + failed.push(accountId); failure ??= error; } } - if (answered > 0 || !failure) return allBooks; + if (failed.length === 0) return allBooks; } catch (error) { console.error('Failed to fetch all address books:', error); return this.getAddressBooks(); } - throw failure; + throw new PartialAccountsError(allBooks, failed, primaryId, failure); } async createAddressBook(name: string): Promise { @@ -8286,13 +8290,17 @@ export class JMAPClient implements IJMAPClient { } } - /** Throws when no account could answer, so a failure doesn't read as an empty address book. */ + /** + * Throws PartialAccountsError when an account couldn't answer, with what the + * others gave, so a failure doesn't read as that account having no cards. + */ async getAllContacts(): Promise { - let answered = 0; + const failed: string[] = []; let failure: unknown; + let allContacts: ContactCard[] = []; + const primaryId = this.getContactsAccountId(); try { - const allContacts: ContactCard[] = []; - const primaryId = this.getContactsAccountId(); + allContacts = []; const accountIds = this.getContactCapableAccountIds(); for (const accountId of accountIds) { @@ -8313,19 +8321,19 @@ export class JMAPClient implements IJMAPClient { isShared: !isPrimary, })); allContacts.push(...contacts); - answered += 1; } catch (error) { console.error(`Failed to fetch contacts for account ${accountId}:`, error); + failed.push(accountId); failure ??= error; } } - if (answered > 0 || !failure) return allContacts; + if (failed.length === 0) return allContacts; } catch (error) { console.error('Failed to fetch all contacts:', error); return this.getContacts(); } - throw failure; + throw new PartialAccountsError(allContacts, failed, primaryId, failure); } async getContact(contactId: string, accountId?: string): Promise { diff --git a/lib/jmap/partial-accounts.ts b/lib/jmap/partial-accounts.ts new file mode 100644 index 000000000..66d3798a8 --- /dev/null +++ b/lib/jmap/partial-accounts.ts @@ -0,0 +1,27 @@ +/** + * A listing across JMAP accounts where at least one account couldn't answer. + * It carries what the others gave. An account that failed isn't an account + * with nothing in it, so the caller keeps what it had for that one. + */ +export class PartialAccountsError extends Error { + constructor( + readonly items: T[], + readonly failedAccountIds: string[], + readonly primaryAccountId: string, + cause: unknown, + ) { + super(cause instanceof Error ? cause.message : 'An account could not answer'); + this.name = 'PartialAccountsError'; + } +} + +/** By name: `instanceof` fails across Next's bundle copies. */ +export function partialAccounts(error: unknown): PartialAccountsError | null { + return error instanceof Error && error.name === 'PartialAccountsError' ? error as PartialAccountsError : null; +} + +/** What to show after a partial listing: the answers, and what's held for the accounts that failed. */ +export function withFailedAccountsKept(partial: PartialAccountsError, held: readonly T[]): T[] { + const failed = new Set(partial.failedAccountIds); + return [...partial.items, ...held.filter((item) => failed.has(item.accountId ?? partial.primaryAccountId))]; +} diff --git a/lib/store/__tests__/client-contacts.test.ts b/lib/store/__tests__/client-contacts.test.ts index 87d516a34..7e97c18f3 100644 --- a/lib/store/__tests__/client-contacts.test.ts +++ b/lib/store/__tests__/client-contacts.test.ts @@ -6,6 +6,8 @@ import { afterEach, describe, expect, it } from 'vitest'; import { JMAPClient } from '@/lib/jmap/client'; import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks'; +import { partialAccounts } from '@/lib/jmap/partial-accounts'; +import type { AddressBook, ContactCard } from '@/lib/jmap/types'; import { DEFAULT_SCOPE } from '../area'; import { contactsArea } from '../contacts'; import { AccountEngine } from '../engine/account-engine'; @@ -80,6 +82,27 @@ describe('the client over the store', () => { expect(books.map((b) => b.id).sort()).toEqual(['book1', `${SHARED_ACCOUNT}:teambook`]); }); + it('says which account failed, with what the others gave', async () => { + const { client, engine } = await offlineClient(seeded()); + const failPrimary: Partial = { + skipNetwork: () => true, + answer: async (_client, calls) => (calls[0][1].accountId === JMAP_ACCOUNT + ? { methodResponses: calls.map(([, , id]) => ['error', { type: 'serverUnavailable' }, id]) } + : engine.answer(calls)), + record: () => {}, + noteNetwork: () => {}, + }; + setOfflineHooks(failPrimary as OfflineHooks); + const contacts = await client.getAllContacts().catch((error: unknown) => error); + const partial = partialAccounts(contacts); + expect(partial?.failedAccountIds).toEqual([JMAP_ACCOUNT]); + expect(partial?.primaryAccountId).toBe(JMAP_ACCOUNT); + expect(partial?.items.map((c) => c.name?.full)).toEqual(['Dora Team']); + const books = partialAccounts(await client.getAllAddressBooks().catch((error: unknown) => error)); + expect(books?.failedAccountIds).toEqual([JMAP_ACCOUNT]); + expect(books?.items.map((b) => b.originalId)).toEqual(['teambook']); + }); + it('searches with its own filter', async () => { const { client } = await offlineClient(seeded()); expect((await client.searchContacts('müll')).map((c) => c.name?.full)).toEqual(['Carla Müller']); diff --git a/stores/__tests__/contact-store.test.ts b/stores/__tests__/contact-store.test.ts index f9dfabcca..f52aa4e11 100644 --- a/stores/__tests__/contact-store.test.ts +++ b/stores/__tests__/contact-store.test.ts @@ -13,6 +13,7 @@ import type { AddressBook, ContactCard, ContactMedia } from '@/lib/jmap/types'; import type { IJMAPClient } from '@/lib/jmap/client-interface'; import { clearContactPhotoCache } from '@/lib/contact-photo-cache'; import { PrincipalsRequestError } from '@/lib/jmap/principals-error'; +import { PartialAccountsError } from '@/lib/jmap/partial-accounts'; import { setClientLookup } from '../client-registry'; vi.stubGlobal('crypto', { randomUUID: () => '00000000-0000-0000-0000-000000000000' }); @@ -1387,6 +1388,46 @@ describe('contact-store', () => { expect(state.addressBooks.map((b) => b.id)).toEqual(['acc-a::booka', 'acc-b::bookb']); }); + it('keeps the primary account\'s cards when only a shared account answered', async () => { + const refusal = new PartialAccountsError([makeContact({ id: 'shared:s2', accountId: 'shared', isShared: true })], ['me'], 'me', new Error('offline')); + const failing = { getAllContacts: vi.fn().mockRejectedValue(refusal) } as unknown as IJMAPClient; + useContactStore.setState({ + contacts: [ + makeContact({ id: 'p1', accountId: 'me' }), + // A card just created has no accountId yet: it's the primary account's. + makeContact({ id: 'p2' }), + makeContact({ id: 'shared:s1', accountId: 'shared', isShared: true }), + ], + }); + await useContactStore.getState().fetchContacts(failing); + const state = useContactStore.getState(); + expect(state.contacts.map((c) => c.id).sort()).toEqual(['p1', 'p2', 'shared:s2']); + expect(state.error).toBe('Failed to fetch contacts'); + expect(state.isLoading).toBe(false); + }); + + it('keeps the primary account\'s books when only a shared account answered', async () => { + const refusal = new PartialAccountsError([{ id: 'shared:b', name: 'B', accountId: 'shared' } as AddressBook], ['me'], 'me', new Error('offline')); + const failing = { getAllAddressBooks: vi.fn().mockRejectedValue(refusal) } as unknown as IJMAPClient; + useContactStore.setState({ addressBooks: [{ id: 'mine', name: 'Mine', accountId: 'me' } as AddressBook] }); + await useContactStore.getState().fetchAddressBooks(failing); + expect(useContactStore.getState().addressBooks.map((b) => b.id).sort()).toEqual(['mine', 'shared:b']); + }); + + it('keeps a failed JMAP account of one login while its others refresh', async () => { + const refusal = new PartialAccountsError([makeContact({ id: 's2', accountId: 'shared' })], ['me'], 'me', new Error('offline')); + const failing = { getAllContacts: vi.fn().mockRejectedValue(refusal) } as unknown as IJMAPClient; + useContactStore.setState({ + contacts: [ + makeContact({ id: 'acc-a::p1', accountId: 'me', localAccountId: 'acc-a' }), + makeContact({ id: 'acc-a::s1', accountId: 'shared', localAccountId: 'acc-a' }), + makeContact({ id: 'acc-b::x', accountId: 'me', localAccountId: 'acc-b' }), + ], + }); + await useContactStore.getState().fetchAllAccountsContacts([{ client: failing, localAccountId: 'acc-a' }]); + expect(useContactStore.getState().contacts.map((c) => c.id).sort()).toEqual(['acc-a::p1', 'acc-a::s2']); + }); + it('keeps the list when the only account fails', async () => { const failing = { getAllContacts: vi.fn().mockRejectedValue(new Error('offline')) } as unknown as IJMAPClient; useContactStore.setState({ contacts: [makeContact()] }); diff --git a/stores/contact-store.ts b/stores/contact-store.ts index 3403088b2..924dfc99a 100644 --- a/stores/contact-store.ts +++ b/stores/contact-store.ts @@ -13,6 +13,7 @@ import { sanitizeDisplayName, splitMailbox } from '@/lib/rfc5322-mailbox'; import { inlineContactMedia, writeContactWithNewPhoto, type ContactSaveOptions } from '@/lib/contact-media'; import { contactToView } from '@/lib/plugin-projection'; import { PrincipalsRequestError } from '@/lib/jmap/principals-error'; +import { PartialAccountsError, partialAccounts, withFailedAccountsKept } from '@/lib/jmap/partial-accounts'; import type { Principal } from '@/lib/jmap/types'; import { currentStoreEpoch } from '@/lib/store-epoch'; @@ -614,7 +615,14 @@ export const useContactStore = create()( } catch (error) { if (currentStoreEpoch() !== epoch) return; console.error('Failed to fetch contacts:', error); - set({ error: 'Failed to fetch contacts', isLoading: false, contactsLoaded: true }); + // An account that couldn't answer keeps its cards; the others are fresh. + const partial = partialAccounts(error); + set((state) => ({ + error: 'Failed to fetch contacts', + isLoading: false, + contactsLoaded: true, + contacts: partial ? withFailedAccountsKept(partial, state.contacts) : state.contacts, + })); } }, @@ -656,7 +664,11 @@ export const useContactStore = create()( } catch (error) { if (currentStoreEpoch() !== epoch) return; console.error('Failed to fetch address books:', error); - set({ error: 'Failed to fetch address books' }); + const partial = partialAccounts(error); + set((state) => ({ + error: 'Failed to fetch address books', + addressBooks: partial ? withFailedAccountsKept(partial, state.addressBooks) : state.addressBooks, + })); } }, @@ -671,8 +683,12 @@ export const useContactStore = create()( return prefixContactsWithLocalAccount(list, localAccountId); } catch (error) { debug.error(`Failed to fetch contacts for account ${localAccountId}:`, error); - // Keep what this account's cards were: it just couldn't answer. - return get().contacts.filter((contact) => contact.localAccountId === localAccountId); + // Keep what this login's cards were: it just couldn't answer. + const held = get().contacts.filter((contact) => contact.localAccountId === localAccountId); + const partial = partialAccounts(error); + if (!partial) return held; + const fresh = prefixContactsWithLocalAccount(partial.items, localAccountId); + return withFailedAccountsKept(new PartialAccountsError(fresh, partial.failedAccountIds, partial.primaryAccountId, error), held); } }), ); @@ -695,7 +711,11 @@ export const useContactStore = create()( return prefixAddressBooksWithLocalAccount(list, localAccountId); } catch (error) { debug.error(`Failed to fetch address books for account ${localAccountId}:`, error); - return get().addressBooks.filter((book) => book.localAccountId === localAccountId); + const held = get().addressBooks.filter((book) => book.localAccountId === localAccountId); + const partial = partialAccounts(error); + if (!partial) return held; + const fresh = prefixAddressBooksWithLocalAccount(partial.items, localAccountId); + return withFailedAccountsKept(new PartialAccountsError(fresh, partial.failedAccountIds, partial.primaryAccountId, error), held); } }), );