From 96a6b3832132e281fbd8b6ca1f310ba7edd0de5e Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:19:21 +0200 Subject: [PATCH] feat: sync the device's contacts and calendars with the server in the background #34 The sync adapters from ed2525e start a headless task for every sync, but the task only reported "disabled". This adds the engine it runs (`src/device-sync/engine`, `src/device-sync/jmap`): for one Android account and authority it downloads the server's changes, uploads what the device changed, and hands a report back to SyncManager. - Downloads follow `/changes` from the state kept in the provider's SyncState. Rows and the state they reach are written in one batch, so a killed run resumes where it stopped. Without a state, after `cannotCalculateChanges` or after the user discarded deletions, a full reconcile compares ids instead. - Uploads go in three phases across accounts (creates, updates, deletions), so moving an item to another account never destroys it before the copy exists. A create carries a uid written to the device first, and the server is asked for that uid before creating, so a lost response or a killed run never makes a duplicate. Updates are patches guarded by `ifInState`; nothing is replaced whole. - More than 50 deletions that are also more than a fifth of the synced items hold every deletion back and report `tooManyDeletions`, so Android asks the user; "undo" restores the items from the server. - An item the server refuses is marked and not sent again until it changes or its back-off (1 hour, doubling to a day) ends. Such refusals and planner errors stay with their item; the rest of the run goes on. - Runs of one account and authority take a lock, stop at checkpoints when cancelled or close to the deadline, and never throw: every failure becomes an outcome, and a rejected sign-in posts one "Sign in again" notification. - The JMAP side uses a client of its own per app account, never the UI's, since a run can share the JS runtime with the app. Turning sync off or signing out runs the same upload first (`teardownAuthority`) and deletes the rows only once nothing is waiting, or when the user agreed to lose it. The planners that map items to rows land in the next commits; until then a run fails before it touches a row. Checked: 94 engine tests against the fake provider and JMAP server (sync flows, failures, crashes and retries, teardown), and `npm run typecheck`, `npm test` and `npm run i18n:check` on this commit. --- locales/rn/en.json | 5 + .../__tests__/engine/calendar-flow.test.ts | 259 ++++ .../__tests__/engine/calendar-passes.test.ts | 119 ++ .../__tests__/engine/contacts-flow.test.ts | 372 +++++ .../__tests__/engine/failures.test.ts | 610 ++++++++ src/device-sync/__tests__/engine/harness.ts | 196 +++ .../__tests__/engine/internals.test.ts | 328 +++++ src/device-sync/__tests__/engine/task.test.ts | 63 + .../__tests__/engine/toy-planners.ts | 657 +++++++++ src/device-sync/engine/batch.ts | 216 +++ src/device-sync/engine/calendar-sync.ts | 663 +++++++++ src/device-sync/engine/checkpoint.ts | 55 + src/device-sync/engine/contacts-sync.ts | 743 ++++++++++ src/device-sync/engine/context.ts | 39 + src/device-sync/engine/deps.ts | 113 ++ src/device-sync/engine/errors.ts | 88 ++ src/device-sync/engine/index.ts | 10 + src/device-sync/engine/item-sync.ts | 1303 +++++++++++++++++ src/device-sync/engine/kinds.ts | 95 ++ src/device-sync/engine/mutex.ts | 70 + src/device-sync/engine/poison.ts | 34 + src/device-sync/engine/provider.ts | 77 + src/device-sync/engine/report.ts | 95 ++ src/device-sync/engine/run.ts | 232 +++ src/device-sync/engine/selection.ts | 36 + src/device-sync/engine/sync-state.ts | 188 +++ src/device-sync/engine/teardown.ts | 127 ++ src/device-sync/jmap/caller.ts | 333 +++++ src/device-sync/jmap/client-port.ts | 87 ++ src/device-sync/jmap/connection.ts | 62 + src/device-sync/jmap/errors.ts | 62 + src/device-sync/jmap/session.ts | 87 ++ src/device-sync/planner.ts | 13 +- src/device-sync/task.ts | 195 ++- 34 files changed, 7608 insertions(+), 24 deletions(-) create mode 100644 src/device-sync/__tests__/engine/calendar-flow.test.ts create mode 100644 src/device-sync/__tests__/engine/calendar-passes.test.ts create mode 100644 src/device-sync/__tests__/engine/contacts-flow.test.ts create mode 100644 src/device-sync/__tests__/engine/failures.test.ts create mode 100644 src/device-sync/__tests__/engine/harness.ts create mode 100644 src/device-sync/__tests__/engine/internals.test.ts create mode 100644 src/device-sync/__tests__/engine/task.test.ts create mode 100644 src/device-sync/__tests__/engine/toy-planners.ts create mode 100644 src/device-sync/engine/batch.ts create mode 100644 src/device-sync/engine/calendar-sync.ts create mode 100644 src/device-sync/engine/checkpoint.ts create mode 100644 src/device-sync/engine/contacts-sync.ts create mode 100644 src/device-sync/engine/context.ts create mode 100644 src/device-sync/engine/deps.ts create mode 100644 src/device-sync/engine/errors.ts create mode 100644 src/device-sync/engine/index.ts create mode 100644 src/device-sync/engine/item-sync.ts create mode 100644 src/device-sync/engine/kinds.ts create mode 100644 src/device-sync/engine/mutex.ts create mode 100644 src/device-sync/engine/poison.ts create mode 100644 src/device-sync/engine/provider.ts create mode 100644 src/device-sync/engine/report.ts create mode 100644 src/device-sync/engine/run.ts create mode 100644 src/device-sync/engine/selection.ts create mode 100644 src/device-sync/engine/sync-state.ts create mode 100644 src/device-sync/engine/teardown.ts create mode 100644 src/device-sync/jmap/caller.ts create mode 100644 src/device-sync/jmap/client-port.ts create mode 100644 src/device-sync/jmap/connection.ts create mode 100644 src/device-sync/jmap/errors.ts create mode 100644 src/device-sync/jmap/session.ts diff --git a/locales/rn/en.json b/locales/rn/en.json index b6c558d8..b880e06a 100644 --- a/locales/rn/en.json +++ b/locales/rn/en.json @@ -252,6 +252,11 @@ "deep_link": { "open_failed": "Couldn't open this" }, + "device_sync": { + "auth_problem_text": "The server no longer accepts the sign-in of {account}, so its contacts and calendars on this device stopped syncing. Sign in again to resume. Changes made on this device are kept.", + "auth_problem_title": "Sign in again to keep syncing", + "problem_channel": "Sync problems" + }, "email_composer": { "attach_camera": "Camera", "attachment_reminder_body": "Your message mentions \"{keyword}\" but no file is attached. Send anyway?", diff --git a/src/device-sync/__tests__/engine/calendar-flow.test.ts b/src/device-sync/__tests__/engine/calendar-flow.test.ts new file mode 100644 index 00000000..9025737e --- /dev/null +++ b/src/device-sync/__tests__/engine/calendar-flow.test.ts @@ -0,0 +1,259 @@ +// Engine runs for calendars, and the teardown the app calls when sync is +// turned off (docs/device-sync.md, "Calendar mapping", "App integration"). + +import { describe, expect, it } from 'vitest'; +import { Events } from '../../android-columns'; +import { parseJsonColumn } from '../../common/json'; +import type { PoisonMarker } from '../../planner'; +import { + addDeviceContact, + addServerCards, + ANDROID_ACCOUNT, + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + createHarness, + renameDeviceContact, + type Harness, +} from './harness'; + +const START = '2026-09-28T09:00:00'; + +function addServerEvent(h: Harness, title: string, calendarId = h.calendar, extra: Record = {}): string { + return h.server.addEvent('a', { + uid: `uid-${title}`, + title, + start: START, + duration: 'PT1H', + timeZone: 'Etc/UTC', + calendarIds: { [calendarId]: true }, + ...extra, + }); +} + +function calendarRow(h: Harness, calendarId: string) { + return h.device.rows('calendars').find((c) => c._sync_id === `a/${calendarId}`); +} + +function titles(h: Harness): string[] { + return h.events().map((e) => String(e[Events.TITLE])).sort(); +} + +describe('device sync engine: calendar', () => { + it('writes calendar rows for the synced calendars and downloads their events, never tasks', async () => { + const h = createHarness(); + const standup = addServerEvent(h, 'Standup'); + h.server.addEvent('a', { uid: 't1', '@type': 'Task', title: 'Buy milk', calendarIds: { [h.calendar]: true } }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { created: 1 } } }); + expect(calendarRow(h, h.calendar)).toMatchObject({ ownerAccount: 'alice@example.com', sync_events: 1, name: 'Calendar' }); + expect(h.events()).toEqual([expect.objectContaining({ title: 'Standup', _sync_id: `a/${standup}`, uid2445: 'uid-Standup' })]); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ + accounts: { a: { selected: [`a/${h.calendar}`], reconcile: null } }, + deviceZone: 'Europe/Berlin', + reminderOwner: 'device', + }); + }); + + it('uploads a new device event, an edit and a deletion', async () => { + const h = createHarness(); + const standup = addServerEvent(h, 'Standup'); + const retro = addServerEvent(h, 'Retro'); + await h.run(CALENDAR_AUTHORITY); + const rowId = Number(calendarRow(h, h.calendar)!._id); + const byTitle = (t: string) => h.events().find((e) => e.title === t)!; + + h.device.user.insertEvent(rowId, { title: 'Lunch', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' }); + h.device.user.updateEvent(Number(byTitle('Standup')._id), { title: 'Daily standup' }); + h.device.user.deleteEvent(Number(byTitle('Retro')._id)); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 1, updated: 1, deleted: 1 } } }); + expect(h.server.get('CalendarEvent', 'a', standup)).toMatchObject({ title: 'Daily standup' }); + expect(h.server.get('CalendarEvent', 'a', retro)).toBeUndefined(); + const lunch = h.server.all('CalendarEvent', 'a').find((e) => e.title === 'Lunch')!; + expect(lunch).toMatchObject({ start: '2026-09-28T12:00:00', calendarIds: { [h.calendar]: true } }); + expect(titles(h)).toEqual(['Daily standup', 'Lunch']); + expect(byTitle('Lunch')).toMatchObject({ _sync_id: `a/${lunch.id}`, dirty: 0 }); + }); + + it('gives a calendar that holds only tasks no row', async () => { + const h = createHarness(); + const todo = h.server.addCalendar('a', { name: 'To do' }); + for (let i = 0; i < 3; i++) h.server.addEvent('a', { uid: `t${i}`, '@type': 'Task', title: `Task ${i}`, calendarIds: { [todo]: true } }); + addServerEvent(h, 'Standup'); + + await h.run(CALENDAR_AUTHORITY); + + expect(calendarRow(h, todo)).toBeUndefined(); + expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.taskOnly).toEqual([todo]); + expect(titles(h)).toEqual(['Standup']); + }); + + it('drops a deselected calendar once its events are uploaded, keeping the others', async () => { + const h = createHarness(); + const work = h.server.addCalendar('a', { name: 'Work' }); + addServerEvent(h, 'Standup'); + const review = addServerEvent(h, 'Review', work); + addServerEvent(h, 'Planning', work); + await h.run(CALENDAR_AUTHORITY); + expect(titles(h)).toEqual(['Planning', 'Review', 'Standup']); + + h.device.user.updateEvent(Number(h.events().find((e) => e.title === 'Review')!._id), { title: 'Code review' }); + h.prefs.calendarSelection[`a/${work}`] = false; + await h.run(CALENDAR_AUTHORITY); + + expect(h.server.get('CalendarEvent', 'a', review)).toMatchObject({ title: 'Code review' }); + expect(calendarRow(h, work)).toBeUndefined(); + expect(titles(h)).toEqual(['Standup']); + expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`]); + }); + + it('keeps a deselected calendar while one of its events could not be uploaded', async () => { + const h = createHarness(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const review = addServerEvent(h, 'Review', work); + addServerEvent(h, 'Planning', work); + await h.run(CALENDAR_AUTHORITY); + h.device.user.updateEvent(Number(h.events().find((e) => e.title === 'Review')!._id), { title: 'Code review' }); + h.server.setErrorFor('CalendarEvent', 'a', review, { type: 'tooLarge' }); + h.prefs.calendarSelection[`a/${work}`] = false; + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${review}`, type: 'tooLarge' })]); + expect(calendarRow(h, work)).toBeDefined(); + expect(titles(h)).toEqual(['Code review']); + expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`, `a/${work}`]); + }); + + it('marks a calendar that mirrors a feed subscription read-only', async () => { + const h = createHarness(); + const feed = h.server.addCalendar('a', { name: 'Holidays' }); + h.deps.subscriptionCalendars = async () => [{ jmapAccountId: null, calendarId: feed }]; + + await h.run(CALENDAR_AUTHORITY); + + expect(calendarRow(h, feed)).toMatchObject({ calendar_access_level: 200 }); + expect(calendarRow(h, h.calendar)).toMatchObject({ calendar_access_level: 700 }); + }); + + it('never gives a device event the identity of an unrelated event that has its uid (event uids are unique per account)', async () => { + const h = createHarness(); + const hidden = h.server.addCalendar('a', { name: 'Hidden' }); + h.prefs.calendarSelection[`a/${hidden}`] = false; + const other = addServerEvent(h, 'Other', hidden, { uid: 'dup' }); + await h.run(CALENDAR_AUTHORITY); + const rowId = Number(calendarRow(h, h.calendar)!._id); + const id = h.device.user.insertEvent(rowId, { title: 'Mine', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' }); + // An app wrote the uid, and the claim keeps it. + await h.device.port(ANDROID_ACCOUNT, CALENDAR_AUTHORITY).applyBatch([ + { + op: 'update', + table: 'events', + id, + values: { _sync_id: '~pending/dup', uid2445: 'dup', sync_data3: JSON.stringify({ uid: 'dup', target: `a/${h.calendar}` }) }, + }, + ]); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.itemErrors).toEqual([expect.objectContaining({ type: 'uidConflict' })]); + expect(h.server.get('CalendarEvent', 'a', other)).toMatchObject({ title: 'Other', calendarIds: { [hidden]: true } }); + const row = h.events().find((e) => Number(e._id) === id)!; + expect(row).toMatchObject({ title: 'Mine', _sync_id: '~pending/dup' }); + expect(parseJsonColumn(row.sync_data5)).toMatchObject({ type: 'uidConflict', n: 1 }); + }); + + it('adopts an event its lost create left on the server, with a lagging uid index', async () => { + const h = createHarness(); + addServerEvent(h, 'Standup'); + await h.run(CALENDAR_AUTHORITY); + h.server.setUidIndexLag(3); + const rowId = Number(calendarRow(h, h.calendar)!._id); + const id = h.device.user.insertEvent(rowId, { title: 'Lunch', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' }); + h.server.applyThenLoseResponse({ match: 'CalendarEvent/set' }); + + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('io'); + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + + expect(h.server.all('CalendarEvent', 'a').map((e) => e.title).sort()).toEqual(['Lunch', 'Standup']); + expect(h.events()).toHaveLength(2); + expect(h.events().find((e) => Number(e._id) === id)).toMatchObject({ dirty: 0, _sync_id: expect.stringMatching(/^a\//) }); + }); +}); + +describe('device sync engine: teardown', () => { + it('uploads waiting changes, then removes every row and the SyncState of the authority', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + addDeviceContact(h, 'Hedy'); + + const result = await h.teardown(CONTACTS_AUTHORITY); + + expect(result).toEqual({ pending: 0 }); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(h.serverNames()).toEqual(['Ada King', 'Hedy']); + expect(h.device.rows('raw_contacts')).toEqual([]); + expect(h.device.rows('groups')).toEqual([]); + expect(h.device.rows('settings')).toEqual([]); + expect(h.device.readSyncState(ANDROID_ACCOUNT, CONTACTS_AUTHORITY)).toBe(''); + }); + + it('deletes nothing while changes could not be uploaded, unless forced', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + addDeviceContact(h, 'Hedy'); + h.deps.jmap = async () => { + const error = new Error('Network request failed'); + error.name = 'NetworkError'; + throw error; + }; + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 1 }); + expect(h.contacts()).toHaveLength(2); + + expect(await h.teardown(CONTACTS_AUTHORITY, { force: true })).toEqual({ pending: 0 }); + expect(h.contacts()).toHaveLength(0); + expect(h.serverNames()).toEqual(['Ada']); + }); + + it('holds mass deletions back like a sync run, and asks', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 80 }, (_, i) => `P${i}`)); + await h.run(); + for (const contact of h.contacts().slice(0, 60)) h.device.user.deleteContact(contact.id); + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 60 }); + expect(h.serverNames()).toHaveLength(80); + }); + + it('removes the calendar rows, and their events with them', async () => { + const h = createHarness(); + addServerEvent(h, 'Standup'); + await h.run(CALENDAR_AUTHORITY); + + expect(await h.teardown(CALENDAR_AUTHORITY)).toEqual({ pending: 0 }); + expect(h.device.rows('calendars')).toEqual([]); + expect(h.device.rows('events')).toEqual([]); + }); + + it('stops a running sync of the account and waits for it', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 200 }, (_, i) => `P${i}`)); + let teardown: Promise<{ pending: number }> | null = null; + h.checkpoints.onCheckpoint = (n) => { + if (n === 3) teardown = h.teardown(CONTACTS_AUTHORITY); + }; + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'cancelled' }); + expect(await teardown).toEqual({ pending: 0 }); + expect(h.contacts()).toHaveLength(0); + }); +}); diff --git a/src/device-sync/__tests__/engine/calendar-passes.test.ts b/src/device-sync/__tests__/engine/calendar-passes.test.ts new file mode 100644 index 00000000..f3cc50f2 --- /dev/null +++ b/src/device-sync/__tests__/engine/calendar-passes.test.ts @@ -0,0 +1,119 @@ +// The calendar passes that need no server writes (device zone change, +// reminder owner change) and the local checks before an upload (a series +// split through CONTENT_EXCEPTION_URI). + +import { describe, expect, it, vi } from 'vitest'; +import { Events } from '../../android-columns'; +import type { CalendarPlanner } from '../../planner'; +import { ANDROID_ACCOUNT, CALENDAR_AUTHORITY, createHarness, type Harness } from './harness'; +import { toyCalendarPlanner } from './toy-planners'; + +function withPlanner(h: Harness, overrides: Partial): CalendarPlanner { + const planner = { ...toyCalendarPlanner, ...overrides } as CalendarPlanner; + h.deps.planners = { ...h.deps.planners, calendar: planner }; + return planner; +} + +function addServerEvent(h: Harness, title: string): string { + return h.server.addEvent('a', { uid: `uid-${title}`, title, start: '2026-09-28T09:00:00', duration: 'PT1H', calendarIds: { [h.calendar]: true } }); +} + +describe('device sync engine: calendar passes', () => { + it('rewrites clean events for a new device zone, and stores the zone with the last chunk', async () => { + const h = createHarness({ tuning: { chunkSize: 2 } }); + for (const title of ['A', 'B', 'C']) addServerEvent(h, title); + await h.run(CALENDAR_AUTHORITY); + const planZoneChange = vi.fn((local, previousZone: string, ctx) => ({ + ref: String(local.syncId), + ops: [{ op: 'update' as const, table: 'events' as const, id: local.eventId, values: { eventTimezone: ctx.deviceZone } }], + previousZone, + })); + withPlanner(h, { planZoneChange }); + h.deps.deviceZone = () => 'America/New_York'; + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.outcome).toBe('ok'); + expect(planZoneChange).toHaveBeenCalledTimes(3); + expect(planZoneChange.mock.calls.every(([, previous]) => previous === 'Europe/Berlin')).toBe(true); + expect(h.events().map((e) => e[Events.EVENT_TIMEZONE])).toEqual(['America/New_York', 'America/New_York', 'America/New_York']); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'America/New_York', deviceZonePending: null }); + }); + + it('finishes an interrupted zone pass in the next run', async () => { + const h = createHarness({ tuning: { chunkSize: 1 } }); + for (const title of ['A', 'B', 'C']) addServerEvent(h, title); + await h.run(CALENDAR_AUTHORITY); + const planZoneChange = vi.fn((local: { eventId: number; syncId: string | null }) => ({ + ref: String(local.syncId), + ops: [{ op: 'update' as const, table: 'events' as const, id: local.eventId, values: { eventTimezone: 'America/New_York' } }], + })); + withPlanner(h, { planZoneChange }); + h.deps.deviceZone = () => 'America/New_York'; + let zoneChunks = 0; + h.checkpoints.onCheckpoint = () => { + if (planZoneChange.mock.calls.length > 0 && ++zoneChunks === 2) h.checkpoints.crashAt = h.checkpoints.count; + }; + + await h.run(CALENDAR_AUTHORITY); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'Europe/Berlin', deviceZonePending: 'America/New_York' }); + + h.checkpoints.crashAt = null; + h.checkpoints.onCheckpoint = undefined; + await h.run(CALENDAR_AUTHORITY); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'America/New_York', deviceZonePending: null }); + expect(h.events().every((e) => e[Events.EVENT_TIMEZONE] === 'America/New_York')).toBe(true); + }); + + it('uploads with the old reminder owner, then rewrites the reminders and the calendar rows for the new one', async () => { + const h = createHarness(); + addServerEvent(h, 'Standup'); + await h.run(CALENDAR_AUTHORITY); + expect(h.device.rows('calendars')[0]).toMatchObject({ maxReminders: 5 }); + const owners: string[] = []; + const planReminderOwnerChange = vi.fn((local: { eventId: number; syncId: string | null }, ctx: { reminderOwner: string }) => { + owners.push(ctx.reminderOwner); + return { ref: String(local.syncId), ops: [{ op: 'delete' as const, table: 'reminders' as const, where: 'event_id = ?', args: [local.eventId] }] }; + }); + const planUpload = vi.fn(toyCalendarPlanner.planUpload); + withPlanner(h, { planReminderOwnerChange, planUpload }); + h.device.user.setReminders(Number(h.events()[0]._id), [{ minutes: 10 }]); + h.device.user.updateEvent(Number(h.events()[0]._id), { title: 'Daily standup' }); + h.prefs.reminderOwner = 'bulwark'; + + await h.run(CALENDAR_AUTHORITY); + + expect(planUpload.mock.calls.map(([, ctx]) => ctx.reminderOwner)).toEqual(['device']); + expect(owners).toEqual(['bulwark']); + expect(h.device.rows('reminders')).toEqual([]); + expect(h.device.rows('calendars')[0]).toMatchObject({ maxReminders: 0 }); + expect(h.state(CALENDAR_AUTHORITY)?.reminderOwner).toBe('bulwark'); + }); + + it('uploads the clone of a CONTENT_EXCEPTION_URI split as a new event after its source', async () => { + const h = createHarness(); + const series = addServerEvent(h, 'Series'); + await h.run(CALENDAR_AUTHORITY); + const master = h.events()[0]; + // A recurring master, as a calendar app sees it. + await h.device.port(ANDROID_ACCOUNT, CALENDAR_AUTHORITY).applyBatch([ + { op: 'update', table: 'events', id: Number(master._id), values: { rrule: 'FREQ=DAILY', duration: 'P3600S', dtend: null } }, + ]); + const planUpload = vi.fn(toyCalendarPlanner.planUpload); + withPlanner(h, { planUpload }); + const clone = h.device.user.splitViaUri(Number(master._id), Date.UTC(2026, 9, 1, 9), '20260930T235959Z', { title: 'Series (later)' }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.outcome).toBe('ok'); + // The source was planned while the split was visible, before its clone was claimed. + const splits = planUpload.mock.calls.map(([local]) => local.split); + expect(splits.indexOf('source')).toBeLessThan(splits.indexOf('clone')); + const events = h.server.all('CalendarEvent', 'a'); + expect(events.map((e) => e.title).sort()).toEqual(['Series', 'Series (later)']); + const cloneRow = h.events().find((e) => Number(e._id) === clone)!; + expect(cloneRow._sync_id).not.toBe(`a/${series}`); + expect(cloneRow._sync_id).toMatch(/^a\//); + expect(new Set(h.events().map((e) => e._sync_id)).size).toBe(2); + }); +}); diff --git a/src/device-sync/__tests__/engine/contacts-flow.test.ts b/src/device-sync/__tests__/engine/contacts-flow.test.ts new file mode 100644 index 00000000..d786b20e --- /dev/null +++ b/src/device-sync/__tests__/engine/contacts-flow.test.ts @@ -0,0 +1,372 @@ +// Engine runs for contacts against the fake provider and the fake JMAP +// server: first sync, incremental /changes, uploads, echoes, selection +// changes and groups (docs/device-sync.md, "A sync run"). + +import { describe, expect, it } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import { runDeviceSync } from '../../engine/run'; +import { + addDeviceContact, + addServerCards, + ANDROID_ACCOUNT, + CONTACTS_AUTHORITY, + createHarness, + REGISTRY_ID, + renameDeviceContact, + RUN_BUDGET, + rowWrites, + type Harness, +} from './harness'; + +function runAs(h: Harness, registryId: string) { + return runDeviceSync( + { runId: 'other', accountName: ANDROID_ACCOUNT, registryId, authority: CONTACTS_AUTHORITY, extras: {}, deadline: h.clock.now + RUN_BUDGET }, + h.deps, + ); +} + +describe('device sync engine: contacts', () => { + it('downloads every card of the synced address books on the first sync (a full reconcile)', async () => { + const h = createHarness(); + addServerCards(h, ['Ada', 'Grace', 'Linus']); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { created: 3, updated: 0, deleted: 0 } } }); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Grace', 'Linus']); + expect(h.contacts().every((c) => c.sourceId?.startsWith('a/') && !c.dirty)).toBe(true); + const state = h.state(); + expect(state?.owner).toEqual({ registryId: REGISTRY_ID, origin: 'https://mail.example.com' }); + expect(state?.accounts.a).toMatchObject({ + itemsState: h.server.state('ContactCard', 'a'), + reconcile: null, + selected: [`a/${h.book}`], + stale: [], + }); + expect(h.device.rows('settings')).toEqual([expect.objectContaining({ ungrouped_visible: 1, should_sync: 1 })]); + expect(h.deps.recordStatus).toHaveBeenCalledWith(REGISTRY_ID, CONTACTS_AUTHORITY, expect.objectContaining({ outcome: 'ok' })); + }); + + it('applies server creates, updates and deletes from /changes', async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada Lovelace' }); + h.server.serverDestroy('ContactCard', 'a', grace); + addServerCards(h, ['Katherine']); + const report = await h.run(); + + expect(report.stats.downloaded).toEqual({ created: 1, updated: 1, deleted: 1 }); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada Lovelace', 'Katherine']); + expect(h.server.calls('ContactCard/changes').length).toBeGreaterThan(0); + expect(h.state()?.accounts.a.itemsState).toBe(h.server.state('ContactCard', 'a')); + }); + + it('pages through /changes and stores the state with every page', async () => { + const h = createHarness(); + await h.run(); + addServerCards(h, Array.from({ length: 300 }, (_, i) => `Person ${i}`)); + const before = h.server.calls('ContactCard/changes').length; + + const report = await h.run(); + + const pages = h.server.calls('ContactCard/changes').slice(before); + expect(pages.map(([, args]) => args.maxChanges)).toEqual([256, 256]); + expect(report.stats.downloaded.created).toBe(300); + expect(h.contacts()).toHaveLength(300); + // One state op per page, each the last op of its batch. + const stateBatches = h.batches.log.filter((ops) => ops.some((op) => op.op === 'syncState')); + for (const ops of stateBatches) expect(ops[ops.length - 1].op).toBe('syncState'); + }); + + it('uploads a device edit as a patch and a new device contact as a create with a client uid', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + const fresh = addDeviceContact(h, 'Hedy'); + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 1, updated: 1, deleted: 0 } } }); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + const created = h.server.all('ContactCard', 'a').find((c) => (c.name as { full: string }).full === 'Hedy')!; + expect(created.uid).toMatch(/^[0-9a-f-]{36}$/); + expect(h.contacts().find((c) => c.id === fresh)).toMatchObject({ sourceId: `a/${created.id}`, dirty: false }); + expect(h.contacts().every((c) => !c.dirty)).toBe(true); + const set = h.server.calls('ContactCard/set'); + expect(set.every(([, args]) => typeof args.ifInState === 'string')).toBe(true); + }); + + it('uploads to an account without any card yet, whose /changes state is spelled differently from its type state', async () => { + const h = createHarness(); + await h.run(); + addDeviceContact(h, 'Hedy'); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.serverNames()).toEqual(['Hedy']); + expect(h.contacts()).toEqual([expect.objectContaining({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) })]); + }); + + it('writes nothing but the state when its own upload comes back through /changes (echo)', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + addDeviceContact(h, 'Hedy'); + await h.run(); + const before = h.batches.log.length; + + const report = await h.run(); + + expect(report.stats.downloaded).toEqual({ created: 0, updated: 0, deleted: 0 }); + expect(rowWrites(h.batches.log.slice(before))).toEqual([]); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Hedy']); + }); + + it('keeps a contact dirty when the user edits it again while its upload is on the way', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + const id = h.contactNamed('Ada')!.id; + renameDeviceContact(h, id, 'Ada King'); + const stop = h.server.onAfterRequest((request) => { + if (request.methods.includes('ContactCard/set')) renameDeviceContact(h, id, 'Ada, Countess'); + }); + + await h.run(); + stop(); + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ name: 'Ada, Countess', dirty: true }); + + await h.run(); + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada, Countess' } }); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ dirty: false }); + }); + + it('lets the server win a conflicting edit of the same unit and counts it', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada (device)'); + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada (server)' }); + + const report = await h.run(); + + expect(report.conflicts).toBe(1); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada (server)']); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada (server)' } }); + }); + + it('deletes on the server what the user deleted, and purges the rows', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + + h.device.user.deleteContact(h.contactNamed('Ada')!.id); + const report = await h.run(); + + expect(report.stats.uploaded.deleted).toBe(1); + expect(h.server.get('ContactCard', 'a', ada)).toBeUndefined(); + expect(h.contacts().map((c) => c.name)).toEqual(['Grace']); + }); + + it('drops only the clean rows of a deselected address book, after uploading its dirty ones', async () => { + const h = createHarness(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + addServerCards(h, ['Ada']); + const [boss, colleague] = addServerCards(h, ['Boss', 'Colleague'], 'a', work); + await h.run(); + expect(h.contacts()).toHaveLength(3); + + renameDeviceContact(h, h.contactNamed('Boss')!.id, 'The Boss'); + h.prefs.contactsSelection[`a/${work}`] = false; + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + expect(h.server.get('ContactCard', 'a', boss)).toMatchObject({ name: { full: 'The Boss' } }); + expect(h.server.get('ContactCard', 'a', colleague)).toBeDefined(); + expect(h.state()?.accounts.a.selected).toEqual([`a/${h.book}`]); + }); + + it("leaves the webmail's Trusted Senders book off the device unless the user picked it", async () => { + const h = createHarness(); + const trusted = h.server.addAddressBook('a', { name: 'Trusted Senders' }); + addServerCards(h, ['Ada']); + addServerCards(h, ['newsletter@example.com'], 'a', trusted); + + await h.run(); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + + h.prefs.contactsSelection[`a/${trusted}`] = true; + await h.run(); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'newsletter@example.com']); + }); + + it('loads a newly selected address book once the account is up to date', async () => { + const h = createHarness({ accounts: 'withShared' }); + addServerCards(h, ['Ada']); + const teamBook = h.server.all('AddressBook', 'team')[0].id as string; + h.server.addCard('team', { uid: 'u-team', name: { full: 'Teammate' }, addressBookIds: { [teamBook]: true } }); + await h.run(); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + + h.prefs.contactsSelection[`team/${teamBook}`] = true; + const report = await h.run(); + + expect(report.stats.downloaded.created).toBe(1); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Teammate']); + expect(h.contactNamed('Teammate')?.sourceId).toMatch(/^team\//); + expect(h.state()?.accounts.team.selected).toEqual([`team/${teamBook}`]); + }); + + it('writes group cards before the contacts that are their members', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string; + h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [adaUid]: true } }); + + await h.run(); + + const groups = h.device.rows('groups'); + expect(groups).toHaveLength(1); + expect(groups[0]).toMatchObject({ title: 'Friends', group_visible: 1 }); + const memberships = h.device.rows('data').filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP); + expect(memberships).toHaveLength(1); + // data1 is a TEXT column: the group row id may read back as text. + expect(String(memberships[0][Data.DATA1])).toBe(String(groups[0]._id)); + expect(memberships[0].group_sourceid).toBe(groups[0].sourceid); + expect(h.state()?.accounts.a.groups).toEqual([groups[0].sourceid]); + }); + + it("updates a member's rows when only the group card's members changed on the server", async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace']); + const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string; + const graceUid = h.server.get('ContactCard', 'a', grace)!.uid as string; + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [adaUid]: true } }); + await h.run(); + const members = () => + h.device + .rows('data') + .filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP) + .map((d) => h.contacts().find((c) => c.id === Number(d.raw_contact_id))?.name) + .sort(); + expect(members()).toEqual(['Ada']); + + // Another client moves the membership: only the group card changes. + h.server.serverUpdate('ContactCard', 'a', group, { members: { [graceUid]: true } }); + await h.run(); + + expect(members()).toEqual(['Grace']); + expect(h.contacts().every((c) => !c.dirty)).toBe(true); + }); + + it('uploads a membership added on the device as a patch of the group card', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: {} }); + await h.run(); + + const groupRow = h.device.rows('groups')[0]; + h.device.user.addToGroup(h.contactNamed('Ada')!.id, Number(groupRow._id)); + await h.run(); + + const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string; + expect(h.server.get('ContactCard', 'a', group)?.members).toEqual({ [adaUid]: true }); + expect(h.contacts().every((c) => !c.dirty)).toBe(true); + }); + + it('skips the network for an upload-only sync with nothing to upload', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + const requests = h.server.requests.length; + + const report = await h.run(CONTACTS_AUTHORITY, { upload: true }); + + expect(report.outcome).toBe('ok'); + expect(h.server.requests.length).toBe(requests); + }); + + it('reports disabled, unsupported and rows of another owner without touching anything', async () => { + const off = createHarness(); + off.deps.isSyncEnabled = async () => false; + expect((await off.run()).outcome).toBe('disabled'); + expect(off.server.requests).toHaveLength(0); + + const bare = createHarness(); + const port = bare.server.port(); + bare.deps.jmap = async () => ({ + origin: 'https://mail.example.com', + port: { ...port, session: () => ({ ...port.session(), primaryAccounts: {}, accounts: {} }) }, + }); + expect((await bare.run()).outcome).toBe('unsupported'); + + const other = createHarness(); + addServerCards(other, ['Ada']); + await other.run(); + other.deps.jmap = async () => ({ port: other.server.port(), origin: 'https://other.example.com' }); + const moved = await other.run(); + expect(moved.outcome).toBe('internal'); + expect(moved.message).toMatch(/another server/); + expect(other.contacts()).toHaveLength(1); + const foreign = await runAs(other, 'bob@mail.example.com'); + expect(foreign.outcome).toBe('internal'); + expect(foreign.message).toMatch(/another account/); + }); + + it('keeps one request in flight and stays within the server limits', async () => { + const h = createHarness(); + h.server.setLimits({ maxObjectsInGet: 20, maxObjectsInSet: 10 }); + addServerCards(h, Array.from({ length: 60 }, (_, i) => `Server ${i}`)); + await h.run(); + for (let i = 0; i < 25; i++) addDeviceContact(h, `Device ${i}`); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.server.maxInFlight).toBe(1); + for (const [, args] of h.server.calls('ContactCard/get')) { + expect(Array.isArray(args.ids)).toBe(true); + expect((args.ids as string[]).length).toBeLessThanOrEqual(20); + } + for (const [, args] of h.server.calls('ContactCard/set')) { + const count = Object.keys(args.create ?? {}).length + Object.keys(args.update ?? {}).length + ((args.destroy as string[]) ?? []).length; + expect(count).toBeLessThanOrEqual(10); + } + expect(h.serverNames()).toHaveLength(85); + expect(h.contacts()).toHaveLength(85); + expect(ANDROID_ACCOUNT).toBe('alice@example.com'); + }); + + it('packs provider batches of at most 400 ops with a yield point at every group and the state op last', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 450 }, (_, i) => `Person ${i}`)); + + await h.run(); + + expect(h.contacts()).toHaveLength(450); + for (const ops of h.batches.log) { + expect(ops.length).toBeLessThanOrEqual(400); + const state = ops.findIndex((op) => op.op === 'syncState'); + if (state >= 0) { + expect(state).toBe(ops.length - 1); + expect((ops[state] as { yieldAllowed?: boolean }).yieldAllowed).toBeFalsy(); + } + // Every download insert starts a group with its "not there yet" guard: a yield point, and nothing inside a group is one. + ops.forEach((op, i) => { + if (op.op === 'insert' && op.table === 'raw_contacts') { + expect(ops[i - 1]).toMatchObject({ op: 'assert', table: 'raw_contacts', expectCount: 0, yieldAllowed: true }); + expect(op.yieldAllowed).toBeFalsy(); + } + }); + } + }); +}); diff --git a/src/device-sync/__tests__/engine/failures.test.ts b/src/device-sync/__tests__/engine/failures.test.ts new file mode 100644 index 00000000..e27ec3f4 --- /dev/null +++ b/src/device-sync/__tests__/engine/failures.test.ts @@ -0,0 +1,610 @@ +// How the engine survives crashes, lost responses and server trouble, and +// its guards (docs/device-sync.md, "Invariants", "Failure matrix", +// "Deletion threshold"). + +import { describe, expect, it } from 'vitest'; +import { parseJsonColumn } from '../../common/json'; +import type { PoisonMarker } from '../../planner'; +import type { JmapInvocation, JmapResponse } from '../../types'; +import { + addDeviceContact, + addServerCards, + CONTACTS_AUTHORITY, + createHarness, + CrashError, + renameDeviceContact, + REGISTRY_ID, + type Harness, +} from './harness'; + +const FINAL = ['Ada King', 'Grace Hopper', 'Hedy', 'Margaret']; + +/** Work on both sides: device edit, create and delete; server update, create and delete. */ +async function scenario(options: Parameters[0] = {}): Promise { + const h = createHarness(options); + const [, grace, , katherine] = addServerCards(h, ['Ada', 'Grace', 'Linus', 'Katherine']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + addDeviceContact(h, 'Hedy'); + h.device.user.deleteContact(h.contactNamed('Linus')!.id); + h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': 'Grace Hopper' }); + addServerCards(h, ['Margaret']); + h.server.serverDestroy('ContactCard', 'a', katherine); + h.checkpoints.count = 0; + h.batches.applied = 0; + return h; +} + +/** Both sides hold exactly the expected contacts, once each, and nothing waits for an upload. */ +function expectConverged(h: Harness, names = FINAL): void { + const device = h.contacts(); + expect(device.map((c) => c.name).sort()).toEqual(names); + expect(device.every((c) => !c.dirty && !c.deleted && c.sourceId)).toBe(true); + expect(h.serverNames()).toEqual(names); + const refs = device.map((c) => c.sourceId); + expect(new Set(refs).size).toBe(refs.length); + for (const contact of device) { + const card = h.server.get('ContactCard', 'a', contact.sourceId!.split('/')[1]); + expect((card?.name as { full: string }).full).toBe(contact.name); + } + const uids = h.server.all('ContactCard', 'a').map((c) => c.uid); + expect(new Set(uids).size).toBe(uids.length); +} + +describe('device sync engine: crashes and lost responses', () => { + it('converges after a crash at any checkpoint, with no duplicates and no lost edits', async () => { + const probe = await scenario(); + await probe.run(); + expectConverged(probe); + const checkpoints = probe.checkpoints.count; + expect(checkpoints).toBeGreaterThan(5); + + for (let n = 1; n <= checkpoints; n++) { + const h = await scenario(); + h.checkpoints.crashAt = n; + const crashed = await h.run(); + expect(crashed.outcome, `crash at checkpoint ${n}`).toBe('internal'); + h.checkpoints.crashAt = null; + const report = await h.run(); + expect(report.outcome, `after a crash at checkpoint ${n}`).toBe('ok'); + expectConverged(h); + } + }); + + it('converges after a crash after any number of provider batches', async () => { + const probe = await scenario(); + await probe.run(); + const batches = probe.batches.applied; + expect(batches).toBeGreaterThan(3); + + for (let n = 0; n < batches; n++) { + const h = await scenario(); + h.batches.crashAfter = n; + await h.run(); + h.batches.crashAfter = null; + const report = await h.run(); + expect(report.outcome, `after a crash following batch ${n}`).toBe('ok'); + expectConverged(h); + } + }); + + it('converges when the server applied any request but its response was lost', async () => { + const probe = await scenario(); + const before = probe.server.requests.length; + await probe.run(); + const requests = probe.server.requests.length - before; + + for (let n = 1; n <= requests; n++) { + const h = await scenario(); + // The uid index lags behind, as on Stalwart: a lookup right after the lost create misses it. + h.server.setUidIndexLag(2); + let seen = 0; + const stop = h.server.onBeforeRequest(() => { + if (++seen === n) h.server.applyThenLoseResponse(); + }); + const lost = await h.run(); + stop(); + expect(lost.outcome, `lost response ${n}`).toBe('io'); + const report = await h.run(); + expect(report.outcome, `after losing response ${n}`).toBe('ok'); + expectConverged(h); + } + }); + + it('converges from a crash anywhere in a first sync (a checkpointed full reconcile)', async () => { + const names = Array.from({ length: 120 }, (_, i) => `P${String(i).padStart(3, '0')}`); + const fresh = () => { + const h = createHarness(); + addServerCards(h, names); + return h; + }; + const probe = fresh(); + await probe.run(); + const checkpoints = probe.checkpoints.count; + + for (let n = 1; n <= checkpoints; n += 2) { + const h = fresh(); + h.checkpoints.crashAt = n; + await h.run(); + h.checkpoints.crashAt = null; + expect((await h.run()).outcome).toBe('ok'); + expect(h.contacts().map((c) => c.name).sort()).toEqual(names); + expect(new Set(h.contacts().map((c) => c.sourceId)).size).toBe(120); + } + }); + + it('resumes a reconcile after its last stored chunk instead of starting over', async () => { + const names = Array.from({ length: 150 }, (_, i) => `P${String(i).padStart(3, '0')}`); + const h = createHarness(); + addServerCards(h, names); + // Batches: the Settings row, the reconcile marker, then one per chunk of 50. Stop after the second chunk. + h.batches.crashAfter = 4; + await h.run(); + const marker = h.state()?.accounts.a.reconcile; + expect(marker).toMatchObject({ phase: 'objects', position: 100 }); + h.batches.crashAfter = null; + const gets = h.server.calls('ContactCard/get').length; + + await h.run(); + + const fetched = h.server + .calls('ContactCard/get') + .slice(gets) + .filter(([, args]) => (args.properties as string[]).includes('name')) + .flatMap(([, args]) => args.ids as string[]); + expect(fetched).toHaveLength(50); + expect(h.contacts()).toHaveLength(150); + expect(h.state()?.accounts.a.reconcile).toBeNull(); + }); + + it('adopts its own create whose response was lost instead of creating it twice', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + h.server.setUidIndexLag(3); + const id = addDeviceContact(h, 'Hedy'); + h.server.applyThenLoseResponse({ match: 'ContactCard/set' }); + + expect((await h.run()).outcome).toBe('io'); + expect(h.serverNames()).toEqual(['Ada', 'Hedy']); + expect(h.contacts().find((c) => c.id === id)?.sourceId).toBeNull(); + + expect((await h.run()).outcome).toBe('ok'); + expect(h.serverNames()).toEqual(['Ada', 'Hedy']); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) }); + expect(h.contacts()).toHaveLength(2); + }); + + it("adopts the card a duplicate-uid error names (the client's automatic retry of a create)", async () => { + const h = createHarness(); + await h.run(); + addDeviceContact(h, 'Hedy'); + // The first attempt was applied; the retry answers with Stalwart's duplicate-uid error. + const port = h.server.port(); + let retried = false; + h.deps.jmap = async () => ({ + origin: 'https://mail.example.com', + port: { + ...port, + request: async (calls: JmapInvocation[], using: string[]): Promise => { + const response = await port.request(calls, using); + const [name, args] = response.methodResponses[0]; + if (retried || name !== 'ContactCard/set' || !args.created) return response; + retried = true; + const [[key, created]] = Object.entries(args.created as Record); + const uid = (calls[0][1].create as Record)[key].uid; + const notCreated = { + [key]: { type: 'invalidProperties', properties: ['uid'], description: `Contact with UID ${uid} already exists with id ${created.id}.` }, + }; + return { ...response, methodResponses: [[name, { ...args, created: undefined, notCreated }, response.methodResponses[0][2]]] }; + }, + }, + }); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(retried).toBe(true); + expect(h.serverNames()).toEqual(['Hedy']); + expect(h.contacts()).toEqual([expect.objectContaining({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) })]); + }); +}); + +describe('device sync engine: server trouble', () => { + it('downloads again and retries the upload after a stateMismatch', async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + let sets = 0; + const stop = h.server.onBeforeRequest((request) => { + // Another client writes right before our first upload. + if (request.methods.includes('ContactCard/set') && ++sets === 1) { + h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': 'Grace Hopper' }); + } + }); + + const report = await h.run(); + stop(); + + expect(report.outcome).toBe('ok'); + const set = h.server.requests.filter((r) => r.methods.includes('ContactCard/set')); + expect(set[0].responses?.[0]).toEqual(['error', { type: 'stateMismatch' }, 's']); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Grace Hopper']); + }); + + it('gives up with io and no progress when the server keeps changing', async () => { + const h = createHarness(); + const [, grace] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + let n = 0; + const stop = h.server.onBeforeRequest((request) => { + if (request.methods.includes('ContactCard/set')) h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': `Grace ${++n}` }); + }); + + const report = await h.run(); + stop(); + + expect(report.outcome).toBe('io'); + expect(report.stats.downloaded).toEqual({ created: 0, updated: 0, deleted: 0 }); + expect(n).toBe(4); + expect(h.contactNamed('Ada King')?.dirty).toBe(true); + }); + + it('holds back the uploads of an account that refuses writes, and uploads the other accounts', async () => { + const h = createHarness({ accounts: 'withShared' }); + const teamBook = h.server.all('AddressBook', 'team')[0].id as string; + h.prefs.contactsSelection[`team/${teamBook}`] = true; + addServerCards(h, ['Ada']); + const [mate] = addServerCards(h, ['Teammate'], 'team', teamBook); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + renameDeviceContact(h, h.contactNamed('Teammate')!.id, 'Team mate'); + h.server.failNextMethod('ContactCard/set', { type: 'accountReadOnly' }, { accountId: 'team' }); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: 'team', type: 'accountReadOnly' })]); + expect(h.serverNames()).toEqual(['Ada King']); + expect(h.server.get('ContactCard', 'team', mate)).toMatchObject({ name: { full: 'Teammate' } }); + expect(h.contactNamed('Team mate')?.dirty).toBe(true); + + await h.run(); + expect(h.server.get('ContactCard', 'team', mate)).toMatchObject({ name: { full: 'Team mate' } }); + }); + + it('reports a 429 as io with delayUntil from Retry-After', async () => { + const h = createHarness(); + await h.run(); + h.server.failNextRequest('rateLimit', { retryAfterMs: 120_000 }); + + const report = await h.run(); + + expect(report.outcome).toBe('io'); + expect(report.delayUntil).toBe(Math.ceil((h.clock.now + 120_000) / 1000)); + }); + + it('rebuilds the client once when the server rejects the credentials, then reports auth', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + h.server.failNextRequest('auth'); + expect((await h.run()).outcome).toBe('ok'); + expect(h.deps.notifyAuthProblem).not.toHaveBeenCalled(); + + h.server.failNextRequest('auth', { times: 2 }); + const report = await h.run(); + + expect(report.outcome).toBe('auth'); + expect(h.deps.notifyAuthProblem).toHaveBeenCalledWith(REGISTRY_ID, 'alice@example.com', CONTACTS_AUTHORITY); + expect(h.contacts()).toHaveLength(1); + }); + + it('reports auth without touching anything when the account has no stored credentials', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + const writes = h.batches.log.length; + h.deps.jmap = async () => { + const error = new Error('No usable credentials for this account'); + error.name = 'AuthenticationError'; + throw error; + }; + + const report = await h.run(); + + expect(report.outcome).toBe('auth'); + expect(h.batches.log.length).toBe(writes); + }); + + it('full-reconciles an account after cannotCalculateChanges', async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace', 'Linus']); + await h.run(); + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada King' }); + h.server.serverDestroy('ContactCard', 'a', grace); + h.server.truncateChangeLog('a', 'contacts'); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.server.requests.some((r) => r.responses?.some(([, args]) => (args as { type?: string }).type === 'cannotCalculateChanges'))).toBe(true); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Linus']); + expect(h.state()?.accounts.a).toMatchObject({ reconcile: null, itemsState: h.server.state('ContactCard', 'a') }); + }); + + it('stops a reconcile that finds an empty server while the device is full (safetyAbort)', async () => { + const h = createHarness(); + const ids = addServerCards(h, Array.from({ length: 12 }, (_, i) => `P${i}`)); + await h.run(); + for (const id of ids) h.server.serverDestroy('ContactCard', 'a', id); + h.server.truncateChangeLog('a', 'contacts'); + + const report = await h.run(); + + expect(report.outcome).toBe('safetyAbort'); + expect(h.contacts()).toHaveLength(12); + }); +}); + +describe('device sync engine: time and cancellation', () => { + it('stops cleanly near the deadline and asks for another sync', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 150 }, (_, i) => `P${i}`)); + h.checkpoints.onCheckpoint = (n) => { + // Time runs out in the middle of the download. + if (n === 4) h.clock.now += 9 * 60_000; + }; + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true }); + expect(h.contacts().length).toBeGreaterThan(0); + expect(h.contacts().length).toBeLessThan(150); + h.checkpoints.onCheckpoint = undefined; + + expect((await h.run()).outcome).toBe('ok'); + expect(h.contacts()).toHaveLength(150); + }); + + it('stops at the next checkpoint when the framework cancels the run', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 150 }, (_, i) => `P${i}`)); + h.deps.isCancelled = async () => true; + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true }); + expect(h.contacts()).toHaveLength(0); + }); + + it('sends Core/echo when a local phase went 40 s without a JMAP request', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 120 }, (_, i) => `P${i}`)); + let n = 0; + const port = h.deps.provider; + h.deps.provider = (name, authority) => { + const inner = port(name, authority); + return { + ...inner, + applyBatch: async (ops) => { + h.clock.now += 45_000; + n++; + return inner.applyBatch(ops); + }, + }; + }; + + await h.run(); + + expect(n).toBeGreaterThan(2); + expect(h.server.calls('Core/echo').length).toBeGreaterThan(0); + }); + + it('runs one sync of an account and authority at a time', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 60 }, (_, i) => `P${i}`)); + + const [first, second] = await Promise.all([h.run(), h.run()]); + + expect([first.outcome, second.outcome]).toEqual(['ok', 'ok']); + expect(h.server.maxInFlight).toBe(1); + expect(h.contacts()).toHaveLength(60); + }); +}); + +describe('device sync engine: poisoned items', () => { + it('backs a refused item off, retries it after the back-off, and at once when it changes', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + const id = h.contactNamed('Ada')!.id; + renameDeviceContact(h, id, 'Ada King'); + h.server.setErrorFor('ContactCard', 'a', (t) => t.op === 'update' && t.id === ada, { + type: 'invalidProperties', + properties: ['name'], + description: 'Invalid property.', + }, 3); + + const first = await h.run(); + + expect(first.outcome).toBe('ok'); + expect(first.stats.skipped).toBe(1); + expect(first.itemErrors).toEqual([ + expect.objectContaining({ ref: `a/${ada}`, side: 'upload', type: 'invalidProperties', retryAt: h.clock.now + 3_600_000 }), + ]); + const marker = parseJsonColumn(h.contacts().find((c) => c.id === id)!.row.sync4); + expect(marker).toMatchObject({ type: 'invalidProperties', n: 1, until: h.clock.now + 3_600_000 }); + + // Not retried hot. + const sets = h.server.calls('ContactCard/set').length; + const again = await h.run(); + expect(h.server.calls('ContactCard/set').length).toBe(sets); + expect(again.itemErrors).toEqual([expect.objectContaining({ type: 'invalidProperties' })]); + + // Retried once the back-off is over; refused again, the back-off doubles. + h.clock.now += 3_600_001; + await h.run(); + expect(h.server.calls('ContactCard/set').length).toBe(sets + 1); + expect(parseJsonColumn(h.contacts().find((c) => c.id === id)!.row.sync4)).toMatchObject({ n: 2, until: h.clock.now + 7_200_000 }); + + // Changed by the user: retried at once (the server refuses once more), then accepted. + renameDeviceContact(h, id, 'Ada, Countess'); + await h.run(); + expect(h.server.calls('ContactCard/set').length).toBe(sets + 2); + renameDeviceContact(h, id, 'Ada Lovelace'); + const fixed = await h.run(); + expect(fixed.itemErrors).toEqual([]); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada Lovelace' } }); + expect(h.contacts().find((c) => c.id === id)!.row.sync4).toBeNull(); + }); + + it('never uploads an item whose rows could not be written until it was fetched again', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada King' }); + // The provider refuses the item's rows (in its batch and alone): it goes on `stale`, the state still moves on. + const provider = h.deps.provider; + let refusals = 2; + h.deps.provider = (name, authority) => { + const port = provider(name, authority); + return { + ...port, + applyBatch: async (ops) => + refusals > 0 && ops.some((op) => op.op === 'update' && op.table === 'data') && refusals-- + ? { ok: false, reason: 'provider', message: 'SQLiteException' } + : port.applyBatch(ops), + }; + }; + const failing = await h.run(); + expect(refusals).toBe(0); + expect(failing.itemErrors).toEqual([expect.objectContaining({ ref: `a/${ada}`, side: 'download', type: 'provider' })]); + expect(h.state()?.accounts.a.stale).toEqual([ada]); + expect(h.state()?.accounts.a.itemsState).toBe(h.server.state('ContactCard', 'a')); + + const next = await h.run(); + + expect(next.outcome).toBe('ok'); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada King']); + expect(h.state()?.accounts.a.stale).toEqual([]); + }); +}); + +describe('device sync engine: planner failures', () => { + it('keeps syncing everything else when a planner throws on one item, and tries that one again next run', async () => { + const h = createHarness(); + const [, grace] = addServerCards(h, ['Ada', 'Grace', 'Linus']); + let broken = true; + const planner = h.deps.planners.contacts; + h.deps.planners = { + ...h.deps.planners, + contacts: { + ...planner, + planDownload: (card, local, ctx) => { + if (broken && card.id === grace) throw new TypeError('cannot read the card'); + return planner.planDownload(card, local, ctx); + }, + planUpload: (local, ctx) => { + if (broken && local.rows.some((r) => r.cells.data1 === 'Hedy')) throw new TypeError('cannot read the rows'); + return planner.planUpload(local, ctx); + }, + }, + }; + addDeviceContact(h, 'Hedy'); + addDeviceContact(h, 'Mary'); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(report.itemErrors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ref: `a/${grace}`, side: 'download', type: 'plannerError' }), + expect.objectContaining({ side: 'upload', type: 'plannerError' }), + ]), + ); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Hedy', 'Linus', 'Mary']); + expect(h.serverNames()).toEqual(['Ada', 'Grace', 'Linus', 'Mary']); + // The reconcile completed past the item that could not be planned. + expect(h.state()?.accounts.a).toMatchObject({ stale: [grace], reconcile: null, itemsState: expect.any(String) }); + + broken = false; + expect((await h.run()).itemErrors).toEqual([]); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Grace', 'Hedy', 'Linus', 'Mary']); + expect(h.serverNames()).toEqual(['Ada', 'Grace', 'Hedy', 'Linus', 'Mary']); + expect(h.state()?.accounts.a.stale).toEqual([]); + }); +}); + +describe('device sync engine: deletion guards', () => { + async function withDeletions(total: number, deleted: number): Promise { + const h = createHarness(); + addServerCards(h, Array.from({ length: total }, (_, i) => `P${String(i).padStart(3, '0')}`)); + await h.run(); + for (const contact of h.contacts().slice(0, deleted)) h.device.user.deleteContact(contact.id); + return h; + } + + it('holds every deletion back above 50 and 20 % (tooManyDeletions)', async () => { + const h = await withDeletions(100, 55); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'tooManyDeletions', tooManyDeletions: { count: 55, threshold: 50 } }); + expect(h.serverNames()).toHaveLength(100); + expect(h.contacts().filter((c) => c.deleted)).toHaveLength(55); + }); + + it('uploads the deletions when the run overrides the threshold', async () => { + const h = await withDeletions(100, 55); + + const report = await h.run(CONTACTS_AUTHORITY, { overrideTooManyDeletions: true }); + + expect(report.outcome).toBe('ok'); + expect(report.stats.uploaded.deleted).toBe(55); + expect(h.serverNames()).toHaveLength(45); + expect(h.contacts()).toHaveLength(45); + }); + + it('restores the deleted contacts from the server when the run discards the deletions', async () => { + const h = await withDeletions(100, 55); + + const discard = await h.run(CONTACTS_AUTHORITY, { discardLocalDeletions: true }); + expect(discard.outcome).toBe('ok'); + expect(h.serverNames()).toHaveLength(100); + + const next = await h.run(CONTACTS_AUTHORITY, { upload: true }); + expect(next.outcome).toBe('ok'); + expect(h.contacts()).toHaveLength(100); + expect(h.contacts().every((c) => !c.deleted && !c.dirty)).toBe(true); + }); + + it('uploads deletions below the threshold', async () => { + const h = await withDeletions(100, 20); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.serverNames()).toHaveLength(80); + }); + + it('counts groups an app hard-deleted, found by their absence', async () => { + const h = createHarness(); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' } }); + addServerCards(h, ['Ada']); + await h.run(); + + h.device.user.fossifyDeleteGroup(Number(h.device.rows('groups')[0]._id)); + const report = await h.run(); + + expect(report.stats.uploaded.deleted).toBe(1); + expect(h.server.get('ContactCard', 'a', group)).toBeUndefined(); + expect(h.state()?.accounts.a.groups).toEqual([]); + }); + + it('fails a crash-free run with CrashError only when injected', () => { + expect(new CrashError('x').name).toBe('CrashError'); + }); +}); diff --git a/src/device-sync/__tests__/engine/harness.ts b/src/device-sync/__tests__/engine/harness.ts new file mode 100644 index 00000000..dc13849e --- /dev/null +++ b/src/device-sync/__tests__/engine/harness.ts @@ -0,0 +1,196 @@ +/** + * The engine under test, wired to the fake providers, the fake JMAP server + * and the toy planners, with helpers to read both sides and inject faults. + */ +import { vi } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import { parseJsonColumn } from '../../common/json'; +import { runDeviceSync } from '../../engine/run'; +import { teardownAuthority, type TeardownOptions } from '../../engine/teardown'; +import type { EngineDeps, JmapConnection, Tuning } from '../../engine/deps'; +import { openWithAuthRebuild } from '../../jmap/connection'; +import type { SyncState } from '../../engine/sync-state'; +import { + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + type AccountSyncPrefs, + type Authority, + type ProviderOp, + type ProviderPort, + type RunExtras, + type RunReport, + type Row, +} from '../../types'; +import { FakeDeviceProviders } from '../fakes/fake-provider'; +import { FakeJmapServer } from '../fakes/fake-jmap-server'; +import { toyCalendarPlanner, toyContactsPlanner } from './toy-planners'; + +export const ANDROID_ACCOUNT = 'alice@example.com'; +export const REGISTRY_ID = 'alice@mail.example.com'; +export const ORIGIN = 'https://mail.example.com'; +export const START = Date.UTC(2026, 8, 27, 10, 0, 0); +export const RUN_BUDGET = 9 * 60_000; + +/** Deterministic randomness: uids and keys differ between items but repeat between test runs. */ +export function seededRandom(seed = 42): () => number { + let s = seed >>> 0; + return () => { + s = (s * 1664525 + 1013904223) >>> 0; + return s / 2 ** 32; + }; +} + +export class CrashError extends Error { + constructor(where: string) { + super(`crash injected ${where}`); + this.name = 'CrashError'; + } +} + +export interface Harness { + server: FakeJmapServer; + device: FakeDeviceProviders; + deps: EngineDeps; + prefs: AccountSyncPrefs; + book: string; + calendar: string; + clock: { now: number }; + /** Checkpoints seen by the last runs (each call of isCancelled). */ + checkpoints: { count: number; crashAt: number | null; onCheckpoint?: (n: number) => void }; + /** Batches applied through the engine's provider port; crash after `crashAfter` of them. */ + batches: { applied: number; crashAfter: number | null; log: ProviderOp[][] }; + run(authority?: Authority, extras?: RunExtras): Promise; + teardown(authority: Authority, options?: TeardownOptions): Promise<{ pending: number }>; + state(authority?: Authority): SyncState | null; + contacts(): Array<{ id: number; sourceId: string | null; name: string | null; dirty: boolean; deleted: boolean; row: Row }>; + contactNamed(name: string): { id: number; sourceId: string | null; dirty: boolean; deleted: boolean } | undefined; + nameDataRow(rawContactId: number): Row | undefined; + events(): Row[]; + serverNames(accountId?: string): string[]; +} + +export function createHarness(options: { tuning?: Partial; accounts?: 'personal' | 'withShared'; rebuild?: boolean } = {}): Harness { + const clock = { now: START }; + const server = new FakeJmapServer({ now: () => clock.now }); + server.addAccount('a', { name: ANDROID_ACCOUNT }); + const book = server.addAddressBook('a', { name: 'Personal', isDefault: true }); + const calendar = server.addCalendar('a', { name: 'Calendar', isDefault: true }); + if (options.accounts === 'withShared') { + server.addAccount('team', { name: 'Team', isPersonal: false }); + server.addAddressBook('team', { name: 'Team contacts' }); + server.addCalendar('team', { name: 'Team calendar' }); + } + const device = new FakeDeviceProviders(); + const prefs: AccountSyncPrefs = { contactsSelection: {}, calendarSelection: {}, reminderOwner: 'device' }; + const checkpoints: Harness['checkpoints'] = { count: 0, crashAt: null }; + const batches: Harness['batches'] = { applied: 0, crashAfter: null, log: [] }; + + const wrap = (port: ProviderPort): ProviderPort => ({ + accountName: port.accountName, + authority: port.authority, + query: (q) => port.query(q), + readSyncState: () => port.readSyncState(), + readPhoto: (id, px) => port.readPhoto(id, px), + applyBatch: async (ops) => { + if (batches.crashAfter !== null && batches.applied >= batches.crashAfter) throw new CrashError(`after ${batches.applied} batches`); + const result = await port.applyBatch(ops); + if (result.ok) { + batches.applied++; + batches.log.push(ops); + } + return result; + }, + }); + + const connection = (): JmapConnection => ({ port: server.port(), origin: ORIGIN }); + const deps: EngineDeps = { + provider: (name, authority) => wrap(device.port(name, authority)), + jmap: async () => (options.rebuild === false ? connection() : openWithAuthRebuild(async () => connection())), + prefs: async () => prefs, + planners: { contacts: toyContactsPlanner, calendar: toyCalendarPlanner }, + now: () => clock.now, + isCancelled: async () => { + checkpoints.count++; + checkpoints.onCheckpoint?.(checkpoints.count); + if (checkpoints.crashAt !== null && checkpoints.count >= checkpoints.crashAt) throw new CrashError(`at checkpoint ${checkpoints.count}`); + return false; + }, + random: seededRandom(), + subscriptionCalendars: async () => [], + isSyncEnabled: async () => true, + deviceZone: () => 'Europe/Berlin', + recordStatus: vi.fn(), + recordKnownState: vi.fn(), + notifyAuthProblem: vi.fn(), + yieldThread: async () => undefined, + sleep: async () => undefined, + tuning: options.tuning, + }; + + let runs = 0; + const harness: Harness = { + server, + device, + deps, + prefs, + book, + calendar, + clock, + checkpoints, + batches, + run: (authority = CONTACTS_AUTHORITY, extras = {}) => + runDeviceSync( + { runId: `run-${++runs}`, accountName: ANDROID_ACCOUNT, registryId: REGISTRY_ID, authority, extras, deadline: clock.now + RUN_BUDGET }, + deps, + ), + teardown: (authority, teardownOptions) => teardownAuthority(deps, REGISTRY_ID, ANDROID_ACCOUNT, authority, teardownOptions), + state: (authority = CONTACTS_AUTHORITY) => parseJsonColumn(device.readSyncState(ANDROID_ACCOUNT, authority)), + contacts: () => + device.rows('raw_contacts').map((row) => { + const id = Number(row._id); + const nameRow = harness.nameDataRow(id); + return { + id, + sourceId: (row.sourceid as string | null) ?? null, + name: (nameRow?.[Data.DATA1] as string | null) ?? null, + dirty: Number(row.dirty) === 1, + deleted: Number(row.deleted) === 1, + row, + }; + }), + contactNamed: (name) => harness.contacts().find((c) => c.name === name), + nameDataRow: (rawContactId) => + device.rows('data').find((d) => Number(d.raw_contact_id) === rawContactId && d.mimetype === MimeType.STRUCTURED_NAME), + events: () => device.rows('events'), + serverNames: (accountId = 'a') => + server + .all('ContactCard', accountId) + .filter((c) => c.kind !== 'group') + .map((c) => String((c.name as { full?: string } | undefined)?.full ?? '')) + .sort(), + }; + return harness; +} + +/** A contact added on the device the way a Contacts app does (DIRTY, no SOURCE_ID). */ +export function addDeviceContact(h: Harness, name: string): number { + return h.device.user.insertContact(ANDROID_ACCOUNT, [{ [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, [Data.DATA1]: name }]); +} + +/** An in-place edit of a contact's name (AOSP Contacts). */ +export function renameDeviceContact(h: Harness, rawContactId: number, name: string): void { + const row = h.nameDataRow(rawContactId); + if (!row) throw new Error(`no name row for ${rawContactId}`); + h.device.user.updateData(Number(row._id), { [Data.DATA1]: name, [Data.DATA2]: null, [Data.DATA3]: null }); +} + +export function addServerCards(h: Harness, names: string[], accountId = 'a', book = h.book): string[] { + return names.map((name, i) => h.server.addCard(accountId, { uid: `uid-${name}-${i}`, name: { full: name }, addressBookIds: { [book]: true } })); +} + +/** Batches that wrote rows (anything but the SyncState op). */ +export function rowWrites(batches: ProviderOp[][]): ProviderOp[][] { + return batches.filter((ops) => ops.some((op) => op.op !== 'syncState' && op.op !== 'assert')); +} + +export { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY }; diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts new file mode 100644 index 00000000..8b58b263 --- /dev/null +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -0,0 +1,328 @@ +// The engine's building blocks on their own: provider batches, the +// SyncState blob, the run lock, SetError handling, query paging, the auth +// rebuild and how failures map to outcomes. + +import { describe, expect, it, vi } from 'vitest'; +import { BatchWriter, buildBatch, prependOps, type Work } from '../../engine/batch'; +import { classifyFailure, RunAbort, StateMismatch, StopRun } from '../../engine/errors'; +import { acquireLock } from '../../engine/mutex'; +import { isBackedOff, nextMarker } from '../../engine/poison'; +import { collectionDefaultOn, isCollectionSelected } from '../../engine/selection'; +import { emptySyncState, parseSyncState, serializeSyncState, StateStore } from '../../engine/sync-state'; +import { classifySetError, existingIdFromUidError, JmapCaller } from '../../jmap/caller'; +import { openWithAuthRebuild } from '../../jmap/connection'; +import { JMAPMethodError } from '../../jmap/errors'; +import { accountsWithCapability, calendarAddresses } from '../../jmap/session'; +import { JMAP_CALENDARS, JMAP_CONTACTS, type BatchResult, type ProviderOp, type ProviderPort } from '../../types'; +import { FakeJmapServer } from '../fakes/fake-jmap-server'; + +function port(apply: (ops: ProviderOp[]) => BatchResult | Promise): ProviderPort & { sent: ProviderOp[][] } { + const sent: ProviderOp[][] = []; + return { + accountName: 'x', + authority: 'com.android.contacts', + sent, + query: async () => ({ columns: [], rows: [] }), + readSyncState: async () => null, + readPhoto: async () => null, + applyBatch: async (ops) => { + sent.push(ops); + return apply(ops); + }, + }; +} + +const ok = (ops: ProviderOp[]): BatchResult => ({ ok: true, results: ops.map(() => ({ count: 1 })) }); +const insert = (table: 'raw_contacts' | 'data', values: Record = {}): ProviderOp => ({ op: 'insert', table, values }); +const writer = (p: ProviderPort, options: Partial<{ maxOps: number; maxBytes: number; maxReplans: number }> = {}) => + new BatchWriter(p, { maxOps: 400, maxBytes: 300_000, maxReplans: 3, ...options }); + +describe('provider batches', () => { + it('rebases refs, makes each group start at a yield point and puts the state op last', () => { + const group = { ref: 'g', ops: [insert('raw_contacts'), { op: 'insert', table: 'data', values: {}, refs: { raw_contact_id: 0 }, yieldAllowed: true } as ProviderOp] }; + + const ops = buildBatch([group, group], { op: 'syncState', value: '{}', yieldAllowed: true } as ProviderOp); + + expect(ops.map((op) => (op as { yieldAllowed?: boolean }).yieldAllowed ?? false)).toEqual([true, false, true, false, false]); + expect((ops[3] as { refs: Record }).refs).toEqual({ raw_contact_id: 2 }); + expect(ops[4]).toEqual({ op: 'syncState', value: '{}' }); + }); + + it('prepends ops to a group, shifting its refs', () => { + const group = prependOps({ ref: 'g', ops: [insert('raw_contacts'), { op: 'insert', table: 'data', values: {}, refs: { raw_contact_id: 0 } }] }, [ + { op: 'assert', table: 'raw_contacts', where: 'sourceid = ?', args: ['a/1'], expectCount: 0 }, + ]); + expect((group.ops[2] as { refs: Record }).refs).toEqual({ raw_contact_id: 1 }); + }); + + it('never splits a group, keeps batches within the op budget and sends photo groups alone', async () => { + const p = port(ok); + const works: Work[] = [ + ...Array.from({ length: 5 }, (_, i) => ({ group: { ref: `g${i}`, ops: [insert('raw_contacts'), insert('data'), insert('data')] } })), + { group: { ref: 'photo', ops: [{ op: 'insert', table: 'data', values: { data15: { b64: 'AAAA' } } }] } }, + { group: { ref: 'g5', ops: [insert('raw_contacts')] } }, + ]; + + await writer(p, { maxOps: 7 }).write(works, { op: () => ({ op: 'syncState', value: 's' }), applied: () => undefined }); + + expect(p.sent.map((b) => b.length)).toEqual([6, 6, 3, 1, 2]); + expect(p.sent[3]).toEqual([expect.objectContaining({ values: { data15: { b64: 'AAAA' } } })]); + expect(p.sent[4][1]).toEqual({ op: 'syncState', value: 's' }); + }); + + it('retries the groups of a failed batch one by one and re-plans a failed assert at most three times', async () => { + let failing = 5; + const p = port((ops) => + ops.some((op) => op.op === 'assert') && failing-- > 0 ? { ok: false, reason: 'assert', message: 'stale' } : ok(ops), + ); + const applied: string[] = []; + const failed = vi.fn(); + let replans = 0; + const stale: Work = { + group: { ref: 'stale', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { version: 1 } }, insert('data')] }, + applied: () => { + applied.push('stale'); + }, + failed, + }; + stale.replan = async () => { + replans++; + return stale; + }; + const fine: Work = { + group: { ref: 'fine', ops: [insert('raw_contacts')] }, + applied: () => { + applied.push('fine'); + }, + }; + const tail = { op: () => ({ op: 'syncState', value: 's' }) as ProviderOp, applied: vi.fn() }; + + await writer(p).write([fine, stale], tail); + + expect(applied).toEqual(['fine']); + expect(replans).toBe(3); + expect(failed).toHaveBeenCalledWith('assert', 'stale'); + expect(tail.applied).toHaveBeenCalledOnce(); + expect(p.sent.at(-1)).toEqual([{ op: 'syncState', value: 's' }]); + }); + + it('splits a batch the Binder refused (tooLarge) and gives up a single group that is too large', async () => { + const p = port((ops) => (ops.length > 2 ? { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' } : ok(ops))); + const failed = vi.fn(); + const works: Work[] = [ + ...Array.from({ length: 4 }, (_, i) => ({ group: { ref: `g${i}`, ops: [insert('raw_contacts')] } })), + { group: { ref: 'big', ops: [insert('raw_contacts'), insert('data'), insert('data')] }, failed }, + ]; + + await writer(p).write(works); + + expect(failed).toHaveBeenCalledWith('tooLarge', 'TransactionTooLargeException'); + expect(p.sent.filter((b) => b.length <= 2).flat()).toHaveLength(4); + }); + + it('ends the run when the permission is gone and refuses to retry an op outside the account', async () => { + const denied = port(() => ({ ok: false, reason: 'permission', message: 'SecurityException' })); + await expect(writer(denied).write([{ group: { ref: 'g', ops: [insert('raw_contacts')] } }])).rejects.toMatchObject({ outcome: 'permission' }); + const scope = port(() => ({ ok: false, reason: 'scope', message: 'not our account' })); + await expect(writer(scope).write([{ group: { ref: 'g', ops: [insert('raw_contacts')] } }])).rejects.toThrow(/not our account/); + }); + + it('skips groups that only assert', async () => { + const p = port(ok); + await writer(p).write([{ group: { ref: 'g', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { dirty: 0 } }] } }]); + expect(p.sent).toEqual([]); + }); +}); + +describe('SyncState', () => { + it('round-trips and tolerates missing parts', () => { + const state = emptySyncState(); + state.owner = { registryId: 'r', origin: 'https://o' }; + state.accounts.a = { collectionsState: 's1', itemsState: 's2', selected: ['a/b'], stale: ['c1'], reconcile: null, groups: ['a/g'] }; + state.deviceZone = 'Europe/Berlin'; + + expect(parseSyncState(serializeSyncState(state))).toEqual({ state, readable: true }); + expect(parseSyncState(JSON.stringify({ v: 1, accounts: { a: { itemsState: 'x' } } })).state.accounts.a).toEqual({ + collectionsState: null, + itemsState: 'x', + selected: [], + stale: [], + reconcile: null, + }); + }); + + it('starts over when the blob is unreadable or of another version', () => { + expect(parseSyncState('{nope').readable).toBe(false); + expect(parseSyncState(JSON.stringify({ v: 2, accounts: {} })).readable).toBe(false); + expect(parseSyncState(null)).toEqual({ state: emptySyncState(), readable: true }); + expect(parseSyncState('')).toEqual({ state: emptySyncState(), readable: true }); + }); + + it('adopts a proposed state only after its batch applied', () => { + const store = new StateStore(emptySyncState(), { registryId: 'r', origin: 'o' }); + const tail = store.tail((next) => { + next.deviceZone = 'UTC'; + }); + const op = tail.op(); + expect(store.committed.deviceZone).toBeUndefined(); + expect(JSON.parse((op as { value: string }).value)).toMatchObject({ deviceZone: 'UTC', owner: { registryId: 'r' } }); + tail.applied(); + expect(store.committed.deviceZone).toBe('UTC'); + }); +}); + +describe('run lock', () => { + it('serialises holders and hands on the turn of a waiter that gave up', async () => { + const order: string[] = []; + const first = await acquireLock('k'); + const second = acquireLock('k', 10); + const third = acquireLock('k'); + expect(await second).toBeNull(); + first!(); + const release = await third; + order.push('third'); + release!(); + expect(order).toEqual(['third']); + const again = await acquireLock('k', 10); + expect(again).not.toBeNull(); + again!(); + }); +}); + +describe('poison markers', () => { + it('backs off an hour, doubling to a day, while the item stays the same', () => { + const first = nextMarker(null, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 }); + const second = nextMarker(first, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 }); + let marker = second; + for (let i = 0; i < 10; i++) marker = nextMarker(marker, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 }); + expect([first.until, second.until, marker.until]).toEqual([3_600_000, 7_200_000, 86_400_000]); + expect(nextMarker(second, 'changed', { type: 'invalidPatch' }, 0, { firstMs: 1, maxMs: 10 }).n).toBe(1); + expect(isBackedOff(first, 'fp', 1)).toBe(true); + expect(isBackedOff(first, 'other', 1)).toBe(false); + expect(isBackedOff(first, 'fp', 3_600_001)).toBe(false); + }); +}); + +describe('JMAP side', () => { + it('classifies SetErrors', () => { + expect(classifySetError({ type: 'notFound' }, 'update')).toBe('notFound'); + expect(classifySetError({ type: 'forbidden' }, 'destroy')).toBe('forbidden'); + expect(classifySetError({ type: 'invalidProperties', properties: ['uid'] }, 'create')).toBe('uidExists'); + expect(classifySetError({ type: 'invalidProperties', properties: ['uid'] }, 'update')).toBe('poison'); + expect(classifySetError({ type: 'overQuota' }, 'create')).toBe('poison'); + expect(existingIdFromUidError({ type: 'invalidProperties', description: 'Contact with UID u1 already exists with id c5.' })).toBe('c5'); + expect(existingIdFromUidError({ type: 'invalidProperties', description: 'An event with UID u1 already exists.' })).toBeNull(); + }); + + it('pages /query by position until the total, with overlapping pages', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + const book = server.addAddressBook('a', { name: 'Personal' }); + for (let i = 0; i < 30; i++) server.addCard('a', { uid: `u${i}` }); + server.setLimits({ maxQueryResults: 8 }); + const caller = new JmapCaller(server.port(), () => 0); + + const ids = await caller.queryAll('ContactCard', 'a', { inAddressBook: book }); + + expect(ids).toHaveLength(30); + expect(server.calls('ContactCard/query').length).toBeGreaterThan(3); + }); + + it('looks uids up in one query and maps them back', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + const book = server.addAddressBook('a', { name: 'Personal' }); + const other = server.addAddressBook('a', { name: 'Other' }); + const c1 = server.addCard('a', { uid: 'u1', addressBookIds: { [book]: true } }); + server.addCard('a', { uid: 'u2', addressBookIds: { [other]: true } }); + const caller = new JmapCaller(server.port(), () => 0); + + const found = await caller.lookupUids('ContactCard', 'a', ['u1', 'u2', 'u3'], book); + + expect([...found]).toEqual([['u1', [c1]]]); + }); + + it('names the accounts with a capability, the primary first', () => { + const server = new FakeJmapServer(); + server.addAccount('team', { name: 'Team', isPersonal: false, capabilities: ['contacts'] }); + server.addAccount('a', { name: 'alice@example.com' }); + const session = server.port().session(); + + expect(accountsWithCapability(session, JMAP_CONTACTS).map((a) => [a.id, a.primary, a.personal])).toEqual([ + ['a', true, true], + ['team', false, false], + ]); + expect(accountsWithCapability(session, JMAP_CALENDARS).map((a) => a.id)).toEqual(['a']); + }); + + it('takes the login as the owner address and survives a server without identities', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'Alice@Example.com' }); + const addresses = await calendarAddresses(new JmapCaller(server.port(), () => 0), 'fallback'); + expect(addresses).toEqual({ ownerAccount: 'alice@example.com', selfAddresses: ['alice@example.com'] }); + }); + + it('rebuilds the connection once when the credentials are rejected', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + const load = vi.fn(async () => ({ port: server.port(), origin: 'o' })); + const connection = await openWithAuthRebuild(load); + server.failNextRequest('auth'); + + await connection.port.request([['Core/echo', {}, '0']], ['urn:ietf:params:jmap:core']); + expect(load).toHaveBeenCalledTimes(2); + + server.failNextRequest('auth'); + await expect(connection.port.request([['Core/echo', {}, '0']], ['urn:ietf:params:jmap:core'])).rejects.toMatchObject({ name: 'AuthenticationError' }); + expect(load).toHaveBeenCalledTimes(2); + }); + + it('opens with a second client when the first has no session, and gives up after that', async () => { + const connection = { port: new FakeJmapServer().port(), origin: 'o' }; + const loads = [null, connection]; + await expect(openWithAuthRebuild(async () => loads.shift() ?? null)).resolves.toMatchObject({ origin: 'o' }); + expect(loads).toEqual([]); + const rejected = Object.assign(new Error('401'), { name: 'AuthenticationError' }); + await expect(openWithAuthRebuild(async () => { throw rejected; })).rejects.toMatchObject({ name: 'AuthenticationError' }); + await expect(openWithAuthRebuild(async () => null)).rejects.toMatchObject({ name: 'AuthenticationError' }); + const offline = Object.assign(new Error('offline'), { name: 'NetworkError' }); + await expect(openWithAuthRebuild(async () => { throw offline; })).rejects.toBe(offline); + }); +}); + +describe('selection defaults', () => { + it("syncs the personal account's collections, not shared ones nor the Trusted Senders book, unless chosen", () => { + expect(collectionDefaultOn(true, 'com.android.contacts', 'Personal')).toBe(true); + expect(collectionDefaultOn(true, 'com.android.contacts', 'Trusted Senders')).toBe(false); + expect(collectionDefaultOn(true, 'com.android.calendar', 'Trusted Senders')).toBe(true); + expect(collectionDefaultOn(false, 'com.android.contacts', 'Team')).toBe(false); + expect(collectionDefaultOn(true, 'com.android.calendar', null)).toBe(true); + + const personal = { id: 'a', personal: true }; + const trusted = { id: 'ab9', name: 'Trusted Senders' }; + expect(isCollectionSelected({}, personal, 'com.android.contacts', trusted)).toBe(false); + expect(isCollectionSelected({ 'a/ab9': true }, personal, 'com.android.contacts', trusted)).toBe(true); + expect(isCollectionSelected({ 'a/ab1': false }, personal, 'com.android.contacts', { id: 'ab1', name: 'Personal' })).toBe(false); + expect(isCollectionSelected({ 't/c1': true }, { id: 't', personal: false }, 'com.android.calendar', { id: 'c1' })).toBe(true); + }); +}); + +describe('outcomes', () => { + it('maps failures to report outcomes', () => { + const named = (name: string, extra: object = {}) => Object.assign(new Error(name), { name, ...extra }); + expect(classifyFailure(new StopRun('deadline'), 0)).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true }); + expect(classifyFailure(new RunAbort('safetyAbort', 'x'), 0)).toMatchObject({ outcome: 'safetyAbort' }); + expect(classifyFailure(new StateMismatch('a'), 0)).toMatchObject({ outcome: 'io', noProgress: true }); + expect(classifyFailure(named('AuthenticationError'), 0)).toMatchObject({ outcome: 'auth', authProblem: true }); + expect(classifyFailure(named('NetworkError'), 0)).toMatchObject({ outcome: 'io' }); + expect(classifyFailure(named('RequestTimeoutError'), 0)).toMatchObject({ outcome: 'io' }); + expect(classifyFailure(named('RateLimitError', { retryAfterMs: 5000 }), 10_000)).toMatchObject({ outcome: 'io', delayUntil: 15 }); + expect(classifyFailure(new JMAPMethodError('serverFail'), 0)).toMatchObject({ outcome: 'io' }); + expect(classifyFailure(new Error('JMAP request failed: 503 - busy'), 0)).toMatchObject({ outcome: 'io' }); + expect( + classifyFailure(new Error('JMAP request failed: 400 - {"type":"urn:ietf:params:jmap:error:limit","limit":"maxCallsInRequest"}'), 0), + ).toMatchObject({ outcome: 'internal' }); + expect(classifyFailure(Object.assign(new Error('denied'), { code: 'permission' }), 0)).toMatchObject({ outcome: 'permission' }); + expect(classifyFailure(new TypeError('x is undefined'), 0)).toMatchObject({ outcome: 'internal' }); + }); +}); diff --git a/src/device-sync/__tests__/engine/task.test.ts b/src/device-sync/__tests__/engine/task.test.ts new file mode 100644 index 00000000..384fc21c --- /dev/null +++ b/src/device-sync/__tests__/engine/task.test.ts @@ -0,0 +1,63 @@ +// The headless task entry: every run ends with finishRun and nothing +// escapes it (docs/device-sync.md, "The Android shell"). + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../native', () => ({ + createProviderPort: vi.fn(), + finishRun: vi.fn(async () => true), + getSyncSettings: vi.fn(async () => ({ + masterAutomatic: true, + authorities: { 'com.android.contacts': { automatic: false, syncable: 1, periodicSeconds: 0, active: false, pending: false } }, + })), + isRunCancelled: vi.fn(async () => false), + showSyncProblem: vi.fn(async () => undefined), +})); + +import * as native from '../../native'; +import { runDeviceSyncTask, teardownAuthority } from '../../task'; +import { useDeviceSyncStore } from '../../../stores/device-sync-store'; +import type { RunPayload } from '../../types'; + +const payload: RunPayload = { + runId: 'r1', + accountName: 'alice@example.com', + registryId: 'alice@mail.example.com', + authority: 'com.android.contacts', + extras: {}, + deadline: Date.now() + 60_000, +}; + +describe('device sync task', () => { + beforeEach(() => { + vi.mocked(native.finishRun).mockClear(); + }); + + it('hands the report to the adapter before settling, and records the status', async () => { + await runDeviceSyncTask(payload); + + expect(native.finishRun).toHaveBeenCalledWith('r1', expect.objectContaining({ v: 1, runId: 'r1', outcome: 'disabled' })); + expect(useDeviceSyncStore.getState().accounts['alice@mail.example.com']?.lastRun?.['com.android.contacts']).toMatchObject({ + outcome: 'disabled', + }); + }); + + it('reports a failure instead of throwing, and survives a refused finishRun', async () => { + vi.mocked(native.getSyncSettings).mockRejectedValueOnce(new Error('module gone')); + vi.mocked(native.finishRun).mockRejectedValueOnce(new Error('bridge down')); + + await expect(runDeviceSyncTask({ ...payload, runId: 'r2' })).resolves.toBeUndefined(); + + expect(native.finishRun).toHaveBeenCalledWith('r2', expect.objectContaining({ outcome: 'internal', message: 'module gone' })); + }); + + it('ignores a payload without a run', async () => { + await runDeviceSyncTask({} as RunPayload); + expect(native.finishRun).not.toHaveBeenCalled(); + }); + + it('exports the teardown the app calls', () => { + expect(teardownAuthority).toBeTypeOf('function'); + expect(teardownAuthority.length).toBe(4); + }); +}); diff --git a/src/device-sync/__tests__/engine/toy-planners.ts b/src/device-sync/__tests__/engine/toy-planners.ts new file mode 100644 index 00000000..8bb7136a --- /dev/null +++ b/src/device-sync/__tests__/engine/toy-planners.ts @@ -0,0 +1,657 @@ +/** + * Small planners for the engine tests, faithful to the planner contract + * (src/device-sync/planner.ts) for a toy mapping: + * + * - contacts: `name.full` ↔ one StructuredName row (DISPLAY_NAME), group + * cards ↔ Groups rows (TITLE), memberships ↔ GroupMembership rows; + * - calendar: `title` ↔ one event row (plus the columns the provider + * requires), calendars ↔ Calendars rows. + * + * Like the real planners they assert what they read first (VERSION and + * DIRTY, or the event projection), keep baselines in DATA_SYNC3 / + * SYNC_DATA4, merge dirty items per unit (server wins a real conflict), + * adopt pending creates by uid, claim new items before creating them, and + * return `effect: 'none'` for echoes. + */ +import { Data, Events, GroupMembership, MimeType } from '../../android-columns'; +import { collectionKey, parseCollectionKey, parseObjectRef, pendingRef, pendingUidOf } from '../../common/ids'; +import { deepEqual, parseJsonColumn } from '../../common/json'; +import { ptr } from '../../common/patch'; +import type { + AcceptedPlan, + CalendarContext, + CalendarEventWire, + CalendarPlanner, + ContactCardWire, + ContactsContext, + ContactsPlanner, + DownloadPlan, + LocalCalendar, + LocalContact, + LocalEvent, + LocalGroup, + OpGroup, + UploadAction, + UploadPlan, +} from '../../planner'; +import type { ProviderOp, Row } from '../../types'; + +const num = (v: unknown) => Number(v ?? 0); +const flag = (v: unknown) => Number(v ?? 0) === 1; +const str = (v: unknown): string | null => (typeof v === 'string' && v ? v : null); +const onIds = (map: Record | undefined | null) => + Object.entries(map ?? {}).filter(([, on]) => on === true).map(([id]) => id); + +interface ToyContactsContext extends ContactsContext { + groupsOf?(uid: string): string[]; +} + +// ─── Contacts ─────────────────────────────────────────── + +function nameOf(card: ContactCardWire | null | undefined): string { + return card?.name?.full ?? ''; +} + +function nameRow(local: LocalContact) { + return local.rows.find((r) => r.mimetype === MimeType.STRUCTURED_NAME) ?? null; +} + +function membershipRows(local: LocalContact) { + return local.rows.filter((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP); +} + +function cardCollections(acct: string, card: ContactCardWire): string { + return onIds(card.addressBookIds).map((id) => collectionKey(acct, id)).join(','); +} + +function shadowOf(card: ContactCardWire): string { + return JSON.stringify(card); +} + +/** Group SOURCE_IDs a card should be a member of, with their row ids. */ +function wantedGroups(card: ContactCardWire, ctx: ToyContactsContext): Array<{ sourceId: string; rowId: number }> { + if (typeof card.uid !== 'string' || !ctx.groupsOf) return []; + return ctx + .groupsOf(card.uid) + .map((sourceId) => ({ sourceId, rowId: ctx.groupRowIdBySourceId(sourceId) })) + .filter((g): g is { sourceId: string; rowId: number } => g.rowId !== null); +} + +function membershipOps(local: LocalContact | null, card: ContactCardWire, ctx: ToyContactsContext, parent: number | null): ProviderOp[] { + const wanted = wantedGroups(card, ctx); + const have = local ? membershipRows(local) : []; + const ops: ProviderOp[] = []; + for (const row of have) { + if (!wanted.some((g) => g.sourceId === row.cells.group_sourceid)) ops.push({ op: 'delete', table: 'data', id: row.id, expectCount: 1 }); + } + for (const group of wanted) { + if (have.some((r) => r.cells.group_sourceid === group.sourceId)) continue; + const values = { [Data.MIMETYPE]: MimeType.GROUP_MEMBERSHIP, [GroupMembership.GROUP_ROW_ID]: group.rowId }; + ops.push( + parent === null + ? { op: 'insert', table: 'data', values, refs: { [Data.RAW_CONTACT_ID]: 0 } } + : { op: 'insert', table: 'data', values: { ...values, [Data.RAW_CONTACT_ID]: parent } }, + ); + } + return ops; +} + +export const toyContactsPlanner: ContactsPlanner = { + rawContactColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4', 'raw_contact_is_read_only'], + dataColumns: ['_id', 'raw_contact_id', 'mimetype', 'data1', 'data_sync1', 'data_sync3', 'group_sourceid'], + groupColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'title', 'sync2', 'sync3', 'sync4'], + + decodeContact(rc: Row, data: Row[]): LocalContact { + return { + rawContactId: num(rc._id), + sourceId: str(rc.sourceid), + version: num(rc.version), + dirty: flag(rc.dirty), + deleted: flag(rc.deleted), + readOnly: flag(rc.raw_contact_is_read_only), + collections: (str(rc.sync1) ?? '').split(',').filter(Boolean), + shadow: parseJsonColumn(rc.sync2), + pending: parseJsonColumn(rc.sync3), + poison: parseJsonColumn(rc.sync4), + rows: data.map((d) => ({ + id: num(d._id), + mimetype: String(d.mimetype), + cells: { data1: d.data1 ?? null, group_sourceid: d.group_sourceid ?? null }, + key: str(d.data_sync1), + photoHash: null, + baseline: parseJsonColumn(d.data_sync3), + })), + }; + }, + + decodeGroup(row: Row): LocalGroup { + return { + groupId: num(row._id), + sourceId: str(row.sourceid), + version: num(row.version), + dirty: flag(row.dirty), + deleted: flag(row.deleted), + title: str(row.title), + shadow: parseJsonColumn(row.sync2), + pending: parseJsonColumn(row.sync3), + poison: parseJsonColumn(row.sync4), + }; + }, + + planDownload(card, local, context): DownloadPlan { + const ctx = context as ToyContactsContext; + const acct = ctx.jmapAccountId; + const ref = `${acct}/${card.id}`; + const name = nameOf(card); + const raw = { sourceid: ref, sync1: cardCollections(acct, card), sync2: shadowOf(card), raw_contact_is_read_only: ctx.isReadOnly(card) ? 1 : 0 }; + if (!local) { + const ops: ProviderOp[] = [ + { op: 'insert', table: 'raw_contacts', values: raw }, + { + op: 'insert', + table: 'data', + values: { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: name, data_sync1: 'name', data_sync3: JSON.stringify({ data1: name }) }, + refs: { [Data.RAW_CONTACT_ID]: 0 }, + }, + ...membershipOps(null, card, ctx, null), + ]; + return { ops: { ref, ops }, conflicts: 0, stillDirty: false, effect: 'insert', writes: ops.length }; + } + const row = nameRow(local); + const guard: ProviderOp = { + op: 'assert', + table: 'raw_contacts', + id: local.rawContactId, + values: { version: local.version, dirty: local.dirty ? 1 : 0 }, + expectCount: 1, + }; + const ops: ProviderOp[] = [guard]; + let conflicts = 0; + let stillDirty = false; + const writeName = (value: string) => { + if (row) { + ops.push({ op: 'update', table: 'data', id: row.id, values: { data1: value, data_sync3: JSON.stringify({ data1: value }) }, expectCount: 1 }); + } else { + ops.push({ + op: 'insert', + table: 'data', + values: { [Data.RAW_CONTACT_ID]: local.rawContactId, [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: value, data_sync1: 'name', data_sync3: JSON.stringify({ data1: value }) }, + }); + } + }; + const adopting = !local.sourceId && local.pending?.uid === card.uid; + if (local.deleted) { + // The device delete wins over a server edit; the destroy is on its way. + return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 }; + } + const localValue = String(row?.cells.data1 ?? ''); + if (adopting) { + // Our own create: identity and shadow; the rows stay the user's, measured against the server's value. + ops.push({ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { ...raw, sync3: null } }); + if (row) ops.push({ op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: name }) }, expectCount: 1 }); + return { ops: { ref, ops }, conflicts: 0, stillDirty: localValue !== name, effect: 'update', writes: ops.length - 1 }; + } + const shadowName = nameOf(local.shadow); + const baseValue = String(row?.baseline?.data1 ?? shadowName); + const localChanged = local.dirty && localValue !== baseValue; + const remoteChanged = name !== shadowName; + if (!localChanged) { + if (localValue !== name) writeName(name); + } else if (remoteChanged) { + if (localValue === name) writeName(name); + else { + writeName(name); + conflicts++; + } + } else { + stillDirty = true; + } + ops.push(...membershipOps(local, card, ctx, local.rawContactId)); + const shadowSame = deepEqual(local.shadow, card) && local.collections.join(',') === raw.sync1 && local.readOnly === !!raw.raw_contact_is_read_only; + if (!shadowSame) ops.push({ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: raw }); + const writes = ops.length - 1; + return { ops: { ref, ops }, conflicts, stillDirty, effect: writes > 0 ? 'update' : 'none', writes }; + }, + + planLocalDelete(local): OpGroup { + return { ref: local.sourceId ?? `row:${local.rawContactId}`, ops: [{ op: 'delete', table: 'raw_contacts', id: local.rawContactId }] }; + }, + + planBaselineHeal(local): OpGroup | null { + const row = nameRow(local); + if (local.dirty || !row || deepEqual(row.baseline, { data1: row.cells.data1 })) return null; + return { + ref: local.sourceId ?? `row:${local.rawContactId}`, + ops: [ + { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version, dirty: 0 }, expectCount: 1 }, + { op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: row.cells.data1 }) }, expectCount: 1 }, + ], + }; + }, + + planUpload(local, ctx): UploadPlan { + const ref = local.sourceId ?? `row:${local.rawContactId}`; + const guard: ProviderOp = { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version }, expectCount: 1 }; + const parsed = parseObjectRef(local.sourceId); + if (local.deleted) { + if (parsed) { + const books = onIds(local.shadow?.addressBookIds); + const synced = books.filter((b) => ctx.isSelected(collectionKey(parsed.accountId, b))); + if (synced.length && synced.length < books.length) { + return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch: Object.fromEntries(synced.map((b) => [`addressBookIds/${b}`, null])) }] }; + } + return { kind: 'upload', actions: [{ kind: 'destroy', id: parsed.id, uid: null }] }; + } + if (local.pending) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: local.pending.uid }] }; + return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'raw_contacts', id: local.rawContactId }] } }; + } + const row = nameRow(local); + const value = String(row?.cells.data1 ?? ''); + if (!parsed) { + if (!local.pending) { + const target = ctx.createTarget(); + if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' }; + return { + kind: 'claim', + ops: { ref, ops: [guard, { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { sync3: JSON.stringify({ uid: ctx.mintUid(), target }) } }] }, + }; + } + const target = parseCollectionKey(local.pending.target); + if (!target) return { kind: 'skip', reason: 'badTarget' }; + return { + kind: 'upload', + actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object: { uid: local.pending.uid, name: { full: value } } }], + }; + } + if (local.readOnly) { + const shadowName = nameOf(local.shadow); + return { + kind: 'revert', + ops: { + ref, + ops: [ + guard, + ...(row ? [{ op: 'update' as const, table: 'data' as const, id: row.id, values: { data1: shadowName, data_sync3: JSON.stringify({ data1: shadowName }) } }] : []), + { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { dirty: 0 } }, + ], + }, + }; + } + const base = String(row?.baseline?.data1 ?? nameOf(local.shadow)); + if (value === base) { + return { kind: 'clean', ops: { ref, ops: [{ ...guard, values: { version: local.version, dirty: 1 } }, { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { dirty: 0 } }] } }; + } + const patch = local.shadow?.name ? { 'name/full': value } : { name: { full: value } }; + return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch }] }; + }, + + planAccepted(local, server, ctx): AcceptedPlan { + const acct = ctx.jmapAccountId; + const ref = `${acct}/${server.id}`; + const row = nameRow(local); + const identity = { sourceid: ref, sync1: cardCollections(acct, server), sync2: shadowOf(server), sync3: null }; + const baseline: ProviderOp[] = row + ? [{ op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: row.cells.data1 }) }, expectCount: 1 }] + : []; + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { ...identity, dirty: 0 } }, + ...baseline, + ], + }, + keepDirtyOps: { ref, ops: [{ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: identity }, ...baseline] }, + }; + }, + + planGroupDownload(card, local, ctx): DownloadPlan { + const acct = ctx.jmapAccountId; + const ref = `${acct}/${card.id}`; + const values = { sourceid: ref, title: nameOf(card), group_visible: 1, sync2: shadowOf(card) }; + if (!local) { + return { ops: { ref, ops: [{ op: 'insert', table: 'groups', values }] }, conflicts: 0, stillDirty: false, effect: 'insert', writes: 1 }; + } + if (local.deleted) return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 }; + const adopting = !local.sourceId && local.pending?.uid === card.uid; + if (!adopting && local.title === values.title && deepEqual(local.shadow, card)) { + return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: local.dirty, effect: 'none', writes: 0 }; + } + const localChanged = local.dirty && local.title !== nameOf(local.shadow); + const keepTitle = adopting || (localChanged && values.title === nameOf(local.shadow)); + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'groups', id: local.groupId, values: { ...values, ...(keepTitle ? { title: local.title } : {}), sync3: null } }, + ], + }, + conflicts: localChanged && !keepTitle && values.title !== local.title ? 1 : 0, + stillDirty: keepTitle && local.dirty, + effect: 'update', + writes: 1, + }; + }, + + planGroupLocalDelete(local): OpGroup { + return { ref: local.sourceId ?? `group:${local.groupId}`, ops: [{ op: 'delete', table: 'groups', id: local.groupId }] }; + }, + + planGroupUpload(local, ctx): UploadPlan { + const ref = local.sourceId ?? `group:${local.groupId}`; + const parsed = parseObjectRef(local.sourceId); + if (local.deleted) { + if (parsed) return { kind: 'upload', actions: [{ kind: 'destroy', id: parsed.id, uid: null }] }; + return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'groups', id: local.groupId }] } }; + } + if (!parsed) { + if (!local.pending) { + const target = ctx.createTarget(); + if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' }; + return { + kind: 'claim', + ops: { + ref, + ops: [ + { op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'groups', id: local.groupId, values: { sync3: JSON.stringify({ uid: ctx.mintUid(), target }) } }, + ], + }, + }; + } + const target = parseCollectionKey(local.pending.target); + if (!target) return { kind: 'skip', reason: 'badTarget' }; + return { + kind: 'upload', + actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object: { uid: local.pending.uid, kind: 'group', name: { full: local.title ?? '' } } }], + }; + } + if ((local.title ?? '') === nameOf(local.shadow)) { + return { + kind: 'clean', + ops: { ref, ops: [{ op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, { op: 'update', table: 'groups', id: local.groupId, values: { dirty: 0 } }] }, + }; + } + return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch: { 'name/full': local.title ?? '' } }] }; + }, + + planGroupAccepted(local, server, ctx): AcceptedPlan { + const ref = `${ctx.jmapAccountId}/${server.id}`; + const values = { sourceid: ref, sync2: shadowOf(server), sync3: null }; + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'groups', id: local.groupId, values: { ...values, dirty: 0 } }, + ], + }, + keepDirtyOps: { ref, ops: [{ op: 'update', table: 'groups', id: local.groupId, values }] }, + }; + }, + + planMembershipUploads(contacts, groups, ctx): UploadAction[] { + const acct = ctx.jmapAccountId; + const patches = new Map>(); + for (const contact of contacts) { + const uid = contact.shadow?.uid ?? contact.pending?.uid; + if (!contact.sourceId || typeof uid !== 'string') continue; + const wanted = new Set(membershipRows(contact).map((r) => String(r.cells.group_sourceid))); + for (const group of groups) { + const parsed = parseObjectRef(group.sourceId); + if (!parsed || parsed.accountId !== acct) continue; + const member = group.shadow?.members?.[uid] === true; + const want = wanted.has(group.sourceId as string); + if (member === want) continue; + const patch = patches.get(parsed.id) ?? {}; + patch[ptr('members', uid)] = want ? true : null; + patches.set(parsed.id, patch); + } + } + return [...patches].map(([id, patch]) => ({ kind: 'update', id, patch })); + }, +}; + +// ─── Calendar ─────────────────────────────────────────── + +const EVENT_HOUR = 3_600_000; + +function startMs(event: CalendarEventWire): number { + return Date.parse(`${String(event.start ?? '2026-01-01T00:00:00').slice(0, 19)}Z`); +} + +function localStart(ms: number): string { + return new Date(ms).toISOString().slice(0, 19); +} + +function eventRef(ctx: CalendarContext, event: CalendarEventWire): string { + return `${ctx.jmapAccountId}/${event.id}`; +} + +export const toyCalendarPlanner: CalendarPlanner = { + calendarColumns: ['_id', '_sync_id', 'name', 'calendar_displayName', 'calendar_access_level', 'maxReminders', 'cal_sync2', 'cal_sync3'], + eventColumns: [ + '_id', 'calendar_id', '_sync_id', 'dirty', 'deleted', 'title', 'dtstart', 'dtend', 'eventTimezone', 'original_id', + 'original_sync_id', 'originalInstanceTime', 'sync_data1', 'sync_data2', 'sync_data3', 'sync_data4', 'sync_data5', 'uid2445', 'mutators', + ], + attendeeColumns: ['_id', 'event_id', 'attendeeEmail'], + reminderColumns: ['_id', 'event_id', 'minutes'], + + decodeCalendar(row: Row): LocalCalendar { + return { + calendarRowId: num(row._id), + syncId: str(row._sync_id), + cells: row, + shadow: parseJsonColumn(row.cal_sync2), + flags: parseJsonColumn(row.cal_sync3), + }; + }, + + decodeEvents(events: Row[], attendees: Row[], reminders: Row[]): LocalEvent[] { + const toRow = (row: Row) => ({ + eventId: num(row._id), + calendarRowId: num(row.calendar_id), + syncId: str(row._sync_id), + dirty: flag(row.dirty), + deleted: flag(row.deleted), + cells: { title: row.title ?? null, dtstart: row.dtstart ?? null, dtend: row.dtend ?? null }, + attendees: attendees.filter((a) => num(a.event_id) === num(row._id)).map((a) => ({ id: num(a._id), cells: { attendeeEmail: a.attendeeEmail ?? null } })), + reminders: reminders.filter((r) => num(r.event_id) === num(row._id)).map((r) => ({ id: num(r._id), cells: { minutes: r.minutes ?? null } })), + baseline: parseJsonColumn<{ cells: Row; attendees: Row[]; reminders: Row[] }>(row.sync_data4), + pending: parseJsonColumn(row.sync_data3), + poison: parseJsonColumn(row.sync_data5), + mutators: str(row.mutators), + }); + const isException = (row: Row) => row.original_id !== null && row.original_id !== undefined || !!str(row.original_sync_id); + const masters = events.filter((r) => !isException(r)).map((row) => ({ + ...toRow(row), + shadow: parseJsonColumn(row.sync_data1), + exceptions: events + .filter((x) => isException(x) && (num(x.original_id) === num(row._id) || (str(x.original_sync_id) !== null && str(x.original_sync_id) === str(row._sync_id)))) + .map((x) => ({ ...toRow(x), recurrenceId: str(x.sync_data2), originalInstanceTime: x.originalInstanceTime === null ? null : num(x.originalInstanceTime) })), + })) as LocalEvent[]; + const bySyncId = new Map(); + for (const m of masters) if (m.syncId) bySyncId.set(m.syncId, [...(bySyncId.get(m.syncId) ?? []), m]); + for (const pair of bySyncId.values()) { + if (pair.length < 2) continue; + pair.sort((a, b) => a.eventId - b.eventId); + pair[0].split = 'source'; + for (const clone of pair.slice(1)) clone.split = 'clone'; + } + return masters; + }, + + planCalendars(selected, local, ctx) { + const groups: OpGroup[] = []; + const keys = new Set(); + for (const { jmapAccountId, calendar, readOnly, accountName } of selected) { + const key = collectionKey(jmapAccountId, calendar.id); + keys.add(key); + const values = { + name: calendar.name, + calendar_displayName: accountName ? `${calendar.name} (${accountName})` : calendar.name, + calendar_access_level: readOnly ? 200 : 700, + maxReminders: ctx.reminderOwner === 'device' ? 5 : 0, + cal_sync2: JSON.stringify(calendar), + cal_sync3: JSON.stringify(readOnly ? { readOnly: 'rights' } : {}), + }; + const existing = local.find((l) => l.syncId === key); + if (!existing) { + groups.push({ + ref: key, + ops: [{ op: 'insert', table: 'calendars', values: { ...values, _sync_id: key, calendar_color: -16776961, ownerAccount: ctx.ownerAccount, sync_events: 1, visible: 1 } }], + }); + } else if (Object.entries(values).some(([k, v]) => String(existing.cells[k] ?? '') !== String(v))) { + groups.push({ ref: key, ops: [{ op: 'update', table: 'calendars', id: existing.calendarRowId, values }] }); + } + } + return { groups, deleteCalendarRows: local.filter((l) => !l.syncId || !keys.has(l.syncId)).map((l) => l.calendarRowId) }; + }, + + planDownload(event, local, ctx): DownloadPlan { + const ref = eventRef(ctx, event); + const calendarRowId = onIds(event.calendarIds).map((id) => ctx.calendarRowId(id)).find((id) => id !== null) ?? null; + const start = startMs(event); + const cells = { title: event.title ?? '', dtstart: start, dtend: start + EVENT_HOUR }; + const baseline = JSON.stringify({ cells: { title: cells.title }, attendees: [], reminders: [] }); + if (!local) { + if (calendarRowId === null) return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: false, effect: 'none', writes: 0 }; + const ops: ProviderOp[] = [{ + op: 'insert', + table: 'events', + // CalendarProvider leaves DIRTY NULL on a sync-adapter insert unless it is written. + values: { calendar_id: calendarRowId, _sync_id: ref, ...cells, eventTimezone: 'UTC', eventStatus: 1, dirty: 0, uid2445: event.uid, sync_data1: JSON.stringify(event), sync_data4: baseline }, + }]; + return { ops: { ref, ops }, conflicts: 0, stillDirty: false, effect: 'insert', writes: 1 }; + } + const guard: ProviderOp = { op: 'assert', table: 'events', id: local.eventId, values: { dirty: local.dirty ? 1 : 0, title: local.cells.title }, expectCount: 1 }; + if (local.deleted) return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 }; + const adopting = !parseObjectRef(local.syncId) && (local.pending?.uid ?? pendingUidOf(local.syncId)) === event.uid; + if (adopting) { + return { + ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { _sync_id: ref, sync_data1: JSON.stringify(event), sync_data3: null, sync_data4: baseline } }] }, + conflicts: 0, + stillDirty: local.cells.title !== event.title, + effect: 'update', + writes: 1, + }; + } + const shadowTitle = local.shadow?.title ?? ''; + const baseTitle = String(local.baseline?.cells.title ?? shadowTitle); + const localChanged = local.dirty && local.cells.title !== baseTitle; + const remoteChanged = (event.title ?? '') !== shadowTitle; + const values: Row = {}; + let conflicts = 0; + if (!localChanged || remoteChanged) { + if (local.cells.title !== cells.title || local.cells.dtstart !== cells.dtstart) Object.assign(values, cells, { sync_data4: baseline }); + if (localChanged && local.cells.title !== cells.title) conflicts = 1; + } + if (!deepEqual(local.shadow, event)) values.sync_data1 = JSON.stringify(event); + if (!Object.keys(values).length) return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: localChanged && !remoteChanged, effect: 'none', writes: 0 }; + return { + ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values, expectCount: 1 }] }, + conflicts, + stillDirty: localChanged && !remoteChanged, + effect: 'update', + writes: 1, + }; + }, + + planLocalDelete(local): OpGroup { + return { + ref: local.syncId ?? `row:${local.eventId}`, + ops: [ + ...local.exceptions.map((x) => ({ op: 'delete' as const, table: 'events' as const, id: x.eventId })), + { op: 'delete', table: 'events', id: local.eventId }, + ], + }; + }, + + planBaselineHeal() { + return null; + }, + + planUpload(local, ctx): UploadPlan { + const ref = local.syncId ?? `row:${local.eventId}`; + const identity = local.split === 'clone' ? null : parseObjectRef(local.syncId); + const pendingUid = local.pending?.uid ?? pendingUidOf(local.syncId); + if (local.deleted) { + if (identity) return { kind: 'upload', actions: [{ kind: 'destroy', id: identity.id, uid: null }] }; + if (pendingUid) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: pendingUid }] }; + return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'events', id: local.eventId }] } }; + } + const guard: ProviderOp = { op: 'assert', table: 'events', id: local.eventId, values: { title: local.cells.title }, expectCount: 1 }; + if (!identity) { + const calendar = ctx.calendarIdOfRow(local.calendarRowId); + if (!calendar) return { kind: 'skip', reason: 'notOurCalendar' }; + if (!pendingUid || local.split === 'clone') { + const uid = ctx.mintUid(); + return { + kind: 'claim', + ops: { + ref, + ops: [guard, { + op: 'update', + table: 'events', + id: local.eventId, + values: { _sync_id: pendingRef(uid), uid2445: uid, sync_data3: JSON.stringify({ uid, target: collectionKey(calendar.jmapAccountId, calendar.calendarId) }) }, + }], + }, + }; + } + const start = Number(local.cells.dtstart ?? 0); + return { + kind: 'upload', + actions: [{ + kind: 'create', + uid: pendingUid, + collectionId: calendar.calendarId, + object: { uid: pendingUid, title: String(local.cells.title ?? ''), start: localStart(start), duration: 'PT1H', timeZone: 'Etc/UTC' }, + }], + }; + } + if (ctx.isReadOnly(onIds(local.shadow?.calendarIds)[0] ?? '')) { + return { kind: 'revert', ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { title: local.shadow?.title ?? '', dirty: 0 } }] } }; + } + const base = String(local.baseline?.cells.title ?? local.shadow?.title ?? ''); + if (local.cells.title === base) { + return { kind: 'clean', ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { dirty: 0 } }] } }; + } + return { kind: 'upload', actions: [{ kind: 'update', id: identity.id, patch: { title: String(local.cells.title ?? '') } }] }; + }, + + planAccepted(local, server, ctx): AcceptedPlan { + const ref = eventRef(ctx, server); + const values = { + _sync_id: ref, + sync_data1: JSON.stringify(server), + sync_data3: null, + sync_data4: JSON.stringify({ cells: { title: local.cells.title }, attendees: [], reminders: [] }), + }; + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'events', id: local.eventId, values: { title: local.cells.title }, expectCount: 1 }, + { op: 'update', table: 'events', id: local.eventId, values: { ...values, dirty: 0 } }, + ], + }, + keepDirtyOps: { ref, ops: [{ op: 'update', table: 'events', id: local.eventId, values }] }, + }; + }, + + planPairs() { + return []; + }, + planZoneChange() { + return null; + }, + planReminderOwnerChange() { + return null; + }, +}; + +/** An Events column list the fake accepts in `where`, for tests that read rows. */ +export const EVENT_TITLE = Events.TITLE; diff --git a/src/device-sync/engine/batch.ts b/src/device-sync/engine/batch.ts new file mode 100644 index 00000000..2b4095ee --- /dev/null +++ b/src/device-sync/engine/batch.ts @@ -0,0 +1,216 @@ +/** + * Packs item op groups into provider batches and applies them + * (docs/device-sync.md, "SyncState", "Native module API", "Failure matrix"): + * + * - a group is never split; `refs` are rebased to the batch and the group's + * first op is its yield point, so the provider may commit between groups + * but never inside one; + * - at most `maxOps` ops and about `maxBytes` of JSON per batch; groups that + * carry photo bytes travel alone (Binder budget); + * - the SyncState op, when there is one, is the last op of the last batch + * and never a yield point: the stored state can't get ahead of its rows. + * + * A failed batch may have been committed up to a yield point the provider + * took, so its groups are retried one by one (their asserts make a retry of + * a committed group fail harmlessly); a group that fails an assert is + * re-read and re-planned up to `maxReplans` times, then given up. `tooLarge` + * splits the batch; `permission` ends the run. + */ +import type { OpGroup } from '../planner'; +import type { BatchFailure, OpResult, ProviderOp, ProviderPort } from '../types'; +import { RunAbort } from './errors'; +import { hasWrites } from './provider'; +import type { Tail } from './sync-state'; + +export interface Work { + group: OpGroup; + /** The group applied; `results` are its ops' results (new row ids for inserts). */ + applied?(results: OpResult[]): void | Promise; + /** After an assert failure: read the item again and plan it again; null when nothing is left to write. */ + replan?(): Promise; + /** Given up: the item's rows stay as they are. */ + failed?(reason: BatchFailure, message: string): void | Promise; +} + +export interface BatchWriterOptions { + maxOps: number; + maxBytes: number; + maxReplans: number; +} + +function sizeOf(ops: readonly ProviderOp[]): number { + return JSON.stringify(ops).length; +} + +function carriesBlob(ops: readonly ProviderOp[]): boolean { + return ops.some( + (op) => + (op.op === 'insert' || op.op === 'update') && + Object.values(op.values).some((v) => v !== null && typeof v === 'object'), + ); +} + +/** A group's ops at `offset` in a batch: refs rebased, the first op a yield point, no other. */ +function placeGroup(ops: readonly ProviderOp[], offset: number): ProviderOp[] { + return ops.map((op, i) => { + const copy = { ...op } as ProviderOp & { yieldAllowed?: boolean }; + if (copy.op === 'insert' && copy.refs) { + const refs: Record = {}; + for (const [column, index] of Object.entries(copy.refs)) refs[column] = index + offset; + copy.refs = refs; + } + if (i === 0) copy.yieldAllowed = true; + else delete copy.yieldAllowed; + return copy; + }); +} + +export function buildBatch(groups: readonly OpGroup[], tailOp?: ProviderOp): ProviderOp[] { + const out: ProviderOp[] = []; + for (const group of groups) out.push(...placeGroup(group.ops, out.length)); + if (tailOp) { + const { yieldAllowed: _never, ...last } = tailOp as ProviderOp & { yieldAllowed?: boolean }; + out.push(last as ProviderOp); + } + return out; +} + +/** Prepends ops to a group (e.g. an engine-side assert), shifting the group's refs. */ +export function prependOps(group: OpGroup, ops: ProviderOp[]): OpGroup { + return { ref: group.ref, ops: [...ops, ...placeGroup(group.ops, ops.length).map(stripYield)] }; +} + +/** One group running `first` then `second` atomically. */ +export function concatGroups(ref: string, first: OpGroup, second: OpGroup): OpGroup { + return { + ref, + ops: [...first.ops.map(stripYield), ...placeGroup(second.ops, first.ops.length).map(stripYield)], + }; +} + +function stripYield(op: ProviderOp): ProviderOp { + const { yieldAllowed: _y, ...rest } = op as ProviderOp & { yieldAllowed?: boolean }; + return rest as ProviderOp; +} + +export class BatchWriter { + /** Batches sent, for tests and logs. */ + batchesSent = 0; + + constructor( + private readonly port: ProviderPort, + private readonly options: BatchWriterOptions, + ) {} + + /** + * Applies the works in order; `tail` (the SyncState op) goes last, after + * every group of `works` was applied or given up. + */ + async write(works: readonly Work[], tail?: Tail): Promise { + const todo = works.filter((w) => hasWrites(w.group.ops)); + const batches = this.pack(todo); + if (batches.length === 0) { + if (tail) await this.applyTail(tail); + return; + } + for (let i = 0; i < batches.length; i++) { + const last = i === batches.length - 1; + const batch = batches[i]; + // A photo batch is already at the Binder budget: its state op goes alone. + const tailHere = last && tail && !(batch.length === 1 && carriesBlob(batch[0].group.ops)) ? tail : undefined; + await this.applyBatch(batch, tailHere); + if (last && tail && !tailHere) await this.applyTail(tail); + } + } + + private pack(works: readonly Work[]): Work[][] { + const batches: Work[][] = []; + let current: Work[] = []; + let ops = 0; + let bytes = 0; + const flush = () => { + if (current.length) batches.push(current); + current = []; + ops = 0; + bytes = 0; + }; + for (const work of works) { + if (carriesBlob(work.group.ops)) { + flush(); + batches.push([work]); + continue; + } + const n = work.group.ops.length; + const b = sizeOf(work.group.ops); + if (current.length && (ops + n > this.options.maxOps || bytes + b > this.options.maxBytes)) flush(); + current.push(work); + ops += n; + bytes += b; + } + flush(); + return batches; + } + + private async send(ops: ProviderOp[]) { + this.batchesSent++; + const result = await this.port.applyBatch(ops); + if (!result.ok) { + if (result.reason === 'permission') throw new RunAbort('permission', result.message); + // The native side refused an op outside our account: an engine bug, never retried. + if (result.reason === 'scope') throw new Error(`Provider refused an op: ${result.message}`); + } + return result; + } + + private async applyBatch(works: Work[], tail?: Tail): Promise { + const tailOp = tail?.op(); + const result = await this.send(buildBatch(works.map((w) => w.group), tailOp)); + if (result.ok) { + let offset = 0; + for (const work of works) { + const n = work.group.ops.length; + await work.applied?.(result.results.slice(offset, offset + n)); + offset += n; + } + tail?.applied(); + return; + } + if (result.reason === 'tooLarge' && works.length > 1) { + const middle = Math.ceil(works.length / 2); + await this.applyBatch(works.slice(0, middle)); + await this.applyBatch(works.slice(middle), tail); + return; + } + if (works.length === 1 && !tail) { + await this.retryAlone(works[0], result.reason, result.message); + return; + } + for (const work of works) await this.applyAlone(work, 0); + if (tail) await this.applyTail(tail); + } + + private async applyAlone(work: Work, attempt: number): Promise { + if (!hasWrites(work.group.ops)) return; + const result = await this.send(buildBatch([work.group])); + if (result.ok) { + await work.applied?.(result.results); + return; + } + await this.retryAlone(work, result.reason, result.message, attempt); + } + + private async retryAlone(work: Work, reason: BatchFailure, message: string, attempt = 0): Promise { + if (reason === 'assert' && work.replan && attempt < this.options.maxReplans) { + const next = await work.replan(); + if (next) await this.applyAlone(next, attempt + 1); + return; + } + await work.failed?.(reason, message); + } + + private async applyTail(tail: Tail): Promise { + const result = await this.send([...buildBatch([], tail.op())]); + if (!result.ok) throw new Error(`Could not store the sync state: ${result.reason} ${result.message}`); + tail.applied(); + } +} diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts new file mode 100644 index 00000000..38997b03 --- /dev/null +++ b/src/device-sync/engine/calendar-sync.ts @@ -0,0 +1,663 @@ +/** + * Calendars: server calendars ↔ Calendars rows, events ↔ Events rows with + * their exceptions, attendees and reminders (docs/device-sync.md, "Calendar + * mapping"). Tasks never sync; a calendar that holds only tasks gets no row. + */ +import { findTasksOnlyCalendarIds, isTaskLikeObject, SCAN_PROPERTIES, type ScannedCalendarObject } from '../../lib/calendar-component-detection'; +import { collectionKey, parseCollectionKey, parseObjectRef, pendingUidOf } from '../common/ids'; +import type { CalendarContext, CalendarEventWire, CalendarLike, CalendarPlanner, LocalCalendar, LocalEvent, OpGroup } from '../planner'; +import { CALENDAR_AUTHORITY, type ReminderOwner, type Row } from '../types'; +import { CALENDAR_EVENT_PROPERTIES, CALENDAR_PROPERTIES } from '../wire'; +import { calendarAddresses, type CalendarAddresses, type CapabilityAccount } from '../jmap/session'; +import type { Work } from './batch'; +import type { RunEnv } from './context'; +import type { SubscriptionCalendar } from './deps'; +import { ItemSync, type Pending } from './item-sync'; +import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds'; +import { poisonFingerprint } from './poison'; +import { flag, num, str } from './provider'; +import { isCollectionSelected } from './selection'; +import { accountOf, type SyncState } from './sync-state'; + +const MASTER = 'original_id IS NULL AND original_sync_id IS NULL'; +const WITH_IDENTITY = "_sync_id IS NOT NULL AND _sync_id NOT LIKE '~pending/%'"; +/** CalendarProvider leaves DIRTY (and may leave DELETED) NULL on sync-adapter inserts: NULL is clean. */ +const CLEAN = '(dirty IS NULL OR dirty = 0) AND (deleted IS NULL OR deleted = 0)'; + +function onIds(map: Record | undefined | null): string[] { + return Object.entries(map ?? {}) + .filter(([, on]) => on === true) + .map(([id]) => id); +} + +function union(a: readonly string[], b: readonly string[]): string[] { + return [...new Set([...a, ...b])]; +} + +export class CalendarSync extends ItemSync { + protected readonly itemType = 'CalendarEvent' as const; + protected readonly itemProperties = CALENDAR_EVENT_PROPERTIES; + protected readonly parentFilter = 'inCalendar' as const; + protected readonly parentProperty = 'calendarIds' as const; + + readonly eventKind: Kind; + private readonly planner: CalendarPlanner; + private readonly calendarColumns: string[]; + private readonly eventColumns: string[]; + private readonly attendeeColumns: string[]; + private readonly reminderColumns: string[]; + + private readonly calendars = new Map(); + private readonly syncedKeys = new Set(); + private readonly readOnlyKeys = new Set(); + private readonly rowByKey = new Map(); + private readonly keyByRow = new Map(); + /** Calendar rows of deselected calendars: dropped after the upload. */ + private deleteRows: number[] = []; + /** Per account: calendars found to hold only tasks (true) or events again (false). */ + private readonly taskOnlyNotes = new Map>(); + private readonly collectionStates = new Map(); + private addresses: CalendarAddresses = { ownerAccount: '', selfAddresses: [] }; + private subscriptions: SubscriptionCalendar[] = []; + private zone = 'UTC'; + /** Who the Reminders rows of this run are written for (the stored owner until the owner-change pass). */ + private reminderOwner: ReminderOwner = 'device'; + + constructor(env: RunEnv) { + super(env); + this.planner = env.deps.planners.calendar; + this.calendarColumns = union(this.planner.calendarColumns, ['_id']); + this.eventColumns = union(this.planner.eventColumns, ['_id', 'calendar_id', '_sync_id', 'original_id', 'original_sync_id']); + this.attendeeColumns = union(this.planner.attendeeColumns, ['_id', 'event_id']); + this.reminderColumns = union(this.planner.reminderColumns, ['_id', 'event_id']); + this.eventKind = this.makeEventKind(); + } + + // ── Kind ── + + private makeEventKind(): Kind { + const planner = this.planner; + return { + name: 'event', + table: 'events', + identityColumn: '_sync_id', + poisonColumn: 'sync_data5', + meta: (e) => { + const identity = parseObjectRef(e.syncId) ? e.syncId : null; + const pendingUid = pendingUidOf(e.syncId); + const calendarKey = this.keyByRow.get(e.calendarRowId) ?? null; + const acct = accountOfRef(identity) ?? accountOfRef(calendarKey); + return { + rowId: e.eventId, + // A split clone shares its source's _SYNC_ID but uploads as a new event. + sourceId: e.split === 'clone' ? null : identity, + pending: e.pending ?? (pendingUid && calendarKey ? { uid: pendingUid, target: calendarKey } : null), + dirty: e.dirty || e.split !== undefined || e.exceptions.some((x) => x.dirty || x.deleted), + deleted: e.deleted, + isNew: !identity || e.split === 'clone', + poison: e.poison, + collections: e.shadow && acct + ? onIds(e.shadow.calendarIds as Record).map((id) => collectionKey(acct, id)) + : calendarKey ? [calendarKey] : [], + }; + }, + fingerprint: (e) => + poisonFingerprint({ + deleted: e.deleted, + shadow: e.shadow, + cells: e.cells, + attendees: e.attendees.map((a) => JSON.stringify(a.cells)).sort(), + reminders: e.reminders.map((r) => JSON.stringify(r.cells)).sort(), + exceptions: e.exceptions + .map((x) => JSON.stringify([x.deleted, x.cells, x.attendees.map((a) => a.cells), x.reminders.map((r) => r.cells)])) + .sort(), + }), + loadByRefs: async (refs) => { + const out = new Map(); + for (const e of await this.loadMastersIn('_sync_id', refs)) { + if (e.split !== 'clone' && e.syncId && parseObjectRef(e.syncId)) out.set(e.syncId, e); + } + return out; + }, + loadByRowIds: (ids) => this.loadMastersIn('_id', ids), + loadNew: () => this.loadMastersWhere("_sync_id IS NULL OR _sync_id LIKE '~pending/%'"), + planDownload: (event, local, acct) => planner.planDownload(event, local, this.ctx(acct)), + planLocalDelete: (e) => planner.planLocalDelete(e), + planBaselineHeal: (e) => planner.planBaselineHeal(e), + planUpload: (e, acct) => planner.planUpload(e, this.ctx(acct)), + planAccepted: (e, event, acct) => planner.planAccepted(e, event, this.ctx(acct)), + }; + } + + protected kinds(): Kind[] { + return [this.eventKind]; + } + + /** Tasks never sync (explicit `@type: "Task"`, or CalDAV VTODOs recognised by their task-only fields). */ + protected kindOf(object: ServerObject): Kind | null { + return isTaskLikeObject(object as ScannedCalendarObject) ? null : this.eventKind; + } + + // ── Loading ── + + private async decodeWithExceptions(masters: Row[]): Promise { + if (!masters.length) return []; + const byId = new Map(); + for (const row of masters) byId.set(num(row._id), row); + const ids = masters.map((r) => num(r._id)); + const syncIds = masters.map((r) => str(r._sync_id)).filter((s): s is string => !!s); + for (const row of await this.env.reader.rowsIn('events', this.eventColumns, 'original_id', ids)) byId.set(num(row._id), row); + if (syncIds.length) { + for (const row of await this.env.reader.rowsIn('events', this.eventColumns, 'original_sync_id', syncIds)) byId.set(num(row._id), row); + } + const all = [...byId.values()]; + const eventIds = all.map((r) => num(r._id)); + const attendees = await this.env.reader.rowsIn('attendees', this.attendeeColumns, 'event_id', eventIds); + const reminders = await this.env.reader.rowsIn('reminders', this.reminderColumns, 'event_id', eventIds); + return this.planner.decodeEvents(all, attendees, reminders); + } + + private async loadMastersIn(column: '_id' | '_sync_id', values: ReadonlyArray): Promise { + if (!values.length) return []; + return this.decodeWithExceptions(await this.env.reader.rowsIn('events', this.eventColumns, column, values, { where: MASTER })); + } + + private async loadMastersWhere(where: string, args: Array = []): Promise { + return this.decodeWithExceptions(await this.env.reader.rows('events', this.eventColumns, `(${where}) AND ${MASTER}`, args)); + } + + private async masterIds(where: string, args: Array = []): Promise { + return (await this.env.reader.rows('events', ['_id'], `(${where}) AND ${MASTER}`, args)).map((r) => num(r._id)); + } + + private async loadCalendars(): Promise { + return (await this.env.reader.rows('calendars', this.calendarColumns)).map((r) => this.planner.decodeCalendar(r)); + } + + private async refreshCalendarRows(): Promise { + this.rowByKey.clear(); + this.keyByRow.clear(); + for (const calendar of await this.loadCalendars()) { + if (!calendar.syncId || !parseCollectionKey(calendar.syncId)) continue; + this.rowByKey.set(calendar.syncId, calendar.calendarRowId); + this.keyByRow.set(calendar.calendarRowId, calendar.syncId); + } + } + + // ── Collections ── + + private isFeed(account: CapabilityAccount, calendarId: string): boolean { + return this.subscriptions.some( + (s) => s.calendarId === calendarId && (s.jmapAccountId === account.id || (s.jmapAccountId === null && account.primary)), + ); + } + + private noteTaskOnly(acct: string, calendarId: string, taskOnly: boolean): void { + let notes = this.taskOnlyNotes.get(acct); + if (!notes) this.taskOnlyNotes.set(acct, (notes = new Map())); + notes.set(calendarId, taskOnly); + } + + private isTaskOnly(acct: string, calendarId: string): boolean { + return this.taskOnlyNotes.get(acct)?.get(calendarId) ?? (this.env.store.account(acct).taskOnly ?? []).includes(calendarId); + } + + /** A calendar whose first objects are all tasks (Stalwart offers no per-calendar component set). */ + private async scanTaskOnly(acct: string, calendarId: string): Promise { + const ids = await this.env.jmap.queryFirst('CalendarEvent', acct, { inCalendar: calendarId }, this.env.tuning.taskScanLimit); + if (!ids.length) return false; + const { list } = await this.env.jmap.get('CalendarEvent', acct, ids, SCAN_PROPERTIES); + return findTasksOnlyCalendarIds(list.map((o) => ({ ...o, calendarIds: { [calendarId]: true } })), [calendarId]).has(calendarId); + } + + protected async collections(write: boolean): Promise { + this.zone = this.env.deps.deviceZone(); + this.reminderOwner = this.env.store.committed.reminderOwner ?? this.env.prefs.reminderOwner; + this.addresses = await calendarAddresses(this.env.jmap, this.env.accountName); + this.subscriptions = await this.env.deps.subscriptionCalendars(); + this.syncedKeys.clear(); + this.readOnlyKeys.clear(); + const selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }> = []; + for (const account of this.env.accounts) { + const { list, state } = await this.env.jmap.getContainers('Calendar', account.id, CALENDAR_PROPERTIES); + this.calendars.set(account.id, list); + if (state) { + this.collectionStates.set(account.id, state); + this.env.recordKnownState(account.id, 'Calendar', state); + } + const stored = this.env.store.account(account.id); + const reconciling = !stored.itemsState || !!stored.reconcile; + for (const calendar of list) { + if (calendar.myRights?.mayReadItems === false) continue; + if (!isCollectionSelected(this.env.prefs.calendarSelection, account, CALENDAR_AUTHORITY, calendar)) continue; + const key = collectionKey(account.id, calendar.id); + // Checked when a calendar comes to the device, and again on every full reconcile. + if (write && (reconciling || (!stored.selected.includes(key) && !this.isTaskOnly(account.id, calendar.id)))) { + this.noteTaskOnly(account.id, calendar.id, await this.scanTaskOnly(account.id, calendar.id)); + } + if (this.isTaskOnly(account.id, calendar.id)) continue; + const rights = calendar.myRights; + const readOnly = this.isFeed(account, calendar.id) || (!!rights && !rights.mayWriteAll && !rights.mayWriteOwn); + this.syncedKeys.add(key); + if (readOnly) this.readOnlyKeys.add(key); + selected.push({ jmapAccountId: account.id, calendar, readOnly, accountName: account.personal ? '' : account.name }); + } + } + await this.refreshCalendarRows(); + this.deleteRows = []; + if (!write) return; + const plan = this.planner.planCalendars(selected, await this.loadCalendars(), this.calendarCtx()); + await this.env.writer.write(plan.groups.map((group) => ({ group }))); + this.deleteRows = plan.deleteCalendarRows; + await this.refreshCalendarRows(); + } + + protected syncedCollections(acct: string): string[] { + return (this.calendars.get(acct) ?? []) + .filter((c) => this.syncedKeys.has(collectionKey(acct, c.id))) + .map((c) => c.id); + } + + protected inSelection(acct: string, object: ServerObject): boolean { + return onIds(object.calendarIds as Record).some((id) => this.syncedKeys.has(collectionKey(acct, id))); + } + + private findCalendar(calendarId: string): CalendarLike | undefined { + for (const account of this.env.accounts) { + const found = this.calendars.get(account.id)?.find((c) => c.id === calendarId); + if (found) return found; + } + return undefined; + } + + private calendarCtx(): Omit { + const env = this.env; + const primary = this.env.accounts.find((a) => a.primary)?.id ?? this.env.accounts[0]?.id ?? ''; + return { + now: env.now(), + mintKey: (taken) => env.mintKey(taken), + mintUid: () => env.mintUid(), + deviceZone: this.zone, + ownerAccount: this.addresses.ownerAccount, + selfAddresses: this.addresses.selfAddresses, + reminderOwner: this.reminderOwner, + calendar: (calendarId) => this.findCalendar(calendarId), + calendarRowId: (calendarId) => this.rowByKey.get(collectionKey(primary, calendarId)) ?? null, + calendarIdOfRow: (rowId) => { + const parsed = parseCollectionKey(this.keyByRow.get(rowId)); + return parsed ? { jmapAccountId: parsed.accountId, calendarId: parsed.id } : null; + }, + isReadOnly: (calendarId) => [...this.readOnlyKeys].some((k) => parseCollectionKey(k)?.id === calendarId), + isSelected: (key) => this.syncedKeys.has(key), + }; + } + + ctx(acct: string): CalendarContext { + return { + ...this.calendarCtx(), + jmapAccountId: acct, + calendar: (calendarId) => this.calendars.get(acct)?.find((c) => c.id === calendarId), + calendarRowId: (calendarId) => this.rowByKey.get(collectionKey(acct, calendarId)) ?? null, + isReadOnly: (calendarId) => this.readOnlyKeys.has(collectionKey(acct, calendarId)), + }; + } + + protected decorateState(acct: string, next: SyncState): () => void { + const account = accountOf(next, acct); + const notes = this.taskOnlyNotes.get(acct); + const consumed = new Map(notes ?? []); + if (consumed.size) { + const taskOnly = new Set(account.taskOnly ?? []); + for (const [id, only] of consumed) { + if (only) taskOnly.add(id); + else taskOnly.delete(id); + } + account.taskOnly = [...taskOnly]; + } + const collections = this.collectionStates.get(acct); + if (collections) account.collectionsState = collections; + return () => { + for (const [id, only] of consumed) if (notes?.get(id) === only) notes.delete(id); + }; + } + + protected async localIdentities(acct: string): Promise> { + const out = new Map(); + const rows = await this.env.reader.rows('events', ['_id', '_sync_id', 'dirty', 'deleted'], `_sync_id LIKE ? AND ${MASTER}`, [`${acct}/%`]); + for (const row of rows) { + const id = idInAccount(str(row._sync_id), acct); + if (id) out.set(id, { dirty: flag(row.dirty), deleted: flag(row.deleted) }); + } + return out; + } + + // ── Uploads ── + + /** + * Masters with local changes: dirty, deleted or new ones, masters of dirty + * or deleted exception rows, and both masters of a CONTENT_EXCEPTION_URI + * split (two masters sharing a _SYNC_ID; the older one's rule changed + * without DIRTY), loaded together so the planner sees the split. + */ + protected async loadUploadItems(): Promise { + const rows = await this.env.reader.rows( + 'events', + ['_id', '_sync_id', 'original_id', 'original_sync_id', 'uid2445'], + "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'", + ); + const ids = new Set(); + const refs = new Set(); + const newUids = new Set(); + for (const row of rows) { + if (row.original_id !== null && row.original_id !== undefined) ids.add(num(row.original_id)); + else if (str(row.original_sync_id)) refs.add(str(row.original_sync_id) as string); + else { + ids.add(num(row._id)); + const uid = str(row.uid2445); + if (uid && !parseObjectRef(str(row._sync_id))) newUids.add(uid); + } + } + const bySyncId = new Map(); + for (const row of await this.env.reader.rows('events', ['_id', '_sync_id'], `_sync_id IS NOT NULL AND ${MASTER}`)) { + const syncId = str(row._sync_id) as string; + bySyncId.set(syncId, [...(bySyncId.get(syncId) ?? []), num(row._id)]); + } + for (const [syncId, masters] of bySyncId) { + if (masters.length > 1 || refs.has(syncId)) for (const id of masters) ids.add(id); + } + // A new master may carry a uid another row has (an app copied an event): those rows come along, so + // the planner sees the collision and gives the new one a fresh uid instead of adopting the other's object. + if (newUids.size) { + for (const row of await this.env.reader.rowsIn('events', ['_id'], 'uid2445', [...newUids], { where: MASTER })) ids.add(num(row._id)); + } + const events = await this.loadMastersIn('_id', [...ids]); + return events.map((local) => ({ kind: this.eventKind as Kind, local })); + } + + /** Event uids are unique per account: only an event in the create's calendar can be the row's own create. */ + protected adoptionTargetMatches(acct: string, object: ServerObject, target: string): boolean { + const parsed = parseCollectionKey(target); + return !!parsed && parsed.accountId === acct && (object.calendarIds as Record | undefined)?.[parsed.id] === true; + } + + protected accountOfItem(held: Held): string | null { + return super.accountOfItem(held) ?? accountOfRef(this.keyByRow.get((held.local as LocalEvent).calendarRowId) ?? null); + } + + protected claimAccount(held: Held): string | null { + return accountOfRef(this.keyByRow.get((held.local as LocalEvent).calendarRowId) ?? null); + } + + /** A split clone is claimed after its source's rule was planned (see prePhase). */ + protected claimable(held: Held): boolean { + return (held.local as LocalEvent).split !== 'clone'; + } + + protected async countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set }> { + let count = 0; + const accounts = new Set(); + for (const held of items) { + const meta = held.kind.meta(held.local); + if (!meta.deleted || !meta.sourceId) continue; + count++; + const acct = accountOfRef(meta.sourceId); + if (acct) accounts.add(acct); + } + const synced = (await this.masterIds(WITH_IDENTITY)).length; + return { count, synced: Math.max(synced, count), accounts }; + } + + /** + * Before the creates: a split's source uploads its rule while the split is + * still visible, then its clone is claimed as a new event; and deleted + + * new masters of the run that are one edit (a move, a series turned + * single) upload as patches of the existing object. + */ + protected async prePhase(acct: string): Promise { + let items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); + const sources = items.filter((h) => (h.local as LocalEvent).split === 'source'); + const clones = items.filter((h) => (h.local as LocalEvent).split === 'clone'); + if (sources.length || clones.length) { + const works: Work[] = []; + const ready: Pending[] = []; + for (const held of sources) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); + } + // A clone (of a split, or a new master whose uid another row carries) needs no source: + // it claims a fresh uid here and is created with the other new events. + for (const held of clones) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); + } + await this.env.writer.write(works); + if (ready.length) await this.send(acct, ready); + items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); + } + const deleted = items.map((h) => h.local as LocalEvent).filter((e) => { + const m = this.eventKind.meta(e); + return m.deleted && !!m.sourceId; + }); + const fresh = items.map((h) => h.local as LocalEvent).filter((e) => { + const m = this.eventKind.meta(e); + return m.isNew && !m.deleted && !!m.pending; + }); + if (!deleted.length || !fresh.length) return; + let pairs: ReturnType; + try { + pairs = this.planner.planPairs(deleted, fresh, this.ctx(acct)); + } catch (error) { + // Unpaired, they upload as ordinary creates and deletions. + this.plannerFailed(acct, `pairs:${acct}`, 'upload', error); + return; + } + if (!pairs.length) return; + await this.env.checkpoints.check('upload:pairs'); + const pending: Pending[] = pairs.map((pair) => { + const meta = this.eventKind.meta(pair.deleted); + const ownId = idInAccount(meta.sourceId, acct); + return { + held: { kind: this.eventKind, local: pair.deleted }, + meta, + fingerprint: this.eventKind.fingerprint(pair.deleted), + actions: pair.actions as Pending['actions'], + accept: async (objects) => { + // The new row takes the identity and exceptions of the old one, which is purged. + await this.env.writer.write([{ group: pair.ops }]); + const [moved] = await this.eventKind.loadByRowIds([pair.fresh.eventId]); + const server = ownId ? objects.get(ownId) : undefined; + if (!moved || !server) { + if (ownId) this.markStale(acct, ownId); + return []; + } + const held: Held = { kind: this.eventKind, local: moved }; + try { + return [this.acceptWork(acct, { held, meta: this.eventKind.meta(moved), fingerprint: '', actions: [] }, server, false)]; + } catch (error) { + this.plannerFailed(acct, refOf(acct, server.id), 'upload', error, server.id); + return []; + } + }, + }; + }); + await this.send(acct, pending); + } + + // ── After the upload ── + + /** A master with nothing waiting for an upload. */ + private isClean(e: LocalEvent): boolean { + const m = this.eventKind.meta(e); + return !m.dirty && !m.deleted && !m.isNew; + } + + /** + * Deselected calendars lose their rows (the provider deletes their events) + * once nothing in them waits for an upload; until then only their clean + * events go. Events that left every synced calendar go when clean. + */ + protected async dropOutsideSelection(acct: string): Promise { + const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id))); + const deselected = this.env.store.account(acct).selected.filter((key) => !synced.has(key)); + const rows = this.deleteRows.filter((row) => accountOfRef(this.keyByRow.get(row) ?? null) === acct); + const outside = [...(this.outside.get(acct) ?? [])]; + if (!deselected.length && !rows.length && !outside.length) return; + const kept = new Set(); + for (const rowId of rows) { + await this.env.checkpoints.check('deselect'); + const key = this.keyByRow.get(rowId) as string; + if (await this.dropCalendarRow(rowId)) continue; + kept.add(key); + } + const works: Work[] = []; + for (const local of (await this.eventKind.loadByRefs(outside)).values()) { + const meta = this.eventKind.meta(local); + const id = idInAccount(meta.sourceId, acct); + if (!id || !meta.collections.length || meta.collections.some((k) => synced.has(k))) continue; + const work = this.removeWork(acct, id, { kind: this.eventKind, local }, 'outside'); + if (work) works.push(work); + } + const selected = [...synced, ...kept]; + await this.env.writer.write(works, this.tail([acct], (next) => { + accountOf(next, acct).selected = selected; + })); + this.outside.delete(acct); + } + + /** Deletes a calendar row when none of its events waits for an upload; else only its clean events. True when the row went. */ + private async dropCalendarRow(rowId: number): Promise { + const waiting = await this.env.reader.rows( + 'events', + ['_id'], + "calendar_id = ? AND (dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%')", + [rowId], + ); + if (!waiting.length) { + await this.env.writer.write([{ group: { ref: `calendar:${rowId}`, ops: [{ op: 'delete', table: 'calendars', id: rowId }] } }]); + return true; + } + const clean = await this.masterIds(`calendar_id = ? AND ${CLEAN} AND ${WITH_IDENTITY}`, [rowId]); + for (let i = 0; i < clean.length; i += this.env.tuning.chunkSize) { + await this.env.checkpoints.check('deselect'); + const works: Work[] = []; + for (const local of await this.eventKind.loadByRowIds(clean.slice(i, i + this.env.tuning.chunkSize))) { + const meta = this.eventKind.meta(local); + const acct = accountOfRef(meta.sourceId); + const id = acct ? idInAccount(meta.sourceId, acct) : null; + if (!acct || !id || !this.isClean(local)) continue; + const work = this.removeWork(acct, id, { kind: this.eventKind, local }, 'outside'); + if (work) works.push(work); + } + await this.env.writer.write(works); + } + return false; + } + + protected async afterSync(): Promise { + // Rows of calendars that belong to no account we know (garbage, or a vanished account). + for (const rowId of this.deleteRows) { + if (this.keyByRow.has(rowId) && accountOfRef(this.keyByRow.get(rowId) ?? null)) continue; + await this.dropCalendarRow(rowId); + } + await this.zonePass(); + await this.reminderOwnerPass(); + } + + /** Every clean master, in chunks, through `plan`; `finish` rides with the last chunk's batch. */ + private async passOverCleanMasters( + label: string, + plan: (e: LocalEvent, acct: string) => OpGroup | null, + finish: (next: SyncState) => void, + ): Promise { + const ids = await this.masterIds(`${CLEAN} AND ${WITH_IDENTITY}`); + const size = this.env.tuning.chunkSize; + if (!ids.length) { + await this.env.writer.write([], this.env.store.tail(finish)); + return; + } + for (let i = 0; i < ids.length; i += size) { + await this.env.checkpoints.check(label); + const works: Work[] = []; + for (const local of await this.eventKind.loadByRowIds(ids.slice(i, i + size))) { + const sourceId = this.eventKind.meta(local).sourceId; + const acct = accountOfRef(sourceId); + if (!acct || !this.isClean(local)) continue; + try { + const group = plan(local, acct); + // A group that fails its assert was edited meanwhile: the upload takes it from here. + if (group) works.push({ group }); + } catch (error) { + this.plannerFailed(acct, sourceId as string, 'download', error); + } + } + await this.env.writer.write(works, i + size >= ids.length ? this.env.store.tail(finish) : undefined); + } + } + + /** + * The device zone changed: clean floating events are rewritten so they + * keep their wall time. `deviceZonePending` marks the pass until its last + * chunk stores the new zone. + */ + private async zonePass(): Promise { + const committed = this.env.store.committed; + const stored = committed.deviceZone ?? null; + if (!stored) { + await this.env.writer.write([], this.env.store.tail((next) => { + next.deviceZone = this.zone; + next.deviceZonePending = null; + })); + return; + } + if (stored === this.zone && !committed.deviceZonePending) return; + if (committed.deviceZonePending !== this.zone) { + await this.env.writer.write([], this.env.store.tail((next) => { + next.deviceZonePending = this.zone; + })); + } + await this.passOverCleanMasters( + 'zone', + (e, acct) => this.planner.planZoneChange(e, stored, this.ctx(acct)), + (next) => { + next.deviceZone = this.zone; + next.deviceZonePending = null; + }, + ); + } + + /** + * The reminder owner changed (the run's uploads used the old one, so + * pending reminder edits went up first): Reminders rows and baselines are + * rewritten for the new owner, and the calendars' MAX_REMINDERS follows. + */ + private async reminderOwnerPass(): Promise { + const stored = this.env.store.committed.reminderOwner ?? null; + const wanted = this.env.prefs.reminderOwner; + if (stored === wanted) return; + this.reminderOwner = wanted; + if (!stored) { + // The first run wrote everything for the wanted owner already. + await this.env.writer.write([], this.env.store.tail((next) => { + next.reminderOwner = wanted; + })); + return; + } + await this.passOverCleanMasters( + 'reminders', + (e, acct) => this.planner.planReminderOwnerChange(e, this.ctx(acct)), + (next) => { + next.reminderOwner = wanted; + }, + ); + const selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }> = []; + for (const account of this.env.accounts) { + for (const calendar of this.calendars.get(account.id) ?? []) { + const key = collectionKey(account.id, calendar.id); + if (!this.syncedKeys.has(key)) continue; + selected.push({ jmapAccountId: account.id, calendar, readOnly: this.readOnlyKeys.has(key), accountName: account.personal ? '' : account.name }); + } + } + const plan = this.planner.planCalendars(selected, await this.loadCalendars(), this.calendarCtx()); + await this.env.writer.write(plan.groups.map((group) => ({ group }))); + } +} diff --git a/src/device-sync/engine/checkpoint.ts b/src/device-sync/engine/checkpoint.ts new file mode 100644 index 00000000..2cd8603c --- /dev/null +++ b/src/device-sync/engine/checkpoint.ts @@ -0,0 +1,55 @@ +/** + * Checkpoints sit between chunks and phases (docs/device-sync.md, "A sync + * run"): the run stops cleanly when the framework cancelled it or its time + * is nearly up, keeps SyncManager's traffic monitor happy during long local + * phases (it cancels a sync with no network traffic for 60 s), and lets the + * JS thread breathe so the UI keeps rendering when it shares the runtime. + */ +import type { JmapCaller } from '../jmap/caller'; +import { StopRun } from './errors'; + +export interface CheckpointOptions { + now(): number; + /** Epoch ms by which the run must have finished. */ + deadline: number; + /** Stop starting work this long before the deadline. */ + marginMs: number; + echoAfterMs: number; + isCancelled(): Promise; + yieldThread(): Promise; +} + +export class Checkpoints { + private jmap: JmapCaller | null = null; + /** Test hook: runs at every checkpoint (crash injection). */ + onCheckpoint?: (label: string) => void | Promise; + + constructor(private readonly options: CheckpointOptions) {} + + attach(jmap: JmapCaller): void { + this.jmap = jmap; + } + + budgetMs(): number { + return this.options.deadline - this.options.now(); + } + + /** Throws StopRun when the run must end here. */ + async check(label = ''): Promise { + await this.onCheckpoint?.(label); + if (this.budgetMs() < this.options.marginMs) throw new StopRun('deadline'); + if (await this.options.isCancelled()) throw new StopRun('cancelled'); + if (this.jmap && this.jmap.idleMs() >= this.options.echoAfterMs) await this.jmap.echo(); + await this.options.yieldThread(); + } + + /** Throws StopRun('deadline') unless at least `ms` of budget remain (a create batch needs two request timeouts). */ + requireBudget(ms: number): void { + if (this.budgetMs() < ms) throw new StopRun('deadline'); + } + + /** Keeps traffic flowing during a long local phase without the other checks. */ + async keepAlive(): Promise { + if (this.jmap && this.jmap.idleMs() >= this.options.echoAfterMs) await this.jmap.echo(); + } +} diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts new file mode 100644 index 00000000..2064a4b4 --- /dev/null +++ b/src/device-sync/engine/contacts-sync.ts @@ -0,0 +1,743 @@ +/** + * Contacts: address books ↔ raw contacts and groups of one Android account + * (docs/device-sync.md, "Contacts mapping"). Group cards (`kind: "group"`) + * are Groups rows and are always written before the contacts of the same + * download, so memberships never name a group that does not exist yet. + */ +import { MimeType } from '../android-columns'; +import { collectionKey, parseCollectionKey } from '../common/ids'; +import { parseJsonColumn } from '../common/json'; +import type { + AddressBookLike, + ContactCardWire, + ContactsContext, + ContactsPlanner, + LocalContact, + LocalGroup, +} from '../planner'; +import { CONTACTS_AUTHORITY, type Row } from '../types'; +import { ADDRESS_BOOK_PROPERTIES, CONTACT_CARD_PROPERTIES } from '../wire'; +import type { Work } from './batch'; +import type { RunEnv } from './context'; +import { ItemSync, type Pending } from './item-sync'; +import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds'; +import { poisonFingerprint } from './poison'; +import { flag, num, str } from './provider'; +import { isCollectionSelected } from './selection'; +import { accountOf, type SyncState } from './sync-state'; + +/** + * The context the engine hands the contacts planner. `groupsOf` is + * not part of `ContactsContext` yet: memberships are stored on the group + * cards (`members` holds member uids), so a contact's GroupMembership rows + * need the reverse lookup. + */ +export interface ContactsRunContext extends ContactsContext { + /** SOURCE_IDs of the group cards whose `members` include this uid. */ + groupsOf(uid: string): string[]; +} + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + +export function toBase64(bytes: Uint8Array): string { + let out = ''; + for (let i = 0; i < bytes.length; i += 3) { + const a = bytes[i]; + const b = i + 1 < bytes.length ? bytes[i + 1] : 0; + const c = i + 2 < bytes.length ? bytes[i + 2] : 0; + const n = (a << 16) | (b << 8) | c; + out += BASE64[(n >> 18) & 63] + BASE64[(n >> 12) & 63]; + out += i + 1 < bytes.length ? BASE64[(n >> 6) & 63] : '='; + out += i + 2 < bytes.length ? BASE64[n & 63] : '='; + } + return out; +} + +/** Base64 of a `data:` URI's bytes, or null. */ +export function dataUriBase64(uri: unknown): string | null { + if (typeof uri !== 'string' || !/^data:/i.test(uri)) return null; + const comma = uri.indexOf(','); + if (comma < 0) return null; + const header = uri.slice(5, comma); + const payload = uri.slice(comma + 1); + if (/;base64$/i.test(header)) return payload.replace(/\s+/g, ''); + const bytes: number[] = []; + for (let i = 0; i < payload.length; i++) { + if (payload[i] === '%' && /^[0-9a-f]{2}$/i.test(payload.slice(i + 1, i + 3))) { + bytes.push(parseInt(payload.slice(i + 1, i + 3), 16)); + i += 2; + } else { + bytes.push(payload.charCodeAt(i) & 0xff); + } + } + return toBase64(Uint8Array.from(bytes)); +} + +/** The name a Relation row shows for a card. */ +export function displayNameOf(card: Pick | null | undefined): string | null { + const full = card?.name?.full; + if (typeof full === 'string' && full.trim()) return full.trim(); + const parts = (card?.name?.components ?? []) + .filter((c) => c && c.kind !== 'separator' && typeof c.value === 'string' && c.value.trim()) + .map((c) => (c.value as string).trim()); + return parts.length ? parts.join(' ') : null; +} + +function onIds(map: Record | undefined | null): string[] { + return Object.entries(map ?? {}) + .filter(([, on]) => on === true) + .map(([id]) => id); +} + +function union(a: readonly string[], b: readonly string[]): string[] { + return [...new Set([...a, ...b])]; +} + +export class ContactsSync extends ItemSync { + protected readonly itemType = 'ContactCard' as const; + protected readonly itemProperties = CONTACT_CARD_PROPERTIES; + protected readonly parentFilter = 'inAddressBook' as const; + protected readonly parentProperty = 'addressBookIds' as const; + + readonly contactKind: Kind; + readonly groupKind: Kind; + private readonly planner: ContactsPlanner; + private readonly rawColumns: string[]; + private readonly dataColumns: string[]; + private readonly groupColumns: string[]; + + private readonly books = new Map(); + private readonly syncedKeys = new Set(); + private readonly collectionStates = new Map(); + /** Group SOURCE_ID → Groups row id, and member uid → group SOURCE_IDs; rebuilt after group writes. */ + private readonly groupRows = new Map(); + private readonly membership = new Map>(); + private groupsStale = true; + /** Per account: group refs that appeared (true) or went (false), for `SyncState.groups`. */ + private readonly groupChanges = new Map>(); + /** Per account: group refs listed in `groups` whose rows an app hard-deleted. */ + private absent = new Map(); + private readonly blobPhotos = new Map(); + private readonly devicePhotos = new Map(); + private uidNames: Map | null = null; + private uidRefs: Map | null = null; + /** Per account: raw contacts the upload phases looked at, for their memberships. */ + private readonly uploadRows = new Map>(); + + constructor(env: RunEnv) { + super(env); + this.planner = env.deps.planners.contacts; + this.rawColumns = union(this.planner.rawContactColumns, ['_id']); + this.dataColumns = union(this.planner.dataColumns, ['_id', 'raw_contact_id', 'mimetype']); + this.groupColumns = union(this.planner.groupColumns, ['_id']); + this.contactKind = this.makeContactKind(); + this.groupKind = this.makeGroupKind(); + } + + // ── Kinds ── + + private makeContactKind(): Kind { + const planner = this.planner; + return { + name: 'contact', + table: 'raw_contacts', + identityColumn: 'sourceid', + poisonColumn: 'sync4', + meta: (c) => ({ + rowId: c.rawContactId, + sourceId: c.sourceId, + pending: c.pending, + dirty: c.dirty, + deleted: c.deleted, + isNew: !c.sourceId, + poison: c.poison, + collections: c.collections, + }), + fingerprint: (c) => + poisonFingerprint({ + deleted: c.deleted, + shadow: c.shadow, + rows: c.rows.map((r) => JSON.stringify([r.mimetype, r.cells])).sort(), + }), + loadByRefs: async (refs) => { + const out = new Map(); + for (const c of await this.loadContactsIn('sourceid', refs)) if (c.sourceId) out.set(c.sourceId, c); + return out; + }, + loadByRowIds: (ids) => this.loadContactsIn('_id', ids), + loadNew: () => this.loadContactsWhere('sourceid IS NULL'), + planDownload: (card, local, acct) => planner.planDownload(card, local, this.ctx(acct)), + planLocalDelete: (c) => planner.planLocalDelete(c), + planBaselineHeal: (c) => planner.planBaselineHeal(c), + planUpload: (c, acct) => planner.planUpload(c, this.ctx(acct)), + planAccepted: (c, card, acct) => planner.planAccepted(c, card, this.ctx(acct)), + }; + } + + private makeGroupKind(): Kind { + const planner = this.planner; + return { + name: 'group', + table: 'groups', + identityColumn: 'sourceid', + poisonColumn: 'sync4', + meta: (g) => ({ + rowId: g.groupId, + sourceId: g.sourceId, + pending: g.pending, + dirty: g.dirty, + deleted: g.deleted, + isNew: !g.sourceId, + poison: g.poison, + collections: this.cardKeys(accountOfRef(g.sourceId), g.shadow), + }), + fingerprint: (g) => poisonFingerprint({ deleted: g.deleted, title: g.title, shadow: g.shadow }), + loadByRefs: async (refs) => { + const out = new Map(); + for (const g of await this.loadGroupsIn('sourceid', refs)) if (g.sourceId) out.set(g.sourceId, g); + return out; + }, + loadByRowIds: (ids) => this.loadGroupsIn('_id', ids), + loadNew: () => this.loadGroupsWhere('sourceid IS NULL'), + planDownload: (card, local, acct) => planner.planGroupDownload(card, local, this.ctx(acct)), + planLocalDelete: (g) => planner.planGroupLocalDelete(g), + planBaselineHeal: () => null, + planUpload: (g, acct) => planner.planGroupUpload(g, this.ctx(acct)), + planAccepted: (g, card, acct) => planner.planGroupAccepted(g, card, this.ctx(acct)), + }; + } + + protected kinds(): Kind[] { + return [this.groupKind, this.contactKind]; + } + + protected kindOf(object: ServerObject): Kind { + return object.kind === 'group' ? this.groupKind : this.contactKind; + } + + // ── Loading ── + + private async decodeContacts(rawRows: Row[]): Promise { + if (!rawRows.length) return []; + const data = await this.env.reader.rowsIn('data', this.dataColumns, 'raw_contact_id', rawRows.map((r) => num(r._id))); + const byContact = new Map(); + for (const row of data) { + const id = num(row.raw_contact_id); + let list = byContact.get(id); + if (!list) byContact.set(id, (list = [])); + list.push(row); + } + return rawRows.map((rc) => this.planner.decodeContact(rc, byContact.get(num(rc._id)) ?? [])); + } + + private async loadContactsIn(column: '_id' | 'sourceid', values: ReadonlyArray): Promise { + if (!values.length) return []; + return this.decodeContacts(await this.env.reader.rowsIn('raw_contacts', this.rawColumns, column, values)); + } + + private async loadContactsWhere(where?: string, args?: Array): Promise { + return this.decodeContacts(await this.env.reader.rows('raw_contacts', this.rawColumns, where, args)); + } + + private async loadGroupsIn(column: '_id' | 'sourceid', values: ReadonlyArray): Promise { + if (!values.length) return []; + const rows = await this.env.reader.rowsIn('groups', this.groupColumns, column, values); + return rows.map((r) => this.planner.decodeGroup(r)); + } + + private async loadGroupsWhere(where?: string, args?: Array): Promise { + const rows = await this.env.reader.rows('groups', this.groupColumns, where, args); + return rows.map((r) => this.planner.decodeGroup(r)); + } + + /** Group rows by SOURCE_ID and who is a member of which group, from the group cards' shadows. */ + private async refreshGroups(): Promise { + this.groupRows.clear(); + this.membership.clear(); + for (const group of await this.loadGroupsWhere()) { + if (!group.sourceId || group.deleted) continue; + this.groupRows.set(group.sourceId, group.groupId); + for (const uid of onIds(group.shadow?.members as Record | undefined)) { + let set = this.membership.get(uid); + if (!set) this.membership.set(uid, (set = new Set())); + set.add(group.sourceId); + } + } + this.groupsStale = false; + } + + // ── Collections ── + + private cardKeys(acct: string | null, card: Pick | null | undefined): string[] { + if (!acct || !card) return []; + return onIds(card.addressBookIds).map((id) => collectionKey(acct, id)); + } + + private book(key: string): AddressBookLike | undefined { + const parsed = parseCollectionKey(key); + return parsed ? this.books.get(parsed.accountId)?.find((b) => b.id === parsed.id) : undefined; + } + + private writable(key: string): boolean { + const book = this.book(key); + return !!book && book.myRights?.mayWrite !== false; + } + + protected async collections(write: boolean): Promise { + this.syncedKeys.clear(); + for (const account of this.env.accounts) { + const { list, state } = await this.env.jmap.getContainers('AddressBook', account.id, ADDRESS_BOOK_PROPERTIES); + this.books.set(account.id, list); + for (const book of list) { + if (isCollectionSelected(this.env.prefs.contactsSelection, account, CONTACTS_AUTHORITY, book)) { + this.syncedKeys.add(collectionKey(account.id, book.id)); + } + } + if (state) { + this.collectionStates.set(account.id, state); + this.env.recordKnownState(account.id, 'AddressBook', state); + } + } + if (write) await this.ensureSettingsRow(); + } + + /** One Settings row per account with UNGROUPED_VISIBLE=1: the default 0 hides every contact without a visible group. */ + private async ensureSettingsRow(): Promise { + const rows = await this.env.reader.rows('settings', ['ungrouped_visible', 'should_sync']); + const values = { ungrouped_visible: 1, should_sync: 1 }; + if (!rows.length) { + await this.env.writer.write([{ group: { ref: 'settings', ops: [{ op: 'insert', table: 'settings', values }] } }]); + } else if (rows.some((r) => num(r.ungrouped_visible) !== 1 || num(r.should_sync) !== 1)) { + await this.env.writer.write([{ group: { ref: 'settings', ops: [{ op: 'update', table: 'settings', values }] } }]); + } + } + + protected syncedCollections(acct: string): string[] { + return (this.books.get(acct) ?? []) + .filter((b) => this.syncedKeys.has(collectionKey(acct, b.id))) + .map((b) => b.id); + } + + protected inSelection(acct: string, object: ServerObject): boolean { + return this.cardKeys(acct, object as ContactCardWire).some((k) => this.syncedKeys.has(k)); + } + + /** + * Where a contact created on the device goes: the chosen book when it + * syncs and is writable, else the personal account's default (or first) + * synced writable book; null when there is none. + */ + createTarget(): string | null { + const chosen = this.env.prefs.newContactsAddressBook; + if (chosen && this.syncedKeys.has(chosen) && this.writable(chosen)) return chosen; + const primary = this.env.accounts.find((a) => a.primary) ?? this.env.accounts[0]; + if (!primary) return null; + const books = [...(this.books.get(primary.id) ?? [])].sort( + (a, b) => Number(!!b.isDefault) - Number(!!a.isDefault) || (a.sortOrder ?? 0) - (b.sortOrder ?? 0) || a.id.localeCompare(b.id), + ); + const book = books.find((b) => { + const key = collectionKey(primary.id, b.id); + return this.syncedKeys.has(key) && this.writable(key); + }); + return book ? collectionKey(primary.id, book.id) : null; + } + + private photoBytes(entry: { uri?: string; blobId?: string }): string | null { + if (entry.blobId) return this.blobPhotos.get(entry.blobId) ?? null; + return dataUriBase64(entry.uri); + } + + ctx(acct: string): ContactsRunContext { + const env = this.env; + return { + jmapAccountId: acct, + now: env.now(), + mintKey: (taken) => env.mintKey(taken), + mintUid: () => env.mintUid(), + selectedCollections: (card) => this.cardKeys(acct, card).filter((k) => this.syncedKeys.has(k)), + isReadOnly: (card) => { + const keys = this.cardKeys(acct, card); + return keys.length > 0 && keys.every((k) => this.book(k)?.myRights?.mayWrite === false); + }, + isSelected: (key) => this.syncedKeys.has(key), + createTarget: () => this.createTarget(), + photoBytes: (entry) => this.photoBytes(entry), + devicePhoto: (rawContactId) => this.devicePhotos.get(rawContactId) ?? null, + nameForUid: (uid) => this.uidNames?.get(uid) ?? null, + groupRowIdBySourceId: (sourceId) => this.groupRows.get(sourceId) ?? null, + groupsOf: (uid) => [...(this.membership.get(uid) ?? [])], + }; + } + + // ── Downloads ── + + /** + * Group cards first: a membership naming an unknown group makes the + * provider create an empty one. Memberships live on the group cards, so + * when a group's `members` changed, its old and new members come along in + * the same download (before its state op): their GroupMembership rows + * change although their own cards did not. + */ + protected async order(acct: string, ids: string[]): Promise<{ first: string[]; rest: string[]; missing: string[] }> { + if (!ids.length) return { first: [], rest: [], missing: [] }; + const { list, notFound } = await this.env.jmap.get<{ id: string; kind?: string; members?: Record }>( + 'ContactCard', + acct, + ids, + ['id', 'kind', 'members'], + ); + const groups = list.filter((c) => c.kind === 'group'); + const groupIds = new Set(groups.map((c) => c.id)); + const found = new Set(list.map((c) => c.id)); + const rest = ids.filter((id) => found.has(id) && !groupIds.has(id)); + const members = (await this.membersToRefresh(acct, groups)).filter((id) => !groupIds.has(id)); + return { + first: ids.filter((id) => groupIds.has(id)), + rest: members.length ? [...new Set([...rest, ...members])].sort() : rest, + missing: notFound, + }; + } + + /** Contacts of `acct` whose membership in one of these group cards flipped since the group's shadow. */ + private async membersToRefresh(acct: string, groups: Array<{ id: string; members?: Record }>): Promise { + if (!groups.length) return []; + const locals = await this.groupKind.loadByRefs(groups.map((g) => refOf(acct, g.id))); + const flipped = new Set(); + for (const group of groups) { + const before = new Set(onIds(locals.get(refOf(acct, group.id))?.shadow?.members as Record | undefined)); + const after = new Set(onIds(group.members)); + for (const uid of before) if (!after.has(uid)) flipped.add(uid); + for (const uid of after) if (!before.has(uid)) flipped.add(uid); + } + if (!flipped.size) return []; + await this.ensureShadowIndex(); + const out: string[] = []; + for (const uid of flipped) { + for (const ref of this.uidRefs?.get(uid) ?? []) { + const id = idInAccount(ref, acct); + if (id) out.push(id); + } + } + return out; + } + + protected async beforePlanning(acct: string, objects: ServerObject[]): Promise { + const cards = objects as ContactCardWire[]; + if (this.groupsStale && cards.some((c) => c.kind !== 'group')) await this.refreshGroups(); + for (const card of cards) { + const photo = Object.values(card.media ?? {}).find((m) => m?.kind === 'photo') as + | { blobId?: string; mediaType?: string } + | undefined; + if (!photo?.blobId || this.blobPhotos.has(photo.blobId)) continue; + try { + this.blobPhotos.set(photo.blobId, toBase64(await this.env.jmap.downloadBlob(acct, photo.blobId, photo.mediaType))); + } catch (error) { + this.env.log('photo download failed', error); + this.blobPhotos.set(photo.blobId, null); + } + } + if (cards.some((c) => Object.keys(c.relatedTo ?? {}).length > 0)) await this.ensureShadowIndex(); + for (const card of cards) this.indexCard(acct, card); + } + + private indexCard(acct: string, card: ContactCardWire & { id?: string }): void { + if (!this.uidNames || !this.uidRefs || typeof card.uid !== 'string' || card.kind === 'group') return; + const name = displayNameOf(card); + if (name) this.uidNames.set(card.uid, name); + if (card.id) { + const ref = refOf(acct, card.id); + const refs = this.uidRefs.get(card.uid) ?? []; + if (!refs.includes(ref)) this.uidRefs.set(card.uid, [...refs, ref]); + } + } + + /** + * The synced cards by uid (their names for Relation rows, their rows for + * membership changes), read from the shadows on first need and kept up to + * date with every card downloaded afterwards. + */ + private async ensureShadowIndex(): Promise { + if (this.uidNames && this.uidRefs) return; + const names = new Map(); + const refs = new Map(); + for (const row of await this.env.reader.rows('raw_contacts', ['sourceid', 'sync2'], 'sourceid IS NOT NULL')) { + const shadow = parseJsonColumn(row.sync2); + const ref = str(row.sourceid); + if (!shadow || typeof shadow.uid !== 'string' || !ref) continue; + const name = displayNameOf(shadow); + if (name) names.set(shadow.uid, name); + refs.set(shadow.uid, [...(refs.get(shadow.uid) ?? []), ref]); + } + this.uidNames = names; + this.uidRefs = refs; + } + + protected adoptionTargetMatches(acct: string, object: ServerObject, target: string): boolean { + return this.cardKeys(acct, object as ContactCardWire).includes(target); + } + + protected lookupScope(_acct: string, collectionId: string): string | null { + // Card uids are unique per address book. + return collectionId; + } + + private groupChange(acct: string, ref: string, present: boolean): void { + let changes = this.groupChanges.get(acct); + if (!changes) this.groupChanges.set(acct, (changes = new Map())); + changes.set(ref, present); + this.groupsStale = true; + } + + protected onDownloaded(kind: Kind, acct: string, object: ServerObject): void { + if (kind === this.groupKind) this.groupChange(acct, refOf(acct, object.id), true); + else this.indexCard(acct, object as ContactCardWire); + } + + protected onRemoved(kind: Kind, acct: string, id: string): void { + if (kind === this.groupKind) this.groupChange(acct, refOf(acct, id), false); + } + + protected onAccepted(kind: Kind, acct: string, server: ServerObject): void { + if (kind === this.groupKind) this.groupChange(acct, refOf(acct, server.id), true); + else this.indexCard(acct, server as ContactCardWire); + } + + protected decorateState(acct: string, next: SyncState): () => void { + const account = accountOf(next, acct); + const changes = this.groupChanges.get(acct); + const consumed = new Map(changes ?? []); + if (consumed.size || account.groups === undefined) { + const groups = new Set(account.groups ?? []); + for (const [ref, present] of consumed) { + if (present) groups.add(ref); + else groups.delete(ref); + } + account.groups = [...groups]; + } + const collections = this.collectionStates.get(acct); + if (collections) account.collectionsState = collections; + return () => { + for (const [ref, present] of consumed) if (changes?.get(ref) === present) changes.delete(ref); + }; + } + + protected async localIdentities(acct: string): Promise> { + const out = new Map(); + for (const table of ['groups', 'raw_contacts'] as const) { + for (const row of await this.env.reader.rows(table, ['_id', 'sourceid', 'dirty', 'deleted'], 'sourceid LIKE ?', [`${acct}/%`])) { + const id = idInAccount(str(row.sourceid), acct); + if (id) out.set(id, { dirty: flag(row.dirty), deleted: flag(row.deleted) }); + } + } + return out; + } + + // ── Uploads ── + + protected async loadUploadItems(): Promise { + const groups = await this.loadGroupsWhere('dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + const contacts = await this.loadContactsWhere('dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + return [ + ...groups.map((local) => ({ kind: this.groupKind as Kind, local })), + ...contacts.map((local) => ({ kind: this.contactKind as Kind, local })), + ]; + } + + protected claimAccount(): string | null { + const target = this.createTarget(); + return accountOfRef(target) ?? this.env.accounts.find((a) => a.primary)?.id ?? this.env.accounts[0]?.id ?? null; + } + + protected noteUploadItems(acct: string, items: Held[]): void { + let rows = this.uploadRows.get(acct); + if (!rows) this.uploadRows.set(acct, (rows = new Set())); + for (const held of items) if (held.kind === this.contactKind) rows.add(held.kind.meta(held.local).rowId); + } + + /** The device photo of every contact about to be planned that has a photo row (JPEG, ≤ 512 px). */ + protected async beforeUploadPlanning(items: Held[]): Promise { + for (const held of items) { + if (held.kind !== this.contactKind) continue; + const contact = held.local as LocalContact; + if (this.devicePhotos.has(contact.rawContactId) || !contact.rows.some((r) => r.mimetype === MimeType.PHOTO)) continue; + try { + const photo = await this.env.reader.port.readPhoto(contact.rawContactId, 512); + this.devicePhotos.set(contact.rawContactId, photo?.jpegBase64 ?? null); + } catch (error) { + this.env.log('photo read failed', error); + this.devicePhotos.set(contact.rawContactId, null); + } + } + } + + /** Group refs the SyncState lists whose rows are gone: an app hard-deleted them (Fossify does). */ + private async absentGroups(): Promise> { + const present = new Set(); + for (const row of await this.env.reader.rows('groups', ['sourceid'], 'sourceid IS NOT NULL')) { + const ref = str(row.sourceid); + if (ref) present.add(ref); + } + const out = new Map(); + for (const [acct, account] of Object.entries(this.env.store.committed.accounts)) { + const changes = this.groupChanges.get(acct); + const refs = (account.groups ?? []).filter((ref) => !present.has(ref) && changes?.get(ref) !== false); + if (refs.length) out.set(acct, refs); + } + return out; + } + + protected async countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set }> { + let count = 0; + const accounts = new Set(); + for (const held of items) { + const meta = held.kind.meta(held.local); + if (!meta.deleted || !meta.sourceId) continue; + count++; + const acct = accountOfRef(meta.sourceId); + if (acct) accounts.add(acct); + } + this.absent = await this.absentGroups(); + for (const [acct, refs] of this.absent) { + count += refs.length; + accounts.add(acct); + } + const synced = + (await this.env.reader.rows('raw_contacts', ['_id'], 'sourceid IS NOT NULL')).length + + (await this.env.reader.rows('groups', ['_id'], 'sourceid IS NOT NULL')).length; + return { count, synced: Math.max(synced, count), accounts }; + } + + protected onDiscard(next: SyncState, accounts: Set): void { + for (const acct of accounts) { + const gone = new Set(this.absent.get(acct) ?? []); + if (!gone.size) continue; + const account = accountOf(next, acct); + account.groups = (account.groups ?? []).filter((ref) => !gone.has(ref)); + } + this.absent = new Map(); + } + + protected async extraPhase(acct: string, deletionsAllowed: boolean): Promise { + if (deletionsAllowed) await this.deleteAbsentGroups(acct); + await this.uploadMemberships(acct); + } + + /** + * A group hard-deleted on the device: destroyed on the server, or, when + * it is also in address books that do not sync, removed from the synced + * ones only. + */ + private async deleteAbsentGroups(acct: string): Promise { + const refs = (this.absent.get(acct) ?? []).filter((ref) => this.groupChanges.get(acct)?.get(ref) !== false); + if (!refs.length) return; + await this.env.checkpoints.check('upload:groups'); + const ids = refs.map((ref) => idInAccount(ref, acct)).filter((id): id is string => !!id); + const { list, notFound } = await this.env.jmap.get('ContactCard', acct, ids, ['id', 'addressBookIds']); + for (const id of notFound) this.groupChange(acct, refOf(acct, id), false); + const update: Record> = {}; + const destroy: string[] = []; + for (const card of list) { + const books = onIds(card.addressBookIds); + const synced = books.filter((id) => this.syncedKeys.has(collectionKey(acct, id))); + if (synced.length && synced.length < books.length) { + update[card.id] = Object.fromEntries(synced.map((id) => [`addressBookIds/${id}`, null])); + } else { + destroy.push(card.id); + } + } + if (!destroy.length && !Object.keys(update).length) return; + const { response } = await this.setChecked(acct, { update, destroy }); + const done = [...(response.destroyed ?? []), ...Object.keys(response.updated ?? {})]; + for (const id of done) { + this.groupChange(acct, refOf(acct, id), false); + this.env.report.stats.uploaded.deleted++; + } + for (const [id, error] of [...Object.entries(response.notDestroyed ?? {}), ...Object.entries(response.notUpdated ?? {})]) { + if (error?.type === 'notFound') this.groupChange(acct, refOf(acct, id), false); + else this.env.report.itemError({ ref: refOf(acct, id), side: 'upload', type: error?.type ?? 'serverFail', description: error?.description }); + } + } + + /** Memberships edited on the device become patches of the group cards' `members`. */ + private async uploadMemberships(acct: string): Promise { + const rows = [...(this.uploadRows.get(acct) ?? [])]; + if (!rows.length) return; + await this.env.checkpoints.check('upload:memberships'); + const contacts = (await this.contactKind.loadByRowIds(rows)).filter((c) => !c.deleted); + const groups = (await this.loadGroupsWhere('sourceid LIKE ?', [`${acct}/%`])).filter( + (g) => accountOfRef(g.sourceId) === acct && !g.deleted, + ); + const actions = this.planner.planMembershipUploads(contacts, groups, this.ctx(acct)); + if (!actions.length) return; + const byGroup = new Map(); + for (const action of actions) { + const id = action.kind === 'create' ? null : action.id; + const group = id ? groups.find((g) => g.sourceId === refOf(acct, id)) : undefined; + if (!group) { + this.env.report.itemError({ ref: id ? refOf(acct, id) : 'membership', side: 'upload', type: 'unknownGroup' }); + continue; + } + let pending = byGroup.get(group.sourceId as string); + if (!pending) { + const held: Held = { kind: this.groupKind, local: group }; + pending = { held, meta: this.groupKind.meta(group), fingerprint: this.groupKind.fingerprint(group), actions: [] }; + byGroup.set(group.sourceId as string, pending); + } + pending.actions.push(action as Pending['actions'][number]); + } + if (byGroup.size) await this.send(acct, [...byGroup.values()]); + } + + // ── Deselection ── + + /** + * After the upload: rows of contacts in no synced address book go, when + * clean (a deselected book, or a card moved away on the server). Dirty + * ones stay until their upload succeeds, and keep their book selected. + */ + protected async dropOutsideSelection(acct: string): Promise { + const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id))); + const committed = this.env.store.account(acct); + const deselected = committed.selected.filter((key) => !synced.has(key)); + const outside = [...(this.outside.get(acct) ?? [])]; + if (!deselected.length && !outside.length) return; + + const kept = new Set(); + const consider = (held: Held, works: Work[]) => { + const meta = held.kind.meta(held.local); + const id = idInAccount(meta.sourceId, acct); + // Rows whose collections are unknown are never dropped. + if (!id || !meta.collections.length || meta.collections.some((k) => synced.has(k))) return; + const work = this.removeWork(acct, id, held, 'outside'); + if (work) works.push(work); + else for (const key of meta.collections) if (deselected.includes(key)) kept.add(key); + }; + if (deselected.length) { + const groupWorks: Work[] = []; + for (const group of await this.loadGroupsWhere('sourceid LIKE ?', [`${acct}/%`])) consider({ kind: this.groupKind, local: group }, groupWorks); + await this.env.writer.write(groupWorks); + const rows = await this.env.reader.rows('raw_contacts', ['_id', 'sourceid'], 'sourceid LIKE ?', [`${acct}/%`]); + const ids = rows.filter((r) => accountOfRef(str(r.sourceid)) === acct).map((r) => num(r._id)); + for (let i = 0; i < ids.length; i += this.env.tuning.chunkSize) { + await this.env.checkpoints.check('deselect'); + const works: Work[] = []; + for (const contact of await this.contactKind.loadByRowIds(ids.slice(i, i + this.env.tuning.chunkSize))) { + consider({ kind: this.contactKind, local: contact }, works); + } + await this.env.writer.write(works); + } + const selected = [...synced, ...kept]; + // The book leaves `selected` with its last dropped row; one still holding a dirty item stays. + await this.env.writer.write([], this.tail([acct], (next) => { + accountOf(next, acct).selected = selected; + })); + } else { + const works: Work[] = []; + for (const kind of this.kinds()) { + for (const local of (await kind.loadByRefs(outside)).values()) consider({ kind, local }, works); + } + await this.env.writer.write(works); + } + this.outside.delete(acct); + } +} diff --git a/src/device-sync/engine/context.ts b/src/device-sync/engine/context.ts new file mode 100644 index 00000000..9162a6fe --- /dev/null +++ b/src/device-sync/engine/context.ts @@ -0,0 +1,39 @@ +/** + * What one run (or teardown) of one Android account and authority carries + * through its phases. + */ +import type { AccountSyncPrefs, Authority, RunExtras } from '../types'; +import type { JmapCaller } from '../jmap/caller'; +import type { CapabilityAccount } from '../jmap/session'; +import type { BatchWriter } from './batch'; +import type { Checkpoints } from './checkpoint'; +import type { EngineDeps, Tuning } from './deps'; +import type { ProviderReader } from './provider'; +import type { ReportBuilder } from './report'; +import type { StateStore } from './sync-state'; + +export interface RunEnv { + readonly deps: EngineDeps; + readonly tuning: Tuning; + /** A teardown only uploads (and never writes calendar rows). */ + readonly mode: 'sync' | 'teardown'; + readonly registryId: string; + readonly accountName: string; + readonly authority: Authority; + readonly extras: RunExtras; + readonly reader: ProviderReader; + readonly writer: BatchWriter; + readonly report: ReportBuilder; + readonly checkpoints: Checkpoints; + readonly store: StateStore; + readonly jmap: JmapCaller; + /** Accounts with the authority's capability, the primary first. */ + readonly accounts: CapabilityAccount[]; + readonly prefs: AccountSyncPrefs; + now(): number; + mintKey(taken?: Iterable): string; + mintUid(): string; + sleep(ms: number): Promise; + log(message: string, detail?: unknown): void; + recordKnownState(jmapAccountId: string, type: string, state: string): void; +} diff --git a/src/device-sync/engine/deps.ts b/src/device-sync/engine/deps.ts new file mode 100644 index 00000000..f7bbf4a2 --- /dev/null +++ b/src/device-sync/engine/deps.ts @@ -0,0 +1,113 @@ +/** + * Everything the sync engine reaches outside itself. The engine is pure + * TypeScript: src/device-sync/task.ts wires these to the native module, a + * detached JMAPClient and the app's stores; the tests wire them to the fake + * provider, the fake JMAP server and test planners. + */ +import type { CalendarPlanner, ContactsPlanner } from '../planner'; +import type { RandomSource } from '../common/ids'; +import type { AccountSyncPrefs, Authority, ProviderPort, RunStatus } from '../types'; +import type { JmapConnection } from '../jmap/connection'; + +export type { JmapConnection }; + +/** A calendar that mirrors an iCal feed (read-only on the device). */ +export interface SubscriptionCalendar { + /** JMAP account of the mirror calendar; null for subscriptions saved before accounts were recorded (the primary). */ + jmapAccountId: string | null; + calendarId: string; +} + +export interface EngineDeps { + /** The provider of one Android account and authority. */ + provider(accountName: string, authority: Authority): ProviderPort; + /** A JMAP connection for the registry account (rebuilt once on an auth failure); throws the client's errors. */ + jmap(registryId: string): Promise; + /** The account's device-sync preferences (the store is hydrated first). */ + prefs(registryId: string): Promise; + planners: { contacts: ContactsPlanner; calendar: CalendarPlanner }; + /** Epoch milliseconds. */ + now(): number; + /** Whether the framework cancelled the run, or nobody waits for it any more. */ + isCancelled(runId: string): Promise; + random: RandomSource; + /** Feed-subscription calendars (read-only on the device). */ + subscriptionCalendars(): Promise; + /** Android's `getSyncAutomatically` for the account and authority. */ + isSyncEnabled(accountName: string, authority: Authority): Promise; + /** The device's IANA time zone (floating events are written in it). */ + deviceZone(): string; + /** The last run's status for the settings UI. */ + recordStatus(registryId: string, authority: Authority, status: RunStatus): void | Promise; + /** The latest state the engine synced for a JMAP account and type, so the app skips the echo of our own writes. */ + recordKnownState(registryId: string, jmapAccountId: string, type: string, state: string): void | Promise; + /** One notification that deep-links to sign-in, after an auth failure the rebuild did not fix. */ + notifyAuthProblem(registryId: string, accountName: string, authority: Authority): void | Promise; + /** Lets the JS thread render between chunks (default: `setTimeout(0)`). */ + yieldThread?(): Promise; + /** Waits before a retry, e.g. the uid lookup's back-off (default: `setTimeout`). */ + sleep?(ms: number): Promise; + log?(message: string, detail?: unknown): void; + /** Overrides for tests. */ + tuning?: Partial; +} + +/** Sizes and limits from docs/device-sync.md ("Limits and performance", "Deletion threshold"). */ +export interface Tuning { + /** Raw contacts or events planned per chunk. */ + chunkSize: number; + /** Provider ops per applyBatch, with a yield point at every item group. */ + maxBatchOps: number; + /** Serialized bytes per applyBatch (the Binder budget is 1 MB for the whole transaction). */ + maxBatchBytes: number; + /** Re-reads and re-plans of an item whose group failed an assert. */ + maxReplans: number; + /** Creates per `/set` (a create batch is not interruptible). */ + maxCreatesPerSet: number; + /** Serialized bytes per `/set` call (Stalwart splits bigger calls into several commits). */ + maxSetBytes: number; + /** `stateMismatch` retries of an account's upload. */ + maxStateMismatchRetries: number; + /** Stop starting work this long before the deadline. */ + deadlineMarginMs: number; + /** A create batch starts only with this much budget left (two request timeouts). */ + createBudgetMs: number; + /** A `Core/echo` when a local phase has gone this long without a JMAP request. */ + echoAfterMs: number; + /** Deletions above this count … */ + deletionThresholdCount: number; + /** … and above this share of the authority's synced objects set `tooManyDeletions`. */ + deletionThresholdRatio: number; + /** A full reconcile that finds nothing while the device holds more than this aborts. */ + safetyAbortRows: number; + /** Back-off of a poisoned item: first delay, doubling up to the cap. */ + poisonBackoffMs: number; + poisonBackoffMaxMs: number; + /** Retries of a uid lookup after a duplicate-uid error, with these delays. */ + uidLookupDelaysMs: number[]; + /** Objects scanned to tell a task-only calendar. */ + taskScanLimit: number; + /** Item errors kept in a report. */ + maxItemErrors: number; +} + +export const DEFAULT_TUNING: Tuning = { + chunkSize: 50, + maxBatchOps: 400, + maxBatchBytes: 300_000, + maxReplans: 3, + maxCreatesPerSet: 50, + maxSetBytes: 1_000_000, + maxStateMismatchRetries: 3, + deadlineMarginMs: 30_000, + createBudgetMs: 60_000, + echoAfterMs: 40_000, + deletionThresholdCount: 50, + deletionThresholdRatio: 0.2, + safetyAbortRows: 10, + poisonBackoffMs: 60 * 60_000, + poisonBackoffMaxMs: 24 * 60 * 60_000, + uidLookupDelaysMs: [500, 1500, 4000], + taskScanLimit: 50, + maxItemErrors: 50, +}; diff --git a/src/device-sync/engine/errors.ts b/src/device-sync/engine/errors.ts new file mode 100644 index 00000000..d2fe7372 --- /dev/null +++ b/src/device-sync/engine/errors.ts @@ -0,0 +1,88 @@ +/** + * How a run stops early, and what each failure means for the report + * (docs/device-sync.md, "Headless payload, report and SyncResult" and + * "Failure matrix"). + */ +import type { RunOutcome } from '../types'; +import { isMethodError, isRequestLimitError, retryAfterMs, transportFailure } from '../jmap/errors'; + +/** Out of time, or cancelled: stop cleanly between chunks, ask for another sync. */ +export class StopRun extends Error { + constructor(readonly reason: 'cancelled' | 'deadline') { + super(reason === 'deadline' ? 'Out of time; the rest follows in the next sync' : 'Cancelled'); + this.name = 'StopRun'; + } +} + +/** A run ends with this outcome. `noProgress` zeroes the counts so SyncManager backs off instead of retrying at once. */ +export class RunAbort extends Error { + constructor( + readonly outcome: RunOutcome, + message: string, + readonly options: { noProgress?: boolean; delayUntil?: number } = {}, + ) { + super(message); + this.name = 'RunAbort'; + } +} + +/** The server changed since the download an upload was planned from. */ +export class StateMismatch extends Error { + constructor(readonly jmapAccountId: string) { + super(`stateMismatch in ${jmapAccountId}`); + this.name = 'StateMismatch'; + } +} + +export interface ClassifiedFailure { + outcome: RunOutcome; + message: string; + delayUntil?: number; + moreRecordsToGet?: boolean; + noProgress?: boolean; + /** Tell the user to sign in again. */ + authProblem?: boolean; +} + +function messageOf(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message.length > 300 ? `${message.slice(0, 300)}…` : message; +} + +/** A provider refusal for a missing runtime permission (the native module rejects with it). */ +export function isPermissionFailure(error: unknown): boolean { + const e = error as { code?: unknown; message?: unknown } | null; + return e?.code === 'permission' || (typeof e?.message === 'string' && /\bpermission\b|SecurityException/i.test(e.message)); +} + +export function classifyFailure(error: unknown, now: number): ClassifiedFailure { + if (error instanceof StopRun) { + return { outcome: 'cancelled', message: error.message, moreRecordsToGet: true }; + } + if (error instanceof RunAbort) { + return { outcome: error.outcome, message: error.message, noProgress: error.options.noProgress, delayUntil: error.options.delayUntil }; + } + if (error instanceof StateMismatch) { + return { outcome: 'io', message: 'The server kept changing during the upload', noProgress: true }; + } + switch (transportFailure(error)) { + case 'auth': + return { outcome: 'auth', message: messageOf(error), authProblem: true }; + case 'network': + case 'timeout': + return { outcome: 'io', message: messageOf(error) }; + case 'rateLimit': { + const wait = retryAfterMs(error) ?? 60_000; + return { outcome: 'io', message: messageOf(error), delayUntil: Math.ceil((now + wait) / 1000) }; + } + default: + break; + } + if (isPermissionFailure(error)) return { outcome: 'permission', message: messageOf(error) }; + if (isRequestLimitError(error)) return { outcome: 'internal', message: messageOf(error) }; + if (isMethodError(error)) return { outcome: 'io', message: `${error.type}: ${messageOf(error)}` }; + if (error instanceof Error && /^(JMAP request failed|Invalid JSON response)/.test(error.message)) { + return { outcome: 'io', message: messageOf(error) }; + } + return { outcome: 'internal', message: messageOf(error) }; +} diff --git a/src/device-sync/engine/index.ts b/src/device-sync/engine/index.ts new file mode 100644 index 00000000..5434709f --- /dev/null +++ b/src/device-sync/engine/index.ts @@ -0,0 +1,10 @@ +/** + * The device sync engine (docs/device-sync.md). Pure TypeScript: every + * dependency comes in through `EngineDeps`; src/device-sync/task.ts wires + * the real ones. + */ +export { runDeviceSync, hasLocalWork } from './run'; +export { teardownAuthority, countPending, type TeardownOptions } from './teardown'; +export { DEFAULT_TUNING, type EngineDeps, type JmapConnection, type SubscriptionCalendar, type Tuning } from './deps'; +export { parseSyncState, serializeSyncState, type SyncState, type AccountState } from './sync-state'; +export { lockKey, isLocked } from './mutex'; diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts new file mode 100644 index 00000000..7f0f731c --- /dev/null +++ b/src/device-sync/engine/item-sync.ts @@ -0,0 +1,1303 @@ +/** + * The download and upload machinery shared by the contacts and calendar sync + * (docs/device-sync.md, "A sync run", "Change detection and merge rules", + * "Uploads"). The authorities plug in their kinds of local items, their + * collections and their extra phases; everything that talks to the server, + * the batching, the retries and the guards is here. + */ +import type { OpGroup, UploadAction } from '../planner'; +import { collectionKey } from '../common/ids'; +import { + classifySetError, + existingIdFromUidError, + type ItemType, + type SetArgs, + type SetErrorShape, + type SetResponse, +} from '../jmap/caller'; +import { isMethodError } from '../jmap/errors'; +import { concatGroups, prependOps, type Work } from './batch'; +import type { RunEnv } from './context'; +import { RunAbort, StateMismatch } from './errors'; +import { + accountOfRef, + idInAccount, + insertGuard, + poisonOp, + refOf, + type Held, + type ItemMeta, + type Kind, + type ServerObject, +} from './kinds'; +import { isBackedOff, nextMarker } from './poison'; +import { hasWrites } from './provider'; +import { accountOf, type SyncState, type Tail } from './sync-state'; + +type Action = UploadAction; + +/** An item's upload, on its way to the server. */ +export interface Pending { + held: Held; + meta: ItemMeta; + fingerprint: string; + actions: Action[]; + /** Resent without scheduling messages after a `forbidden`. */ + noScheduling?: boolean; + /** Custom handling once every action was accepted (pairs, memberships); replaces `planAccepted`. */ + accept?(objects: Map): Promise; +} + +type Outcome = + | { action: Action; ok: true; id: string } + | { action: Action; ok: false; error: SetErrorShape; scheduling: boolean }; + +interface SetCallPlan { + args: { create: Record>; update: Record>; destroy: string[]; sendSchedulingMessages: boolean }; + entries: Array<{ pending: Pending; action: Action; key: string }>; + touched: Set; + updateIds: string[]; + creates: number; + bytes: number; +} + +type Removal = 'destroyed' | 'outside'; + +/** Method-level refusals of a whole `/set` that concern the account, not the objects. */ +const ACCOUNT_REFUSALS = new Set(['accountReadOnly', 'forbidden', 'accountNotFound', 'accountNotSupportedByMethod']); + +export abstract class ItemSync { + /** Per account: object ids put on `stale` (true) or fetched fine again (false), for the next state op. */ + private readonly staleMarks = new Map>(); + /** Per account: refs of dirty items whose object left every synced collection; dropped once uploaded. */ + protected readonly outside = new Map>(); + /** Per account: objects our uploads created for other rows (a series split), adopted by the download. */ + private readonly extraCreated = new Map>(); + /** Per account: `ifInState` of the next `/set` (the download's state, then our last `newState`). */ + private readonly setStates = new Map(); + + constructor(protected readonly env: RunEnv) {} + + // ── What the authority provides ── + + protected abstract readonly itemType: ItemType; + protected abstract readonly itemProperties: readonly string[]; + protected abstract readonly parentFilter: 'inAddressBook' | 'inCalendar'; + protected abstract readonly parentProperty: 'addressBookIds' | 'calendarIds'; + + protected abstract kinds(): Kind[]; + /** The kind whose rows hold the object; null when it does not sync (a task). */ + protected abstract kindOf(object: ServerObject): Kind | null; + /** Whether the object is in a collection that syncs. */ + protected abstract inSelection(jmapAccountId: string, object: ServerObject): boolean; + /** Ids of the collections of the account that sync now. */ + protected abstract syncedCollections(jmapAccountId: string): string[]; + /** Reads the collections, and (`write`) brings the collection rows up to date. */ + protected abstract collections(write: boolean): Promise; + /** Local identities of the account: object id → local flags, for reconcile deletions. */ + protected abstract localIdentities(jmapAccountId: string): Promise>; + /** Every item with local changes: dirty, deleted, or new. */ + protected abstract loadUploadItems(): Promise; + /** The JMAP account a new item without a pending create is claimed in. */ + protected abstract claimAccount(held: Held): string | null; + /** Local deletions waiting for an upload, synced objects, and the accounts concerned. */ + protected abstract countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set }>; + /** Drops the clean rows of deselected collections and of objects that left the selection. */ + protected abstract dropOutsideSelection(jmapAccountId: string): Promise; + + // ── Hooks ── + + /** Processing order of a download: `first` completely before `rest` (group cards before contacts); `missing` are gone. */ + protected async order(_acct: string, ids: string[]): Promise<{ first: string[]; rest: string[]; missing: string[] }> { + return { first: [], rest: ids, missing: [] }; + } + protected async beforePlanning(_acct: string, _objects: ServerObject[]): Promise {} + protected async beforeUploadPlanning(_items: Held[]): Promise {} + protected async beforeReconcile(_acct: string): Promise {} + /** Authority changes folded into a state op of the account; the callback runs once that op is stored. */ + protected decorateState(_acct: string, _next: SyncState): (() => void) | void {} + protected onDownloaded(_kind: Kind, _acct: string, _object: ServerObject): void {} + protected onRemoved(_kind: Kind, _acct: string, _id: string): void {} + protected onAccepted(_kind: Kind, _acct: string, _server: ServerObject): void {} + protected onDiscard(_next: SyncState, _accounts: Set): void {} + /** Whether a pending create of `target` may adopt this object (contacts: the target book holds it). */ + protected adoptionTargetMatches(acct: string, _object: ServerObject, target: string): boolean { + return accountOfRef(target) === acct; + } + /** Where a uid is looked up for a create in `collectionId`: that collection, or null for the whole account. */ + protected lookupScope(_acct: string, _collectionId: string): string | null { + return null; + } + /** Whether a new item is claimed with the others (calendar: a split clone waits for its source). */ + protected claimable(_held: Held): boolean { + return true; + } + /** The JMAP account an item uploads to. */ + protected accountOfItem(held: Held): string | null { + const meta = held.kind.meta(held.local); + return accountOfRef(meta.sourceId) ?? accountOfRef(meta.pending?.target ?? null); + } + /** Before the creates (calendar: split series and pairs). */ + protected async prePhase(_acct: string): Promise {} + /** After the deletions (contacts: groups deleted by absence, memberships). */ + protected async extraPhase(_acct: string, _deletionsAllowed: boolean): Promise {} + /** After uploads and deselection (calendar: zone and reminder-owner passes). */ + protected async afterSync(): Promise {} + + // ── Runs ── + + /** A sync run's phases after the preflight. */ + async sync(): Promise { + await this.collections(true); + for (const acct of this.accountIds()) await this.refetchStale(acct); + for (const acct of this.accountIds()) { + await this.changesLoop(acct); + await this.loadNewlySelected(acct); + } + await this.uploadAll(); + for (const acct of this.knownAccountIds()) { + await this.env.checkpoints.check('deselect'); + await this.dropOutsideSelection(acct); + } + await this.afterSync(); + await this.flushState(); + } + + /** A teardown's upload: everything waiting on the device, nothing else. */ + async uploadOnly(): Promise { + await this.collections(false); + await this.uploadAll(); + await this.flushState(); + } + + protected accountIds(): string[] { + return this.env.accounts.map((a) => a.id); + } + + /** The session's accounts plus accounts the rows were written for that are gone from it. */ + protected knownAccountIds(): string[] { + return [...new Set([...this.accountIds(), ...Object.keys(this.env.store.committed.accounts)])]; + } + + // ── State ── + + private marks(acct: string): Map { + let marks = this.staleMarks.get(acct); + if (!marks) this.staleMarks.set(acct, (marks = new Map())); + return marks; + } + + protected markStale(acct: string, id: string): void { + this.marks(acct).set(id, true); + } + + protected markResolved(acct: string, id: string): void { + if (this.marks(acct).has(id) || this.env.store.account(acct).stale.includes(id)) this.marks(acct).set(id, false); + } + + protected isStale(acct: string, sourceId: string | null): boolean { + const id = idInAccount(sourceId, acct); + if (!id) return false; + return this.marks(acct).get(id) ?? this.env.store.account(acct).stale.includes(id); + } + + /** + * A planner threw on one item (a bug, or data it cannot handle): that + * item is reported and left alone, the rest of the run goes on. A + * download's object goes on `stale`, so the state still moves and the + * object is tried again next run. + */ + protected plannerFailed(acct: string, ref: string, side: 'download' | 'upload', error: unknown, staleId?: string): void { + if (staleId) this.markStale(acct, staleId); + if (side === 'upload') this.env.report.stats.skipped++; + const description = error instanceof Error ? error.message : String(error); + this.env.report.itemError({ ref, side, type: 'plannerError', description: description.slice(0, 300) }); + this.env.log(`planner failed on ${ref}`, error); + } + + protected outsideOf(acct: string): Set { + let set = this.outside.get(acct); + if (!set) this.outside.set(acct, (set = new Set())); + return set; + } + + /** A state op for `accts`, carrying their stale marks and the authority's changes along with `mutate`. */ + protected tail(accts: readonly string[], mutate?: (next: SyncState) => void): Tail { + const consumed: Array<[string, string, boolean]> = []; + const commits: Array<() => void> = []; + const base = this.env.store.tail((next) => { + consumed.length = 0; + commits.length = 0; + mutate?.(next); + for (const acct of accts) { + const account = accountOf(next, acct); + const marks = this.staleMarks.get(acct); + if (marks?.size) { + const stale = new Set(account.stale); + for (const [id, isStale] of marks) { + consumed.push([acct, id, isStale]); + if (isStale) stale.add(id); + else stale.delete(id); + } + account.stale = [...stale]; + } + const commit = this.decorateState(acct, next); + if (commit) commits.push(commit); + } + }); + return { + op: base.op, + applied: () => { + base.applied(); + for (const [acct, id, isStale] of consumed) { + const marks = this.staleMarks.get(acct); + if (marks?.get(id) === isStale) marks.delete(id); + } + for (const commit of commits) commit(); + }, + }; + } + + /** Stores what is still pending (stale marks, collection states, …) at the end of a run. */ + protected async flushState(): Promise { + const accts = this.knownAccountIds(); + const probe = this.tail(accts); + const before = JSON.stringify(this.env.store.committed); + const op = probe.op(); + if (op.op === 'syncState' && op.value === before) return; + await this.env.writer.write([], { op: () => op, applied: probe.applied }); + } + + // ── Downloads ── + + /** Objects put on `stale` by an earlier run: fetched again first (invariant 10). */ + protected async refetchStale(acct: string): Promise { + const ids = [...this.env.store.account(acct).stale]; + if (ids.length) await this.processIds(acct, ids, [], () => undefined); + } + + /** `/changes` pages from the stored state; a full reconcile without one or after `cannotCalculateChanges`. */ + protected async changesLoop(acct: string): Promise { + let reconciled = false; + for (let round = 0; round < 10_000; round++) { + const account = this.env.store.account(acct); + if (!account.itemsState || account.reconcile) { + if (reconciled) return; + await this.fullReconcile(acct); + reconciled = true; + continue; + } + const since = account.itemsState; + await this.env.checkpoints.check('changes'); + let page; + try { + page = await this.env.jmap.changes(this.itemType, acct, since); + } catch (error) { + if (isMethodError(error, 'cannotCalculateChanges') && !reconciled) { + this.env.log(`cannotCalculateChanges for ${acct}: full reconcile`); + await this.requestReconcile(acct); + continue; + } + throw error; + } + const ids = [...new Set([...page.created, ...page.updated])]; + const newState = page.newState; + if (!ids.length && !page.destroyed.length && newState === since) return; + await this.processIds(acct, ids, page.destroyed, (next) => { + accountOf(next, acct).itemsState = newState; + }); + if (!page.hasMoreChanges || newState === since) return; + } + } + + protected async requestReconcile(acct: string): Promise { + await this.env.writer.write( + [], + this.tail([acct], (next) => { + accountOf(next, acct).reconcile = { from: null, phase: 'ids', position: 0, after: null }; + }), + ); + } + + /** + * A full reconcile, checkpointed in `reconcile`: the state first, then the + * ids of every synced collection, then the objects in chunks; deletions + * only once the complete id list is known. Zero objects on the server + * against a full device stops the run (`safetyAbort`). + */ + protected async fullReconcile(acct: string): Promise { + let marker = this.env.store.account(acct).reconcile; + if (!marker?.from) { + await this.env.checkpoints.check('reconcile'); + const from = await this.env.jmap.takeState(this.itemType, acct); + const started = { from, phase: 'ids' as const, position: 0, after: null }; + await this.env.writer.write([], this.tail([acct], (next) => { + accountOf(next, acct).reconcile = { ...started }; + })); + marker = started; + } + const from = marker.from as string; + await this.beforeReconcile(acct); + const collections = this.syncedCollections(acct); + const listed = new Set(); + for (const collection of collections) { + await this.env.checkpoints.check('reconcile:ids'); + for (const id of await this.env.jmap.queryAll(this.itemType, acct, { [this.parentFilter]: collection })) listed.add(id); + } + const local = await this.localIdentities(acct); + if (listed.size === 0 && collections.length > 0 && local.size > this.env.tuning.safetyAbortRows) { + throw new RunAbort( + 'safetyAbort', + `The server lists no objects in ${acct} while this device holds ${local.size}; nothing was changed`, + ); + } + const { first, rest, missing } = await this.order(acct, [...listed].sort()); + const size = this.env.tuning.chunkSize; + for (let i = 0; i < first.length; i += size) { + await this.env.checkpoints.check('reconcile:objects'); + await this.processChunk(acct, first.slice(i, i + size), [], undefined); + } + const resumeAfter = marker.after; + const todo = resumeAfter ? rest.filter((id) => id > resumeAfter) : rest; + let position = rest.length - todo.length; + for (let i = 0; i < todo.length; i += size) { + await this.env.checkpoints.check('reconcile:objects'); + const part = todo.slice(i, i + size); + position += part.length; + const mark = { from, phase: 'objects' as const, position, after: part[part.length - 1] }; + await this.processChunk(acct, part, [], (next) => { + accountOf(next, acct).reconcile = { ...mark }; + }); + } + const gone = new Set(missing); + const verify: string[] = []; + for (const [id, flags] of local) { + if (listed.has(id)) continue; + if (flags.dirty || flags.deleted) verify.push(id); + else gone.add(id); + } + if (verify.length) { + // Destroyed, or moved out of the synced collections? Only a destroy takes dirty rows. + const { list, notFound } = await this.env.jmap.get(this.itemType, acct, verify, ['id']); + for (const id of notFound) gone.add(id); + for (const object of list) this.outsideOf(acct).add(refOf(acct, object.id)); + } + const selected = collections.map((c) => collectionKey(acct, c)); + await this.processIds(acct, [], [...gone], (next) => { + const account = accountOf(next, acct); + account.itemsState = from; + account.reconcile = null; + account.selected = selected; + }); + } + + /** Collections selected since the rows were written: loaded once the account is up to date. */ + protected async loadNewlySelected(acct: string): Promise { + const account = this.env.store.account(acct); + if (!account.itemsState || account.reconcile) return; + for (const collection of this.syncedCollections(acct)) { + const key = collectionKey(acct, collection); + if (this.env.store.account(acct).selected.includes(key)) continue; + await this.env.checkpoints.check('select'); + const ids = await this.env.jmap.queryAll(this.itemType, acct, { [this.parentFilter]: collection }); + await this.processIds(acct, ids, [], (next) => { + const a = accountOf(next, acct); + if (!a.selected.includes(key)) a.selected.push(key); + }); + } + } + + /** Downloads `ids` and removes the rows of `destroyed`, in chunks; `mutate` rides with the last one. */ + protected async processIds( + acct: string, + ids: readonly string[], + destroyed: readonly string[], + mutate?: (next: SyncState) => void, + ): Promise { + const { first, rest, missing } = ids.length ? await this.order(acct, [...ids]) : { first: [], rest: [], missing: [] }; + const size = this.env.tuning.chunkSize; + const chunks: Array<{ ids: string[]; gone: string[] }> = []; + for (const part of [first, rest]) { + for (let i = 0; i < part.length; i += size) chunks.push({ ids: part.slice(i, i + size), gone: [] }); + } + const gone = [...new Set([...destroyed, ...missing])]; + for (let i = 0; i < gone.length; i += size) chunks.push({ ids: [], gone: gone.slice(i, i + size) }); + if (chunks.length === 0) { + if (mutate) await this.env.writer.write([], this.tail([acct], mutate)); + return; + } + for (let i = 0; i < chunks.length; i++) { + await this.env.checkpoints.check('download'); + await this.processChunk(acct, chunks[i].ids, chunks[i].gone, i === chunks.length - 1 ? mutate : undefined); + } + } + + /** One chunk: fetch, find the local items (adopting pending creates by uid), plan, write, read back. */ + protected async processChunk( + acct: string, + ids: readonly string[], + gone: readonly string[], + mutate?: (next: SyncState) => void, + ): Promise { + let objects: ServerObject[] = []; + let notFound: string[] = []; + if (ids.length) { + ({ list: objects, notFound } = await this.env.jmap.get(this.itemType, acct, ids, this.itemProperties)); + } + await this.beforePlanning(acct, objects); + const allGone = [...new Set([...gone, ...notFound])]; + const held = await this.heldByRefs([...objects.map((o) => refOf(acct, o.id)), ...allGone.map((id) => refOf(acct, id))]); + let candidates: Held[] | null = null; + const works: Work[] = []; + const written = new Map>(); + for (const object of objects) { + this.env.report.stats.entries++; + const kind = this.kindOf(object); + let found = held.get(refOf(acct, object.id)) ?? null; + if (!found && kind && this.inSelection(acct, object)) { + candidates ??= await this.adoptionCandidates(acct); + found = this.takeAdoptable(candidates, kind, acct, object); + } + let work: Work | null; + try { + work = this.downloadDecision(acct, object, kind, found, written); + } catch (error) { + this.plannerFailed(acct, refOf(acct, object.id), 'download', error, object.id); + continue; + } + if (work) works.push(work); + else this.markResolved(acct, object.id); + } + for (const id of allGone) { + const found = held.get(refOf(acct, id)); + let work: Work | null; + try { + work = found ? this.removeWork(acct, id, found, 'destroyed') : null; + } catch (error) { + this.plannerFailed(acct, refOf(acct, id), 'download', error, id); + continue; + } + if (work) works.push(work); + else this.markResolved(acct, id); + } + await this.env.writer.write(works, mutate ? this.tail([acct], mutate) : undefined); + await this.readBack(written); + } + + private async heldByRefs(refs: readonly string[]): Promise> { + const out = new Map(); + if (!refs.length) return out; + for (const kind of this.kinds()) { + for (const [ref, local] of await kind.loadByRefs(refs)) if (!out.has(ref)) out.set(ref, { kind, local }); + } + return out; + } + + private async adoptionCandidates(acct: string): Promise { + const out: Held[] = []; + for (const kind of this.kinds()) { + for (const local of await kind.loadNew()) { + const meta = kind.meta(local); + if (meta.pending && accountOfRef(meta.pending.target) === acct) out.push({ kind, local }); + } + } + return out; + } + + /** A new local item whose pending create carries the object's uid: our own create whose identity was never written. */ + private takeAdoptable(candidates: Held[], kind: Kind, acct: string, object: ServerObject): Held | null { + if (typeof object.uid !== 'string') return null; + const index = candidates.findIndex((c) => { + if (c.kind !== kind) return false; + const pending = kind.meta(c.local).pending; + return !!pending && pending.uid === object.uid && this.adoptionTargetMatches(acct, object, pending.target); + }); + return index < 0 ? null : candidates.splice(index, 1)[0]; + } + + private downloadDecision( + acct: string, + object: ServerObject, + kind: Kind | null, + found: Held | null, + written: Map> | null, + ): Work | null { + if (!kind || !this.inSelection(acct, object)) { + return found ? this.removeWork(acct, object.id, found, 'outside') : null; + } + if (found && found.kind !== kind) { + const meta = found.kind.meta(found.local); + if (meta.dirty || meta.deleted) { + this.env.report.itemError({ ref: refOf(acct, object.id), side: 'download', type: 'kindChanged' }); + return null; + } + return this.replaceWork(acct, object, kind, found); + } + return this.downloadWork(kind, acct, object, found ? found.local : null, written); + } + + /** The object merged into its rows (inserted, updated, or adopted); an echo writes nothing but heals the baselines. */ + protected downloadWork( + kind: Kind, + acct: string, + object: ServerObject, + local: unknown, + written: Map> | null, + ): Work | null { + const ref = refOf(acct, object.id); + const plan = kind.planDownload(object, local ?? null, acct); + if (plan.effect === 'none' || !hasWrites(plan.ops.ops)) { + if (!local || kind.meta(local).dirty) return null; + const heal = kind.planBaselineHeal(local); + return heal && hasWrites(heal.ops) ? { group: heal } : null; + } + const group = local ? plan.ops : prependOps(plan.ops, [insertGuard(kind, ref)]); + return { + group, + applied: () => { + if (plan.effect === 'insert') this.env.report.stats.downloaded.created++; + else this.env.report.stats.downloaded.updated++; + this.env.report.conflicts += plan.conflicts; + this.markResolved(acct, object.id); + this.onDownloaded(kind, acct, object); + if (written) { + let refs = written.get(kind); + if (!refs) written.set(kind, (refs = new Set())); + refs.add(ref); + } + }, + replan: async () => { + const current = local ? await this.reload(kind, local) : ((await kind.loadByRefs([ref])).get(ref) ?? null); + try { + return this.downloadWork(kind, acct, object, current, written); + } catch (error) { + this.plannerFailed(acct, ref, 'download', error, object.id); + return null; + } + }, + failed: (reason, message) => { + this.markStale(acct, object.id); + this.env.report.itemError({ ref, side: 'download', type: reason, description: message }); + }, + }; + } + + /** An object that changed kind (a card that became a group): its old rows go, new ones come, atomically. */ + private replaceWork(acct: string, object: ServerObject, kind: Kind, found: Held): Work { + const ref = refOf(acct, object.id); + const plan = kind.planDownload(object, null, acct); + return { + group: concatGroups(ref, found.kind.planLocalDelete(found.local), plan.ops), + applied: () => { + this.env.report.stats.downloaded.updated++; + this.markResolved(acct, object.id); + this.onRemoved(found.kind, acct, object.id); + this.onDownloaded(kind, acct, object); + }, + failed: (reason, message) => { + this.markStale(acct, object.id); + this.env.report.itemError({ ref, side: 'download', type: reason, description: message }); + }, + }; + } + + /** + * Rows of an object the server destroyed (the server delete wins over a + * local edit: a conflict), or that left every synced collection (only + * clean rows go; dirty ones upload first and are dropped afterwards). + */ + protected removeWork(acct: string, id: string, found: Held, reason: Removal): Work | null { + const ref = refOf(acct, id); + const meta = found.kind.meta(found.local); + if (reason === 'outside' && (meta.dirty || meta.deleted || meta.isNew)) { + this.outsideOf(acct).add(ref); + return null; + } + const conflict = reason === 'destroyed' && meta.dirty && !meta.deleted; + return { + group: found.kind.planLocalDelete(found.local), + applied: () => { + this.env.report.stats.downloaded.deleted++; + if (conflict) this.env.report.conflicts++; + this.markResolved(acct, id); + this.onRemoved(found.kind, acct, id); + }, + replan: async () => { + const again = (await found.kind.loadByRefs([ref])).get(ref); + try { + return again === undefined ? null : this.removeWork(acct, id, { kind: found.kind, local: again }, reason); + } catch (error) { + this.plannerFailed(acct, ref, 'download', error, id); + return null; + } + }, + failed: (failure, message) => { + this.markStale(acct, id); + this.env.report.itemError({ ref, side: 'download', type: failure, description: message }); + }, + }; + } + + protected async reload(kind: Kind, local: unknown): Promise { + const meta = kind.meta(local); + if (meta.sourceId) return (await kind.loadByRefs([meta.sourceId])).get(meta.sourceId) ?? null; + const [again] = await kind.loadByRowIds([meta.rowId]); + return again ?? null; + } + + /** Rows as the provider stored them: clean items whose baselines drifted (normalisation) get them rewritten. */ + private async readBack(written: Map>): Promise { + const works: Work[] = []; + for (const [kind, refs] of written) { + for (const local of (await kind.loadByRefs([...refs])).values()) { + if (kind.meta(local).dirty) continue; + const heal = kind.planBaselineHeal(local); + if (heal && hasWrites(heal.ops)) works.push({ group: heal }); + } + } + if (works.length) await this.env.writer.write(works); + } + + // ── Uploads ── + + /** + * Local changes to the server, per phase across the accounts: pairs and + * splits, creates, updates, deletions, then memberships. Creates of every + * account come before deletions of any, so a move between accounts never + * destroys first. A `stateMismatch` downloads the account again and + * repeats the phase, at most three times. + */ + protected async uploadAll(): Promise { + await this.claimNewItems(); + const deletionsAllowed = await this.deletionPolicy(await this.loadUploadItems()); + const accounts = this.accountIds(); + for (const acct of accounts) await this.withRetries(acct, () => this.prePhase(acct)); + for (const phase of ['create', 'update', 'delete'] as const) { + for (const acct of accounts) await this.withRetries(acct, () => this.phase(acct, phase, deletionsAllowed)); + } + for (const acct of accounts) await this.withRetries(acct, () => this.extraPhase(acct, deletionsAllowed)); + for (const acct of accounts) await this.adoptExtraCreated(acct); + } + + private async withRetries(acct: string, run: () => Promise): Promise { + for (let attempt = 0; ; attempt++) { + try { + await run(); + return; + } catch (error) { + if (isMethodError(error) && ACCOUNT_REFUSALS.has(error.type)) { + // The account takes no writes (a shared account turned read-only, or gone): its items wait, the others go on. + this.env.report.itemError({ ref: acct, side: 'upload', type: error.type, description: error.message }); + return; + } + if (!(error instanceof StateMismatch) || error.jmapAccountId !== acct) throw error; + if (attempt >= this.env.tuning.maxStateMismatchRetries) { + throw new RunAbort('io', 'The server kept changing while this device uploaded', { noProgress: true }); + } + await this.catchUp(acct); + } + } + } + + /** + * After a `stateMismatch`: download what changed, then guard the next + * `/set` with the type's current state when nothing changed after the + * stored one. Stalwart can name one point of the history two ways (a + * `/changes` state against the type's state, e.g. `s…` against `n` before + * the first item change), and `ifInState` compares them as text. + */ + private async catchUp(acct: string): Promise { + this.setStates.delete(acct); + await this.changesLoop(acct); + const since = this.env.store.account(acct).itemsState; + if (!since) return; + const current = await this.env.jmap.takeState(this.itemType, acct); + if (!current || current === since) return; + const page = await this.env.jmap.changes(this.itemType, acct, since); + // Anything new since `since` is downloaded by the next attempt; `current` may predate it. + if (!page.created.length && !page.updated.length && !page.destroyed.length && !page.hasMoreChanges) { + this.setStates.set(acct, current); + } + } + + protected ifInState(acct: string): string | null { + if (!this.setStates.has(acct)) this.setStates.set(acct, this.env.store.account(acct).itemsState); + return this.setStates.get(acct) ?? null; + } + + /** New items get their uid and target written before anything is sent. */ + private async claimNewItems(): Promise { + const works: Work[] = []; + for (const held of await this.loadUploadItems()) { + const meta = held.kind.meta(held.local); + if (!meta.isNew || meta.pending || meta.deleted || !this.claimable(held)) continue; + const ref = `row:${meta.rowId}`; + const acct = this.claimAccount(held); + if (!acct) { + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: 'noTargetCollection' }); + continue; + } + let plan; + try { + plan = held.kind.planUpload(held.local, acct); + } catch (error) { + this.plannerFailed(acct, ref, 'upload', error); + continue; + } + if (plan.kind === 'skip') { + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: plan.reason }); + } else if (plan.kind === 'claim' || plan.kind === 'purge' || plan.kind === 'clean') { + works.push(this.planWork(acct, held, plan.kind, plan.ops)); + } + } + await this.env.writer.write(works); + } + + /** + * The deletion threshold: more than 50 local deletions and more than 20 % + * of the synced objects hold every deletion back (`tooManyDeletions`) + * unless the run overrides it; `discardLocalDeletions` restores them. + */ + private async deletionPolicy(items: Held[]): Promise { + const { count, synced, accounts } = await this.countDeletions(items); + if (count === 0) return true; + if (this.env.extras.discardLocalDeletions) { + await this.discardDeletions(items, accounts); + return false; + } + if (this.env.extras.overrideTooManyDeletions) return true; + const t = this.env.tuning; + if (count > t.deletionThresholdCount && count > synced * t.deletionThresholdRatio) { + this.env.report.tooManyDeletions = { + count, + threshold: Math.max(t.deletionThresholdCount, Math.floor(synced * t.deletionThresholdRatio)), + }; + return false; + } + return true; + } + + /** + * "Undo" in the too-many-deletions notification: the deleted rows go and + * the accounts get a reconcile marker, so the next runs download the + * objects again whatever their extras. The marker is stored first: a crash + * in between only asks the question again. + */ + private async discardDeletions(items: Held[], accounts: Set): Promise { + await this.env.writer.write([], this.tail([...accounts], (next) => { + for (const acct of accounts) accountOf(next, acct).reconcile = { from: null, phase: 'ids', position: 0, after: null }; + this.onDiscard(next, accounts); + })); + const works: Work[] = []; + for (const held of items) { + const meta = held.kind.meta(held.local); + if (!meta.deleted || meta.isNew) continue; + const acct = accountOfRef(meta.sourceId); + if (acct) works.push(this.purgeWork(acct, held, false)); + } + await this.env.writer.write(works); + this.env.report.note('local deletions discarded; the objects are downloaded again'); + } + + private async phase(acct: string, phase: 'create' | 'update' | 'delete', deletionsAllowed: boolean): Promise { + const items = (await this.loadUploadItems()).filter((held) => { + if (this.accountOfItem(held) !== acct) return false; + const m = held.kind.meta(held.local); + if (phase === 'create') return m.isNew && !m.deleted && !!m.pending; + if (phase === 'update') return !m.isNew && !m.deleted && m.dirty; + return m.deleted && (deletionsAllowed || m.isNew); + }); + this.noteUploadItems(acct, items); + const size = this.env.tuning.chunkSize; + for (let i = 0; i < items.length; i += size) { + await this.env.checkpoints.check(`upload:${phase}`); + // From a create's /set to its identity write nothing is interruptible; start only with time for it. + if (phase === 'create') this.env.checkpoints.requireBudget(this.env.tuning.createBudgetMs); + const ready = await this.prepare(acct, items.slice(i, i + size)); + const toSend = phase === 'create' ? await this.lookupBeforeCreate(acct, ready) : ready; + if (toSend.length) await this.send(acct, toSend); + } + } + + /** Items an upload phase looked at (contacts: their memberships are uploaded afterwards). */ + protected noteUploadItems(_acct: string, _items: Held[]): void {} + + /** Plans the items' uploads; plans that need no request are applied here. */ + protected async prepare(acct: string, items: Held[]): Promise { + await this.beforeUploadPlanning(items); + const works: Work[] = []; + const ready: Pending[] = []; + for (const held of items) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); + } + await this.env.writer.write(works); + return ready; + } + + protected async planItem(acct: string, held: Held, works: Work[]): Promise { + const { kind, local } = held; + const meta = kind.meta(local); + const ref = meta.sourceId ?? `row:${meta.rowId}`; + const report = this.env.report; + report.stats.entries++; + if (this.isStale(acct, meta.sourceId)) { + report.stats.skipped++; + report.itemError({ ref, side: 'upload', type: 'stale', description: 'Waits until the server version could be stored' }); + return null; + } + const fingerprint = kind.fingerprint(local); + if (meta.poison && isBackedOff(meta.poison, fingerprint, this.env.now())) { + report.stats.skipped++; + report.itemError({ ref, side: 'upload', type: meta.poison.type, description: meta.poison.description, retryAt: meta.poison.until }); + return null; + } + let plan; + try { + plan = kind.planUpload(local, acct); + } catch (error) { + this.plannerFailed(acct, ref, 'upload', error); + return null; + } + switch (plan.kind) { + case 'upload': { + const actions = await this.resolveDestroysByUid(acct, meta, plan.actions); + if (actions.length === 0) { + works.push(this.purgeWork(acct, held, false)); + return null; + } + return { held, meta, fingerprint, actions }; + } + case 'revert': + report.stats.skipped++; + works.push(this.planWork(acct, held, 'revert', plan.ops)); + if (plan.refetch) { + const id = idInAccount(meta.sourceId, acct); + if (id) this.markStale(acct, id); + } + return null; + case 'skip': + report.stats.skipped++; + report.itemError({ ref, side: 'upload', type: plan.reason }); + return null; + default: + works.push(this.planWork(acct, held, plan.kind, plan.ops)); + return null; + } + } + + /** A plan applied locally (clean, claim, purge, revert); replanned once more after an assert failure. */ + private planWork(acct: string, held: Held, kind: 'clean' | 'claim' | 'purge' | 'revert', group: OpGroup): Work { + return { + group, + replan: async () => { + const again = await this.reload(held.kind, held.local); + if (again === null) return null; + const plan = held.kind.planUpload(again, acct); + return plan.kind === kind && 'ops' in plan ? this.planWork(acct, { kind: held.kind, local: again }, kind, plan.ops) : null; + }, + }; + } + + /** The rows of an item whose deletion reached the server (or never had to). */ + protected purgeWork(acct: string, held: Held, uploaded: boolean): Work { + const meta = held.kind.meta(held.local); + const id = idInAccount(meta.sourceId, acct); + return { + group: held.kind.planLocalDelete(held.local), + applied: () => { + if (uploaded) this.env.report.stats.uploaded.deleted++; + if (id) this.onRemoved(held.kind, acct, id); + }, + replan: async () => { + const again = await this.reload(held.kind, held.local); + return again === null ? null : this.purgeWork(acct, { kind: held.kind, local: again }, uploaded); + }, + }; + } + + /** A destroy of "whatever carries this uid" (the create's outcome is unknown) becomes destroys of what the lookup finds. */ + private async resolveDestroysByUid(acct: string, meta: ItemMeta, actions: Action[]): Promise { + const out: Action[] = []; + for (const action of actions) { + if (action.kind !== 'destroy' || action.id !== null) { + out.push(action); + continue; + } + if (!action.uid) continue; + const collection = meta.pending ? meta.pending.target.slice(meta.pending.target.indexOf('/') + 1) : ''; + const found = await this.env.jmap.lookupUids(this.itemType, acct, [action.uid], this.lookupScope(acct, collection)); + for (const id of found.get(action.uid) ?? []) out.push({ ...action, id }); + } + return out; + } + + /** The uid lookup before a create: a hit is adopted (the row takes its identity and uploads as an update). */ + private async lookupBeforeCreate(acct: string, ready: Pending[]): Promise { + const scopes = new Map>(); + for (const p of ready) { + for (const a of p.actions) { + if (a.kind !== 'create') continue; + const scope = this.lookupScope(acct, a.collectionId); + let uids = scopes.get(scope); + if (!uids) scopes.set(scope, (uids = new Set())); + uids.add(a.uid); + } + } + const found = new Map(); + const keyOf = (collectionId: string, uid: string) => `${this.lookupScope(acct, collectionId) ?? ''}\u0000${uid}`; + for (const [scope, uids] of scopes) { + const hits = await this.env.jmap.lookupUids(this.itemType, acct, [...uids], scope); + for (const [uid, ids] of hits) found.set(`${scope ?? ''}\u0000${uid}`, ids[0]); + } + if (!found.size) return ready; + const out: Pending[] = []; + const works: Work[] = []; + for (const p of ready) { + const own = p.actions.find((a) => a.kind === 'create' && a.uid === p.meta.pending?.uid); + const ownId = own?.kind === 'create' ? found.get(keyOf(own.collectionId, own.uid)) : undefined; + if (ownId) { + works.push(...(await this.adoptWorks(acct, p, ownId))); + continue; + } + const actions = p.actions.filter((a) => a.kind !== 'create' || !found.has(keyOf(a.collectionId, a.uid))); + if (actions.length) out.push({ ...p, actions }); + } + await this.env.writer.write(works); + return out; + } + + /** + * The row takes the server object's identity and is merged as a dirty + * item (the planner adopts by pending uid). Only an object in the + * create's target collection can be the row's own earlier create: event + * uids are unique per account, and an unrelated event elsewhere with the + * same uid must not receive this row's content (the item is poisoned). + */ + private async adoptWorks(acct: string, p: Pending, id: string): Promise { + const { list } = await this.env.jmap.get(this.itemType, acct, [id], this.itemProperties); + const object = list[0]; + if (!object) return []; + const target = p.meta.pending?.target; + if (!target || !this.adoptionTargetMatches(acct, object, target)) { + return [this.poisonWork(p, { type: 'uidConflict', description: `Another object (${refOf(acct, id)}) has this uid` })]; + } + await this.beforePlanning(acct, [object]); + const work = this.downloadWork(p.held.kind, acct, object, p.held.local, null); + return work ? [work] : []; + } + + /** Sends the items' actions in `/set` calls and settles each item once all its actions have an answer. */ + protected async send(acct: string, pending: Pending[]): Promise { + const calls = this.packCalls(pending); + const objects = new Map(); + const outcomes = new Map(); + const remaining = new Map(pending.map((p) => [p, p.actions.length])); + const retry: Pending[] = []; + for (const call of calls) { + const response = await this.setCall(acct, call, objects); + const settled: Pending[] = []; + for (const entry of call.entries) { + const outcome = this.outcomeOf(response, entry.action, entry.key, call.args.sendSchedulingMessages); + // No answer for it at all: left as it is, the next run looks again. + if (!outcome) continue; + let list = outcomes.get(entry.pending); + if (!list) outcomes.set(entry.pending, (list = [])); + list.push(outcome); + const left = (remaining.get(entry.pending) ?? 1) - 1; + remaining.set(entry.pending, left); + if (left === 0) settled.push(entry.pending); + } + // Identities of this call's creates are written before anything else runs. + const works: Work[] = []; + for (const p of settled) works.push(...(await this.settle(acct, p, outcomes.get(p) ?? [], objects, retry))); + await this.env.writer.write(works); + } + if (retry.length) await this.send(acct, retry); + } + + private packCalls(pending: Pending[]): SetCallPlan[] { + const limits = this.env.jmap.limits(); + const maxObjects = Math.max(1, limits.maxObjectsInSet); + const maxCreates = Math.max(1, Math.min(this.env.tuning.maxCreatesPerSet, maxObjects)); + const maxBytes = Math.min(this.env.tuning.maxSetBytes, Math.max(1, limits.maxSizeRequest - 10_000)); + const calls: SetCallPlan[] = []; + const open = new Map(); + let creation = 0; + const fresh = (scheduling: boolean): SetCallPlan => ({ + args: { create: {}, update: {}, destroy: [], sendSchedulingMessages: scheduling }, + entries: [], + touched: new Set(), + updateIds: [], + creates: 0, + bytes: 0, + }); + for (const p of pending) { + for (const action of p.actions) { + // `sendSchedulingMessages` is a request-level flag: one value per call. + const scheduling = !p.noScheduling && !!action.sendSchedulingMessages; + const size = JSON.stringify(action).length; + const id = action.kind === 'create' ? null : action.id; + let call = open.get(scheduling); + const fits = + !!call && + call.entries.length + 1 <= maxObjects && + (action.kind !== 'create' || call.creates + 1 <= maxCreates) && + call.bytes + size <= maxBytes && + (id === null || !call.touched.has(id)); + if (!call || !fits) { + call = fresh(scheduling); + calls.push(call); + open.set(scheduling, call); + } + let key: string; + if (action.kind === 'create') { + key = `k${++creation}`; + call.args.create[key] = this.createObject(action); + call.creates++; + } else if (action.kind === 'update') { + key = action.id; + call.args.update[key] = action.patch; + call.updateIds.push(key); + } else { + key = action.id as string; + call.args.destroy.push(key); + } + call.touched.add(key); + call.bytes += size; + call.entries.push({ pending: p, action, key }); + } + } + return calls; + } + + private createObject(action: Extract): Record { + const object: Record = { ...(action.object as Record), uid: action.uid }; + if (!object[this.parentProperty]) object[this.parentProperty] = { [action.collectionId]: true }; + return object; + } + + /** + * One `/set` of the authority's type, guarded by `ifInState`; our + * `newState` guards the next one. `stateMismatch` (and Stalwart's + * `serverUnavailable`, a concurrent write that failed the whole call) + * throws StateMismatch: the phase downloads again and repeats. + */ + protected async setChecked( + acct: string, + args: SetArgs, + thenGet?: { ids: string[]; properties: readonly string[] }, + ): Promise<{ response: SetResponse; got: ServerObject[] | null }> { + let result; + try { + result = await this.env.jmap.set(this.itemType, acct, args, this.ifInState(acct), thenGet); + } catch (error) { + if (isMethodError(error, 'stateMismatch') || isMethodError(error, 'serverUnavailable')) throw new StateMismatch(acct); + throw error; + } + if (result.response.newState) { + this.setStates.set(acct, result.response.newState); + this.env.recordKnownState(acct, this.itemType, result.response.newState); + } + return result; + } + + private async setCall(acct: string, call: SetCallPlan, objects: Map): Promise { + const { response, got } = await this.setChecked( + acct, + { + create: call.args.create, + update: call.args.update, + destroy: call.args.destroy, + sendSchedulingMessages: call.args.sendSchedulingMessages, + }, + call.updateIds.length ? { ids: call.updateIds, properties: this.itemProperties } : undefined, + ); + for (const object of got ?? []) objects.set(object.id, object); + // Stalwart resolves no result reference to a /set: created objects are fetched in a request of their own. + const createdIds = Object.values(response.created ?? {}).map((c) => c?.id).filter((id): id is string => !!id); + if (createdIds.length) { + const { list } = await this.env.jmap.get(this.itemType, acct, createdIds, this.itemProperties); + for (const object of list) objects.set(object.id, object); + } + return response; + } + + private outcomeOf(response: SetResponse, action: Action, key: string, scheduling: boolean): Outcome | null { + if (action.kind === 'create') { + const created = response.created?.[key]; + if (created?.id) return { action, ok: true, id: created.id }; + const error = response.notCreated?.[key]; + return error ? { action, ok: false, error, scheduling } : null; + } + if (action.kind === 'update') { + if (response.updated && key in response.updated) return { action, ok: true, id: key }; + const error = response.notUpdated?.[key]; + return error ? { action, ok: false, error, scheduling } : null; + } + if (response.destroyed?.includes(key)) return { action, ok: true, id: key }; + const error = response.notDestroyed?.[key]; + return error ? { action, ok: false, error, scheduling } : null; + } + + /** What the server's answers mean for one item (docs/device-sync.md, "Uploads", SetErrors). */ + private async settle( + acct: string, + p: Pending, + outcomes: Outcome[], + objects: Map, + retry: Pending[], + ): Promise { + const failure = outcomes.find((o): o is Extract => !o.ok); + if (!failure) return this.succeeded(acct, p, outcomes, objects); + switch (classifySetError(failure.error, failure.action.kind)) { + case 'notFound': + if (failure.action.kind === 'destroy') { + // Gone already: the deletion is done. + const done = outcomes.map((o): Outcome => (o === failure ? { action: o.action, ok: true, id: (o.action as { id: string }).id } : o)); + return this.succeeded(acct, p, done, objects); + } + if (failure.action.kind === 'update') { + // Deleted on the server meanwhile: the server delete wins, the rows go. + const id = idInAccount(p.meta.sourceId, acct); + const work = id ? this.removeWork(acct, id, p.held, 'destroyed') : null; + return work ? [work] : []; + } + return [this.poisonWork(p, failure.error)]; + case 'forbidden': + if (failure.scheduling && !p.noScheduling) { + // Scheduling is off, the account has no calendar address, or no permission: store the change without it. + this.env.report.note('invitations not sent'); + retry.push({ ...p, noScheduling: true }); + return []; + } + if (failure.action.kind === 'create') return [this.poisonWork(p, failure.error)]; + return this.revertWorks(acct, p, failure.error); + case 'uidExists': + return this.adoptAfterUidError(acct, p, failure); + default: + return [this.poisonWork(p, failure.error)]; + } + } + + private async succeeded(acct: string, p: Pending, outcomes: Outcome[], objects: Map): Promise { + const ownUid = p.meta.pending?.uid; + let ownId = idInAccount(p.meta.sourceId, acct); + let created = false; + for (const o of outcomes) { + if (!o.ok || o.action.kind !== 'create') continue; + if (!ownId && o.action.uid === ownUid) { + ownId = o.id; + created = true; + } else { + let set = this.extraCreated.get(acct); + if (!set) this.extraCreated.set(acct, (set = new Set())); + set.add(o.id); + } + } + if (p.accept) return p.accept(objects); + const server = ownId ? objects.get(ownId) : undefined; + if (!p.meta.deleted && !server) { + // Accepted, but its new version could not be read: fetched again next run, nothing uploads for it meanwhile. + if (ownId) this.markStale(acct, ownId); + return []; + } + try { + return [p.meta.deleted ? this.purgeWork(acct, p.held, true) : this.acceptWork(acct, p, server as ServerObject, created)]; + } catch (error) { + // Uploaded, but the rows could not be planned: the shadow is behind, fetch it again. + this.plannerFailed(acct, p.meta.sourceId ?? `row:${p.meta.rowId}`, 'upload', error, ownId ?? undefined); + return []; + } + } + + /** + * The accepted version becomes the shadow and the baselines, DIRTY is + * cleared, behind the assert the upload was read under; edited again + * meanwhile, only identity, shadow and the uploaded baselines are written + * and DIRTY stays (the newer edit uploads next). + */ + protected acceptWork(acct: string, p: Pending, server: ServerObject, created: boolean): Work { + const { kind, local } = p.held; + const plan = kind.planAccepted(local, server, acct); + const extra = p.meta.poison ? [poisonOp(kind, p.meta.rowId, null)] : []; + const withExtra = (group: OpGroup): OpGroup => (extra.length ? { ref: group.ref, ops: [...group.ops, ...extra] } : group); + const counted = () => { + if (created) this.env.report.stats.uploaded.created++; + else this.env.report.stats.uploaded.updated++; + this.onAccepted(kind, acct, server); + }; + const failed = (reason: string, message: string) => { + this.markStale(acct, server.id); + this.env.report.itemError({ ref: refOf(acct, server.id), side: 'upload', type: reason, description: message }); + }; + return { + group: withExtra(plan.ops), + applied: counted, + replan: async () => ({ group: withExtra(plan.keepDirtyOps), applied: counted, failed }), + failed, + }; + } + + private poisonWork(p: Pending, error: SetErrorShape): Work { + const marker = nextMarker(p.meta.poison, p.fingerprint, error, this.env.now(), { + firstMs: this.env.tuning.poisonBackoffMs, + maxMs: this.env.tuning.poisonBackoffMaxMs, + }); + const ref = p.meta.sourceId ?? `row:${p.meta.rowId}`; + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: error.type, description: error.description, retryAt: marker.until }); + return { group: { ref, ops: [poisonOp(p.held.kind, p.meta.rowId, marker)] } }; + } + + /** Not allowed after all (read-only on the server): the rows are rewritten from the server's version. */ + private async revertWorks(acct: string, p: Pending, error: SetErrorShape): Promise { + const { kind, local } = p.held; + const ref = p.meta.sourceId ?? `row:${p.meta.rowId}`; + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: error.type, description: error.description }); + const id = idInAccount(p.meta.sourceId, acct); + const remove = kind.planLocalDelete(local); + const object = id ? (await this.env.jmap.get(this.itemType, acct, [id], this.itemProperties)).list[0] : undefined; + const objectKind = object ? this.kindOf(object) : null; + if (!object || !objectKind || !this.inSelection(acct, object)) { + return [{ group: remove, applied: () => (id ? this.onRemoved(kind, acct, id) : undefined) }]; + } + await this.beforePlanning(acct, [object]); + const insert = objectKind.planDownload(object, null, acct); + return [{ group: concatGroups(ref, remove, insert.ops), applied: () => this.onDownloaded(objectKind, acct, object) }]; + } + + /** + * `invalidProperties` on `uid`: the object exists (a lost response, or the + * client's automatic retry of the create). Cards name it in the error; + * events are looked up again with back-off (the uid index lags), else the + * next download adopts the row by uid. + */ + private async adoptAfterUidError(acct: string, p: Pending, failure: Extract): Promise { + const action = failure.action as Extract; + if (action.uid !== p.meta.pending?.uid) return []; + let id = existingIdFromUidError(failure.error); + if (!id) { + const scope = this.lookupScope(acct, action.collectionId); + for (const delay of [0, ...this.env.tuning.uidLookupDelaysMs]) { + if (delay) await this.env.sleep(delay); + const ids = (await this.env.jmap.lookupUids(this.itemType, acct, [action.uid], scope)).get(action.uid); + if (ids?.length) { + id = ids[0]; + break; + } + } + } + return id ? this.adoptWorks(acct, p, id) : []; + } + + /** Objects our uploads created for other rows come back through the download, which adopts them by uid. */ + private async adoptExtraCreated(acct: string): Promise { + const ids = [...(this.extraCreated.get(acct) ?? [])]; + this.extraCreated.delete(acct); + if (ids.length) await this.processIds(acct, ids, []); + } +} diff --git a/src/device-sync/engine/kinds.ts b/src/device-sync/engine/kinds.ts new file mode 100644 index 00000000..7ef6538f --- /dev/null +++ b/src/device-sync/engine/kinds.ts @@ -0,0 +1,95 @@ +/** + * The engine's view of one kind of local item (a contact, a group, an event + * master with its exceptions): what it needs to know about an item and the + * planner calls it makes for it. The contacts and calendar sync plug their + * planners in through this, so downloads, uploads, retries and the guards + * are written once (engine/item-sync.ts). + */ +import type { AcceptedPlan, DownloadPlan, OpGroup, PendingCreate, PoisonMarker, UploadPlan } from '../planner'; +import type { ProviderOp } from '../types'; + +export type KindName = 'contact' | 'group' | 'event'; + +/** A server object as `/get` returns it. */ +export interface ServerObject { + id: string; + uid?: string; + [property: string]: unknown; +} + +export interface ItemMeta { + rowId: number; + /** `/` of the server object; null until created (events: a pending marker is no identity). */ + sourceId: string | null; + pending: PendingCreate | null; + /** Local changes wait for an upload (events: the master or one of its exceptions). */ + dirty: boolean; + deleted: boolean; + /** Never reached the server: no identity (docs/device-sync.md, "Identity"). */ + isNew: boolean; + poison: PoisonMarker | null; + /** Collection keys the server object was in when last seen. */ + collections: string[]; +} + +/** A decoded local item and its kind. */ +export interface Held { + kind: Kind; + local: unknown; +} + +export interface Kind { + readonly name: KindName; + readonly table: 'raw_contacts' | 'groups' | 'events'; + /** The column holding the identity, for the "not inserted yet" guard of a download insert. */ + readonly identityColumn: 'sourceid' | '_sync_id'; + readonly poisonColumn: 'sync4' | 'sync_data5'; + meta(local: L): ItemMeta; + /** What an upload is computed from, for poison markers. */ + fingerprint(local: L): string; + /** Items by identity, one per ref. */ + loadByRefs(refs: readonly string[]): Promise>; + loadByRowIds(rowIds: readonly number[]): Promise; + /** Items without identity (new ones), for adoption by pending uid. */ + loadNew(): Promise; + planDownload(object: O, local: L | null, jmapAccountId: string): DownloadPlan; + planLocalDelete(local: L): OpGroup; + planBaselineHeal(local: L): OpGroup | null; + planUpload(local: L, jmapAccountId: string): UploadPlan; + planAccepted(local: L, server: O, jmapAccountId: string): AcceptedPlan; +} + +export function refOf(jmapAccountId: string, id: string): string { + return `${jmapAccountId}/${id}`; +} + +/** The id part of a `/` ref of `jmapAccountId`, else null. */ +export function idInAccount(ref: string | null, jmapAccountId: string): string | null { + if (!ref) return null; + const prefix = `${jmapAccountId}/`; + return ref.startsWith(prefix) && !ref.includes('#') ? ref.slice(prefix.length) : null; +} + +export function accountOfRef(ref: string | null): string | null { + if (!ref) return null; + const slash = ref.indexOf('/'); + return slash > 0 ? ref.slice(0, slash) : null; +} + +/** + * A download insert first asserts that no row has the identity yet: when a + * failed batch was committed up to a yield point, the retry must not insert + * the item twice. + */ +export function insertGuard(kind: Kind, ref: string): ProviderOp { + return { op: 'assert', table: kind.table, where: `${kind.identityColumn} = ?`, args: [ref], expectCount: 0 }; +} + +export function poisonOp(kind: Kind, rowId: number, marker: PoisonMarker | null): ProviderOp { + return { + op: 'update', + table: kind.table, + id: rowId, + values: { [kind.poisonColumn]: marker ? JSON.stringify(marker) : null }, + }; +} diff --git a/src/device-sync/engine/mutex.ts b/src/device-sync/engine/mutex.ts new file mode 100644 index 00000000..405a19cd --- /dev/null +++ b/src/device-sync/engine/mutex.ts @@ -0,0 +1,70 @@ +/** + * One sync run or app-side operation (turning sync off, sign-out) at a time + * per registry account and authority. Module level on purpose: a headless + * run and the UI share one JS runtime (docs/device-sync.md, "Architecture"), + * so a module-level lock covers both. + */ +import type { Authority } from '../types'; + +const tails = new Map>(); +const stopRequests = new Map(); + +export function lockKey(registryId: string, authority: Authority): string { + return `${registryId}\u0000${authority}`; +} + +/** + * Waits for the lock; resolves with its release function, or null when + * `waitMs` passed first (the place in the queue is then handed on). + */ +export function acquireLock(key: string, waitMs = Infinity): Promise<(() => void) | null> { + const previous = tails.get(key) ?? Promise.resolve(); + let pass!: () => void; + const mine = new Promise((resolve) => { + pass = resolve; + }); + const tail = previous.then(() => mine); + tails.set(key, tail); + void tail.then(() => { + if (tails.get(key) === tail) tails.delete(key); + }); + return new Promise((resolve) => { + let settled = false; + const timer = Number.isFinite(waitMs) + ? setTimeout(() => { + if (settled) return; + settled = true; + resolve(null); + void previous.then(pass); + }, Math.max(0, waitMs)) + : null; + void previous.then(() => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + let released = false; + resolve(() => { + if (released) return; + released = true; + pass(); + }); + }); + }); +} + +export function isLocked(key: string): boolean { + return tails.has(key); +} + +/** Asks the run holding the lock to stop at its next checkpoint (it reports `cancelled`). */ +export function requestStop(key: string): void { + stopRequests.set(key, (stopRequests.get(key) ?? 0) + 1); +} + +export function clearStopRequest(key: string): void { + stopRequests.delete(key); +} + +export function stopRequested(key: string): boolean { + return (stopRequests.get(key) ?? 0) > 0; +} diff --git a/src/device-sync/engine/poison.ts b/src/device-sync/engine/poison.ts new file mode 100644 index 00000000..cc1ffecc --- /dev/null +++ b/src/device-sync/engine/poison.ts @@ -0,0 +1,34 @@ +/** + * Poison markers (docs/device-sync.md, "Uploads", SetErrors): an item the + * server refused (`invalidProperties`, `invalidPatch`, `tooLarge`, + * `overQuota`, …) is not retried until its back-off ends (1 h, doubling to + * a day) or it changes. The engine writes the marker (SYNC4 / SYNC_DATA5) + * and checks it before planning an upload; the fingerprint covers what the + * item would upload from: its mapped rows, its flags and its shadow. + */ +import { jsonHash } from '../common/json'; +import type { PoisonMarker } from '../planner'; + +export function poisonFingerprint(content: unknown): string { + return jsonHash(content); +} + +/** True while the marker holds: same fingerprint and the back-off not over. */ +export function isBackedOff(marker: PoisonMarker | null, fingerprint: string, now: number): boolean { + return !!marker && marker.fp === fingerprint && marker.until > now; +} + +/** The next marker after a refusal: attempts count up while the item stays the same. */ +export function nextMarker( + previous: PoisonMarker | null, + fingerprint: string, + error: { type: string; description?: string }, + now: number, + backoff: { firstMs: number; maxMs: number }, +): PoisonMarker { + const n = previous && previous.fp === fingerprint ? previous.n + 1 : 1; + const delay = Math.min(backoff.firstMs * 2 ** (n - 1), backoff.maxMs); + const marker: PoisonMarker = { fp: fingerprint, type: error.type, n, until: now + delay }; + if (error.description) marker.description = error.description.slice(0, 500); + return marker; +} diff --git a/src/device-sync/engine/provider.ts b/src/device-sync/engine/provider.ts new file mode 100644 index 00000000..9fc7370b --- /dev/null +++ b/src/device-sync/engine/provider.ts @@ -0,0 +1,77 @@ +/** + * Reads through the provider port as row objects. Every query is scoped to + * our account by the native side; the SQL stays within the subset the fake + * provider evaluates (src/device-sync/__tests__/fakes/fake-sql.ts). + */ +import type { ProviderOp, ProviderPort, ProviderTable, Row } from '../types'; +import { RunAbort, isPermissionFailure } from './errors'; + +/** Host parameters per `IN (…)`: well under SQLite's 999. */ +const IN_CHUNK = 400; + +export class ProviderReader { + constructor(readonly port: ProviderPort) {} + + async rows(table: ProviderTable, columns: readonly string[], where?: string, args?: Array): Promise { + let result; + try { + result = await this.port.query({ table, columns: [...columns], ...(where ? { where, args: args ?? [] } : {}) }); + } catch (error) { + if (isPermissionFailure(error)) throw new RunAbort('permission', (error as Error).message); + throw error; + } + const { columns: names, rows } = result; + return rows.map((cells) => { + const row: Row = {}; + names.forEach((name, i) => { + row[name] = cells[i] ?? null; + }); + return row; + }); + } + + /** Rows whose `column` is one of `values` (in chunks), optionally narrowed by `extra`. */ + async rowsIn( + table: ProviderTable, + columns: readonly string[], + column: string, + values: ReadonlyArray, + extra?: { where: string; args?: Array }, + ): Promise { + const out: Row[] = []; + const unique = [...new Set(values)]; + for (let i = 0; i < unique.length; i += IN_CHUNK) { + const part = unique.slice(i, i + IN_CHUNK); + const inClause = `${column} IN (${part.map(() => '?').join(', ')})`; + const where = extra ? `(${inClause}) AND (${extra.where})` : inClause; + out.push(...(await this.rows(table, columns, where, [...part, ...(extra?.args ?? [])]))); + } + return out; + } + + async readSyncState(): Promise { + try { + return await this.port.readSyncState(); + } catch (error) { + if (isPermissionFailure(error)) throw new RunAbort('permission', (error as Error).message); + throw error; + } + } +} + +export function num(value: unknown): number { + return typeof value === 'number' ? value : Number(value ?? 0); +} + +export function flag(value: unknown): boolean { + return value !== null && value !== undefined && value !== '' && Number(value) !== 0; +} + +export function str(value: unknown): string | null { + return typeof value === 'string' && value ? value : value === null || value === undefined || value === '' ? null : String(value); +} + +/** True when an op group writes something (asserts alone change nothing). */ +export function hasWrites(ops: readonly ProviderOp[]): boolean { + return ops.some((op) => op.op !== 'assert'); +} diff --git a/src/device-sync/engine/report.ts b/src/device-sync/engine/report.ts new file mode 100644 index 00000000..6f810607 --- /dev/null +++ b/src/device-sync/engine/report.ts @@ -0,0 +1,95 @@ +/** + * The run report (`RunReport`, docs/device-sync.md "Headless payload, report + * and SyncResult"). Counts are objects actually written or sent, so a run + * that repeated a page without writing anything is no progress to + * SyncManager. + */ +import type { Authority, ItemError, RunOutcome, RunReport, RunStats, RunStatus } from '../types'; + +export function emptyStats(): RunStats { + return { + downloaded: { created: 0, updated: 0, deleted: 0 }, + uploaded: { created: 0, updated: 0, deleted: 0 }, + entries: 0, + skipped: 0, + }; +} + +export class ReportBuilder { + readonly stats = emptyStats(); + conflicts = 0; + private readonly errors: ItemError[] = []; + private errorCount = 0; + private readonly notes = new Set(); + tooManyDeletions?: { count: number; threshold: number }; + + constructor( + readonly runId: string, + readonly authority: Authority, + readonly startedAt: number, + private readonly maxItemErrors = 50, + ) {} + + itemError(error: ItemError): void { + this.errorCount++; + // One entry per item and side: a retried item is reported once, with its latest error. + const existing = this.errors.findIndex((e) => e.ref === error.ref && e.side === error.side); + if (existing >= 0) { + this.errors[existing] = error; + this.errorCount--; + return; + } + if (this.errors.length < this.maxItemErrors) this.errors.push(error); + } + + /** A note for the report's message ("invitations not sent", …). */ + note(text: string): void { + this.notes.add(text); + } + + get itemErrorCount(): number { + return this.errorCount; + } + + build( + outcome: RunOutcome, + now: number, + extra: { message?: string; delayUntil?: number; moreRecordsToGet?: boolean; noProgress?: boolean } = {}, + ): RunReport { + const stats: RunStats = JSON.parse(JSON.stringify(this.stats)); + if (extra.noProgress) { + stats.downloaded = { created: 0, updated: 0, deleted: 0 }; + stats.uploaded = { created: 0, updated: 0, deleted: 0 }; + } + const message = [extra.message, ...this.notes].filter(Boolean).join('; '); + const report: RunReport = { + v: 1, + runId: this.runId, + authority: this.authority, + outcome, + startedAt: this.startedAt, + durationMs: Math.max(0, now - this.startedAt), + stats, + conflicts: this.conflicts, + itemErrors: this.errors.slice(), + }; + if (message) report.message = message; + if (outcome === 'tooManyDeletions' && this.tooManyDeletions) report.tooManyDeletions = this.tooManyDeletions; + if (extra.delayUntil !== undefined) report.delayUntil = extra.delayUntil; + if (extra.moreRecordsToGet) report.moreRecordsToGet = true; + return report; + } +} + +export function statusOf(report: RunReport, itemErrorCount = report.itemErrors.length): RunStatus { + const status: RunStatus = { + at: report.startedAt + report.durationMs, + outcome: report.outcome, + durationMs: report.durationMs, + conflicts: report.conflicts, + itemErrors: itemErrorCount, + stats: report.stats, + }; + if (report.message) status.message = report.message; + return status; +} diff --git a/src/device-sync/engine/run.ts b/src/device-sync/engine/run.ts new file mode 100644 index 00000000..d037c332 --- /dev/null +++ b/src/device-sync/engine/run.ts @@ -0,0 +1,232 @@ +/** + * One sync run of one Android account and authority (docs/device-sync.md, + * "A sync run"): preflight, then the authority's phases, then the report. + * Never throws: every failure becomes an outcome. + */ +import { makeKeyMinter, uuidFrom } from '../common/ids'; +import { JmapCaller } from '../jmap/caller'; +import type { JmapConnection } from '../jmap/connection'; +import { accountsWithCapability } from '../jmap/session'; +import { + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + JMAP_CALENDARS, + JMAP_CONTACTS, + type Authority, + type RunExtras, + type RunPayload, + type RunReport, +} from '../types'; +import { BatchWriter } from './batch'; +import { CalendarSync } from './calendar-sync'; +import { Checkpoints } from './checkpoint'; +import { ContactsSync } from './contacts-sync'; +import type { RunEnv } from './context'; +import { DEFAULT_TUNING, type EngineDeps, type Tuning } from './deps'; +import { classifyFailure, RunAbort } from './errors'; +import { acquireLock, lockKey, stopRequested } from './mutex'; +import { ProviderReader, str } from './provider'; +import { ReportBuilder, statusOf } from './report'; +import { emptySyncState, parseSyncState, StateStore, type SyncState } from './sync-state'; +import type { ItemSync } from './item-sync'; + +export function tuningOf(deps: EngineDeps): Tuning { + return { ...DEFAULT_TUNING, ...deps.tuning }; +} + +const defaultYield = () => new Promise((resolve) => setTimeout(resolve, 0)); +const defaultSleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function capabilityOf(authority: Authority): string { + return authority === CONTACTS_AUTHORITY ? JMAP_CONTACTS : JMAP_CALENDARS; +} + +function itemTypeOf(authority: Authority): string { + return authority === CONTACTS_AUTHORITY ? 'ContactCard' : 'CalendarEvent'; +} + +/** Rows waiting for an upload (dirty, deleted, new), or state work a run must do whatever its extras. */ +export async function hasLocalWork(reader: ProviderReader, authority: Authority, state: SyncState): Promise { + for (const account of Object.values(state.accounts)) { + if (account.reconcile || account.stale.length) return true; + } + if (authority === CALENDAR_AUTHORITY) { + const rows = await reader.rows('events', ['_id'], "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'"); + return rows.length > 0; + } + const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + if (contacts.length) return true; + const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']); + const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean)); + if (groups.some((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1)) return true; + // A group an app hard-deleted is only found by its absence. + return Object.values(state.accounts).some((a) => (a.groups ?? []).some((ref) => !present.has(ref))); +} + +export interface EnvOptions { + deps: EngineDeps; + tuning: Tuning; + mode: 'sync' | 'teardown'; + registryId: string; + accountName: string; + authority: Authority; + extras: RunExtras; + deadline: number; + isCancelled(): Promise; + reader: ProviderReader; + report: ReportBuilder; + stored: SyncState; + connection: JmapConnection; +} + +/** + * The run's environment once the preflight passed: the JMAP connection is + * open and its accounts known. Throws RunAbort('unsupported') when no + * account offers the authority's capability. + */ +export async function createEnv(options: EnvOptions): Promise<{ env: RunEnv; sync: ItemSync }> { + const { deps, tuning, registryId, authority, connection } = options; + const jmap = new JmapCaller(connection.port, () => deps.now()); + const accounts = accountsWithCapability(jmap.session(), capabilityOf(authority)); + if (!accounts.length) { + throw new RunAbort('unsupported', `The server offers no ${authority === CONTACTS_AUTHORITY ? 'contacts' : 'calendars'} for this account`); + } + const prefs = await deps.prefs(registryId); + const recordKnownState = (jmapAccountId: string, type: string, state: string) => { + try { + void Promise.resolve(deps.recordKnownState(registryId, jmapAccountId, type, state)).catch(() => undefined); + } catch { + // Only an echo hint for the app. + } + }; + const store = new StateStore(options.stored, { registryId, origin: connection.origin }, (before, after) => { + for (const [acct, account] of Object.entries(after.accounts)) { + const state = account.itemsState; + if (state && state !== before.accounts[acct]?.itemsState) recordKnownState(acct, itemTypeOf(authority), state); + } + }); + const checkpoints = new Checkpoints({ + now: () => deps.now(), + deadline: options.deadline, + marginMs: tuning.deadlineMarginMs, + echoAfterMs: tuning.echoAfterMs, + isCancelled: options.isCancelled, + yieldThread: deps.yieldThread ?? defaultYield, + }); + checkpoints.attach(jmap); + const mintKey = makeKeyMinter(deps.random); + const env: RunEnv = { + deps, + tuning, + mode: options.mode, + registryId, + accountName: options.accountName, + authority, + extras: options.extras, + reader: options.reader, + writer: new BatchWriter(options.reader.port, { + maxOps: tuning.maxBatchOps, + maxBytes: tuning.maxBatchBytes, + maxReplans: tuning.maxReplans, + }), + report: options.report, + checkpoints, + store, + jmap, + accounts, + prefs, + now: () => deps.now(), + mintKey: (taken) => mintKey(taken), + mintUid: () => uuidFrom(deps.random), + sleep: deps.sleep ?? defaultSleep, + log: (message, detail) => deps.log?.(message, detail), + recordKnownState, + }; + const sync = authority === CONTACTS_AUTHORITY ? new ContactsSync(env) : new CalendarSync(env); + return { env, sync }; +} + +/** + * Runs one sync and returns its report. Holds the (registry account, + * authority) lock for the whole run; a run that cannot get it before its + * deadline reports `cancelled` and asks for another sync. + */ +export async function runDeviceSync(payload: RunPayload, deps: EngineDeps): Promise { + const tuning = tuningOf(deps); + const report = new ReportBuilder(payload.runId, payload.authority, deps.now(), tuning.maxItemErrors); + const key = lockKey(payload.registryId, payload.authority); + const release = await acquireLock(key, Math.max(0, payload.deadline - deps.now() - tuning.deadlineMarginMs)); + let result: RunReport; + if (!release) { + result = report.build('cancelled', deps.now(), { message: 'Another sync of this account is still running', moreRecordsToGet: true }); + } else { + try { + result = await runLocked(payload, deps, tuning, report, key); + } finally { + release(); + } + } + try { + await deps.recordStatus(payload.registryId, payload.authority, statusOf(result, report.itemErrorCount)); + } catch (error) { + deps.log?.('recording the run status failed', error); + } + return result; +} + +async function runLocked(payload: RunPayload, deps: EngineDeps, tuning: Tuning, report: ReportBuilder, key: string): Promise { + const finish = (outcome: RunReport['outcome'], extra: Parameters[2] = {}) => + report.build(outcome, deps.now(), extra); + const extras = payload.extras ?? {}; + try { + if (!(await deps.isSyncEnabled(payload.accountName, payload.authority))) { + return finish('disabled', { message: 'Sync is off for this account' }); + } + const reader = new ProviderReader(deps.provider(payload.accountName, payload.authority)); + const { state: stored, readable } = parseSyncState(await reader.readSyncState()); + if (!readable) deps.log?.('unreadable SyncState: starting over with a full reconcile'); + if (stored.owner && stored.owner.registryId !== payload.registryId) { + return finish('internal', { message: 'The rows on this device were written for another account' }); + } + // Providers schedule an upload sync for every account 30 s after any app write. + if (extras.upload && !(await hasLocalWork(reader, payload.authority, stored))) return finish('ok'); + + const connection = await deps.jmap(payload.registryId); + if (stored.owner && stored.owner.origin !== connection.origin) { + return finish('internal', { message: 'The rows on this device were written for another server' }); + } + const { sync } = await createEnv({ + deps, + tuning, + mode: 'sync', + registryId: payload.registryId, + accountName: payload.accountName, + authority: payload.authority, + extras, + deadline: payload.deadline, + isCancelled: async () => stopRequested(key) || (await deps.isCancelled(payload.runId)), + reader, + report, + stored: readable ? stored : emptySyncState(), + connection, + }); + await sync.sync(); + return finish(report.tooManyDeletions ? 'tooManyDeletions' : 'ok'); + } catch (error) { + const failure = classifyFailure(error, deps.now()); + if (failure.authProblem) { + try { + await deps.notifyAuthProblem(payload.registryId, payload.accountName, payload.authority); + } catch (notifyError) { + deps.log?.('auth notification failed', notifyError); + } + } + if (failure.outcome === 'internal') deps.log?.('device sync failed', error); + return finish(failure.outcome, { + message: failure.message, + delayUntil: failure.delayUntil, + moreRecordsToGet: failure.moreRecordsToGet, + noProgress: failure.noProgress, + }); + } +} diff --git a/src/device-sync/engine/selection.ts b/src/device-sync/engine/selection.ts new file mode 100644 index 00000000..0072086c --- /dev/null +++ b/src/device-sync/engine/selection.ts @@ -0,0 +1,36 @@ +/** + * Which collections sync. The store keeps explicit choices only; a + * collection the user never touched follows the default: the personal + * account's collections on, shared accounts' off (docs/device-sync.md, + * "App integration"), except the webmail's allow-list address book. + */ +import { collectionKey } from '../common/ids'; +import { CONTACTS_AUTHORITY, type Authority } from '../types'; + +/** + * The address book the webmail keeps its allow-listed senders in. Mirrors + * TRUSTED_SENDERS_BOOK_NAME in src/stores/contacts-store.ts, which the pure + * engine cannot import. + */ +export const TRUSTED_SENDERS_BOOK_NAME = 'Trusted Senders'; + +/** + * Whether a collection the user never chose syncs: those of the personal + * account do, shared ones don't; nor does the "Trusted Senders" address + * book, a mail filter rather than people to call. + */ +export function collectionDefaultOn(personal: boolean, authority: Authority, name: string | null | undefined): boolean { + if (!personal) return false; + return !(authority === CONTACTS_AUTHORITY && name === TRUSTED_SENDERS_BOOK_NAME); +} + +/** An explicit choice wins; otherwise the default. */ +export function isCollectionSelected( + explicit: Readonly>, + account: { id: string; personal: boolean }, + authority: Authority, + collection: { id: string; name?: string | null }, +): boolean { + const choice = explicit[collectionKey(account.id, collection.id)]; + return typeof choice === 'boolean' ? choice : collectionDefaultOn(account.personal, authority, collection.name); +} diff --git a/src/device-sync/engine/sync-state.ts b/src/device-sync/engine/sync-state.ts new file mode 100644 index 00000000..f9df3106 --- /dev/null +++ b/src/device-sync/engine/sync-state.ts @@ -0,0 +1,188 @@ +/** + * The SyncState blob (docs/device-sync.md, "SyncState"): one per Android + * account and authority, versioned JSON the provider stores next to the + * rows. It is only ever written as the last op of a batch whose rows it + * describes, so a crash never leaves it ahead of them. + */ +import { clone } from '../common/json'; +import type { ProviderOp, ReminderOwner } from '../types'; + +export const SYNC_STATE_VERSION = 1; + +/** A full reconcile in progress. */ +export interface ReconcileMarker { + /** The item state taken before listing anything; null when a reconcile was asked for and has not started. */ + from: string | null; + phase: 'ids' | 'objects'; + /** Objects processed so far, in the reconcile's order. */ + position: number; + /** The last id processed in that (sorted) order: a resumed reconcile continues after it. */ + after: string | null; +} + +export interface AccountState { + /** AddressBook / Calendar state of the last collections read. */ + collectionsState: string | null; + /** ContactCard / CalendarEvent state the rows describe. */ + itemsState: string | null; + /** Collection keys the rows were written for. */ + selected: string[]; + /** Object ids whose rows could not be written; fetched again at the start of every run, nothing uploads for them meanwhile. */ + stale: string[]; + reconcile: ReconcileMarker | null; + /** Contacts: refs of the group cards present on the device (a hard-deleted group is found by its absence). */ + groups?: string[]; + /** Calendar: ids of calendars found to hold only tasks. */ + taskOnly?: string[]; +} + +export interface SyncState { + v: typeof SYNC_STATE_VERSION; + /** The registry account and server origin the rows belong to. */ + owner: { registryId: string; origin: string } | null; + accounts: Record; + /** Calendar: the zone floating events were written in. */ + deviceZone?: string | null; + /** Calendar: the zone a zone-change pass is moving floating events to. */ + deviceZonePending?: string | null; + /** Calendar: who the Reminders rows were written for (a change rewrites them). */ + reminderOwner?: ReminderOwner | null; +} + +export function emptySyncState(): SyncState { + return { v: SYNC_STATE_VERSION, owner: null, accounts: {} }; +} + +export function emptyAccountState(): AccountState { + return { collectionsState: null, itemsState: null, selected: [], stale: [], reconcile: null }; +} + +const strings = (value: unknown): string[] => + Array.isArray(value) ? value.filter((v): v is string => typeof v === 'string') : []; +const stringOrNull = (value: unknown): string | null => (typeof value === 'string' && value ? value : null); + +function parseReconcile(value: unknown): ReconcileMarker | null { + if (!value || typeof value !== 'object') return null; + const r = value as Record; + return { + from: stringOrNull(r.from), + phase: r.phase === 'objects' ? 'objects' : 'ids', + position: typeof r.position === 'number' && r.position >= 0 ? r.position : 0, + after: stringOrNull(r.after), + }; +} + +function parseAccount(value: unknown): AccountState { + const a = (value && typeof value === 'object' ? value : {}) as Record; + const out: AccountState = { + collectionsState: stringOrNull(a.collectionsState), + itemsState: stringOrNull(a.itemsState), + selected: strings(a.selected), + stale: strings(a.stale), + reconcile: parseReconcile(a.reconcile), + }; + if (a.groups !== undefined) out.groups = strings(a.groups); + if (a.taskOnly !== undefined) out.taskOnly = strings(a.taskOnly); + return out; +} + +/** + * The stored blob. `readable` is false when there was one that could not be + * understood (another version, garbage): the rows are then treated as ours + * and every account starts over with a full reconcile. + */ +export function parseSyncState(text: string | null): { state: SyncState; readable: boolean } { + if (!text) return { state: emptySyncState(), readable: true }; + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return { state: emptySyncState(), readable: false }; + } + const r = (raw && typeof raw === 'object' ? raw : null) as Record | null; + if (!r || r.v !== SYNC_STATE_VERSION) return { state: emptySyncState(), readable: false }; + const owner = r.owner as Record | null | undefined; + const state: SyncState = { + v: SYNC_STATE_VERSION, + owner: + owner && typeof owner.registryId === 'string' && typeof owner.origin === 'string' + ? { registryId: owner.registryId, origin: owner.origin } + : null, + accounts: {}, + }; + if (r.accounts && typeof r.accounts === 'object') { + for (const [id, account] of Object.entries(r.accounts as Record)) { + state.accounts[id] = parseAccount(account); + } + } + if ('deviceZone' in r) state.deviceZone = stringOrNull(r.deviceZone); + if ('deviceZonePending' in r) state.deviceZonePending = stringOrNull(r.deviceZonePending); + if (r.reminderOwner === 'device' || r.reminderOwner === 'bulwark') state.reminderOwner = r.reminderOwner; + return { state, readable: true }; +} + +export function serializeSyncState(state: SyncState): string { + return JSON.stringify(state); +} + +/** The op that ends a batch and moves the stored state along with its rows. */ +export interface Tail { + op(): ProviderOp; + /** Called once the batch carrying `op()` applied. */ + applied(): void; +} + +/** + * The committed SyncState of a run. Changes are proposed as a tail: the + * batch writer calls `op()` right before sending the last batch (so the + * change sees what earlier batches decided, e.g. items put on `stale`), and + * the state is only adopted after that batch applied. + */ +export class StateStore { + private state: SyncState; + + constructor( + initial: SyncState, + private readonly owner: { registryId: string; origin: string } | null, + private readonly onCommit?: (before: SyncState, after: SyncState) => void, + ) { + this.state = clone(initial); + } + + get committed(): Readonly { + return this.state; + } + + account(jmapAccountId: string): Readonly { + return this.state.accounts[jmapAccountId] ?? emptyAccountState(); + } + + /** A tail applying `mutate` to a copy of the committed state (plus the owner). */ + tail(mutate: (next: SyncState) => void): Tail { + let next: SyncState | null = null; + return { + op: () => { + next = clone(this.state); + if (this.owner) next.owner = { ...this.owner }; + mutate(next); + return { op: 'syncState', value: serializeSyncState(next) }; + }, + applied: () => { + if (!next) return; + const before = this.state; + this.state = next; + this.onCommit?.(before, next); + }, + }; + } +} + +/** The account's entry in a state being built, created when missing. */ +export function accountOf(state: SyncState, jmapAccountId: string): AccountState { + let account = state.accounts[jmapAccountId]; + if (!account) { + account = emptyAccountState(); + state.accounts[jmapAccountId] = account; + } + return account; +} diff --git a/src/device-sync/engine/teardown.ts b/src/device-sync/engine/teardown.ts new file mode 100644 index 00000000..507280a9 --- /dev/null +++ b/src/device-sync/engine/teardown.ts @@ -0,0 +1,127 @@ +/** + * Turning device sync off for an authority, or signing out + * (docs/device-sync.md, "App integration", Lifecycle): under the run lock, + * the device's dirty, deleted and new items are uploaded first (bounded in + * time, failures ignored), then every row of the authority and its SyncState + * go. When changes are still waiting, nothing is deleted unless the user + * confirmed losing them (`force`). + */ +import { CALENDAR_AUTHORITY, type Authority, type ProviderOp } from '../types'; +import { BatchWriter } from './batch'; +import type { EngineDeps } from './deps'; +import { acquireLock, clearStopRequest, lockKey, requestStop } from './mutex'; +import { ProviderReader, str } from './provider'; +import { ReportBuilder } from './report'; +import { createEnv, tuningOf } from './run'; +import { parseSyncState } from './sync-state'; + +export interface TeardownOptions { + /** Upload waiting changes first (default true). */ + uploadFirst?: boolean; + /** Time for the upload (default 30 s). */ + timeoutMs?: number; + /** Delete even when changes could not be uploaded (the user confirmed). */ + force?: boolean; +} + +/** Items still waiting for an upload: dirty, deleted, new, and groups deleted by absence. */ +export async function countPending(reader: ProviderReader, authority: Authority): Promise { + const { state } = parseSyncState(await reader.readSyncState()); + if (authority === CALENDAR_AUTHORITY) { + const rows = await reader.rows( + 'events', + ['_id', 'original_id', 'original_sync_id'], + "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'", + ); + const masters = new Set(rows.map((r) => str(r.original_id) ?? str(r.original_sync_id) ?? `row:${String(r._id)}`)); + return masters.size; + } + const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']); + const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean)); + const waitingGroups = groups.filter((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1).length; + const absent = Object.values(state.accounts).reduce((n, a) => n + (a.groups ?? []).filter((ref) => !present.has(ref)).length, 0); + return contacts.length + waitingGroups + absent; +} + +function deleteEverything(authority: Authority): ProviderOp[] { + // Sync-adapter deletes: raw contacts take their data rows along, calendar rows their events. + return authority === CALENDAR_AUTHORITY + ? [{ op: 'delete', table: 'calendars' }] + : [ + { op: 'delete', table: 'raw_contacts' }, + { op: 'delete', table: 'groups' }, + { op: 'delete', table: 'settings' }, + ]; +} + +export async function teardownAuthority( + deps: EngineDeps, + registryId: string, + accountName: string, + authority: Authority, + options: TeardownOptions = {}, +): Promise<{ pending: number }> { + const tuning = tuningOf(deps); + const key = lockKey(registryId, authority); + // A sync holding the lock stops at its next checkpoint instead of running to its deadline. + requestStop(key); + const release = await acquireLock(key); + clearStopRequest(key); + try { + const reader = new ProviderReader(deps.provider(accountName, authority)); + if (options.uploadFirst !== false) { + const started = deps.now(); + try { + const { state: stored, readable } = parseSyncState(await reader.readSyncState()); + // Rows written for another registry account are never uploaded to this one. + if (!stored.owner || stored.owner.registryId === registryId) { + const connection = await deps.jmap(registryId); + if (!stored.owner || stored.owner.origin === connection.origin) { + const report = new ReportBuilder('teardown', authority, started, tuning.maxItemErrors); + const { sync } = await createEnv({ + deps, + tuning: { ...tuning, deadlineMarginMs: 0, createBudgetMs: 0 }, + mode: 'teardown', + registryId, + accountName, + authority, + extras: {}, + deadline: started + (options.timeoutMs ?? 30_000), + isCancelled: async () => false, + reader, + report, + stored: readable ? stored : parseSyncState(null).state, + connection, + }); + await sync.uploadOnly(); + } + } + } catch (error) { + // Offline, signed out, out of time: whatever did not go up is counted below. + deps.log?.('upload before teardown stopped', error); + } + } + const pending = await countPending(reader, authority); + if (pending > 0 && !options.force) return { pending }; + const writer = new BatchWriter(reader.port, { + maxOps: tuning.maxBatchOps, + maxBytes: tuning.maxBatchBytes, + maxReplans: 0, + }); + await writer.write( + [ + { + group: { ref: 'teardown', ops: deleteEverything(authority) }, + failed: (reason, message) => { + throw new Error(`Could not remove the synced rows: ${reason} ${message}`); + }, + }, + ], + { op: () => ({ op: 'syncState', value: '' }), applied: () => undefined }, + ); + return { pending: 0 }; + } finally { + release?.(); + } +} diff --git a/src/device-sync/jmap/caller.ts b/src/device-sync/jmap/caller.ts new file mode 100644 index 00000000..52f34e08 --- /dev/null +++ b/src/device-sync/jmap/caller.ts @@ -0,0 +1,333 @@ +/** + * The engine's JMAP calls on one registry account's `JmapPort`: one request + * in flight at a time, the time of the last request (for the keep-alive + * echo), and the method helpers device sync needs, each sized by the + * session's limits (docs/device-sync.md, "A sync run" and "Limits"). + * + * Pure: the port is a detached JMAPClient on the device and the fake server + * in tests. + */ +import { requireMethodResult } from '../../api/jmap-result'; +import type { JMAPResponseBody } from '../../api/types'; +import { + JMAP_CALENDARS, + JMAP_CONTACTS, + JMAP_CORE, + type JmapCoreLimits, + type JmapInvocation, + type JmapPort, + type JmapResponse, + type JmapSessionView, +} from '../types'; +import type { PatchObject } from '../common/patch'; + +export type ItemType = 'ContactCard' | 'CalendarEvent'; +export type ContainerType = 'AddressBook' | 'Calendar'; +type DataType = ItemType | ContainerType; + +/** `/changes` pages: Stalwart caps at 5000, a small page keeps every page a quick checkpoint. */ +export const MAX_CHANGES = 256; +/** `/query` pages (Stalwart's `query_max_results`). */ +export const QUERY_PAGE = 5000; +/** Ids re-read at the start of the next `/query` page, so a deletion that shifts the list skips nothing. */ +const QUERY_OVERLAP = 100; + +export function capabilityOf(type: DataType): string { + return type === 'ContactCard' || type === 'AddressBook' ? JMAP_CONTACTS : JMAP_CALENDARS; +} + +function using(type: DataType): string[] { + return [JMAP_CORE, capabilityOf(type)]; +} + +export interface ChangesPage { + oldState: string; + newState: string; + hasMoreChanges: boolean; + created: string[]; + updated: string[]; + destroyed: string[]; +} + +export interface SetErrorShape { + type: string; + description?: string; + properties?: string[]; +} + +export interface SetResponse { + oldState?: string; + newState: string; + created?: Record>; + updated?: Record; + destroyed?: string[]; + notCreated?: Record; + notUpdated?: Record; + notDestroyed?: Record; +} + +export interface SetArgs { + create?: Record>; + update?: Record; + destroy?: string[]; + sendSchedulingMessages?: boolean; +} + +export class JmapCaller { + private chain: Promise = Promise.resolve(); + private lastRequestAt: number; + /** Requests sent, for tests and logs. */ + requestCount = 0; + + constructor( + readonly port: JmapPort, + private readonly now: () => number, + ) { + this.lastRequestAt = now(); + } + + session(): JmapSessionView { + return this.port.session(); + } + + limits(): JmapCoreLimits { + return this.port.limits(); + } + + /** Milliseconds since the last request started or ended. */ + idleMs(): number { + return this.now() - this.lastRequestAt; + } + + /** + * One request at a time: the server's `maxConcurrentRequests` is a per-user + * budget the UI shares, and a sync run has no reason to spend more than one. + */ + request(calls: JmapInvocation[], usingCaps: string[]): Promise { + const run = async () => { + this.lastRequestAt = this.now(); + this.requestCount++; + try { + return await this.port.request(calls, usingCaps); + } finally { + this.lastRequestAt = this.now(); + } + }; + const result = this.chain.then(run, run); + this.chain = result.catch(() => undefined); + return result; + } + + /** One method call; a method-level error throws `JMAPMethodError` with its `type`. */ + async call(method: string, args: Record, usingCaps: string[]): Promise { + const response = await this.request([[method, args, '0']], usingCaps); + return requireMethodResult(response as unknown as JMAPResponseBody, '0', method); + } + + /** Network traffic for SyncManager's "no traffic for 60 s" monitor during long local phases. */ + async echo(): Promise { + await this.request([['Core/echo', { ping: this.now() }, '0']], [JMAP_CORE]); + } + + /** Bytes of a blob (a blob-backed photo), in turn with the requests. */ + downloadBlob(accountId: string, blobId: string, type?: string): Promise { + const run = async () => { + this.lastRequestAt = this.now(); + try { + return await this.port.downloadBlob(accountId, blobId, type); + } finally { + this.lastRequestAt = this.now(); + } + }; + const result = this.chain.then(run, run); + this.chain = result.catch(() => undefined); + return result; + } + + // ── Reads ── + + /** + * Objects by id with explicit properties, in calls of at most + * `maxObjectsInGet` ids (never `ids: null`, which Stalwart silently caps). + */ + async get( + type: DataType, + accountId: string, + ids: readonly string[], + properties: readonly string[], + ): Promise<{ list: T[]; notFound: string[]; state: string | null }> { + const list: T[] = []; + const notFound: string[] = []; + let state: string | null = null; + const size = Math.max(1, this.limits().maxObjectsInGet); + for (let i = 0; i < ids.length; i += size) { + const body = await this.call<{ list?: T[]; notFound?: string[]; state?: string }>( + `${type}/get`, + { accountId, ids: ids.slice(i, i + size), properties: [...properties] }, + using(type), + ); + list.push(...(body.list ?? [])); + notFound.push(...(body.notFound ?? [])); + state = body.state ?? state; + } + return { list, notFound, state }; + } + + /** Every container of the account (there are few; Stalwart allows 250 per account). */ + async getContainers(type: ContainerType, accountId: string, properties: readonly string[]): Promise<{ list: T[]; state: string | null }> { + const body = await this.call<{ list?: T[]; state?: string }>( + `${type}/get`, + { accountId, ids: null, properties: [...properties] }, + using(type), + ); + return { list: body.list ?? [], state: body.state ?? null }; + } + + /** The current state of a type, taken before a full reconcile lists anything. */ + async takeState(type: ItemType, accountId: string): Promise { + const body = await this.call<{ state?: string }>(`${type}/get`, { accountId, ids: [], properties: ['id'] }, using(type)); + return body.state ?? ''; + } + + async changes(type: ItemType, accountId: string, sinceState: string, maxChanges = MAX_CHANGES): Promise { + const body = await this.call>( + `${type}/changes`, + { accountId, sinceState, maxChanges }, + using(type), + ); + return { + oldState: body.oldState ?? sinceState, + newState: body.newState ?? sinceState, + hasMoreChanges: Boolean(body.hasMoreChanges), + created: body.created ?? [], + updated: body.updated ?? [], + destroyed: body.destroyed ?? [], + }; + } + + /** + * Every id matching `filter`, paged by `position` in pages of at most 5000. + * Pages overlap a little: a deletion between two pages shifts the list and + * would otherwise skip an object. + */ + async queryAll(type: ItemType, accountId: string, filter: Record, pageSize = QUERY_PAGE): Promise { + const seen = new Set(); + let position = 0; + for (let round = 0; round < 1000; round++) { + const body = await this.call<{ ids?: string[]; total?: number; position?: number }>( + `${type}/query`, + { accountId, filter, position, limit: pageSize, calculateTotal: true }, + using(type), + ); + const ids = body.ids ?? []; + for (const id of ids) seen.add(id); + const total = typeof body.total === 'number' ? body.total : null; + const end = position + ids.length; + if (ids.length === 0) break; + // A server may cap the page below `pageSize`; the total says whether more follow. + if (total !== null ? end >= total : ids.length < pageSize) break; + position = end - Math.min(QUERY_OVERLAP, Math.floor(ids.length / 4)); + } + return [...seen]; + } + + /** The first `limit` ids matching `filter` (task-only calendar detection). */ + async queryFirst(type: ItemType, accountId: string, filter: Record, limit: number): Promise { + const body = await this.call<{ ids?: string[] }>(`${type}/query`, { accountId, filter, limit }, using(type)); + return body.ids ?? []; + } + + /** + * Ids of the objects carrying each uid, in `collectionId` when given + * (contact uids are unique per address book, event uids per account). The + * uid filter reads Stalwart's asynchronous search index, so a just-created + * object can be missing. + */ + async lookupUids( + type: ItemType, + accountId: string, + uids: readonly string[], + collectionId: string | null, + ): Promise> { + const found = new Map(); + if (uids.length === 0) return found; + const byUid: Record = + uids.length === 1 ? { uid: uids[0] } : { operator: 'OR', conditions: uids.map((uid) => ({ uid })) }; + const parentFilter = type === 'ContactCard' ? 'inAddressBook' : 'inCalendar'; + const filter = collectionId === null + ? byUid + : { operator: 'AND', conditions: [{ [parentFilter]: collectionId }, byUid] }; + const ids = await this.queryAll(type, accountId, filter); + if (ids.length === 0) return found; + const { list } = await this.get<{ id: string; uid?: string }>(type, accountId, ids, ['id', 'uid']); + for (const object of list) { + if (typeof object.uid !== 'string' || !uids.includes(object.uid)) continue; + found.set(object.uid, [...(found.get(object.uid) ?? []), object.id]); + } + return found; + } + + // ── Writes ── + + /** + * One `/set` call, guarded by `ifInState` when there is one; with + * `thenGet`, a `/get` of those ids in the same request (updates only: + * Stalwart resolves no result reference to a `/set`, and a create's id is + * only known from the response). A method-level error (`stateMismatch`, …) + * throws `JMAPMethodError`. + */ + async set( + type: ItemType, + accountId: string, + args: SetArgs, + ifInState: string | null, + thenGet?: { ids: string[]; properties: readonly string[] }, + ): Promise<{ response: SetResponse; got: T[] | null }> { + const setArgs: Record = { accountId }; + if (ifInState) setArgs.ifInState = ifInState; + if (args.create && Object.keys(args.create).length) setArgs.create = args.create; + if (args.update && Object.keys(args.update).length) setArgs.update = args.update; + if (args.destroy?.length) setArgs.destroy = args.destroy; + if (args.sendSchedulingMessages) setArgs.sendSchedulingMessages = true; + const calls: JmapInvocation[] = [[`${type}/set`, setArgs, 's']]; + const withGet = !!thenGet && thenGet.ids.length > 0 && thenGet.ids.length <= this.limits().maxObjectsInGet; + if (withGet) { + calls.push([`${type}/get`, { accountId, ids: thenGet!.ids, properties: [...thenGet!.properties] }, 'g']); + } + const res = (await this.request(calls, using(type))) as unknown as JMAPResponseBody; + const response = requireMethodResult(res, 's', `${type}/set`); + let got: T[] | null = null; + if (withGet) { + got = requireMethodResult<{ list?: T[] }>(res, 'g', `${type}/get`).list ?? []; + } else if (thenGet && thenGet.ids.length > 0) { + got = (await this.get(type, accountId, thenGet.ids, thenGet.properties)).list; + } + return { response, got }; + } +} + +// ── SetErrors ── + +/** + * What a SetError means for the item (docs/device-sync.md, "Uploads"): + * - `notFound`: gone on the server (an update: a remote delete; a destroy: done); + * - `forbidden`: not allowed (retried without scheduling messages, else reverted); + * - `uidExists`: `invalidProperties` on `uid`: the object exists already (a + * lost response, or the client's automatic retry of a create); + * - `poison`: the object itself is refused; back off until it changes. + */ +export type SetErrorClass = 'notFound' | 'forbidden' | 'uidExists' | 'poison'; + +export function classifySetError(error: SetErrorShape | undefined, op: 'create' | 'update' | 'destroy'): SetErrorClass { + const type = error?.type; + if (type === 'notFound') return 'notFound'; + if (type === 'forbidden') return 'forbidden'; + if (op === 'create' && type === 'invalidProperties' && error?.properties?.includes('uid')) return 'uidExists'; + return 'poison'; +} + +/** The existing card Stalwart names in a duplicate-uid error ("… already exists with id c5."). */ +export function existingIdFromUidError(error: SetErrorShape | undefined): string | null { + const m = /already exists with id ([A-Za-z0-9_-]+)/.exec(error?.description ?? ''); + return m ? m[1] : null; +} diff --git a/src/device-sync/jmap/client-port.ts b/src/device-sync/jmap/client-port.ts new file mode 100644 index 00000000..6fbe93f6 --- /dev/null +++ b/src/device-sync/jmap/client-port.ts @@ -0,0 +1,87 @@ +/** + * The JMAP side of a device sync run on the device: a `JmapPort` over a + * detached `JMAPClient` of one registry account. Never the UI's `jmapClient` + * singleton: a run can share the JS runtime with the UI (the headless task + * starts with `allowedInForeground`), and re-binding the singleton would send + * the UI's requests with another account's credentials (the widgets and the + * push task avoid it for the same reason). + * + * This file is the React Native boundary of `src/device-sync/jmap`; the rest + * of the folder is pure and runs against the fake server in tests. + */ +import { JMAPClient } from '../../api/jmap-client'; +import { JMAP_CORE, type JmapCoreLimits, type JmapPort, type JmapResponse, type JmapSessionView } from '../types'; +import { openWithAuthRebuild, type JmapConnection } from './connection'; + +/** RFC 8620 fallbacks when the server advertises no core limit. */ +const FALLBACK_LIMITS: JmapCoreLimits = { + maxObjectsInGet: 500, + maxObjectsInSet: 500, + maxCallsInRequest: 16, + maxConcurrentRequests: 4, + maxSizeRequest: 10_000_000, +}; + +function positive(value: unknown, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : fallback; +} + +export function coreLimits(capabilities: Record | undefined): JmapCoreLimits { + const core = (capabilities?.[JMAP_CORE] ?? {}) as Partial>; + return { + maxObjectsInGet: positive(core.maxObjectsInGet, FALLBACK_LIMITS.maxObjectsInGet), + maxObjectsInSet: positive(core.maxObjectsInSet, FALLBACK_LIMITS.maxObjectsInSet), + maxCallsInRequest: positive(core.maxCallsInRequest, FALLBACK_LIMITS.maxCallsInRequest), + maxConcurrentRequests: positive(core.maxConcurrentRequests, FALLBACK_LIMITS.maxConcurrentRequests), + maxSizeRequest: positive(core.maxSizeRequest, FALLBACK_LIMITS.maxSizeRequest), + }; +} + +function originOf(url: string | null): string { + const m = /^(https?:\/\/[^/?#]+)/i.exec(url ?? ''); + return m ? m[1].toLowerCase() : (url ?? ''); +} + +function sessionView(client: JMAPClient): JmapSessionView { + const session = client.currentSession; + if (!session) throw new Error('JMAP session not loaded'); + const accounts: JmapSessionView['accounts'] = {}; + for (const [id, account] of Object.entries(session.accounts ?? {})) { + accounts[id] = { + name: account.name, + isPersonal: account.isPersonal, + isReadOnly: account.isReadOnly, + accountCapabilities: account.accountCapabilities ?? {}, + }; + } + return { + username: session.username || client.username || '', + accounts, + primaryAccounts: session.primaryAccounts ?? {}, + capabilities: session.capabilities ?? {}, + }; +} + +function clientPort(client: JMAPClient): JmapPort { + return { + session: () => sessionView(client), + limits: () => coreLimits(client.currentSession?.capabilities), + request: async (calls, using) => (await client.request(calls, using)) as unknown as JmapResponse, + downloadBlob: async (accountId, blobId, type) => + new Uint8Array(await client.fetchBlobArrayBuffer(blobId, undefined, type, accountId)), + }; +} + +/** + * A port for the registry account `registryId` (`AccountEntry.id`). No + * stored credentials or rejected ones are retried once with a fresh client, + * then fail with an AuthenticationError; an unreachable server throws a + * NetworkError and touches nothing. + */ +export function openJmapPort(registryId: string): Promise { + return openWithAuthRebuild(async () => { + const client = new JMAPClient(); + if (!(await client.loadAccount(registryId))) return null; + return { port: clientPort(client), origin: originOf(client.serverUrl) }; + }); +} diff --git a/src/device-sync/jmap/connection.ts b/src/device-sync/jmap/connection.ts new file mode 100644 index 00000000..a36fd255 --- /dev/null +++ b/src/device-sync/jmap/connection.ts @@ -0,0 +1,62 @@ +/** + * A registry account's JMAP connection for one sync run, with the one + * rebuild the design allows when the server rejects the credentials: the UI + * (or another run) may have rotated the OAuth refresh token this client + * still holds, and a fresh client reads the rotated one from secure storage + * (docs/device-sync.md, "A sync run", preflight). + */ +import type { JmapPort } from '../types'; +import { authError, transportFailure } from './errors'; + +export interface JmapConnection { + port: JmapPort; + /** Origin of the server (`https://host[:port]`), recorded as the SyncState owner. */ + origin: string; +} + +/** Loads a client; null when the account has no usable stored credentials. */ +export type ConnectionLoader = () => Promise; + +async function loadOnce(load: ConnectionLoader): Promise { + try { + return await load(); + } catch (error) { + if (transportFailure(error) === 'auth') return null; + throw error; + } +} + +/** + * Opens the connection: a missing session or an AuthenticationError is + * retried once with a fresh client; still failing, it throws an + * AuthenticationError. Network trouble propagates as it came. Later + * requests rebuild once more the same way when the server answers 401. + */ +export async function openWithAuthRebuild(load: ConnectionLoader): Promise { + const first = await loadOnce(load); + const opened = first ?? (await load()); + if (!opened) throw authError('No usable credentials for this account'); + + let current = opened.port; + let rebuilt = !first; + const retryOnAuth = async (send: (port: JmapPort) => Promise): Promise => { + try { + return await send(current); + } catch (error) { + // A 401 comes back before the server ran anything, so replaying is safe. + if (transportFailure(error) !== 'auth' || rebuilt) throw error; + rebuilt = true; + const next = await load(); + if (!next) throw error; + current = next.port; + return send(current); + } + }; + const port: JmapPort = { + session: () => current.session(), + limits: () => current.limits(), + request: (calls, using) => retryOnAuth((p) => p.request(calls, using)), + downloadBlob: (accountId, blobId, type) => retryOnAuth((p) => p.downloadBlob(accountId, blobId, type)), + }; + return { port, origin: opened.origin }; +} diff --git a/src/device-sync/jmap/errors.ts b/src/device-sync/jmap/errors.ts new file mode 100644 index 00000000..faaeff72 --- /dev/null +++ b/src/device-sync/jmap/errors.ts @@ -0,0 +1,62 @@ +/** + * Transport failures of the JMAP side, recognised by `error.name` as + * jmap-client.ts sets it (AuthenticationError, NetworkError, + * RequestTimeoutError, RateLimitError), so the engine never imports the + * client: it stays pure and runs against the fake server in tests. + */ +import { JMAPMethodError } from '../../api/jmap-result'; + +export { JMAPMethodError }; + +/** + * - `auth`: the server rejected the credentials (a 401 even after a token refresh); + * - `network` / `timeout`: no response; the server may have applied the request; + * - `rateLimit`: a 429, with `retryAfterMs`; + * - `other`: anything else (a non-OK HTTP status, a body that is not JSON, a bug). + */ +export type TransportFailure = 'auth' | 'network' | 'timeout' | 'rateLimit' | 'other'; + +export function transportFailure(error: unknown): TransportFailure { + switch ((error as { name?: unknown } | null)?.name) { + case 'AuthenticationError': + case 'TotpRequiredError': + return 'auth'; + case 'NetworkError': + return 'network'; + case 'RequestTimeoutError': + return 'timeout'; + case 'RateLimitError': + return 'rateLimit'; + default: + return 'other'; + } +} + +/** Milliseconds a RateLimitError asks us to wait, or null. */ +export function retryAfterMs(error: unknown): number | null { + const ms = (error as { retryAfterMs?: unknown } | null)?.retryAfterMs; + return typeof ms === 'number' && Number.isFinite(ms) && ms >= 0 ? ms : null; +} + +/** A method-level error (`['error', {type}]`) of the given type, e.g. `cannotCalculateChanges`. */ +export function isMethodError(error: unknown, type?: string): error is JMAPMethodError { + return error instanceof JMAPMethodError && (type === undefined || error.type === type); +} + +/** + * jmap-client.ts turns Stalwart's HTTP 400 request-level limit error into a + * plain `JMAP request failed: 400 - {…limit…}`. The engine sizes every + * request by the session's limits, so one of those is a bug, not an outage. + */ +export function isRequestLimitError(error: unknown): boolean { + const message = (error as { message?: unknown } | null)?.message; + return typeof message === 'string' + && /JMAP request failed: 400/.test(message) + && message.includes('urn:ietf:params:jmap:error:limit'); +} + +export function authError(message: string): Error { + const error = new Error(message); + error.name = 'AuthenticationError'; + return error; +} diff --git a/src/device-sync/jmap/session.ts b/src/device-sync/jmap/session.ts new file mode 100644 index 00000000..caf93550 --- /dev/null +++ b/src/device-sync/jmap/session.ts @@ -0,0 +1,87 @@ +/** + * What device sync reads from a JMAP session: the accounts that hold address + * books or calendars, and the user's own calendar addresses. + */ +import { JMAP_CORE, type JmapSessionView } from '../types'; +import type { JmapCaller } from './caller'; + +const JMAP_SUBMISSION = 'urn:ietf:params:jmap:submission'; + +export interface CapabilityAccount { + id: string; + name: string; + /** The capability's primary account (the user's own). */ + primary: boolean; + /** Collections of personal accounts are selected by default, shared ones are not. */ + personal: boolean; + readOnly: boolean; +} + +/** + * The primary account of the capability (`primaryAccounts[capability]`) + * first, then every other account that advertises it (shared accounts). + */ +export function accountsWithCapability(session: JmapSessionView, capability: string): CapabilityAccount[] { + const out: CapabilityAccount[] = []; + const primary = session.primaryAccounts?.[capability]; + const add = (id: string, isPrimary: boolean) => { + const account = session.accounts?.[id]; + if (!account || out.some((a) => a.id === id)) return; + out.push({ + id, + name: account.name || id, + primary: isPrimary, + personal: isPrimary || account.isPersonal, + readOnly: !!account.isReadOnly, + }); + }; + if (primary) add(primary, true); + for (const [id, account] of Object.entries(session.accounts ?? {})) { + if (id !== primary && account.accountCapabilities && capability in account.accountCapabilities) add(id, false); + } + return out; +} + +export interface CalendarAddresses { + /** `OWNER_ACCOUNT` of our calendars: never empty (Etar crashes on a null owner). */ + ownerAccount: string; + /** Every address of the user, lowercase, without `mailto:`. */ + selfAddresses: string[]; +} + +function normalizeAddress(value: unknown): string | null { + if (typeof value !== 'string') return null; + const address = value.trim().replace(/^mailto:/i, '').toLowerCase(); + return address.includes('@') ? address : null; +} + +/** + * The login when it is an email address, else the first identity's address; + * aliases from identities count as "me" too (docs/device-sync.md, + * "Decisions"). Identities are read only when the session offers + * submission; a server without `Identity/get` leaves the login. + */ +export async function calendarAddresses(caller: JmapCaller, fallbackName: string): Promise { + const session = caller.session(); + const login = normalizeAddress(session.username); + const identities: string[] = []; + const submissionAccount = session.primaryAccounts?.[JMAP_SUBMISSION]; + if (submissionAccount) { + try { + const body = await caller.call<{ list?: Array<{ email?: string }> }>( + 'Identity/get', + { accountId: submissionAccount, properties: ['email'] }, + [JMAP_CORE, JMAP_SUBMISSION], + ); + for (const identity of body.list ?? []) { + const address = normalizeAddress(identity.email); + if (address) identities.push(address); + } + } catch { + // Without identities the login is the only address we know. + } + } + const selfAddresses = [...new Set([...(login ? [login] : []), ...identities])]; + const ownerAccount = login ?? identities[0] ?? (session.username || fallbackName).toLowerCase(); + return { ownerAccount, selfAddresses: selfAddresses.length ? selfAddresses : [ownerAccount] }; +} diff --git a/src/device-sync/planner.ts b/src/device-sync/planner.ts index ebf16a05..5f429838 100644 --- a/src/device-sync/planner.ts +++ b/src/device-sync/planner.ts @@ -182,6 +182,13 @@ export interface ContactsContext extends MapContext { nameForUid(uid: string): string | null; /** Group rows by SOURCE_ID, for memberships. */ groupRowIdBySourceId(sourceId: string): number | null; + /** + * SOURCE_IDs of the synced group cards whose `members` list this uid: the + * server keeps memberships on the group cards, so a contact's + * GroupMembership rows need the reverse lookup. The engine always provides + * it; without it the planner leaves memberships as they are. + */ + groupsOf?(uid: string): string[]; } export interface ContactsPlanner { @@ -332,7 +339,11 @@ export interface CalendarPlanner { /** Deletes the master and its exception rows (a sync-adapter delete does not cascade). */ planLocalDelete(local: LocalEvent): OpGroup; planBaselineHeal(local: LocalEvent): OpGroup | null; - /** May return several actions: a "this and following" split is a rule patch plus a create. */ + /** + * May return several actions. For a "this and following" split, the + * source's plan is the rule patch plus its pruned overrides; the new series + * is a row of its own and goes through claim and create by itself. + */ planUpload(local: LocalEvent, ctx: CalendarContext): UploadPlan; planAccepted(local: LocalEvent, server: CalendarEventWire, ctx: CalendarContext): AcceptedPlan; /** diff --git a/src/device-sync/task.ts b/src/device-sync/task.ts index 95e473d7..54d18436 100644 --- a/src/device-sync/task.ts +++ b/src/device-sync/task.ts @@ -1,41 +1,190 @@ /** * Entry of the BulwarkDeviceSync headless task, which the contacts and - * calendar sync adapters start for every sync (index.ts registers it). See - * docs/device-sync.md. + * calendar sync adapters start for every sync (index.ts registers it), and + * the teardown the app calls when sync is turned off or an account signs + * out. See docs/device-sync.md. * - * The run must always end with `finishRun`, called before the task's promise + * This is the React Native boundary of the engine: it wires the native + * module, a detached JMAPClient per registry account (never the UI's + * singleton) and the device-sync store into the pure engine. + * + * A run always ends with `finishRun`, called before the task's promise * settles: JS timers stop once the task is over, and a sync adapter waiting - * for a report that never comes holds its sync until the deadline. - * - * The sync engine is not wired in yet, so a run only reports that device - * sync is off; nothing in the app can turn it on at this point. + * for a report that never comes holds its sync until the deadline. Nothing + * here throws: every failure becomes a report. */ -import { NativeModules } from 'react-native'; -import type { DeviceSyncNativeModule, RunPayload, RunReport } from './types'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { detectDeviceLocale, isSupportedLocale, translate, type LocaleCode } from '../i18n'; +import { randomBytes } from '../lib/random'; +import { accountSyncPrefs, useDeviceSyncStore, waitForDeviceSyncHydration } from '../stores/device-sync-store'; +import type { RandomSource } from './common/ids'; +import { + runDeviceSync, + teardownAuthority as teardownWithDeps, + type EngineDeps, + type SubscriptionCalendar, + type TeardownOptions, +} from './engine'; +import { emptyStats } from './engine/report'; +import { openJmapPort } from './jmap/client-port'; +import { createProviderPort, finishRun, getSyncSettings, isRunCancelled, showSyncProblem } from './native'; +import type { CalendarPlanner, ContactsPlanner } from './planner'; +import type { Authority, RunPayload, RunReport } from './types'; -export async function runDeviceSyncTask(payload: RunPayload): Promise { - const native = NativeModules.BulwarkDeviceSync as DeviceSyncNativeModule | undefined; - if (!native || !payload?.runId) return; - const report: RunReport = { +// Mirrors the persist names of the locale and feed-subscription stores; read +// straight from storage so a headless run loads neither store. +const LOCALE_STORAGE_KEY = 'webmail:locale:v1'; +const SUBSCRIPTIONS_STORAGE_KEY = 'calendar-subscriptions'; +/** Where "sign in again" leads (the accounts pane), as for a registry eviction. */ +const SIGN_IN_URI = 'bulwarkmobile://settings/account'; + +// The contacts and calendar planners land in the next +// commits. Until then a run fails before it touches a row. +function notWired(): never { + throw new Error('Device sync planners are not wired into this build yet'); +} +const contactsPlanner = new Proxy({}, { get: notWired }) as ContactsPlanner; +const calendarPlanner = new Proxy({}, { get: notWired }) as CalendarPlanner; + +/** Random numbers from the platform CSPRNG, buffered (uids and map keys of device-made items). */ +function secureRandom(): RandomSource { + let buffer: Uint8Array = new Uint8Array(0); + let offset = 0; + return () => { + if (offset + 4 > buffer.length) { + buffer = randomBytes(256); + offset = 0; + } + const n = ((buffer[offset] << 24) >>> 0) + (buffer[offset + 1] << 16) + (buffer[offset + 2] << 8) + buffer[offset + 3]; + offset += 4; + return n / 2 ** 32; + }; +} + +/** Feed-subscription calendars (read-only on the device), from the subscriptions store's storage. */ +async function subscriptionCalendars(): Promise { + try { + const raw = await AsyncStorage.getItem(SUBSCRIPTIONS_STORAGE_KEY); + const subscriptions = raw ? (JSON.parse(raw) as { state?: { subscriptions?: unknown } }).state?.subscriptions : null; + if (!Array.isArray(subscriptions)) return []; + return subscriptions + .filter((s): s is { calendarId: string; accountId?: unknown } => typeof s?.calendarId === 'string') + .map((s) => ({ calendarId: s.calendarId, jmapAccountId: typeof s.accountId === 'string' ? s.accountId : null })); + } catch { + return []; + } +} + +/** The app's language: the in-app override, else the device's. */ +async function appLocale(): Promise { + try { + const raw = await AsyncStorage.getItem(LOCALE_STORAGE_KEY); + const override = raw ? (JSON.parse(raw) as { override?: unknown }).override : null; + if (typeof override === 'string' && isSupportedLocale(override)) return override; + } catch { + // The device language it is. + } + return detectDeviceLocale(); +} + +function deviceZone(): string { + try { + return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC'; + } catch { + return 'UTC'; + } +} + +async function notifyAuthProblem(_registryId: string, accountName: string): Promise { + const locale = await appLocale(); + await showSyncProblem( + accountName, + translate(locale, 'device_sync.auth_problem_title', 'Sign in again to keep syncing'), + translate( + locale, + 'device_sync.auth_problem_text', + 'The server no longer accepts the sign-in of {account}, so its contacts and calendars on this device stopped syncing. Sign in again to resume. Changes made on this device are kept.', + { account: accountName }, + ), + SIGN_IN_URI, + translate(locale, 'device_sync.problem_channel', 'Sync problems'), + ); +} + +export function createTaskDeps(): EngineDeps { + return { + provider: createProviderPort, + jmap: openJmapPort, + prefs: async (registryId) => { + await waitForDeviceSyncHydration(); + return accountSyncPrefs(registryId); + }, + planners: { contacts: contactsPlanner, calendar: calendarPlanner }, + now: () => Date.now(), + isCancelled: isRunCancelled, + random: secureRandom(), + subscriptionCalendars, + isSyncEnabled: async (accountName, authority) => (await getSyncSettings(accountName)).authorities?.[authority]?.automatic === true, + deviceZone, + // zustand 5 persists on every set(), hydrated or not: wait, or the stored preferences are overwritten. + recordStatus: async (registryId, authority, status) => { + await waitForDeviceSyncHydration(); + useDeviceSyncStore.getState().recordRunStatus(registryId, authority, status); + }, + recordKnownState: async (registryId, jmapAccountId, type, state) => { + await waitForDeviceSyncHydration(); + useDeviceSyncStore.getState().recordKnownState(registryId, jmapAccountId, type, state); + }, + notifyAuthProblem, + log: (message, detail) => { + console.warn(`[device-sync] ${message}`, detail instanceof Error ? detail.message : (detail ?? '')); + }, + }; +} + +function failureReport(payload: RunPayload, error: unknown): RunReport { + return { v: 1, runId: payload.runId, authority: payload.authority, - outcome: 'disabled', - message: 'Device sync is not available in this build', + outcome: 'internal', + message: error instanceof Error ? error.message : String(error), startedAt: Date.now(), durationMs: 0, - stats: { - downloaded: { created: 0, updated: 0, deleted: 0 }, - uploaded: { created: 0, updated: 0, deleted: 0 }, - entries: 0, - skipped: 0, - }, + stats: emptyStats(), conflicts: 0, itemErrors: [], }; +} + +export async function runDeviceSyncTask(payload: RunPayload): Promise { + if (!payload?.runId) return; + let report: RunReport; try { - await native.finishRun(payload.runId, JSON.stringify(report)); - } catch { + report = await runDeviceSync(payload, createTaskDeps()); + } catch (error) { + report = failureReport(payload, error); + } + try { + await finishRun(payload.runId, report); + } catch (error) { // The adapter ends the run on its own when the task finishes. + console.warn('[device-sync] finishRun failed', error instanceof Error ? error.message : error); } } + +/** + * Turning sync off for an authority (or signing out): uploads what the + * device changed (bounded by `timeoutMs`, default 30 s), then removes the + * authority's rows and SyncState. When changes are still waiting + * (`pending > 0`) nothing is deleted unless `force` is set, once the user + * confirmed losing them. Runs under the same lock as sync runs. + */ +export function teardownAuthority( + registryId: string, + accountName: string, + authority: Authority, + options?: TeardownOptions, +): Promise<{ pending: number }> { + return teardownWithDeps(createTaskDeps(), registryId, accountName, authority, options); +}