mirror of
https://github.com/bulwarkmail/native.git
synced 2026-10-31 16:06:32 +00:00
feat: sync the device's contacts and calendars with the server in the background #34
The sync adapters from ed2525e start a headless task for every sync, but
the task only reported "disabled". This adds the engine it runs
(`src/device-sync/engine`, `src/device-sync/jmap`): for one Android
account and authority it downloads the server's changes, uploads what
the device changed, and hands a report back to SyncManager.
- Downloads follow `/changes` from the state kept in the provider's
SyncState. Rows and the state they reach are written in one batch, so
a killed run resumes where it stopped. Without a state, after
`cannotCalculateChanges` or after the user discarded deletions, a full
reconcile compares ids instead.
- Uploads go in three phases across accounts (creates, updates,
deletions), so moving an item to another account never destroys it
before the copy exists. A create carries a uid written to the device
first, and the server is asked for that uid before creating, so a lost
response or a killed run never makes a duplicate. Updates are patches
guarded by `ifInState`; nothing is replaced whole.
- More than 50 deletions that are also more than a fifth of the synced
items hold every deletion back and report `tooManyDeletions`, so
Android asks the user; "undo" restores the items from the server.
- An item the server refuses is marked and not sent again until it
changes or its back-off (1 hour, doubling to a day) ends. Such
refusals and planner errors stay with their item; the rest of the run
goes on.
- Runs of one account and authority take a lock, stop at checkpoints
when cancelled or close to the deadline, and never throw: every
failure becomes an outcome, and a rejected sign-in posts one "Sign in
again" notification.
- The JMAP side uses a client of its own per app account, never the
UI's, since a run can share the JS runtime with the app.
Turning sync off or signing out runs the same upload first
(`teardownAuthority`) and deletes the rows only once nothing is waiting,
or when the user agreed to lose it.
The planners that map items to rows land in the next commits; until
then a run fails before it touches a row.
Checked: 94 engine tests against the fake provider and JMAP server
(sync flows, failures, crashes and retries, teardown), and `npm run
typecheck`, `npm test` and `npm run i18n:check` on this commit.
This commit is contained in:
@@ -252,6 +252,11 @@
|
||||
"deep_link": {
|
||||
"open_failed": "Couldn't open this"
|
||||
},
|
||||
"device_sync": {
|
||||
"auth_problem_text": "The server no longer accepts the sign-in of {account}, so its contacts and calendars on this device stopped syncing. Sign in again to resume. Changes made on this device are kept.",
|
||||
"auth_problem_title": "Sign in again to keep syncing",
|
||||
"problem_channel": "Sync problems"
|
||||
},
|
||||
"email_composer": {
|
||||
"attach_camera": "Camera",
|
||||
"attachment_reminder_body": "Your message mentions \"{keyword}\" but no file is attached. Send anyway?",
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
// Engine runs for calendars, and the teardown the app calls when sync is
|
||||
// turned off (docs/device-sync.md, "Calendar mapping", "App integration").
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { Events } from '../../android-columns';
|
||||
import { parseJsonColumn } from '../../common/json';
|
||||
import type { PoisonMarker } from '../../planner';
|
||||
import {
|
||||
addDeviceContact,
|
||||
addServerCards,
|
||||
ANDROID_ACCOUNT,
|
||||
CALENDAR_AUTHORITY,
|
||||
CONTACTS_AUTHORITY,
|
||||
createHarness,
|
||||
renameDeviceContact,
|
||||
type Harness,
|
||||
} from './harness';
|
||||
|
||||
const START = '2026-09-28T09:00:00';
|
||||
|
||||
function addServerEvent(h: Harness, title: string, calendarId = h.calendar, extra: Record<string, unknown> = {}): string {
|
||||
return h.server.addEvent('a', {
|
||||
uid: `uid-${title}`,
|
||||
title,
|
||||
start: START,
|
||||
duration: 'PT1H',
|
||||
timeZone: 'Etc/UTC',
|
||||
calendarIds: { [calendarId]: true },
|
||||
...extra,
|
||||
});
|
||||
}
|
||||
|
||||
function calendarRow(h: Harness, calendarId: string) {
|
||||
return h.device.rows('calendars').find((c) => c._sync_id === `a/${calendarId}`);
|
||||
}
|
||||
|
||||
function titles(h: Harness): string[] {
|
||||
return h.events().map((e) => String(e[Events.TITLE])).sort();
|
||||
}
|
||||
|
||||
describe('device sync engine: calendar', () => {
|
||||
it('writes calendar rows for the synced calendars and downloads their events, never tasks', async () => {
|
||||
const h = createHarness();
|
||||
const standup = addServerEvent(h, 'Standup');
|
||||
h.server.addEvent('a', { uid: 't1', '@type': 'Task', title: 'Buy milk', calendarIds: { [h.calendar]: true } });
|
||||
|
||||
const report = await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { created: 1 } } });
|
||||
expect(calendarRow(h, h.calendar)).toMatchObject({ ownerAccount: 'alice@example.com', sync_events: 1, name: 'Calendar' });
|
||||
expect(h.events()).toEqual([expect.objectContaining({ title: 'Standup', _sync_id: `a/${standup}`, uid2445: 'uid-Standup' })]);
|
||||
expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({
|
||||
accounts: { a: { selected: [`a/${h.calendar}`], reconcile: null } },
|
||||
deviceZone: 'Europe/Berlin',
|
||||
reminderOwner: 'device',
|
||||
});
|
||||
});
|
||||
|
||||
it('uploads a new device event, an edit and a deletion', async () => {
|
||||
const h = createHarness();
|
||||
const standup = addServerEvent(h, 'Standup');
|
||||
const retro = addServerEvent(h, 'Retro');
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
const rowId = Number(calendarRow(h, h.calendar)!._id);
|
||||
const byTitle = (t: string) => h.events().find((e) => e.title === t)!;
|
||||
|
||||
h.device.user.insertEvent(rowId, { title: 'Lunch', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' });
|
||||
h.device.user.updateEvent(Number(byTitle('Standup')._id), { title: 'Daily standup' });
|
||||
h.device.user.deleteEvent(Number(byTitle('Retro')._id));
|
||||
const report = await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 1, updated: 1, deleted: 1 } } });
|
||||
expect(h.server.get('CalendarEvent', 'a', standup)).toMatchObject({ title: 'Daily standup' });
|
||||
expect(h.server.get('CalendarEvent', 'a', retro)).toBeUndefined();
|
||||
const lunch = h.server.all('CalendarEvent', 'a').find((e) => e.title === 'Lunch')!;
|
||||
expect(lunch).toMatchObject({ start: '2026-09-28T12:00:00', calendarIds: { [h.calendar]: true } });
|
||||
expect(titles(h)).toEqual(['Daily standup', 'Lunch']);
|
||||
expect(byTitle('Lunch')).toMatchObject({ _sync_id: `a/${lunch.id}`, dirty: 0 });
|
||||
});
|
||||
|
||||
it('gives a calendar that holds only tasks no row', async () => {
|
||||
const h = createHarness();
|
||||
const todo = h.server.addCalendar('a', { name: 'To do' });
|
||||
for (let i = 0; i < 3; i++) h.server.addEvent('a', { uid: `t${i}`, '@type': 'Task', title: `Task ${i}`, calendarIds: { [todo]: true } });
|
||||
addServerEvent(h, 'Standup');
|
||||
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(calendarRow(h, todo)).toBeUndefined();
|
||||
expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.taskOnly).toEqual([todo]);
|
||||
expect(titles(h)).toEqual(['Standup']);
|
||||
});
|
||||
|
||||
it('drops a deselected calendar once its events are uploaded, keeping the others', async () => {
|
||||
const h = createHarness();
|
||||
const work = h.server.addCalendar('a', { name: 'Work' });
|
||||
addServerEvent(h, 'Standup');
|
||||
const review = addServerEvent(h, 'Review', work);
|
||||
addServerEvent(h, 'Planning', work);
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
expect(titles(h)).toEqual(['Planning', 'Review', 'Standup']);
|
||||
|
||||
h.device.user.updateEvent(Number(h.events().find((e) => e.title === 'Review')!._id), { title: 'Code review' });
|
||||
h.prefs.calendarSelection[`a/${work}`] = false;
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(h.server.get('CalendarEvent', 'a', review)).toMatchObject({ title: 'Code review' });
|
||||
expect(calendarRow(h, work)).toBeUndefined();
|
||||
expect(titles(h)).toEqual(['Standup']);
|
||||
expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`]);
|
||||
});
|
||||
|
||||
it('keeps a deselected calendar while one of its events could not be uploaded', async () => {
|
||||
const h = createHarness();
|
||||
const work = h.server.addCalendar('a', { name: 'Work' });
|
||||
const review = addServerEvent(h, 'Review', work);
|
||||
addServerEvent(h, 'Planning', work);
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
h.device.user.updateEvent(Number(h.events().find((e) => e.title === 'Review')!._id), { title: 'Code review' });
|
||||
h.server.setErrorFor('CalendarEvent', 'a', review, { type: 'tooLarge' });
|
||||
h.prefs.calendarSelection[`a/${work}`] = false;
|
||||
|
||||
const report = await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${review}`, type: 'tooLarge' })]);
|
||||
expect(calendarRow(h, work)).toBeDefined();
|
||||
expect(titles(h)).toEqual(['Code review']);
|
||||
expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`, `a/${work}`]);
|
||||
});
|
||||
|
||||
it('marks a calendar that mirrors a feed subscription read-only', async () => {
|
||||
const h = createHarness();
|
||||
const feed = h.server.addCalendar('a', { name: 'Holidays' });
|
||||
h.deps.subscriptionCalendars = async () => [{ jmapAccountId: null, calendarId: feed }];
|
||||
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(calendarRow(h, feed)).toMatchObject({ calendar_access_level: 200 });
|
||||
expect(calendarRow(h, h.calendar)).toMatchObject({ calendar_access_level: 700 });
|
||||
});
|
||||
|
||||
it('never gives a device event the identity of an unrelated event that has its uid (event uids are unique per account)', async () => {
|
||||
const h = createHarness();
|
||||
const hidden = h.server.addCalendar('a', { name: 'Hidden' });
|
||||
h.prefs.calendarSelection[`a/${hidden}`] = false;
|
||||
const other = addServerEvent(h, 'Other', hidden, { uid: 'dup' });
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
const rowId = Number(calendarRow(h, h.calendar)!._id);
|
||||
const id = h.device.user.insertEvent(rowId, { title: 'Mine', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' });
|
||||
// An app wrote the uid, and the claim keeps it.
|
||||
await h.device.port(ANDROID_ACCOUNT, CALENDAR_AUTHORITY).applyBatch([
|
||||
{
|
||||
op: 'update',
|
||||
table: 'events',
|
||||
id,
|
||||
values: { _sync_id: '~pending/dup', uid2445: 'dup', sync_data3: JSON.stringify({ uid: 'dup', target: `a/${h.calendar}` }) },
|
||||
},
|
||||
]);
|
||||
|
||||
const report = await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(report.itemErrors).toEqual([expect.objectContaining({ type: 'uidConflict' })]);
|
||||
expect(h.server.get('CalendarEvent', 'a', other)).toMatchObject({ title: 'Other', calendarIds: { [hidden]: true } });
|
||||
const row = h.events().find((e) => Number(e._id) === id)!;
|
||||
expect(row).toMatchObject({ title: 'Mine', _sync_id: '~pending/dup' });
|
||||
expect(parseJsonColumn<PoisonMarker>(row.sync_data5)).toMatchObject({ type: 'uidConflict', n: 1 });
|
||||
});
|
||||
|
||||
it('adopts an event its lost create left on the server, with a lagging uid index', async () => {
|
||||
const h = createHarness();
|
||||
addServerEvent(h, 'Standup');
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
h.server.setUidIndexLag(3);
|
||||
const rowId = Number(calendarRow(h, h.calendar)!._id);
|
||||
const id = h.device.user.insertEvent(rowId, { title: 'Lunch', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' });
|
||||
h.server.applyThenLoseResponse({ match: 'CalendarEvent/set' });
|
||||
|
||||
expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('io');
|
||||
expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok');
|
||||
|
||||
expect(h.server.all('CalendarEvent', 'a').map((e) => e.title).sort()).toEqual(['Lunch', 'Standup']);
|
||||
expect(h.events()).toHaveLength(2);
|
||||
expect(h.events().find((e) => Number(e._id) === id)).toMatchObject({ dirty: 0, _sync_id: expect.stringMatching(/^a\//) });
|
||||
});
|
||||
});
|
||||
|
||||
describe('device sync engine: teardown', () => {
|
||||
it('uploads waiting changes, then removes every row and the SyncState of the authority', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King');
|
||||
addDeviceContact(h, 'Hedy');
|
||||
|
||||
const result = await h.teardown(CONTACTS_AUTHORITY);
|
||||
|
||||
expect(result).toEqual({ pending: 0 });
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } });
|
||||
expect(h.serverNames()).toEqual(['Ada King', 'Hedy']);
|
||||
expect(h.device.rows('raw_contacts')).toEqual([]);
|
||||
expect(h.device.rows('groups')).toEqual([]);
|
||||
expect(h.device.rows('settings')).toEqual([]);
|
||||
expect(h.device.readSyncState(ANDROID_ACCOUNT, CONTACTS_AUTHORITY)).toBe('');
|
||||
});
|
||||
|
||||
it('deletes nothing while changes could not be uploaded, unless forced', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
addDeviceContact(h, 'Hedy');
|
||||
h.deps.jmap = async () => {
|
||||
const error = new Error('Network request failed');
|
||||
error.name = 'NetworkError';
|
||||
throw error;
|
||||
};
|
||||
|
||||
expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 1 });
|
||||
expect(h.contacts()).toHaveLength(2);
|
||||
|
||||
expect(await h.teardown(CONTACTS_AUTHORITY, { force: true })).toEqual({ pending: 0 });
|
||||
expect(h.contacts()).toHaveLength(0);
|
||||
expect(h.serverNames()).toEqual(['Ada']);
|
||||
});
|
||||
|
||||
it('holds mass deletions back like a sync run, and asks', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: 80 }, (_, i) => `P${i}`));
|
||||
await h.run();
|
||||
for (const contact of h.contacts().slice(0, 60)) h.device.user.deleteContact(contact.id);
|
||||
|
||||
expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 60 });
|
||||
expect(h.serverNames()).toHaveLength(80);
|
||||
});
|
||||
|
||||
it('removes the calendar rows, and their events with them', async () => {
|
||||
const h = createHarness();
|
||||
addServerEvent(h, 'Standup');
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(await h.teardown(CALENDAR_AUTHORITY)).toEqual({ pending: 0 });
|
||||
expect(h.device.rows('calendars')).toEqual([]);
|
||||
expect(h.device.rows('events')).toEqual([]);
|
||||
});
|
||||
|
||||
it('stops a running sync of the account and waits for it', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: 200 }, (_, i) => `P${i}`));
|
||||
let teardown: Promise<{ pending: number }> | null = null;
|
||||
h.checkpoints.onCheckpoint = (n) => {
|
||||
if (n === 3) teardown = h.teardown(CONTACTS_AUTHORITY);
|
||||
};
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'cancelled' });
|
||||
expect(await teardown).toEqual({ pending: 0 });
|
||||
expect(h.contacts()).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
// The calendar passes that need no server writes (device zone change,
|
||||
// reminder owner change) and the local checks before an upload (a series
|
||||
// split through CONTENT_EXCEPTION_URI).
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { Events } from '../../android-columns';
|
||||
import type { CalendarPlanner } from '../../planner';
|
||||
import { ANDROID_ACCOUNT, CALENDAR_AUTHORITY, createHarness, type Harness } from './harness';
|
||||
import { toyCalendarPlanner } from './toy-planners';
|
||||
|
||||
function withPlanner(h: Harness, overrides: Partial<CalendarPlanner>): CalendarPlanner {
|
||||
const planner = { ...toyCalendarPlanner, ...overrides } as CalendarPlanner;
|
||||
h.deps.planners = { ...h.deps.planners, calendar: planner };
|
||||
return planner;
|
||||
}
|
||||
|
||||
function addServerEvent(h: Harness, title: string): string {
|
||||
return h.server.addEvent('a', { uid: `uid-${title}`, title, start: '2026-09-28T09:00:00', duration: 'PT1H', calendarIds: { [h.calendar]: true } });
|
||||
}
|
||||
|
||||
describe('device sync engine: calendar passes', () => {
|
||||
it('rewrites clean events for a new device zone, and stores the zone with the last chunk', async () => {
|
||||
const h = createHarness({ tuning: { chunkSize: 2 } });
|
||||
for (const title of ['A', 'B', 'C']) addServerEvent(h, title);
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
const planZoneChange = vi.fn((local, previousZone: string, ctx) => ({
|
||||
ref: String(local.syncId),
|
||||
ops: [{ op: 'update' as const, table: 'events' as const, id: local.eventId, values: { eventTimezone: ctx.deviceZone } }],
|
||||
previousZone,
|
||||
}));
|
||||
withPlanner(h, { planZoneChange });
|
||||
h.deps.deviceZone = () => 'America/New_York';
|
||||
|
||||
const report = await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(planZoneChange).toHaveBeenCalledTimes(3);
|
||||
expect(planZoneChange.mock.calls.every(([, previous]) => previous === 'Europe/Berlin')).toBe(true);
|
||||
expect(h.events().map((e) => e[Events.EVENT_TIMEZONE])).toEqual(['America/New_York', 'America/New_York', 'America/New_York']);
|
||||
expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'America/New_York', deviceZonePending: null });
|
||||
});
|
||||
|
||||
it('finishes an interrupted zone pass in the next run', async () => {
|
||||
const h = createHarness({ tuning: { chunkSize: 1 } });
|
||||
for (const title of ['A', 'B', 'C']) addServerEvent(h, title);
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
const planZoneChange = vi.fn((local: { eventId: number; syncId: string | null }) => ({
|
||||
ref: String(local.syncId),
|
||||
ops: [{ op: 'update' as const, table: 'events' as const, id: local.eventId, values: { eventTimezone: 'America/New_York' } }],
|
||||
}));
|
||||
withPlanner(h, { planZoneChange });
|
||||
h.deps.deviceZone = () => 'America/New_York';
|
||||
let zoneChunks = 0;
|
||||
h.checkpoints.onCheckpoint = () => {
|
||||
if (planZoneChange.mock.calls.length > 0 && ++zoneChunks === 2) h.checkpoints.crashAt = h.checkpoints.count;
|
||||
};
|
||||
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'Europe/Berlin', deviceZonePending: 'America/New_York' });
|
||||
|
||||
h.checkpoints.crashAt = null;
|
||||
h.checkpoints.onCheckpoint = undefined;
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'America/New_York', deviceZonePending: null });
|
||||
expect(h.events().every((e) => e[Events.EVENT_TIMEZONE] === 'America/New_York')).toBe(true);
|
||||
});
|
||||
|
||||
it('uploads with the old reminder owner, then rewrites the reminders and the calendar rows for the new one', async () => {
|
||||
const h = createHarness();
|
||||
addServerEvent(h, 'Standup');
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
expect(h.device.rows('calendars')[0]).toMatchObject({ maxReminders: 5 });
|
||||
const owners: string[] = [];
|
||||
const planReminderOwnerChange = vi.fn((local: { eventId: number; syncId: string | null }, ctx: { reminderOwner: string }) => {
|
||||
owners.push(ctx.reminderOwner);
|
||||
return { ref: String(local.syncId), ops: [{ op: 'delete' as const, table: 'reminders' as const, where: 'event_id = ?', args: [local.eventId] }] };
|
||||
});
|
||||
const planUpload = vi.fn(toyCalendarPlanner.planUpload);
|
||||
withPlanner(h, { planReminderOwnerChange, planUpload });
|
||||
h.device.user.setReminders(Number(h.events()[0]._id), [{ minutes: 10 }]);
|
||||
h.device.user.updateEvent(Number(h.events()[0]._id), { title: 'Daily standup' });
|
||||
h.prefs.reminderOwner = 'bulwark';
|
||||
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(planUpload.mock.calls.map(([, ctx]) => ctx.reminderOwner)).toEqual(['device']);
|
||||
expect(owners).toEqual(['bulwark']);
|
||||
expect(h.device.rows('reminders')).toEqual([]);
|
||||
expect(h.device.rows('calendars')[0]).toMatchObject({ maxReminders: 0 });
|
||||
expect(h.state(CALENDAR_AUTHORITY)?.reminderOwner).toBe('bulwark');
|
||||
});
|
||||
|
||||
it('uploads the clone of a CONTENT_EXCEPTION_URI split as a new event after its source', async () => {
|
||||
const h = createHarness();
|
||||
const series = addServerEvent(h, 'Series');
|
||||
await h.run(CALENDAR_AUTHORITY);
|
||||
const master = h.events()[0];
|
||||
// A recurring master, as a calendar app sees it.
|
||||
await h.device.port(ANDROID_ACCOUNT, CALENDAR_AUTHORITY).applyBatch([
|
||||
{ op: 'update', table: 'events', id: Number(master._id), values: { rrule: 'FREQ=DAILY', duration: 'P3600S', dtend: null } },
|
||||
]);
|
||||
const planUpload = vi.fn(toyCalendarPlanner.planUpload);
|
||||
withPlanner(h, { planUpload });
|
||||
const clone = h.device.user.splitViaUri(Number(master._id), Date.UTC(2026, 9, 1, 9), '20260930T235959Z', { title: 'Series (later)' });
|
||||
|
||||
const report = await h.run(CALENDAR_AUTHORITY);
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
// The source was planned while the split was visible, before its clone was claimed.
|
||||
const splits = planUpload.mock.calls.map(([local]) => local.split);
|
||||
expect(splits.indexOf('source')).toBeLessThan(splits.indexOf('clone'));
|
||||
const events = h.server.all('CalendarEvent', 'a');
|
||||
expect(events.map((e) => e.title).sort()).toEqual(['Series', 'Series (later)']);
|
||||
const cloneRow = h.events().find((e) => Number(e._id) === clone)!;
|
||||
expect(cloneRow._sync_id).not.toBe(`a/${series}`);
|
||||
expect(cloneRow._sync_id).toMatch(/^a\//);
|
||||
expect(new Set(h.events().map((e) => e._sync_id)).size).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,372 @@
|
||||
// Engine runs for contacts against the fake provider and the fake JMAP
|
||||
// server: first sync, incremental /changes, uploads, echoes, selection
|
||||
// changes and groups (docs/device-sync.md, "A sync run").
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { Data, MimeType } from '../../android-columns';
|
||||
import { runDeviceSync } from '../../engine/run';
|
||||
import {
|
||||
addDeviceContact,
|
||||
addServerCards,
|
||||
ANDROID_ACCOUNT,
|
||||
CONTACTS_AUTHORITY,
|
||||
createHarness,
|
||||
REGISTRY_ID,
|
||||
renameDeviceContact,
|
||||
RUN_BUDGET,
|
||||
rowWrites,
|
||||
type Harness,
|
||||
} from './harness';
|
||||
|
||||
function runAs(h: Harness, registryId: string) {
|
||||
return runDeviceSync(
|
||||
{ runId: 'other', accountName: ANDROID_ACCOUNT, registryId, authority: CONTACTS_AUTHORITY, extras: {}, deadline: h.clock.now + RUN_BUDGET },
|
||||
h.deps,
|
||||
);
|
||||
}
|
||||
|
||||
describe('device sync engine: contacts', () => {
|
||||
it('downloads every card of the synced address books on the first sync (a full reconcile)', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, ['Ada', 'Grace', 'Linus']);
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { created: 3, updated: 0, deleted: 0 } } });
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Grace', 'Linus']);
|
||||
expect(h.contacts().every((c) => c.sourceId?.startsWith('a/') && !c.dirty)).toBe(true);
|
||||
const state = h.state();
|
||||
expect(state?.owner).toEqual({ registryId: REGISTRY_ID, origin: 'https://mail.example.com' });
|
||||
expect(state?.accounts.a).toMatchObject({
|
||||
itemsState: h.server.state('ContactCard', 'a'),
|
||||
reconcile: null,
|
||||
selected: [`a/${h.book}`],
|
||||
stale: [],
|
||||
});
|
||||
expect(h.device.rows('settings')).toEqual([expect.objectContaining({ ungrouped_visible: 1, should_sync: 1 })]);
|
||||
expect(h.deps.recordStatus).toHaveBeenCalledWith(REGISTRY_ID, CONTACTS_AUTHORITY, expect.objectContaining({ outcome: 'ok' }));
|
||||
});
|
||||
|
||||
it('applies server creates, updates and deletes from /changes', async () => {
|
||||
const h = createHarness();
|
||||
const [ada, grace] = addServerCards(h, ['Ada', 'Grace']);
|
||||
await h.run();
|
||||
|
||||
h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada Lovelace' });
|
||||
h.server.serverDestroy('ContactCard', 'a', grace);
|
||||
addServerCards(h, ['Katherine']);
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.stats.downloaded).toEqual({ created: 1, updated: 1, deleted: 1 });
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada Lovelace', 'Katherine']);
|
||||
expect(h.server.calls('ContactCard/changes').length).toBeGreaterThan(0);
|
||||
expect(h.state()?.accounts.a.itemsState).toBe(h.server.state('ContactCard', 'a'));
|
||||
});
|
||||
|
||||
it('pages through /changes and stores the state with every page', async () => {
|
||||
const h = createHarness();
|
||||
await h.run();
|
||||
addServerCards(h, Array.from({ length: 300 }, (_, i) => `Person ${i}`));
|
||||
const before = h.server.calls('ContactCard/changes').length;
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
const pages = h.server.calls('ContactCard/changes').slice(before);
|
||||
expect(pages.map(([, args]) => args.maxChanges)).toEqual([256, 256]);
|
||||
expect(report.stats.downloaded.created).toBe(300);
|
||||
expect(h.contacts()).toHaveLength(300);
|
||||
// One state op per page, each the last op of its batch.
|
||||
const stateBatches = h.batches.log.filter((ops) => ops.some((op) => op.op === 'syncState'));
|
||||
for (const ops of stateBatches) expect(ops[ops.length - 1].op).toBe('syncState');
|
||||
});
|
||||
|
||||
it('uploads a device edit as a patch and a new device contact as a create with a client uid', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King');
|
||||
const fresh = addDeviceContact(h, 'Hedy');
|
||||
const report = await h.run();
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 1, updated: 1, deleted: 0 } } });
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } });
|
||||
const created = h.server.all('ContactCard', 'a').find((c) => (c.name as { full: string }).full === 'Hedy')!;
|
||||
expect(created.uid).toMatch(/^[0-9a-f-]{36}$/);
|
||||
expect(h.contacts().find((c) => c.id === fresh)).toMatchObject({ sourceId: `a/${created.id}`, dirty: false });
|
||||
expect(h.contacts().every((c) => !c.dirty)).toBe(true);
|
||||
const set = h.server.calls('ContactCard/set');
|
||||
expect(set.every(([, args]) => typeof args.ifInState === 'string')).toBe(true);
|
||||
});
|
||||
|
||||
it('uploads to an account without any card yet, whose /changes state is spelled differently from its type state', async () => {
|
||||
const h = createHarness();
|
||||
await h.run();
|
||||
addDeviceContact(h, 'Hedy');
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(h.serverNames()).toEqual(['Hedy']);
|
||||
expect(h.contacts()).toEqual([expect.objectContaining({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) })]);
|
||||
});
|
||||
|
||||
it('writes nothing but the state when its own upload comes back through /changes (echo)', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King');
|
||||
addDeviceContact(h, 'Hedy');
|
||||
await h.run();
|
||||
const before = h.batches.log.length;
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.stats.downloaded).toEqual({ created: 0, updated: 0, deleted: 0 });
|
||||
expect(rowWrites(h.batches.log.slice(before))).toEqual([]);
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Hedy']);
|
||||
});
|
||||
|
||||
it('keeps a contact dirty when the user edits it again while its upload is on the way', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
const id = h.contactNamed('Ada')!.id;
|
||||
renameDeviceContact(h, id, 'Ada King');
|
||||
const stop = h.server.onAfterRequest((request) => {
|
||||
if (request.methods.includes('ContactCard/set')) renameDeviceContact(h, id, 'Ada, Countess');
|
||||
});
|
||||
|
||||
await h.run();
|
||||
stop();
|
||||
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } });
|
||||
expect(h.contacts().find((c) => c.id === id)).toMatchObject({ name: 'Ada, Countess', dirty: true });
|
||||
|
||||
await h.run();
|
||||
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada, Countess' } });
|
||||
expect(h.contacts().find((c) => c.id === id)).toMatchObject({ dirty: false });
|
||||
});
|
||||
|
||||
it('lets the server win a conflicting edit of the same unit and counts it', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada (device)');
|
||||
h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada (server)' });
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.conflicts).toBe(1);
|
||||
expect(h.contacts().map((c) => c.name)).toEqual(['Ada (server)']);
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada (server)' } });
|
||||
});
|
||||
|
||||
it('deletes on the server what the user deleted, and purges the rows', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada', 'Grace']);
|
||||
await h.run();
|
||||
|
||||
h.device.user.deleteContact(h.contactNamed('Ada')!.id);
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.stats.uploaded.deleted).toBe(1);
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toBeUndefined();
|
||||
expect(h.contacts().map((c) => c.name)).toEqual(['Grace']);
|
||||
});
|
||||
|
||||
it('drops only the clean rows of a deselected address book, after uploading its dirty ones', async () => {
|
||||
const h = createHarness();
|
||||
const work = h.server.addAddressBook('a', { name: 'Work' });
|
||||
addServerCards(h, ['Ada']);
|
||||
const [boss, colleague] = addServerCards(h, ['Boss', 'Colleague'], 'a', work);
|
||||
await h.run();
|
||||
expect(h.contacts()).toHaveLength(3);
|
||||
|
||||
renameDeviceContact(h, h.contactNamed('Boss')!.id, 'The Boss');
|
||||
h.prefs.contactsSelection[`a/${work}`] = false;
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(h.contacts().map((c) => c.name)).toEqual(['Ada']);
|
||||
expect(h.server.get('ContactCard', 'a', boss)).toMatchObject({ name: { full: 'The Boss' } });
|
||||
expect(h.server.get('ContactCard', 'a', colleague)).toBeDefined();
|
||||
expect(h.state()?.accounts.a.selected).toEqual([`a/${h.book}`]);
|
||||
});
|
||||
|
||||
it("leaves the webmail's Trusted Senders book off the device unless the user picked it", async () => {
|
||||
const h = createHarness();
|
||||
const trusted = h.server.addAddressBook('a', { name: 'Trusted Senders' });
|
||||
addServerCards(h, ['Ada']);
|
||||
addServerCards(h, ['newsletter@example.com'], 'a', trusted);
|
||||
|
||||
await h.run();
|
||||
expect(h.contacts().map((c) => c.name)).toEqual(['Ada']);
|
||||
|
||||
h.prefs.contactsSelection[`a/${trusted}`] = true;
|
||||
await h.run();
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'newsletter@example.com']);
|
||||
});
|
||||
|
||||
it('loads a newly selected address book once the account is up to date', async () => {
|
||||
const h = createHarness({ accounts: 'withShared' });
|
||||
addServerCards(h, ['Ada']);
|
||||
const teamBook = h.server.all('AddressBook', 'team')[0].id as string;
|
||||
h.server.addCard('team', { uid: 'u-team', name: { full: 'Teammate' }, addressBookIds: { [teamBook]: true } });
|
||||
await h.run();
|
||||
expect(h.contacts().map((c) => c.name)).toEqual(['Ada']);
|
||||
|
||||
h.prefs.contactsSelection[`team/${teamBook}`] = true;
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.stats.downloaded.created).toBe(1);
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Teammate']);
|
||||
expect(h.contactNamed('Teammate')?.sourceId).toMatch(/^team\//);
|
||||
expect(h.state()?.accounts.team.selected).toEqual([`team/${teamBook}`]);
|
||||
});
|
||||
|
||||
it('writes group cards before the contacts that are their members', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada']);
|
||||
const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string;
|
||||
h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [adaUid]: true } });
|
||||
|
||||
await h.run();
|
||||
|
||||
const groups = h.device.rows('groups');
|
||||
expect(groups).toHaveLength(1);
|
||||
expect(groups[0]).toMatchObject({ title: 'Friends', group_visible: 1 });
|
||||
const memberships = h.device.rows('data').filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP);
|
||||
expect(memberships).toHaveLength(1);
|
||||
// data1 is a TEXT column: the group row id may read back as text.
|
||||
expect(String(memberships[0][Data.DATA1])).toBe(String(groups[0]._id));
|
||||
expect(memberships[0].group_sourceid).toBe(groups[0].sourceid);
|
||||
expect(h.state()?.accounts.a.groups).toEqual([groups[0].sourceid]);
|
||||
});
|
||||
|
||||
it("updates a member's rows when only the group card's members changed on the server", async () => {
|
||||
const h = createHarness();
|
||||
const [ada, grace] = addServerCards(h, ['Ada', 'Grace']);
|
||||
const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string;
|
||||
const graceUid = h.server.get('ContactCard', 'a', grace)!.uid as string;
|
||||
const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [adaUid]: true } });
|
||||
await h.run();
|
||||
const members = () =>
|
||||
h.device
|
||||
.rows('data')
|
||||
.filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP)
|
||||
.map((d) => h.contacts().find((c) => c.id === Number(d.raw_contact_id))?.name)
|
||||
.sort();
|
||||
expect(members()).toEqual(['Ada']);
|
||||
|
||||
// Another client moves the membership: only the group card changes.
|
||||
h.server.serverUpdate('ContactCard', 'a', group, { members: { [graceUid]: true } });
|
||||
await h.run();
|
||||
|
||||
expect(members()).toEqual(['Grace']);
|
||||
expect(h.contacts().every((c) => !c.dirty)).toBe(true);
|
||||
});
|
||||
|
||||
it('uploads a membership added on the device as a patch of the group card', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada']);
|
||||
const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: {} });
|
||||
await h.run();
|
||||
|
||||
const groupRow = h.device.rows('groups')[0];
|
||||
h.device.user.addToGroup(h.contactNamed('Ada')!.id, Number(groupRow._id));
|
||||
await h.run();
|
||||
|
||||
const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string;
|
||||
expect(h.server.get('ContactCard', 'a', group)?.members).toEqual({ [adaUid]: true });
|
||||
expect(h.contacts().every((c) => !c.dirty)).toBe(true);
|
||||
});
|
||||
|
||||
it('skips the network for an upload-only sync with nothing to upload', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
const requests = h.server.requests.length;
|
||||
|
||||
const report = await h.run(CONTACTS_AUTHORITY, { upload: true });
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(h.server.requests.length).toBe(requests);
|
||||
});
|
||||
|
||||
it('reports disabled, unsupported and rows of another owner without touching anything', async () => {
|
||||
const off = createHarness();
|
||||
off.deps.isSyncEnabled = async () => false;
|
||||
expect((await off.run()).outcome).toBe('disabled');
|
||||
expect(off.server.requests).toHaveLength(0);
|
||||
|
||||
const bare = createHarness();
|
||||
const port = bare.server.port();
|
||||
bare.deps.jmap = async () => ({
|
||||
origin: 'https://mail.example.com',
|
||||
port: { ...port, session: () => ({ ...port.session(), primaryAccounts: {}, accounts: {} }) },
|
||||
});
|
||||
expect((await bare.run()).outcome).toBe('unsupported');
|
||||
|
||||
const other = createHarness();
|
||||
addServerCards(other, ['Ada']);
|
||||
await other.run();
|
||||
other.deps.jmap = async () => ({ port: other.server.port(), origin: 'https://other.example.com' });
|
||||
const moved = await other.run();
|
||||
expect(moved.outcome).toBe('internal');
|
||||
expect(moved.message).toMatch(/another server/);
|
||||
expect(other.contacts()).toHaveLength(1);
|
||||
const foreign = await runAs(other, 'bob@mail.example.com');
|
||||
expect(foreign.outcome).toBe('internal');
|
||||
expect(foreign.message).toMatch(/another account/);
|
||||
});
|
||||
|
||||
it('keeps one request in flight and stays within the server limits', async () => {
|
||||
const h = createHarness();
|
||||
h.server.setLimits({ maxObjectsInGet: 20, maxObjectsInSet: 10 });
|
||||
addServerCards(h, Array.from({ length: 60 }, (_, i) => `Server ${i}`));
|
||||
await h.run();
|
||||
for (let i = 0; i < 25; i++) addDeviceContact(h, `Device ${i}`);
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(h.server.maxInFlight).toBe(1);
|
||||
for (const [, args] of h.server.calls('ContactCard/get')) {
|
||||
expect(Array.isArray(args.ids)).toBe(true);
|
||||
expect((args.ids as string[]).length).toBeLessThanOrEqual(20);
|
||||
}
|
||||
for (const [, args] of h.server.calls('ContactCard/set')) {
|
||||
const count = Object.keys(args.create ?? {}).length + Object.keys(args.update ?? {}).length + ((args.destroy as string[]) ?? []).length;
|
||||
expect(count).toBeLessThanOrEqual(10);
|
||||
}
|
||||
expect(h.serverNames()).toHaveLength(85);
|
||||
expect(h.contacts()).toHaveLength(85);
|
||||
expect(ANDROID_ACCOUNT).toBe('alice@example.com');
|
||||
});
|
||||
|
||||
it('packs provider batches of at most 400 ops with a yield point at every group and the state op last', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: 450 }, (_, i) => `Person ${i}`));
|
||||
|
||||
await h.run();
|
||||
|
||||
expect(h.contacts()).toHaveLength(450);
|
||||
for (const ops of h.batches.log) {
|
||||
expect(ops.length).toBeLessThanOrEqual(400);
|
||||
const state = ops.findIndex((op) => op.op === 'syncState');
|
||||
if (state >= 0) {
|
||||
expect(state).toBe(ops.length - 1);
|
||||
expect((ops[state] as { yieldAllowed?: boolean }).yieldAllowed).toBeFalsy();
|
||||
}
|
||||
// Every download insert starts a group with its "not there yet" guard: a yield point, and nothing inside a group is one.
|
||||
ops.forEach((op, i) => {
|
||||
if (op.op === 'insert' && op.table === 'raw_contacts') {
|
||||
expect(ops[i - 1]).toMatchObject({ op: 'assert', table: 'raw_contacts', expectCount: 0, yieldAllowed: true });
|
||||
expect(op.yieldAllowed).toBeFalsy();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,610 @@
|
||||
// How the engine survives crashes, lost responses and server trouble, and
|
||||
// its guards (docs/device-sync.md, "Invariants", "Failure matrix",
|
||||
// "Deletion threshold").
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseJsonColumn } from '../../common/json';
|
||||
import type { PoisonMarker } from '../../planner';
|
||||
import type { JmapInvocation, JmapResponse } from '../../types';
|
||||
import {
|
||||
addDeviceContact,
|
||||
addServerCards,
|
||||
CONTACTS_AUTHORITY,
|
||||
createHarness,
|
||||
CrashError,
|
||||
renameDeviceContact,
|
||||
REGISTRY_ID,
|
||||
type Harness,
|
||||
} from './harness';
|
||||
|
||||
const FINAL = ['Ada King', 'Grace Hopper', 'Hedy', 'Margaret'];
|
||||
|
||||
/** Work on both sides: device edit, create and delete; server update, create and delete. */
|
||||
async function scenario(options: Parameters<typeof createHarness>[0] = {}): Promise<Harness> {
|
||||
const h = createHarness(options);
|
||||
const [, grace, , katherine] = addServerCards(h, ['Ada', 'Grace', 'Linus', 'Katherine']);
|
||||
await h.run();
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King');
|
||||
addDeviceContact(h, 'Hedy');
|
||||
h.device.user.deleteContact(h.contactNamed('Linus')!.id);
|
||||
h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': 'Grace Hopper' });
|
||||
addServerCards(h, ['Margaret']);
|
||||
h.server.serverDestroy('ContactCard', 'a', katherine);
|
||||
h.checkpoints.count = 0;
|
||||
h.batches.applied = 0;
|
||||
return h;
|
||||
}
|
||||
|
||||
/** Both sides hold exactly the expected contacts, once each, and nothing waits for an upload. */
|
||||
function expectConverged(h: Harness, names = FINAL): void {
|
||||
const device = h.contacts();
|
||||
expect(device.map((c) => c.name).sort()).toEqual(names);
|
||||
expect(device.every((c) => !c.dirty && !c.deleted && c.sourceId)).toBe(true);
|
||||
expect(h.serverNames()).toEqual(names);
|
||||
const refs = device.map((c) => c.sourceId);
|
||||
expect(new Set(refs).size).toBe(refs.length);
|
||||
for (const contact of device) {
|
||||
const card = h.server.get('ContactCard', 'a', contact.sourceId!.split('/')[1]);
|
||||
expect((card?.name as { full: string }).full).toBe(contact.name);
|
||||
}
|
||||
const uids = h.server.all('ContactCard', 'a').map((c) => c.uid);
|
||||
expect(new Set(uids).size).toBe(uids.length);
|
||||
}
|
||||
|
||||
describe('device sync engine: crashes and lost responses', () => {
|
||||
it('converges after a crash at any checkpoint, with no duplicates and no lost edits', async () => {
|
||||
const probe = await scenario();
|
||||
await probe.run();
|
||||
expectConverged(probe);
|
||||
const checkpoints = probe.checkpoints.count;
|
||||
expect(checkpoints).toBeGreaterThan(5);
|
||||
|
||||
for (let n = 1; n <= checkpoints; n++) {
|
||||
const h = await scenario();
|
||||
h.checkpoints.crashAt = n;
|
||||
const crashed = await h.run();
|
||||
expect(crashed.outcome, `crash at checkpoint ${n}`).toBe('internal');
|
||||
h.checkpoints.crashAt = null;
|
||||
const report = await h.run();
|
||||
expect(report.outcome, `after a crash at checkpoint ${n}`).toBe('ok');
|
||||
expectConverged(h);
|
||||
}
|
||||
});
|
||||
|
||||
it('converges after a crash after any number of provider batches', async () => {
|
||||
const probe = await scenario();
|
||||
await probe.run();
|
||||
const batches = probe.batches.applied;
|
||||
expect(batches).toBeGreaterThan(3);
|
||||
|
||||
for (let n = 0; n < batches; n++) {
|
||||
const h = await scenario();
|
||||
h.batches.crashAfter = n;
|
||||
await h.run();
|
||||
h.batches.crashAfter = null;
|
||||
const report = await h.run();
|
||||
expect(report.outcome, `after a crash following batch ${n}`).toBe('ok');
|
||||
expectConverged(h);
|
||||
}
|
||||
});
|
||||
|
||||
it('converges when the server applied any request but its response was lost', async () => {
|
||||
const probe = await scenario();
|
||||
const before = probe.server.requests.length;
|
||||
await probe.run();
|
||||
const requests = probe.server.requests.length - before;
|
||||
|
||||
for (let n = 1; n <= requests; n++) {
|
||||
const h = await scenario();
|
||||
// The uid index lags behind, as on Stalwart: a lookup right after the lost create misses it.
|
||||
h.server.setUidIndexLag(2);
|
||||
let seen = 0;
|
||||
const stop = h.server.onBeforeRequest(() => {
|
||||
if (++seen === n) h.server.applyThenLoseResponse();
|
||||
});
|
||||
const lost = await h.run();
|
||||
stop();
|
||||
expect(lost.outcome, `lost response ${n}`).toBe('io');
|
||||
const report = await h.run();
|
||||
expect(report.outcome, `after losing response ${n}`).toBe('ok');
|
||||
expectConverged(h);
|
||||
}
|
||||
});
|
||||
|
||||
it('converges from a crash anywhere in a first sync (a checkpointed full reconcile)', async () => {
|
||||
const names = Array.from({ length: 120 }, (_, i) => `P${String(i).padStart(3, '0')}`);
|
||||
const fresh = () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, names);
|
||||
return h;
|
||||
};
|
||||
const probe = fresh();
|
||||
await probe.run();
|
||||
const checkpoints = probe.checkpoints.count;
|
||||
|
||||
for (let n = 1; n <= checkpoints; n += 2) {
|
||||
const h = fresh();
|
||||
h.checkpoints.crashAt = n;
|
||||
await h.run();
|
||||
h.checkpoints.crashAt = null;
|
||||
expect((await h.run()).outcome).toBe('ok');
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(names);
|
||||
expect(new Set(h.contacts().map((c) => c.sourceId)).size).toBe(120);
|
||||
}
|
||||
});
|
||||
|
||||
it('resumes a reconcile after its last stored chunk instead of starting over', async () => {
|
||||
const names = Array.from({ length: 150 }, (_, i) => `P${String(i).padStart(3, '0')}`);
|
||||
const h = createHarness();
|
||||
addServerCards(h, names);
|
||||
// Batches: the Settings row, the reconcile marker, then one per chunk of 50. Stop after the second chunk.
|
||||
h.batches.crashAfter = 4;
|
||||
await h.run();
|
||||
const marker = h.state()?.accounts.a.reconcile;
|
||||
expect(marker).toMatchObject({ phase: 'objects', position: 100 });
|
||||
h.batches.crashAfter = null;
|
||||
const gets = h.server.calls('ContactCard/get').length;
|
||||
|
||||
await h.run();
|
||||
|
||||
const fetched = h.server
|
||||
.calls('ContactCard/get')
|
||||
.slice(gets)
|
||||
.filter(([, args]) => (args.properties as string[]).includes('name'))
|
||||
.flatMap(([, args]) => args.ids as string[]);
|
||||
expect(fetched).toHaveLength(50);
|
||||
expect(h.contacts()).toHaveLength(150);
|
||||
expect(h.state()?.accounts.a.reconcile).toBeNull();
|
||||
});
|
||||
|
||||
it('adopts its own create whose response was lost instead of creating it twice', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
h.server.setUidIndexLag(3);
|
||||
const id = addDeviceContact(h, 'Hedy');
|
||||
h.server.applyThenLoseResponse({ match: 'ContactCard/set' });
|
||||
|
||||
expect((await h.run()).outcome).toBe('io');
|
||||
expect(h.serverNames()).toEqual(['Ada', 'Hedy']);
|
||||
expect(h.contacts().find((c) => c.id === id)?.sourceId).toBeNull();
|
||||
|
||||
expect((await h.run()).outcome).toBe('ok');
|
||||
expect(h.serverNames()).toEqual(['Ada', 'Hedy']);
|
||||
expect(h.contacts().find((c) => c.id === id)).toMatchObject({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) });
|
||||
expect(h.contacts()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("adopts the card a duplicate-uid error names (the client's automatic retry of a create)", async () => {
|
||||
const h = createHarness();
|
||||
await h.run();
|
||||
addDeviceContact(h, 'Hedy');
|
||||
// The first attempt was applied; the retry answers with Stalwart's duplicate-uid error.
|
||||
const port = h.server.port();
|
||||
let retried = false;
|
||||
h.deps.jmap = async () => ({
|
||||
origin: 'https://mail.example.com',
|
||||
port: {
|
||||
...port,
|
||||
request: async (calls: JmapInvocation[], using: string[]): Promise<JmapResponse> => {
|
||||
const response = await port.request(calls, using);
|
||||
const [name, args] = response.methodResponses[0];
|
||||
if (retried || name !== 'ContactCard/set' || !args.created) return response;
|
||||
retried = true;
|
||||
const [[key, created]] = Object.entries(args.created as Record<string, { id: string }>);
|
||||
const uid = (calls[0][1].create as Record<string, { uid: string }>)[key].uid;
|
||||
const notCreated = {
|
||||
[key]: { type: 'invalidProperties', properties: ['uid'], description: `Contact with UID ${uid} already exists with id ${created.id}.` },
|
||||
};
|
||||
return { ...response, methodResponses: [[name, { ...args, created: undefined, notCreated }, response.methodResponses[0][2]]] };
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(retried).toBe(true);
|
||||
expect(h.serverNames()).toEqual(['Hedy']);
|
||||
expect(h.contacts()).toEqual([expect.objectContaining({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) })]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('device sync engine: server trouble', () => {
|
||||
it('downloads again and retries the upload after a stateMismatch', async () => {
|
||||
const h = createHarness();
|
||||
const [ada, grace] = addServerCards(h, ['Ada', 'Grace']);
|
||||
await h.run();
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King');
|
||||
let sets = 0;
|
||||
const stop = h.server.onBeforeRequest((request) => {
|
||||
// Another client writes right before our first upload.
|
||||
if (request.methods.includes('ContactCard/set') && ++sets === 1) {
|
||||
h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': 'Grace Hopper' });
|
||||
}
|
||||
});
|
||||
|
||||
const report = await h.run();
|
||||
stop();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
const set = h.server.requests.filter((r) => r.methods.includes('ContactCard/set'));
|
||||
expect(set[0].responses?.[0]).toEqual(['error', { type: 'stateMismatch' }, 's']);
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } });
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Grace Hopper']);
|
||||
});
|
||||
|
||||
it('gives up with io and no progress when the server keeps changing', async () => {
|
||||
const h = createHarness();
|
||||
const [, grace] = addServerCards(h, ['Ada', 'Grace']);
|
||||
await h.run();
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King');
|
||||
let n = 0;
|
||||
const stop = h.server.onBeforeRequest((request) => {
|
||||
if (request.methods.includes('ContactCard/set')) h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': `Grace ${++n}` });
|
||||
});
|
||||
|
||||
const report = await h.run();
|
||||
stop();
|
||||
|
||||
expect(report.outcome).toBe('io');
|
||||
expect(report.stats.downloaded).toEqual({ created: 0, updated: 0, deleted: 0 });
|
||||
expect(n).toBe(4);
|
||||
expect(h.contactNamed('Ada King')?.dirty).toBe(true);
|
||||
});
|
||||
|
||||
it('holds back the uploads of an account that refuses writes, and uploads the other accounts', async () => {
|
||||
const h = createHarness({ accounts: 'withShared' });
|
||||
const teamBook = h.server.all('AddressBook', 'team')[0].id as string;
|
||||
h.prefs.contactsSelection[`team/${teamBook}`] = true;
|
||||
addServerCards(h, ['Ada']);
|
||||
const [mate] = addServerCards(h, ['Teammate'], 'team', teamBook);
|
||||
await h.run();
|
||||
renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King');
|
||||
renameDeviceContact(h, h.contactNamed('Teammate')!.id, 'Team mate');
|
||||
h.server.failNextMethod('ContactCard/set', { type: 'accountReadOnly' }, { accountId: 'team' });
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(report.itemErrors).toEqual([expect.objectContaining({ ref: 'team', type: 'accountReadOnly' })]);
|
||||
expect(h.serverNames()).toEqual(['Ada King']);
|
||||
expect(h.server.get('ContactCard', 'team', mate)).toMatchObject({ name: { full: 'Teammate' } });
|
||||
expect(h.contactNamed('Team mate')?.dirty).toBe(true);
|
||||
|
||||
await h.run();
|
||||
expect(h.server.get('ContactCard', 'team', mate)).toMatchObject({ name: { full: 'Team mate' } });
|
||||
});
|
||||
|
||||
it('reports a 429 as io with delayUntil from Retry-After', async () => {
|
||||
const h = createHarness();
|
||||
await h.run();
|
||||
h.server.failNextRequest('rateLimit', { retryAfterMs: 120_000 });
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('io');
|
||||
expect(report.delayUntil).toBe(Math.ceil((h.clock.now + 120_000) / 1000));
|
||||
});
|
||||
|
||||
it('rebuilds the client once when the server rejects the credentials, then reports auth', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, ['Ada']);
|
||||
h.server.failNextRequest('auth');
|
||||
expect((await h.run()).outcome).toBe('ok');
|
||||
expect(h.deps.notifyAuthProblem).not.toHaveBeenCalled();
|
||||
|
||||
h.server.failNextRequest('auth', { times: 2 });
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('auth');
|
||||
expect(h.deps.notifyAuthProblem).toHaveBeenCalledWith(REGISTRY_ID, 'alice@example.com', CONTACTS_AUTHORITY);
|
||||
expect(h.contacts()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('reports auth without touching anything when the account has no stored credentials', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
const writes = h.batches.log.length;
|
||||
h.deps.jmap = async () => {
|
||||
const error = new Error('No usable credentials for this account');
|
||||
error.name = 'AuthenticationError';
|
||||
throw error;
|
||||
};
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('auth');
|
||||
expect(h.batches.log.length).toBe(writes);
|
||||
});
|
||||
|
||||
it('full-reconciles an account after cannotCalculateChanges', async () => {
|
||||
const h = createHarness();
|
||||
const [ada, grace] = addServerCards(h, ['Ada', 'Grace', 'Linus']);
|
||||
await h.run();
|
||||
h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada King' });
|
||||
h.server.serverDestroy('ContactCard', 'a', grace);
|
||||
h.server.truncateChangeLog('a', 'contacts');
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(h.server.requests.some((r) => r.responses?.some(([, args]) => (args as { type?: string }).type === 'cannotCalculateChanges'))).toBe(true);
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Linus']);
|
||||
expect(h.state()?.accounts.a).toMatchObject({ reconcile: null, itemsState: h.server.state('ContactCard', 'a') });
|
||||
});
|
||||
|
||||
it('stops a reconcile that finds an empty server while the device is full (safetyAbort)', async () => {
|
||||
const h = createHarness();
|
||||
const ids = addServerCards(h, Array.from({ length: 12 }, (_, i) => `P${i}`));
|
||||
await h.run();
|
||||
for (const id of ids) h.server.serverDestroy('ContactCard', 'a', id);
|
||||
h.server.truncateChangeLog('a', 'contacts');
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('safetyAbort');
|
||||
expect(h.contacts()).toHaveLength(12);
|
||||
});
|
||||
});
|
||||
|
||||
describe('device sync engine: time and cancellation', () => {
|
||||
it('stops cleanly near the deadline and asks for another sync', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: 150 }, (_, i) => `P${i}`));
|
||||
h.checkpoints.onCheckpoint = (n) => {
|
||||
// Time runs out in the middle of the download.
|
||||
if (n === 4) h.clock.now += 9 * 60_000;
|
||||
};
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true });
|
||||
expect(h.contacts().length).toBeGreaterThan(0);
|
||||
expect(h.contacts().length).toBeLessThan(150);
|
||||
h.checkpoints.onCheckpoint = undefined;
|
||||
|
||||
expect((await h.run()).outcome).toBe('ok');
|
||||
expect(h.contacts()).toHaveLength(150);
|
||||
});
|
||||
|
||||
it('stops at the next checkpoint when the framework cancels the run', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: 150 }, (_, i) => `P${i}`));
|
||||
h.deps.isCancelled = async () => true;
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true });
|
||||
expect(h.contacts()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('sends Core/echo when a local phase went 40 s without a JMAP request', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: 120 }, (_, i) => `P${i}`));
|
||||
let n = 0;
|
||||
const port = h.deps.provider;
|
||||
h.deps.provider = (name, authority) => {
|
||||
const inner = port(name, authority);
|
||||
return {
|
||||
...inner,
|
||||
applyBatch: async (ops) => {
|
||||
h.clock.now += 45_000;
|
||||
n++;
|
||||
return inner.applyBatch(ops);
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
await h.run();
|
||||
|
||||
expect(n).toBeGreaterThan(2);
|
||||
expect(h.server.calls('Core/echo').length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('runs one sync of an account and authority at a time', async () => {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: 60 }, (_, i) => `P${i}`));
|
||||
|
||||
const [first, second] = await Promise.all([h.run(), h.run()]);
|
||||
|
||||
expect([first.outcome, second.outcome]).toEqual(['ok', 'ok']);
|
||||
expect(h.server.maxInFlight).toBe(1);
|
||||
expect(h.contacts()).toHaveLength(60);
|
||||
});
|
||||
});
|
||||
|
||||
describe('device sync engine: poisoned items', () => {
|
||||
it('backs a refused item off, retries it after the back-off, and at once when it changes', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada', 'Grace']);
|
||||
await h.run();
|
||||
const id = h.contactNamed('Ada')!.id;
|
||||
renameDeviceContact(h, id, 'Ada King');
|
||||
h.server.setErrorFor('ContactCard', 'a', (t) => t.op === 'update' && t.id === ada, {
|
||||
type: 'invalidProperties',
|
||||
properties: ['name'],
|
||||
description: 'Invalid property.',
|
||||
}, 3);
|
||||
|
||||
const first = await h.run();
|
||||
|
||||
expect(first.outcome).toBe('ok');
|
||||
expect(first.stats.skipped).toBe(1);
|
||||
expect(first.itemErrors).toEqual([
|
||||
expect.objectContaining({ ref: `a/${ada}`, side: 'upload', type: 'invalidProperties', retryAt: h.clock.now + 3_600_000 }),
|
||||
]);
|
||||
const marker = parseJsonColumn<PoisonMarker>(h.contacts().find((c) => c.id === id)!.row.sync4);
|
||||
expect(marker).toMatchObject({ type: 'invalidProperties', n: 1, until: h.clock.now + 3_600_000 });
|
||||
|
||||
// Not retried hot.
|
||||
const sets = h.server.calls('ContactCard/set').length;
|
||||
const again = await h.run();
|
||||
expect(h.server.calls('ContactCard/set').length).toBe(sets);
|
||||
expect(again.itemErrors).toEqual([expect.objectContaining({ type: 'invalidProperties' })]);
|
||||
|
||||
// Retried once the back-off is over; refused again, the back-off doubles.
|
||||
h.clock.now += 3_600_001;
|
||||
await h.run();
|
||||
expect(h.server.calls('ContactCard/set').length).toBe(sets + 1);
|
||||
expect(parseJsonColumn<PoisonMarker>(h.contacts().find((c) => c.id === id)!.row.sync4)).toMatchObject({ n: 2, until: h.clock.now + 7_200_000 });
|
||||
|
||||
// Changed by the user: retried at once (the server refuses once more), then accepted.
|
||||
renameDeviceContact(h, id, 'Ada, Countess');
|
||||
await h.run();
|
||||
expect(h.server.calls('ContactCard/set').length).toBe(sets + 2);
|
||||
renameDeviceContact(h, id, 'Ada Lovelace');
|
||||
const fixed = await h.run();
|
||||
expect(fixed.itemErrors).toEqual([]);
|
||||
expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada Lovelace' } });
|
||||
expect(h.contacts().find((c) => c.id === id)!.row.sync4).toBeNull();
|
||||
});
|
||||
|
||||
it('never uploads an item whose rows could not be written until it was fetched again', async () => {
|
||||
const h = createHarness();
|
||||
const [ada] = addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada King' });
|
||||
// The provider refuses the item's rows (in its batch and alone): it goes on `stale`, the state still moves on.
|
||||
const provider = h.deps.provider;
|
||||
let refusals = 2;
|
||||
h.deps.provider = (name, authority) => {
|
||||
const port = provider(name, authority);
|
||||
return {
|
||||
...port,
|
||||
applyBatch: async (ops) =>
|
||||
refusals > 0 && ops.some((op) => op.op === 'update' && op.table === 'data') && refusals--
|
||||
? { ok: false, reason: 'provider', message: 'SQLiteException' }
|
||||
: port.applyBatch(ops),
|
||||
};
|
||||
};
|
||||
const failing = await h.run();
|
||||
expect(refusals).toBe(0);
|
||||
expect(failing.itemErrors).toEqual([expect.objectContaining({ ref: `a/${ada}`, side: 'download', type: 'provider' })]);
|
||||
expect(h.state()?.accounts.a.stale).toEqual([ada]);
|
||||
expect(h.state()?.accounts.a.itemsState).toBe(h.server.state('ContactCard', 'a'));
|
||||
|
||||
const next = await h.run();
|
||||
|
||||
expect(next.outcome).toBe('ok');
|
||||
expect(h.contacts().map((c) => c.name)).toEqual(['Ada King']);
|
||||
expect(h.state()?.accounts.a.stale).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('device sync engine: planner failures', () => {
|
||||
it('keeps syncing everything else when a planner throws on one item, and tries that one again next run', async () => {
|
||||
const h = createHarness();
|
||||
const [, grace] = addServerCards(h, ['Ada', 'Grace', 'Linus']);
|
||||
let broken = true;
|
||||
const planner = h.deps.planners.contacts;
|
||||
h.deps.planners = {
|
||||
...h.deps.planners,
|
||||
contacts: {
|
||||
...planner,
|
||||
planDownload: (card, local, ctx) => {
|
||||
if (broken && card.id === grace) throw new TypeError('cannot read the card');
|
||||
return planner.planDownload(card, local, ctx);
|
||||
},
|
||||
planUpload: (local, ctx) => {
|
||||
if (broken && local.rows.some((r) => r.cells.data1 === 'Hedy')) throw new TypeError('cannot read the rows');
|
||||
return planner.planUpload(local, ctx);
|
||||
},
|
||||
},
|
||||
};
|
||||
addDeviceContact(h, 'Hedy');
|
||||
addDeviceContact(h, 'Mary');
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(report.itemErrors).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ ref: `a/${grace}`, side: 'download', type: 'plannerError' }),
|
||||
expect.objectContaining({ side: 'upload', type: 'plannerError' }),
|
||||
]),
|
||||
);
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Hedy', 'Linus', 'Mary']);
|
||||
expect(h.serverNames()).toEqual(['Ada', 'Grace', 'Linus', 'Mary']);
|
||||
// The reconcile completed past the item that could not be planned.
|
||||
expect(h.state()?.accounts.a).toMatchObject({ stale: [grace], reconcile: null, itemsState: expect.any(String) });
|
||||
|
||||
broken = false;
|
||||
expect((await h.run()).itemErrors).toEqual([]);
|
||||
expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Grace', 'Hedy', 'Linus', 'Mary']);
|
||||
expect(h.serverNames()).toEqual(['Ada', 'Grace', 'Hedy', 'Linus', 'Mary']);
|
||||
expect(h.state()?.accounts.a.stale).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('device sync engine: deletion guards', () => {
|
||||
async function withDeletions(total: number, deleted: number): Promise<Harness> {
|
||||
const h = createHarness();
|
||||
addServerCards(h, Array.from({ length: total }, (_, i) => `P${String(i).padStart(3, '0')}`));
|
||||
await h.run();
|
||||
for (const contact of h.contacts().slice(0, deleted)) h.device.user.deleteContact(contact.id);
|
||||
return h;
|
||||
}
|
||||
|
||||
it('holds every deletion back above 50 and 20 % (tooManyDeletions)', async () => {
|
||||
const h = await withDeletions(100, 55);
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report).toMatchObject({ outcome: 'tooManyDeletions', tooManyDeletions: { count: 55, threshold: 50 } });
|
||||
expect(h.serverNames()).toHaveLength(100);
|
||||
expect(h.contacts().filter((c) => c.deleted)).toHaveLength(55);
|
||||
});
|
||||
|
||||
it('uploads the deletions when the run overrides the threshold', async () => {
|
||||
const h = await withDeletions(100, 55);
|
||||
|
||||
const report = await h.run(CONTACTS_AUTHORITY, { overrideTooManyDeletions: true });
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(report.stats.uploaded.deleted).toBe(55);
|
||||
expect(h.serverNames()).toHaveLength(45);
|
||||
expect(h.contacts()).toHaveLength(45);
|
||||
});
|
||||
|
||||
it('restores the deleted contacts from the server when the run discards the deletions', async () => {
|
||||
const h = await withDeletions(100, 55);
|
||||
|
||||
const discard = await h.run(CONTACTS_AUTHORITY, { discardLocalDeletions: true });
|
||||
expect(discard.outcome).toBe('ok');
|
||||
expect(h.serverNames()).toHaveLength(100);
|
||||
|
||||
const next = await h.run(CONTACTS_AUTHORITY, { upload: true });
|
||||
expect(next.outcome).toBe('ok');
|
||||
expect(h.contacts()).toHaveLength(100);
|
||||
expect(h.contacts().every((c) => !c.deleted && !c.dirty)).toBe(true);
|
||||
});
|
||||
|
||||
it('uploads deletions below the threshold', async () => {
|
||||
const h = await withDeletions(100, 20);
|
||||
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.outcome).toBe('ok');
|
||||
expect(h.serverNames()).toHaveLength(80);
|
||||
});
|
||||
|
||||
it('counts groups an app hard-deleted, found by their absence', async () => {
|
||||
const h = createHarness();
|
||||
const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' } });
|
||||
addServerCards(h, ['Ada']);
|
||||
await h.run();
|
||||
|
||||
h.device.user.fossifyDeleteGroup(Number(h.device.rows('groups')[0]._id));
|
||||
const report = await h.run();
|
||||
|
||||
expect(report.stats.uploaded.deleted).toBe(1);
|
||||
expect(h.server.get('ContactCard', 'a', group)).toBeUndefined();
|
||||
expect(h.state()?.accounts.a.groups).toEqual([]);
|
||||
});
|
||||
|
||||
it('fails a crash-free run with CrashError only when injected', () => {
|
||||
expect(new CrashError('x').name).toBe('CrashError');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* The engine under test, wired to the fake providers, the fake JMAP server
|
||||
* and the toy planners, with helpers to read both sides and inject faults.
|
||||
*/
|
||||
import { vi } from 'vitest';
|
||||
import { Data, MimeType } from '../../android-columns';
|
||||
import { parseJsonColumn } from '../../common/json';
|
||||
import { runDeviceSync } from '../../engine/run';
|
||||
import { teardownAuthority, type TeardownOptions } from '../../engine/teardown';
|
||||
import type { EngineDeps, JmapConnection, Tuning } from '../../engine/deps';
|
||||
import { openWithAuthRebuild } from '../../jmap/connection';
|
||||
import type { SyncState } from '../../engine/sync-state';
|
||||
import {
|
||||
CALENDAR_AUTHORITY,
|
||||
CONTACTS_AUTHORITY,
|
||||
type AccountSyncPrefs,
|
||||
type Authority,
|
||||
type ProviderOp,
|
||||
type ProviderPort,
|
||||
type RunExtras,
|
||||
type RunReport,
|
||||
type Row,
|
||||
} from '../../types';
|
||||
import { FakeDeviceProviders } from '../fakes/fake-provider';
|
||||
import { FakeJmapServer } from '../fakes/fake-jmap-server';
|
||||
import { toyCalendarPlanner, toyContactsPlanner } from './toy-planners';
|
||||
|
||||
export const ANDROID_ACCOUNT = 'alice@example.com';
|
||||
export const REGISTRY_ID = 'alice@mail.example.com';
|
||||
export const ORIGIN = 'https://mail.example.com';
|
||||
export const START = Date.UTC(2026, 8, 27, 10, 0, 0);
|
||||
export const RUN_BUDGET = 9 * 60_000;
|
||||
|
||||
/** Deterministic randomness: uids and keys differ between items but repeat between test runs. */
|
||||
export function seededRandom(seed = 42): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s * 1664525 + 1013904223) >>> 0;
|
||||
return s / 2 ** 32;
|
||||
};
|
||||
}
|
||||
|
||||
export class CrashError extends Error {
|
||||
constructor(where: string) {
|
||||
super(`crash injected ${where}`);
|
||||
this.name = 'CrashError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface Harness {
|
||||
server: FakeJmapServer;
|
||||
device: FakeDeviceProviders;
|
||||
deps: EngineDeps;
|
||||
prefs: AccountSyncPrefs;
|
||||
book: string;
|
||||
calendar: string;
|
||||
clock: { now: number };
|
||||
/** Checkpoints seen by the last runs (each call of isCancelled). */
|
||||
checkpoints: { count: number; crashAt: number | null; onCheckpoint?: (n: number) => void };
|
||||
/** Batches applied through the engine's provider port; crash after `crashAfter` of them. */
|
||||
batches: { applied: number; crashAfter: number | null; log: ProviderOp[][] };
|
||||
run(authority?: Authority, extras?: RunExtras): Promise<RunReport>;
|
||||
teardown(authority: Authority, options?: TeardownOptions): Promise<{ pending: number }>;
|
||||
state(authority?: Authority): SyncState | null;
|
||||
contacts(): Array<{ id: number; sourceId: string | null; name: string | null; dirty: boolean; deleted: boolean; row: Row }>;
|
||||
contactNamed(name: string): { id: number; sourceId: string | null; dirty: boolean; deleted: boolean } | undefined;
|
||||
nameDataRow(rawContactId: number): Row | undefined;
|
||||
events(): Row[];
|
||||
serverNames(accountId?: string): string[];
|
||||
}
|
||||
|
||||
export function createHarness(options: { tuning?: Partial<Tuning>; accounts?: 'personal' | 'withShared'; rebuild?: boolean } = {}): Harness {
|
||||
const clock = { now: START };
|
||||
const server = new FakeJmapServer({ now: () => clock.now });
|
||||
server.addAccount('a', { name: ANDROID_ACCOUNT });
|
||||
const book = server.addAddressBook('a', { name: 'Personal', isDefault: true });
|
||||
const calendar = server.addCalendar('a', { name: 'Calendar', isDefault: true });
|
||||
if (options.accounts === 'withShared') {
|
||||
server.addAccount('team', { name: 'Team', isPersonal: false });
|
||||
server.addAddressBook('team', { name: 'Team contacts' });
|
||||
server.addCalendar('team', { name: 'Team calendar' });
|
||||
}
|
||||
const device = new FakeDeviceProviders();
|
||||
const prefs: AccountSyncPrefs = { contactsSelection: {}, calendarSelection: {}, reminderOwner: 'device' };
|
||||
const checkpoints: Harness['checkpoints'] = { count: 0, crashAt: null };
|
||||
const batches: Harness['batches'] = { applied: 0, crashAfter: null, log: [] };
|
||||
|
||||
const wrap = (port: ProviderPort): ProviderPort => ({
|
||||
accountName: port.accountName,
|
||||
authority: port.authority,
|
||||
query: (q) => port.query(q),
|
||||
readSyncState: () => port.readSyncState(),
|
||||
readPhoto: (id, px) => port.readPhoto(id, px),
|
||||
applyBatch: async (ops) => {
|
||||
if (batches.crashAfter !== null && batches.applied >= batches.crashAfter) throw new CrashError(`after ${batches.applied} batches`);
|
||||
const result = await port.applyBatch(ops);
|
||||
if (result.ok) {
|
||||
batches.applied++;
|
||||
batches.log.push(ops);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
const connection = (): JmapConnection => ({ port: server.port(), origin: ORIGIN });
|
||||
const deps: EngineDeps = {
|
||||
provider: (name, authority) => wrap(device.port(name, authority)),
|
||||
jmap: async () => (options.rebuild === false ? connection() : openWithAuthRebuild(async () => connection())),
|
||||
prefs: async () => prefs,
|
||||
planners: { contacts: toyContactsPlanner, calendar: toyCalendarPlanner },
|
||||
now: () => clock.now,
|
||||
isCancelled: async () => {
|
||||
checkpoints.count++;
|
||||
checkpoints.onCheckpoint?.(checkpoints.count);
|
||||
if (checkpoints.crashAt !== null && checkpoints.count >= checkpoints.crashAt) throw new CrashError(`at checkpoint ${checkpoints.count}`);
|
||||
return false;
|
||||
},
|
||||
random: seededRandom(),
|
||||
subscriptionCalendars: async () => [],
|
||||
isSyncEnabled: async () => true,
|
||||
deviceZone: () => 'Europe/Berlin',
|
||||
recordStatus: vi.fn(),
|
||||
recordKnownState: vi.fn(),
|
||||
notifyAuthProblem: vi.fn(),
|
||||
yieldThread: async () => undefined,
|
||||
sleep: async () => undefined,
|
||||
tuning: options.tuning,
|
||||
};
|
||||
|
||||
let runs = 0;
|
||||
const harness: Harness = {
|
||||
server,
|
||||
device,
|
||||
deps,
|
||||
prefs,
|
||||
book,
|
||||
calendar,
|
||||
clock,
|
||||
checkpoints,
|
||||
batches,
|
||||
run: (authority = CONTACTS_AUTHORITY, extras = {}) =>
|
||||
runDeviceSync(
|
||||
{ runId: `run-${++runs}`, accountName: ANDROID_ACCOUNT, registryId: REGISTRY_ID, authority, extras, deadline: clock.now + RUN_BUDGET },
|
||||
deps,
|
||||
),
|
||||
teardown: (authority, teardownOptions) => teardownAuthority(deps, REGISTRY_ID, ANDROID_ACCOUNT, authority, teardownOptions),
|
||||
state: (authority = CONTACTS_AUTHORITY) => parseJsonColumn<SyncState>(device.readSyncState(ANDROID_ACCOUNT, authority)),
|
||||
contacts: () =>
|
||||
device.rows('raw_contacts').map((row) => {
|
||||
const id = Number(row._id);
|
||||
const nameRow = harness.nameDataRow(id);
|
||||
return {
|
||||
id,
|
||||
sourceId: (row.sourceid as string | null) ?? null,
|
||||
name: (nameRow?.[Data.DATA1] as string | null) ?? null,
|
||||
dirty: Number(row.dirty) === 1,
|
||||
deleted: Number(row.deleted) === 1,
|
||||
row,
|
||||
};
|
||||
}),
|
||||
contactNamed: (name) => harness.contacts().find((c) => c.name === name),
|
||||
nameDataRow: (rawContactId) =>
|
||||
device.rows('data').find((d) => Number(d.raw_contact_id) === rawContactId && d.mimetype === MimeType.STRUCTURED_NAME),
|
||||
events: () => device.rows('events'),
|
||||
serverNames: (accountId = 'a') =>
|
||||
server
|
||||
.all('ContactCard', accountId)
|
||||
.filter((c) => c.kind !== 'group')
|
||||
.map((c) => String((c.name as { full?: string } | undefined)?.full ?? ''))
|
||||
.sort(),
|
||||
};
|
||||
return harness;
|
||||
}
|
||||
|
||||
/** A contact added on the device the way a Contacts app does (DIRTY, no SOURCE_ID). */
|
||||
export function addDeviceContact(h: Harness, name: string): number {
|
||||
return h.device.user.insertContact(ANDROID_ACCOUNT, [{ [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, [Data.DATA1]: name }]);
|
||||
}
|
||||
|
||||
/** An in-place edit of a contact's name (AOSP Contacts). */
|
||||
export function renameDeviceContact(h: Harness, rawContactId: number, name: string): void {
|
||||
const row = h.nameDataRow(rawContactId);
|
||||
if (!row) throw new Error(`no name row for ${rawContactId}`);
|
||||
h.device.user.updateData(Number(row._id), { [Data.DATA1]: name, [Data.DATA2]: null, [Data.DATA3]: null });
|
||||
}
|
||||
|
||||
export function addServerCards(h: Harness, names: string[], accountId = 'a', book = h.book): string[] {
|
||||
return names.map((name, i) => h.server.addCard(accountId, { uid: `uid-${name}-${i}`, name: { full: name }, addressBookIds: { [book]: true } }));
|
||||
}
|
||||
|
||||
/** Batches that wrote rows (anything but the SyncState op). */
|
||||
export function rowWrites(batches: ProviderOp[][]): ProviderOp[][] {
|
||||
return batches.filter((ops) => ops.some((op) => op.op !== 'syncState' && op.op !== 'assert'));
|
||||
}
|
||||
|
||||
export { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY };
|
||||
@@ -0,0 +1,328 @@
|
||||
// The engine's building blocks on their own: provider batches, the
|
||||
// SyncState blob, the run lock, SetError handling, query paging, the auth
|
||||
// rebuild and how failures map to outcomes.
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { BatchWriter, buildBatch, prependOps, type Work } from '../../engine/batch';
|
||||
import { classifyFailure, RunAbort, StateMismatch, StopRun } from '../../engine/errors';
|
||||
import { acquireLock } from '../../engine/mutex';
|
||||
import { isBackedOff, nextMarker } from '../../engine/poison';
|
||||
import { collectionDefaultOn, isCollectionSelected } from '../../engine/selection';
|
||||
import { emptySyncState, parseSyncState, serializeSyncState, StateStore } from '../../engine/sync-state';
|
||||
import { classifySetError, existingIdFromUidError, JmapCaller } from '../../jmap/caller';
|
||||
import { openWithAuthRebuild } from '../../jmap/connection';
|
||||
import { JMAPMethodError } from '../../jmap/errors';
|
||||
import { accountsWithCapability, calendarAddresses } from '../../jmap/session';
|
||||
import { JMAP_CALENDARS, JMAP_CONTACTS, type BatchResult, type ProviderOp, type ProviderPort } from '../../types';
|
||||
import { FakeJmapServer } from '../fakes/fake-jmap-server';
|
||||
|
||||
function port(apply: (ops: ProviderOp[]) => BatchResult | Promise<BatchResult>): ProviderPort & { sent: ProviderOp[][] } {
|
||||
const sent: ProviderOp[][] = [];
|
||||
return {
|
||||
accountName: 'x',
|
||||
authority: 'com.android.contacts',
|
||||
sent,
|
||||
query: async () => ({ columns: [], rows: [] }),
|
||||
readSyncState: async () => null,
|
||||
readPhoto: async () => null,
|
||||
applyBatch: async (ops) => {
|
||||
sent.push(ops);
|
||||
return apply(ops);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const ok = (ops: ProviderOp[]): BatchResult => ({ ok: true, results: ops.map(() => ({ count: 1 })) });
|
||||
const insert = (table: 'raw_contacts' | 'data', values: Record<string, string> = {}): ProviderOp => ({ op: 'insert', table, values });
|
||||
const writer = (p: ProviderPort, options: Partial<{ maxOps: number; maxBytes: number; maxReplans: number }> = {}) =>
|
||||
new BatchWriter(p, { maxOps: 400, maxBytes: 300_000, maxReplans: 3, ...options });
|
||||
|
||||
describe('provider batches', () => {
|
||||
it('rebases refs, makes each group start at a yield point and puts the state op last', () => {
|
||||
const group = { ref: 'g', ops: [insert('raw_contacts'), { op: 'insert', table: 'data', values: {}, refs: { raw_contact_id: 0 }, yieldAllowed: true } as ProviderOp] };
|
||||
|
||||
const ops = buildBatch([group, group], { op: 'syncState', value: '{}', yieldAllowed: true } as ProviderOp);
|
||||
|
||||
expect(ops.map((op) => (op as { yieldAllowed?: boolean }).yieldAllowed ?? false)).toEqual([true, false, true, false, false]);
|
||||
expect((ops[3] as { refs: Record<string, number> }).refs).toEqual({ raw_contact_id: 2 });
|
||||
expect(ops[4]).toEqual({ op: 'syncState', value: '{}' });
|
||||
});
|
||||
|
||||
it('prepends ops to a group, shifting its refs', () => {
|
||||
const group = prependOps({ ref: 'g', ops: [insert('raw_contacts'), { op: 'insert', table: 'data', values: {}, refs: { raw_contact_id: 0 } }] }, [
|
||||
{ op: 'assert', table: 'raw_contacts', where: 'sourceid = ?', args: ['a/1'], expectCount: 0 },
|
||||
]);
|
||||
expect((group.ops[2] as { refs: Record<string, number> }).refs).toEqual({ raw_contact_id: 1 });
|
||||
});
|
||||
|
||||
it('never splits a group, keeps batches within the op budget and sends photo groups alone', async () => {
|
||||
const p = port(ok);
|
||||
const works: Work[] = [
|
||||
...Array.from({ length: 5 }, (_, i) => ({ group: { ref: `g${i}`, ops: [insert('raw_contacts'), insert('data'), insert('data')] } })),
|
||||
{ group: { ref: 'photo', ops: [{ op: 'insert', table: 'data', values: { data15: { b64: 'AAAA' } } }] } },
|
||||
{ group: { ref: 'g5', ops: [insert('raw_contacts')] } },
|
||||
];
|
||||
|
||||
await writer(p, { maxOps: 7 }).write(works, { op: () => ({ op: 'syncState', value: 's' }), applied: () => undefined });
|
||||
|
||||
expect(p.sent.map((b) => b.length)).toEqual([6, 6, 3, 1, 2]);
|
||||
expect(p.sent[3]).toEqual([expect.objectContaining({ values: { data15: { b64: 'AAAA' } } })]);
|
||||
expect(p.sent[4][1]).toEqual({ op: 'syncState', value: 's' });
|
||||
});
|
||||
|
||||
it('retries the groups of a failed batch one by one and re-plans a failed assert at most three times', async () => {
|
||||
let failing = 5;
|
||||
const p = port((ops) =>
|
||||
ops.some((op) => op.op === 'assert') && failing-- > 0 ? { ok: false, reason: 'assert', message: 'stale' } : ok(ops),
|
||||
);
|
||||
const applied: string[] = [];
|
||||
const failed = vi.fn();
|
||||
let replans = 0;
|
||||
const stale: Work = {
|
||||
group: { ref: 'stale', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { version: 1 } }, insert('data')] },
|
||||
applied: () => {
|
||||
applied.push('stale');
|
||||
},
|
||||
failed,
|
||||
};
|
||||
stale.replan = async () => {
|
||||
replans++;
|
||||
return stale;
|
||||
};
|
||||
const fine: Work = {
|
||||
group: { ref: 'fine', ops: [insert('raw_contacts')] },
|
||||
applied: () => {
|
||||
applied.push('fine');
|
||||
},
|
||||
};
|
||||
const tail = { op: () => ({ op: 'syncState', value: 's' }) as ProviderOp, applied: vi.fn() };
|
||||
|
||||
await writer(p).write([fine, stale], tail);
|
||||
|
||||
expect(applied).toEqual(['fine']);
|
||||
expect(replans).toBe(3);
|
||||
expect(failed).toHaveBeenCalledWith('assert', 'stale');
|
||||
expect(tail.applied).toHaveBeenCalledOnce();
|
||||
expect(p.sent.at(-1)).toEqual([{ op: 'syncState', value: 's' }]);
|
||||
});
|
||||
|
||||
it('splits a batch the Binder refused (tooLarge) and gives up a single group that is too large', async () => {
|
||||
const p = port((ops) => (ops.length > 2 ? { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' } : ok(ops)));
|
||||
const failed = vi.fn();
|
||||
const works: Work[] = [
|
||||
...Array.from({ length: 4 }, (_, i) => ({ group: { ref: `g${i}`, ops: [insert('raw_contacts')] } })),
|
||||
{ group: { ref: 'big', ops: [insert('raw_contacts'), insert('data'), insert('data')] }, failed },
|
||||
];
|
||||
|
||||
await writer(p).write(works);
|
||||
|
||||
expect(failed).toHaveBeenCalledWith('tooLarge', 'TransactionTooLargeException');
|
||||
expect(p.sent.filter((b) => b.length <= 2).flat()).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('ends the run when the permission is gone and refuses to retry an op outside the account', async () => {
|
||||
const denied = port(() => ({ ok: false, reason: 'permission', message: 'SecurityException' }));
|
||||
await expect(writer(denied).write([{ group: { ref: 'g', ops: [insert('raw_contacts')] } }])).rejects.toMatchObject({ outcome: 'permission' });
|
||||
const scope = port(() => ({ ok: false, reason: 'scope', message: 'not our account' }));
|
||||
await expect(writer(scope).write([{ group: { ref: 'g', ops: [insert('raw_contacts')] } }])).rejects.toThrow(/not our account/);
|
||||
});
|
||||
|
||||
it('skips groups that only assert', async () => {
|
||||
const p = port(ok);
|
||||
await writer(p).write([{ group: { ref: 'g', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { dirty: 0 } }] } }]);
|
||||
expect(p.sent).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SyncState', () => {
|
||||
it('round-trips and tolerates missing parts', () => {
|
||||
const state = emptySyncState();
|
||||
state.owner = { registryId: 'r', origin: 'https://o' };
|
||||
state.accounts.a = { collectionsState: 's1', itemsState: 's2', selected: ['a/b'], stale: ['c1'], reconcile: null, groups: ['a/g'] };
|
||||
state.deviceZone = 'Europe/Berlin';
|
||||
|
||||
expect(parseSyncState(serializeSyncState(state))).toEqual({ state, readable: true });
|
||||
expect(parseSyncState(JSON.stringify({ v: 1, accounts: { a: { itemsState: 'x' } } })).state.accounts.a).toEqual({
|
||||
collectionsState: null,
|
||||
itemsState: 'x',
|
||||
selected: [],
|
||||
stale: [],
|
||||
reconcile: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('starts over when the blob is unreadable or of another version', () => {
|
||||
expect(parseSyncState('{nope').readable).toBe(false);
|
||||
expect(parseSyncState(JSON.stringify({ v: 2, accounts: {} })).readable).toBe(false);
|
||||
expect(parseSyncState(null)).toEqual({ state: emptySyncState(), readable: true });
|
||||
expect(parseSyncState('')).toEqual({ state: emptySyncState(), readable: true });
|
||||
});
|
||||
|
||||
it('adopts a proposed state only after its batch applied', () => {
|
||||
const store = new StateStore(emptySyncState(), { registryId: 'r', origin: 'o' });
|
||||
const tail = store.tail((next) => {
|
||||
next.deviceZone = 'UTC';
|
||||
});
|
||||
const op = tail.op();
|
||||
expect(store.committed.deviceZone).toBeUndefined();
|
||||
expect(JSON.parse((op as { value: string }).value)).toMatchObject({ deviceZone: 'UTC', owner: { registryId: 'r' } });
|
||||
tail.applied();
|
||||
expect(store.committed.deviceZone).toBe('UTC');
|
||||
});
|
||||
});
|
||||
|
||||
describe('run lock', () => {
|
||||
it('serialises holders and hands on the turn of a waiter that gave up', async () => {
|
||||
const order: string[] = [];
|
||||
const first = await acquireLock('k');
|
||||
const second = acquireLock('k', 10);
|
||||
const third = acquireLock('k');
|
||||
expect(await second).toBeNull();
|
||||
first!();
|
||||
const release = await third;
|
||||
order.push('third');
|
||||
release!();
|
||||
expect(order).toEqual(['third']);
|
||||
const again = await acquireLock('k', 10);
|
||||
expect(again).not.toBeNull();
|
||||
again!();
|
||||
});
|
||||
});
|
||||
|
||||
describe('poison markers', () => {
|
||||
it('backs off an hour, doubling to a day, while the item stays the same', () => {
|
||||
const first = nextMarker(null, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 });
|
||||
const second = nextMarker(first, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 });
|
||||
let marker = second;
|
||||
for (let i = 0; i < 10; i++) marker = nextMarker(marker, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 });
|
||||
expect([first.until, second.until, marker.until]).toEqual([3_600_000, 7_200_000, 86_400_000]);
|
||||
expect(nextMarker(second, 'changed', { type: 'invalidPatch' }, 0, { firstMs: 1, maxMs: 10 }).n).toBe(1);
|
||||
expect(isBackedOff(first, 'fp', 1)).toBe(true);
|
||||
expect(isBackedOff(first, 'other', 1)).toBe(false);
|
||||
expect(isBackedOff(first, 'fp', 3_600_001)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('JMAP side', () => {
|
||||
it('classifies SetErrors', () => {
|
||||
expect(classifySetError({ type: 'notFound' }, 'update')).toBe('notFound');
|
||||
expect(classifySetError({ type: 'forbidden' }, 'destroy')).toBe('forbidden');
|
||||
expect(classifySetError({ type: 'invalidProperties', properties: ['uid'] }, 'create')).toBe('uidExists');
|
||||
expect(classifySetError({ type: 'invalidProperties', properties: ['uid'] }, 'update')).toBe('poison');
|
||||
expect(classifySetError({ type: 'overQuota' }, 'create')).toBe('poison');
|
||||
expect(existingIdFromUidError({ type: 'invalidProperties', description: 'Contact with UID u1 already exists with id c5.' })).toBe('c5');
|
||||
expect(existingIdFromUidError({ type: 'invalidProperties', description: 'An event with UID u1 already exists.' })).toBeNull();
|
||||
});
|
||||
|
||||
it('pages /query by position until the total, with overlapping pages', async () => {
|
||||
const server = new FakeJmapServer();
|
||||
server.addAccount('a', { name: 'alice@example.com' });
|
||||
const book = server.addAddressBook('a', { name: 'Personal' });
|
||||
for (let i = 0; i < 30; i++) server.addCard('a', { uid: `u${i}` });
|
||||
server.setLimits({ maxQueryResults: 8 });
|
||||
const caller = new JmapCaller(server.port(), () => 0);
|
||||
|
||||
const ids = await caller.queryAll('ContactCard', 'a', { inAddressBook: book });
|
||||
|
||||
expect(ids).toHaveLength(30);
|
||||
expect(server.calls('ContactCard/query').length).toBeGreaterThan(3);
|
||||
});
|
||||
|
||||
it('looks uids up in one query and maps them back', async () => {
|
||||
const server = new FakeJmapServer();
|
||||
server.addAccount('a', { name: 'alice@example.com' });
|
||||
const book = server.addAddressBook('a', { name: 'Personal' });
|
||||
const other = server.addAddressBook('a', { name: 'Other' });
|
||||
const c1 = server.addCard('a', { uid: 'u1', addressBookIds: { [book]: true } });
|
||||
server.addCard('a', { uid: 'u2', addressBookIds: { [other]: true } });
|
||||
const caller = new JmapCaller(server.port(), () => 0);
|
||||
|
||||
const found = await caller.lookupUids('ContactCard', 'a', ['u1', 'u2', 'u3'], book);
|
||||
|
||||
expect([...found]).toEqual([['u1', [c1]]]);
|
||||
});
|
||||
|
||||
it('names the accounts with a capability, the primary first', () => {
|
||||
const server = new FakeJmapServer();
|
||||
server.addAccount('team', { name: 'Team', isPersonal: false, capabilities: ['contacts'] });
|
||||
server.addAccount('a', { name: 'alice@example.com' });
|
||||
const session = server.port().session();
|
||||
|
||||
expect(accountsWithCapability(session, JMAP_CONTACTS).map((a) => [a.id, a.primary, a.personal])).toEqual([
|
||||
['a', true, true],
|
||||
['team', false, false],
|
||||
]);
|
||||
expect(accountsWithCapability(session, JMAP_CALENDARS).map((a) => a.id)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('takes the login as the owner address and survives a server without identities', async () => {
|
||||
const server = new FakeJmapServer();
|
||||
server.addAccount('a', { name: 'Alice@Example.com' });
|
||||
const addresses = await calendarAddresses(new JmapCaller(server.port(), () => 0), 'fallback');
|
||||
expect(addresses).toEqual({ ownerAccount: 'alice@example.com', selfAddresses: ['alice@example.com'] });
|
||||
});
|
||||
|
||||
it('rebuilds the connection once when the credentials are rejected', async () => {
|
||||
const server = new FakeJmapServer();
|
||||
server.addAccount('a', { name: 'alice@example.com' });
|
||||
const load = vi.fn(async () => ({ port: server.port(), origin: 'o' }));
|
||||
const connection = await openWithAuthRebuild(load);
|
||||
server.failNextRequest('auth');
|
||||
|
||||
await connection.port.request([['Core/echo', {}, '0']], ['urn:ietf:params:jmap:core']);
|
||||
expect(load).toHaveBeenCalledTimes(2);
|
||||
|
||||
server.failNextRequest('auth');
|
||||
await expect(connection.port.request([['Core/echo', {}, '0']], ['urn:ietf:params:jmap:core'])).rejects.toMatchObject({ name: 'AuthenticationError' });
|
||||
expect(load).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('opens with a second client when the first has no session, and gives up after that', async () => {
|
||||
const connection = { port: new FakeJmapServer().port(), origin: 'o' };
|
||||
const loads = [null, connection];
|
||||
await expect(openWithAuthRebuild(async () => loads.shift() ?? null)).resolves.toMatchObject({ origin: 'o' });
|
||||
expect(loads).toEqual([]);
|
||||
const rejected = Object.assign(new Error('401'), { name: 'AuthenticationError' });
|
||||
await expect(openWithAuthRebuild(async () => { throw rejected; })).rejects.toMatchObject({ name: 'AuthenticationError' });
|
||||
await expect(openWithAuthRebuild(async () => null)).rejects.toMatchObject({ name: 'AuthenticationError' });
|
||||
const offline = Object.assign(new Error('offline'), { name: 'NetworkError' });
|
||||
await expect(openWithAuthRebuild(async () => { throw offline; })).rejects.toBe(offline);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selection defaults', () => {
|
||||
it("syncs the personal account's collections, not shared ones nor the Trusted Senders book, unless chosen", () => {
|
||||
expect(collectionDefaultOn(true, 'com.android.contacts', 'Personal')).toBe(true);
|
||||
expect(collectionDefaultOn(true, 'com.android.contacts', 'Trusted Senders')).toBe(false);
|
||||
expect(collectionDefaultOn(true, 'com.android.calendar', 'Trusted Senders')).toBe(true);
|
||||
expect(collectionDefaultOn(false, 'com.android.contacts', 'Team')).toBe(false);
|
||||
expect(collectionDefaultOn(true, 'com.android.calendar', null)).toBe(true);
|
||||
|
||||
const personal = { id: 'a', personal: true };
|
||||
const trusted = { id: 'ab9', name: 'Trusted Senders' };
|
||||
expect(isCollectionSelected({}, personal, 'com.android.contacts', trusted)).toBe(false);
|
||||
expect(isCollectionSelected({ 'a/ab9': true }, personal, 'com.android.contacts', trusted)).toBe(true);
|
||||
expect(isCollectionSelected({ 'a/ab1': false }, personal, 'com.android.contacts', { id: 'ab1', name: 'Personal' })).toBe(false);
|
||||
expect(isCollectionSelected({ 't/c1': true }, { id: 't', personal: false }, 'com.android.calendar', { id: 'c1' })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('outcomes', () => {
|
||||
it('maps failures to report outcomes', () => {
|
||||
const named = (name: string, extra: object = {}) => Object.assign(new Error(name), { name, ...extra });
|
||||
expect(classifyFailure(new StopRun('deadline'), 0)).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true });
|
||||
expect(classifyFailure(new RunAbort('safetyAbort', 'x'), 0)).toMatchObject({ outcome: 'safetyAbort' });
|
||||
expect(classifyFailure(new StateMismatch('a'), 0)).toMatchObject({ outcome: 'io', noProgress: true });
|
||||
expect(classifyFailure(named('AuthenticationError'), 0)).toMatchObject({ outcome: 'auth', authProblem: true });
|
||||
expect(classifyFailure(named('NetworkError'), 0)).toMatchObject({ outcome: 'io' });
|
||||
expect(classifyFailure(named('RequestTimeoutError'), 0)).toMatchObject({ outcome: 'io' });
|
||||
expect(classifyFailure(named('RateLimitError', { retryAfterMs: 5000 }), 10_000)).toMatchObject({ outcome: 'io', delayUntil: 15 });
|
||||
expect(classifyFailure(new JMAPMethodError('serverFail'), 0)).toMatchObject({ outcome: 'io' });
|
||||
expect(classifyFailure(new Error('JMAP request failed: 503 - busy'), 0)).toMatchObject({ outcome: 'io' });
|
||||
expect(
|
||||
classifyFailure(new Error('JMAP request failed: 400 - {"type":"urn:ietf:params:jmap:error:limit","limit":"maxCallsInRequest"}'), 0),
|
||||
).toMatchObject({ outcome: 'internal' });
|
||||
expect(classifyFailure(Object.assign(new Error('denied'), { code: 'permission' }), 0)).toMatchObject({ outcome: 'permission' });
|
||||
expect(classifyFailure(new TypeError('x is undefined'), 0)).toMatchObject({ outcome: 'internal' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
// The headless task entry: every run ends with finishRun and nothing
|
||||
// escapes it (docs/device-sync.md, "The Android shell").
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../native', () => ({
|
||||
createProviderPort: vi.fn(),
|
||||
finishRun: vi.fn(async () => true),
|
||||
getSyncSettings: vi.fn(async () => ({
|
||||
masterAutomatic: true,
|
||||
authorities: { 'com.android.contacts': { automatic: false, syncable: 1, periodicSeconds: 0, active: false, pending: false } },
|
||||
})),
|
||||
isRunCancelled: vi.fn(async () => false),
|
||||
showSyncProblem: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
import * as native from '../../native';
|
||||
import { runDeviceSyncTask, teardownAuthority } from '../../task';
|
||||
import { useDeviceSyncStore } from '../../../stores/device-sync-store';
|
||||
import type { RunPayload } from '../../types';
|
||||
|
||||
const payload: RunPayload = {
|
||||
runId: 'r1',
|
||||
accountName: 'alice@example.com',
|
||||
registryId: 'alice@mail.example.com',
|
||||
authority: 'com.android.contacts',
|
||||
extras: {},
|
||||
deadline: Date.now() + 60_000,
|
||||
};
|
||||
|
||||
describe('device sync task', () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(native.finishRun).mockClear();
|
||||
});
|
||||
|
||||
it('hands the report to the adapter before settling, and records the status', async () => {
|
||||
await runDeviceSyncTask(payload);
|
||||
|
||||
expect(native.finishRun).toHaveBeenCalledWith('r1', expect.objectContaining({ v: 1, runId: 'r1', outcome: 'disabled' }));
|
||||
expect(useDeviceSyncStore.getState().accounts['alice@mail.example.com']?.lastRun?.['com.android.contacts']).toMatchObject({
|
||||
outcome: 'disabled',
|
||||
});
|
||||
});
|
||||
|
||||
it('reports a failure instead of throwing, and survives a refused finishRun', async () => {
|
||||
vi.mocked(native.getSyncSettings).mockRejectedValueOnce(new Error('module gone'));
|
||||
vi.mocked(native.finishRun).mockRejectedValueOnce(new Error('bridge down'));
|
||||
|
||||
await expect(runDeviceSyncTask({ ...payload, runId: 'r2' })).resolves.toBeUndefined();
|
||||
|
||||
expect(native.finishRun).toHaveBeenCalledWith('r2', expect.objectContaining({ outcome: 'internal', message: 'module gone' }));
|
||||
});
|
||||
|
||||
it('ignores a payload without a run', async () => {
|
||||
await runDeviceSyncTask({} as RunPayload);
|
||||
expect(native.finishRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('exports the teardown the app calls', () => {
|
||||
expect(teardownAuthority).toBeTypeOf('function');
|
||||
expect(teardownAuthority.length).toBe(4);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,657 @@
|
||||
/**
|
||||
* Small planners for the engine tests, faithful to the planner contract
|
||||
* (src/device-sync/planner.ts) for a toy mapping:
|
||||
*
|
||||
* - contacts: `name.full` ↔ one StructuredName row (DISPLAY_NAME), group
|
||||
* cards ↔ Groups rows (TITLE), memberships ↔ GroupMembership rows;
|
||||
* - calendar: `title` ↔ one event row (plus the columns the provider
|
||||
* requires), calendars ↔ Calendars rows.
|
||||
*
|
||||
* Like the real planners they assert what they read first (VERSION and
|
||||
* DIRTY, or the event projection), keep baselines in DATA_SYNC3 /
|
||||
* SYNC_DATA4, merge dirty items per unit (server wins a real conflict),
|
||||
* adopt pending creates by uid, claim new items before creating them, and
|
||||
* return `effect: 'none'` for echoes.
|
||||
*/
|
||||
import { Data, Events, GroupMembership, MimeType } from '../../android-columns';
|
||||
import { collectionKey, parseCollectionKey, parseObjectRef, pendingRef, pendingUidOf } from '../../common/ids';
|
||||
import { deepEqual, parseJsonColumn } from '../../common/json';
|
||||
import { ptr } from '../../common/patch';
|
||||
import type {
|
||||
AcceptedPlan,
|
||||
CalendarContext,
|
||||
CalendarEventWire,
|
||||
CalendarPlanner,
|
||||
ContactCardWire,
|
||||
ContactsContext,
|
||||
ContactsPlanner,
|
||||
DownloadPlan,
|
||||
LocalCalendar,
|
||||
LocalContact,
|
||||
LocalEvent,
|
||||
LocalGroup,
|
||||
OpGroup,
|
||||
UploadAction,
|
||||
UploadPlan,
|
||||
} from '../../planner';
|
||||
import type { ProviderOp, Row } from '../../types';
|
||||
|
||||
const num = (v: unknown) => Number(v ?? 0);
|
||||
const flag = (v: unknown) => Number(v ?? 0) === 1;
|
||||
const str = (v: unknown): string | null => (typeof v === 'string' && v ? v : null);
|
||||
const onIds = (map: Record<string, boolean> | undefined | null) =>
|
||||
Object.entries(map ?? {}).filter(([, on]) => on === true).map(([id]) => id);
|
||||
|
||||
interface ToyContactsContext extends ContactsContext {
|
||||
groupsOf?(uid: string): string[];
|
||||
}
|
||||
|
||||
// ─── Contacts ───────────────────────────────────────────
|
||||
|
||||
function nameOf(card: ContactCardWire | null | undefined): string {
|
||||
return card?.name?.full ?? '';
|
||||
}
|
||||
|
||||
function nameRow(local: LocalContact) {
|
||||
return local.rows.find((r) => r.mimetype === MimeType.STRUCTURED_NAME) ?? null;
|
||||
}
|
||||
|
||||
function membershipRows(local: LocalContact) {
|
||||
return local.rows.filter((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP);
|
||||
}
|
||||
|
||||
function cardCollections(acct: string, card: ContactCardWire): string {
|
||||
return onIds(card.addressBookIds).map((id) => collectionKey(acct, id)).join(',');
|
||||
}
|
||||
|
||||
function shadowOf(card: ContactCardWire): string {
|
||||
return JSON.stringify(card);
|
||||
}
|
||||
|
||||
/** Group SOURCE_IDs a card should be a member of, with their row ids. */
|
||||
function wantedGroups(card: ContactCardWire, ctx: ToyContactsContext): Array<{ sourceId: string; rowId: number }> {
|
||||
if (typeof card.uid !== 'string' || !ctx.groupsOf) return [];
|
||||
return ctx
|
||||
.groupsOf(card.uid)
|
||||
.map((sourceId) => ({ sourceId, rowId: ctx.groupRowIdBySourceId(sourceId) }))
|
||||
.filter((g): g is { sourceId: string; rowId: number } => g.rowId !== null);
|
||||
}
|
||||
|
||||
function membershipOps(local: LocalContact | null, card: ContactCardWire, ctx: ToyContactsContext, parent: number | null): ProviderOp[] {
|
||||
const wanted = wantedGroups(card, ctx);
|
||||
const have = local ? membershipRows(local) : [];
|
||||
const ops: ProviderOp[] = [];
|
||||
for (const row of have) {
|
||||
if (!wanted.some((g) => g.sourceId === row.cells.group_sourceid)) ops.push({ op: 'delete', table: 'data', id: row.id, expectCount: 1 });
|
||||
}
|
||||
for (const group of wanted) {
|
||||
if (have.some((r) => r.cells.group_sourceid === group.sourceId)) continue;
|
||||
const values = { [Data.MIMETYPE]: MimeType.GROUP_MEMBERSHIP, [GroupMembership.GROUP_ROW_ID]: group.rowId };
|
||||
ops.push(
|
||||
parent === null
|
||||
? { op: 'insert', table: 'data', values, refs: { [Data.RAW_CONTACT_ID]: 0 } }
|
||||
: { op: 'insert', table: 'data', values: { ...values, [Data.RAW_CONTACT_ID]: parent } },
|
||||
);
|
||||
}
|
||||
return ops;
|
||||
}
|
||||
|
||||
export const toyContactsPlanner: ContactsPlanner = {
|
||||
rawContactColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4', 'raw_contact_is_read_only'],
|
||||
dataColumns: ['_id', 'raw_contact_id', 'mimetype', 'data1', 'data_sync1', 'data_sync3', 'group_sourceid'],
|
||||
groupColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'title', 'sync2', 'sync3', 'sync4'],
|
||||
|
||||
decodeContact(rc: Row, data: Row[]): LocalContact {
|
||||
return {
|
||||
rawContactId: num(rc._id),
|
||||
sourceId: str(rc.sourceid),
|
||||
version: num(rc.version),
|
||||
dirty: flag(rc.dirty),
|
||||
deleted: flag(rc.deleted),
|
||||
readOnly: flag(rc.raw_contact_is_read_only),
|
||||
collections: (str(rc.sync1) ?? '').split(',').filter(Boolean),
|
||||
shadow: parseJsonColumn<ContactCardWire>(rc.sync2),
|
||||
pending: parseJsonColumn(rc.sync3),
|
||||
poison: parseJsonColumn(rc.sync4),
|
||||
rows: data.map((d) => ({
|
||||
id: num(d._id),
|
||||
mimetype: String(d.mimetype),
|
||||
cells: { data1: d.data1 ?? null, group_sourceid: d.group_sourceid ?? null },
|
||||
key: str(d.data_sync1),
|
||||
photoHash: null,
|
||||
baseline: parseJsonColumn<Row>(d.data_sync3),
|
||||
})),
|
||||
};
|
||||
},
|
||||
|
||||
decodeGroup(row: Row): LocalGroup {
|
||||
return {
|
||||
groupId: num(row._id),
|
||||
sourceId: str(row.sourceid),
|
||||
version: num(row.version),
|
||||
dirty: flag(row.dirty),
|
||||
deleted: flag(row.deleted),
|
||||
title: str(row.title),
|
||||
shadow: parseJsonColumn<ContactCardWire>(row.sync2),
|
||||
pending: parseJsonColumn(row.sync3),
|
||||
poison: parseJsonColumn(row.sync4),
|
||||
};
|
||||
},
|
||||
|
||||
planDownload(card, local, context): DownloadPlan {
|
||||
const ctx = context as ToyContactsContext;
|
||||
const acct = ctx.jmapAccountId;
|
||||
const ref = `${acct}/${card.id}`;
|
||||
const name = nameOf(card);
|
||||
const raw = { sourceid: ref, sync1: cardCollections(acct, card), sync2: shadowOf(card), raw_contact_is_read_only: ctx.isReadOnly(card) ? 1 : 0 };
|
||||
if (!local) {
|
||||
const ops: ProviderOp[] = [
|
||||
{ op: 'insert', table: 'raw_contacts', values: raw },
|
||||
{
|
||||
op: 'insert',
|
||||
table: 'data',
|
||||
values: { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: name, data_sync1: 'name', data_sync3: JSON.stringify({ data1: name }) },
|
||||
refs: { [Data.RAW_CONTACT_ID]: 0 },
|
||||
},
|
||||
...membershipOps(null, card, ctx, null),
|
||||
];
|
||||
return { ops: { ref, ops }, conflicts: 0, stillDirty: false, effect: 'insert', writes: ops.length };
|
||||
}
|
||||
const row = nameRow(local);
|
||||
const guard: ProviderOp = {
|
||||
op: 'assert',
|
||||
table: 'raw_contacts',
|
||||
id: local.rawContactId,
|
||||
values: { version: local.version, dirty: local.dirty ? 1 : 0 },
|
||||
expectCount: 1,
|
||||
};
|
||||
const ops: ProviderOp[] = [guard];
|
||||
let conflicts = 0;
|
||||
let stillDirty = false;
|
||||
const writeName = (value: string) => {
|
||||
if (row) {
|
||||
ops.push({ op: 'update', table: 'data', id: row.id, values: { data1: value, data_sync3: JSON.stringify({ data1: value }) }, expectCount: 1 });
|
||||
} else {
|
||||
ops.push({
|
||||
op: 'insert',
|
||||
table: 'data',
|
||||
values: { [Data.RAW_CONTACT_ID]: local.rawContactId, [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: value, data_sync1: 'name', data_sync3: JSON.stringify({ data1: value }) },
|
||||
});
|
||||
}
|
||||
};
|
||||
const adopting = !local.sourceId && local.pending?.uid === card.uid;
|
||||
if (local.deleted) {
|
||||
// The device delete wins over a server edit; the destroy is on its way.
|
||||
return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 };
|
||||
}
|
||||
const localValue = String(row?.cells.data1 ?? '');
|
||||
if (adopting) {
|
||||
// Our own create: identity and shadow; the rows stay the user's, measured against the server's value.
|
||||
ops.push({ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { ...raw, sync3: null } });
|
||||
if (row) ops.push({ op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: name }) }, expectCount: 1 });
|
||||
return { ops: { ref, ops }, conflicts: 0, stillDirty: localValue !== name, effect: 'update', writes: ops.length - 1 };
|
||||
}
|
||||
const shadowName = nameOf(local.shadow);
|
||||
const baseValue = String(row?.baseline?.data1 ?? shadowName);
|
||||
const localChanged = local.dirty && localValue !== baseValue;
|
||||
const remoteChanged = name !== shadowName;
|
||||
if (!localChanged) {
|
||||
if (localValue !== name) writeName(name);
|
||||
} else if (remoteChanged) {
|
||||
if (localValue === name) writeName(name);
|
||||
else {
|
||||
writeName(name);
|
||||
conflicts++;
|
||||
}
|
||||
} else {
|
||||
stillDirty = true;
|
||||
}
|
||||
ops.push(...membershipOps(local, card, ctx, local.rawContactId));
|
||||
const shadowSame = deepEqual(local.shadow, card) && local.collections.join(',') === raw.sync1 && local.readOnly === !!raw.raw_contact_is_read_only;
|
||||
if (!shadowSame) ops.push({ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: raw });
|
||||
const writes = ops.length - 1;
|
||||
return { ops: { ref, ops }, conflicts, stillDirty, effect: writes > 0 ? 'update' : 'none', writes };
|
||||
},
|
||||
|
||||
planLocalDelete(local): OpGroup {
|
||||
return { ref: local.sourceId ?? `row:${local.rawContactId}`, ops: [{ op: 'delete', table: 'raw_contacts', id: local.rawContactId }] };
|
||||
},
|
||||
|
||||
planBaselineHeal(local): OpGroup | null {
|
||||
const row = nameRow(local);
|
||||
if (local.dirty || !row || deepEqual(row.baseline, { data1: row.cells.data1 })) return null;
|
||||
return {
|
||||
ref: local.sourceId ?? `row:${local.rawContactId}`,
|
||||
ops: [
|
||||
{ op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version, dirty: 0 }, expectCount: 1 },
|
||||
{ op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: row.cells.data1 }) }, expectCount: 1 },
|
||||
],
|
||||
};
|
||||
},
|
||||
|
||||
planUpload(local, ctx): UploadPlan<ContactCardWire> {
|
||||
const ref = local.sourceId ?? `row:${local.rawContactId}`;
|
||||
const guard: ProviderOp = { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version }, expectCount: 1 };
|
||||
const parsed = parseObjectRef(local.sourceId);
|
||||
if (local.deleted) {
|
||||
if (parsed) {
|
||||
const books = onIds(local.shadow?.addressBookIds);
|
||||
const synced = books.filter((b) => ctx.isSelected(collectionKey(parsed.accountId, b)));
|
||||
if (synced.length && synced.length < books.length) {
|
||||
return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch: Object.fromEntries(synced.map((b) => [`addressBookIds/${b}`, null])) }] };
|
||||
}
|
||||
return { kind: 'upload', actions: [{ kind: 'destroy', id: parsed.id, uid: null }] };
|
||||
}
|
||||
if (local.pending) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: local.pending.uid }] };
|
||||
return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'raw_contacts', id: local.rawContactId }] } };
|
||||
}
|
||||
const row = nameRow(local);
|
||||
const value = String(row?.cells.data1 ?? '');
|
||||
if (!parsed) {
|
||||
if (!local.pending) {
|
||||
const target = ctx.createTarget();
|
||||
if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' };
|
||||
return {
|
||||
kind: 'claim',
|
||||
ops: { ref, ops: [guard, { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { sync3: JSON.stringify({ uid: ctx.mintUid(), target }) } }] },
|
||||
};
|
||||
}
|
||||
const target = parseCollectionKey(local.pending.target);
|
||||
if (!target) return { kind: 'skip', reason: 'badTarget' };
|
||||
return {
|
||||
kind: 'upload',
|
||||
actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object: { uid: local.pending.uid, name: { full: value } } }],
|
||||
};
|
||||
}
|
||||
if (local.readOnly) {
|
||||
const shadowName = nameOf(local.shadow);
|
||||
return {
|
||||
kind: 'revert',
|
||||
ops: {
|
||||
ref,
|
||||
ops: [
|
||||
guard,
|
||||
...(row ? [{ op: 'update' as const, table: 'data' as const, id: row.id, values: { data1: shadowName, data_sync3: JSON.stringify({ data1: shadowName }) } }] : []),
|
||||
{ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { dirty: 0 } },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
const base = String(row?.baseline?.data1 ?? nameOf(local.shadow));
|
||||
if (value === base) {
|
||||
return { kind: 'clean', ops: { ref, ops: [{ ...guard, values: { version: local.version, dirty: 1 } }, { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { dirty: 0 } }] } };
|
||||
}
|
||||
const patch = local.shadow?.name ? { 'name/full': value } : { name: { full: value } };
|
||||
return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch }] };
|
||||
},
|
||||
|
||||
planAccepted(local, server, ctx): AcceptedPlan {
|
||||
const acct = ctx.jmapAccountId;
|
||||
const ref = `${acct}/${server.id}`;
|
||||
const row = nameRow(local);
|
||||
const identity = { sourceid: ref, sync1: cardCollections(acct, server), sync2: shadowOf(server), sync3: null };
|
||||
const baseline: ProviderOp[] = row
|
||||
? [{ op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: row.cells.data1 }) }, expectCount: 1 }]
|
||||
: [];
|
||||
return {
|
||||
ops: {
|
||||
ref,
|
||||
ops: [
|
||||
{ op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version }, expectCount: 1 },
|
||||
{ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { ...identity, dirty: 0 } },
|
||||
...baseline,
|
||||
],
|
||||
},
|
||||
keepDirtyOps: { ref, ops: [{ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: identity }, ...baseline] },
|
||||
};
|
||||
},
|
||||
|
||||
planGroupDownload(card, local, ctx): DownloadPlan {
|
||||
const acct = ctx.jmapAccountId;
|
||||
const ref = `${acct}/${card.id}`;
|
||||
const values = { sourceid: ref, title: nameOf(card), group_visible: 1, sync2: shadowOf(card) };
|
||||
if (!local) {
|
||||
return { ops: { ref, ops: [{ op: 'insert', table: 'groups', values }] }, conflicts: 0, stillDirty: false, effect: 'insert', writes: 1 };
|
||||
}
|
||||
if (local.deleted) return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 };
|
||||
const adopting = !local.sourceId && local.pending?.uid === card.uid;
|
||||
if (!adopting && local.title === values.title && deepEqual(local.shadow, card)) {
|
||||
return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: local.dirty, effect: 'none', writes: 0 };
|
||||
}
|
||||
const localChanged = local.dirty && local.title !== nameOf(local.shadow);
|
||||
const keepTitle = adopting || (localChanged && values.title === nameOf(local.shadow));
|
||||
return {
|
||||
ops: {
|
||||
ref,
|
||||
ops: [
|
||||
{ op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 },
|
||||
{ op: 'update', table: 'groups', id: local.groupId, values: { ...values, ...(keepTitle ? { title: local.title } : {}), sync3: null } },
|
||||
],
|
||||
},
|
||||
conflicts: localChanged && !keepTitle && values.title !== local.title ? 1 : 0,
|
||||
stillDirty: keepTitle && local.dirty,
|
||||
effect: 'update',
|
||||
writes: 1,
|
||||
};
|
||||
},
|
||||
|
||||
planGroupLocalDelete(local): OpGroup {
|
||||
return { ref: local.sourceId ?? `group:${local.groupId}`, ops: [{ op: 'delete', table: 'groups', id: local.groupId }] };
|
||||
},
|
||||
|
||||
planGroupUpload(local, ctx): UploadPlan<ContactCardWire> {
|
||||
const ref = local.sourceId ?? `group:${local.groupId}`;
|
||||
const parsed = parseObjectRef(local.sourceId);
|
||||
if (local.deleted) {
|
||||
if (parsed) return { kind: 'upload', actions: [{ kind: 'destroy', id: parsed.id, uid: null }] };
|
||||
return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'groups', id: local.groupId }] } };
|
||||
}
|
||||
if (!parsed) {
|
||||
if (!local.pending) {
|
||||
const target = ctx.createTarget();
|
||||
if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' };
|
||||
return {
|
||||
kind: 'claim',
|
||||
ops: {
|
||||
ref,
|
||||
ops: [
|
||||
{ op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 },
|
||||
{ op: 'update', table: 'groups', id: local.groupId, values: { sync3: JSON.stringify({ uid: ctx.mintUid(), target }) } },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
const target = parseCollectionKey(local.pending.target);
|
||||
if (!target) return { kind: 'skip', reason: 'badTarget' };
|
||||
return {
|
||||
kind: 'upload',
|
||||
actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object: { uid: local.pending.uid, kind: 'group', name: { full: local.title ?? '' } } }],
|
||||
};
|
||||
}
|
||||
if ((local.title ?? '') === nameOf(local.shadow)) {
|
||||
return {
|
||||
kind: 'clean',
|
||||
ops: { ref, ops: [{ op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, { op: 'update', table: 'groups', id: local.groupId, values: { dirty: 0 } }] },
|
||||
};
|
||||
}
|
||||
return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch: { 'name/full': local.title ?? '' } }] };
|
||||
},
|
||||
|
||||
planGroupAccepted(local, server, ctx): AcceptedPlan {
|
||||
const ref = `${ctx.jmapAccountId}/${server.id}`;
|
||||
const values = { sourceid: ref, sync2: shadowOf(server), sync3: null };
|
||||
return {
|
||||
ops: {
|
||||
ref,
|
||||
ops: [
|
||||
{ op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 },
|
||||
{ op: 'update', table: 'groups', id: local.groupId, values: { ...values, dirty: 0 } },
|
||||
],
|
||||
},
|
||||
keepDirtyOps: { ref, ops: [{ op: 'update', table: 'groups', id: local.groupId, values }] },
|
||||
};
|
||||
},
|
||||
|
||||
planMembershipUploads(contacts, groups, ctx): UploadAction<ContactCardWire>[] {
|
||||
const acct = ctx.jmapAccountId;
|
||||
const patches = new Map<string, Record<string, unknown>>();
|
||||
for (const contact of contacts) {
|
||||
const uid = contact.shadow?.uid ?? contact.pending?.uid;
|
||||
if (!contact.sourceId || typeof uid !== 'string') continue;
|
||||
const wanted = new Set(membershipRows(contact).map((r) => String(r.cells.group_sourceid)));
|
||||
for (const group of groups) {
|
||||
const parsed = parseObjectRef(group.sourceId);
|
||||
if (!parsed || parsed.accountId !== acct) continue;
|
||||
const member = group.shadow?.members?.[uid] === true;
|
||||
const want = wanted.has(group.sourceId as string);
|
||||
if (member === want) continue;
|
||||
const patch = patches.get(parsed.id) ?? {};
|
||||
patch[ptr('members', uid)] = want ? true : null;
|
||||
patches.set(parsed.id, patch);
|
||||
}
|
||||
}
|
||||
return [...patches].map(([id, patch]) => ({ kind: 'update', id, patch }));
|
||||
},
|
||||
};
|
||||
|
||||
// ─── Calendar ───────────────────────────────────────────
|
||||
|
||||
const EVENT_HOUR = 3_600_000;
|
||||
|
||||
function startMs(event: CalendarEventWire): number {
|
||||
return Date.parse(`${String(event.start ?? '2026-01-01T00:00:00').slice(0, 19)}Z`);
|
||||
}
|
||||
|
||||
function localStart(ms: number): string {
|
||||
return new Date(ms).toISOString().slice(0, 19);
|
||||
}
|
||||
|
||||
function eventRef(ctx: CalendarContext, event: CalendarEventWire): string {
|
||||
return `${ctx.jmapAccountId}/${event.id}`;
|
||||
}
|
||||
|
||||
export const toyCalendarPlanner: CalendarPlanner = {
|
||||
calendarColumns: ['_id', '_sync_id', 'name', 'calendar_displayName', 'calendar_access_level', 'maxReminders', 'cal_sync2', 'cal_sync3'],
|
||||
eventColumns: [
|
||||
'_id', 'calendar_id', '_sync_id', 'dirty', 'deleted', 'title', 'dtstart', 'dtend', 'eventTimezone', 'original_id',
|
||||
'original_sync_id', 'originalInstanceTime', 'sync_data1', 'sync_data2', 'sync_data3', 'sync_data4', 'sync_data5', 'uid2445', 'mutators',
|
||||
],
|
||||
attendeeColumns: ['_id', 'event_id', 'attendeeEmail'],
|
||||
reminderColumns: ['_id', 'event_id', 'minutes'],
|
||||
|
||||
decodeCalendar(row: Row): LocalCalendar {
|
||||
return {
|
||||
calendarRowId: num(row._id),
|
||||
syncId: str(row._sync_id),
|
||||
cells: row,
|
||||
shadow: parseJsonColumn(row.cal_sync2),
|
||||
flags: parseJsonColumn(row.cal_sync3),
|
||||
};
|
||||
},
|
||||
|
||||
decodeEvents(events: Row[], attendees: Row[], reminders: Row[]): LocalEvent[] {
|
||||
const toRow = (row: Row) => ({
|
||||
eventId: num(row._id),
|
||||
calendarRowId: num(row.calendar_id),
|
||||
syncId: str(row._sync_id),
|
||||
dirty: flag(row.dirty),
|
||||
deleted: flag(row.deleted),
|
||||
cells: { title: row.title ?? null, dtstart: row.dtstart ?? null, dtend: row.dtend ?? null },
|
||||
attendees: attendees.filter((a) => num(a.event_id) === num(row._id)).map((a) => ({ id: num(a._id), cells: { attendeeEmail: a.attendeeEmail ?? null } })),
|
||||
reminders: reminders.filter((r) => num(r.event_id) === num(row._id)).map((r) => ({ id: num(r._id), cells: { minutes: r.minutes ?? null } })),
|
||||
baseline: parseJsonColumn<{ cells: Row; attendees: Row[]; reminders: Row[] }>(row.sync_data4),
|
||||
pending: parseJsonColumn(row.sync_data3),
|
||||
poison: parseJsonColumn(row.sync_data5),
|
||||
mutators: str(row.mutators),
|
||||
});
|
||||
const isException = (row: Row) => row.original_id !== null && row.original_id !== undefined || !!str(row.original_sync_id);
|
||||
const masters = events.filter((r) => !isException(r)).map((row) => ({
|
||||
...toRow(row),
|
||||
shadow: parseJsonColumn<CalendarEventWire>(row.sync_data1),
|
||||
exceptions: events
|
||||
.filter((x) => isException(x) && (num(x.original_id) === num(row._id) || (str(x.original_sync_id) !== null && str(x.original_sync_id) === str(row._sync_id))))
|
||||
.map((x) => ({ ...toRow(x), recurrenceId: str(x.sync_data2), originalInstanceTime: x.originalInstanceTime === null ? null : num(x.originalInstanceTime) })),
|
||||
})) as LocalEvent[];
|
||||
const bySyncId = new Map<string, LocalEvent[]>();
|
||||
for (const m of masters) if (m.syncId) bySyncId.set(m.syncId, [...(bySyncId.get(m.syncId) ?? []), m]);
|
||||
for (const pair of bySyncId.values()) {
|
||||
if (pair.length < 2) continue;
|
||||
pair.sort((a, b) => a.eventId - b.eventId);
|
||||
pair[0].split = 'source';
|
||||
for (const clone of pair.slice(1)) clone.split = 'clone';
|
||||
}
|
||||
return masters;
|
||||
},
|
||||
|
||||
planCalendars(selected, local, ctx) {
|
||||
const groups: OpGroup[] = [];
|
||||
const keys = new Set<string>();
|
||||
for (const { jmapAccountId, calendar, readOnly, accountName } of selected) {
|
||||
const key = collectionKey(jmapAccountId, calendar.id);
|
||||
keys.add(key);
|
||||
const values = {
|
||||
name: calendar.name,
|
||||
calendar_displayName: accountName ? `${calendar.name} (${accountName})` : calendar.name,
|
||||
calendar_access_level: readOnly ? 200 : 700,
|
||||
maxReminders: ctx.reminderOwner === 'device' ? 5 : 0,
|
||||
cal_sync2: JSON.stringify(calendar),
|
||||
cal_sync3: JSON.stringify(readOnly ? { readOnly: 'rights' } : {}),
|
||||
};
|
||||
const existing = local.find((l) => l.syncId === key);
|
||||
if (!existing) {
|
||||
groups.push({
|
||||
ref: key,
|
||||
ops: [{ op: 'insert', table: 'calendars', values: { ...values, _sync_id: key, calendar_color: -16776961, ownerAccount: ctx.ownerAccount, sync_events: 1, visible: 1 } }],
|
||||
});
|
||||
} else if (Object.entries(values).some(([k, v]) => String(existing.cells[k] ?? '') !== String(v))) {
|
||||
groups.push({ ref: key, ops: [{ op: 'update', table: 'calendars', id: existing.calendarRowId, values }] });
|
||||
}
|
||||
}
|
||||
return { groups, deleteCalendarRows: local.filter((l) => !l.syncId || !keys.has(l.syncId)).map((l) => l.calendarRowId) };
|
||||
},
|
||||
|
||||
planDownload(event, local, ctx): DownloadPlan {
|
||||
const ref = eventRef(ctx, event);
|
||||
const calendarRowId = onIds(event.calendarIds).map((id) => ctx.calendarRowId(id)).find((id) => id !== null) ?? null;
|
||||
const start = startMs(event);
|
||||
const cells = { title: event.title ?? '', dtstart: start, dtend: start + EVENT_HOUR };
|
||||
const baseline = JSON.stringify({ cells: { title: cells.title }, attendees: [], reminders: [] });
|
||||
if (!local) {
|
||||
if (calendarRowId === null) return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: false, effect: 'none', writes: 0 };
|
||||
const ops: ProviderOp[] = [{
|
||||
op: 'insert',
|
||||
table: 'events',
|
||||
// CalendarProvider leaves DIRTY NULL on a sync-adapter insert unless it is written.
|
||||
values: { calendar_id: calendarRowId, _sync_id: ref, ...cells, eventTimezone: 'UTC', eventStatus: 1, dirty: 0, uid2445: event.uid, sync_data1: JSON.stringify(event), sync_data4: baseline },
|
||||
}];
|
||||
return { ops: { ref, ops }, conflicts: 0, stillDirty: false, effect: 'insert', writes: 1 };
|
||||
}
|
||||
const guard: ProviderOp = { op: 'assert', table: 'events', id: local.eventId, values: { dirty: local.dirty ? 1 : 0, title: local.cells.title }, expectCount: 1 };
|
||||
if (local.deleted) return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 };
|
||||
const adopting = !parseObjectRef(local.syncId) && (local.pending?.uid ?? pendingUidOf(local.syncId)) === event.uid;
|
||||
if (adopting) {
|
||||
return {
|
||||
ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { _sync_id: ref, sync_data1: JSON.stringify(event), sync_data3: null, sync_data4: baseline } }] },
|
||||
conflicts: 0,
|
||||
stillDirty: local.cells.title !== event.title,
|
||||
effect: 'update',
|
||||
writes: 1,
|
||||
};
|
||||
}
|
||||
const shadowTitle = local.shadow?.title ?? '';
|
||||
const baseTitle = String(local.baseline?.cells.title ?? shadowTitle);
|
||||
const localChanged = local.dirty && local.cells.title !== baseTitle;
|
||||
const remoteChanged = (event.title ?? '') !== shadowTitle;
|
||||
const values: Row = {};
|
||||
let conflicts = 0;
|
||||
if (!localChanged || remoteChanged) {
|
||||
if (local.cells.title !== cells.title || local.cells.dtstart !== cells.dtstart) Object.assign(values, cells, { sync_data4: baseline });
|
||||
if (localChanged && local.cells.title !== cells.title) conflicts = 1;
|
||||
}
|
||||
if (!deepEqual(local.shadow, event)) values.sync_data1 = JSON.stringify(event);
|
||||
if (!Object.keys(values).length) return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: localChanged && !remoteChanged, effect: 'none', writes: 0 };
|
||||
return {
|
||||
ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values, expectCount: 1 }] },
|
||||
conflicts,
|
||||
stillDirty: localChanged && !remoteChanged,
|
||||
effect: 'update',
|
||||
writes: 1,
|
||||
};
|
||||
},
|
||||
|
||||
planLocalDelete(local): OpGroup {
|
||||
return {
|
||||
ref: local.syncId ?? `row:${local.eventId}`,
|
||||
ops: [
|
||||
...local.exceptions.map((x) => ({ op: 'delete' as const, table: 'events' as const, id: x.eventId })),
|
||||
{ op: 'delete', table: 'events', id: local.eventId },
|
||||
],
|
||||
};
|
||||
},
|
||||
|
||||
planBaselineHeal() {
|
||||
return null;
|
||||
},
|
||||
|
||||
planUpload(local, ctx): UploadPlan<CalendarEventWire> {
|
||||
const ref = local.syncId ?? `row:${local.eventId}`;
|
||||
const identity = local.split === 'clone' ? null : parseObjectRef(local.syncId);
|
||||
const pendingUid = local.pending?.uid ?? pendingUidOf(local.syncId);
|
||||
if (local.deleted) {
|
||||
if (identity) return { kind: 'upload', actions: [{ kind: 'destroy', id: identity.id, uid: null }] };
|
||||
if (pendingUid) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: pendingUid }] };
|
||||
return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'events', id: local.eventId }] } };
|
||||
}
|
||||
const guard: ProviderOp = { op: 'assert', table: 'events', id: local.eventId, values: { title: local.cells.title }, expectCount: 1 };
|
||||
if (!identity) {
|
||||
const calendar = ctx.calendarIdOfRow(local.calendarRowId);
|
||||
if (!calendar) return { kind: 'skip', reason: 'notOurCalendar' };
|
||||
if (!pendingUid || local.split === 'clone') {
|
||||
const uid = ctx.mintUid();
|
||||
return {
|
||||
kind: 'claim',
|
||||
ops: {
|
||||
ref,
|
||||
ops: [guard, {
|
||||
op: 'update',
|
||||
table: 'events',
|
||||
id: local.eventId,
|
||||
values: { _sync_id: pendingRef(uid), uid2445: uid, sync_data3: JSON.stringify({ uid, target: collectionKey(calendar.jmapAccountId, calendar.calendarId) }) },
|
||||
}],
|
||||
},
|
||||
};
|
||||
}
|
||||
const start = Number(local.cells.dtstart ?? 0);
|
||||
return {
|
||||
kind: 'upload',
|
||||
actions: [{
|
||||
kind: 'create',
|
||||
uid: pendingUid,
|
||||
collectionId: calendar.calendarId,
|
||||
object: { uid: pendingUid, title: String(local.cells.title ?? ''), start: localStart(start), duration: 'PT1H', timeZone: 'Etc/UTC' },
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (ctx.isReadOnly(onIds(local.shadow?.calendarIds)[0] ?? '')) {
|
||||
return { kind: 'revert', ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { title: local.shadow?.title ?? '', dirty: 0 } }] } };
|
||||
}
|
||||
const base = String(local.baseline?.cells.title ?? local.shadow?.title ?? '');
|
||||
if (local.cells.title === base) {
|
||||
return { kind: 'clean', ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { dirty: 0 } }] } };
|
||||
}
|
||||
return { kind: 'upload', actions: [{ kind: 'update', id: identity.id, patch: { title: String(local.cells.title ?? '') } }] };
|
||||
},
|
||||
|
||||
planAccepted(local, server, ctx): AcceptedPlan {
|
||||
const ref = eventRef(ctx, server);
|
||||
const values = {
|
||||
_sync_id: ref,
|
||||
sync_data1: JSON.stringify(server),
|
||||
sync_data3: null,
|
||||
sync_data4: JSON.stringify({ cells: { title: local.cells.title }, attendees: [], reminders: [] }),
|
||||
};
|
||||
return {
|
||||
ops: {
|
||||
ref,
|
||||
ops: [
|
||||
{ op: 'assert', table: 'events', id: local.eventId, values: { title: local.cells.title }, expectCount: 1 },
|
||||
{ op: 'update', table: 'events', id: local.eventId, values: { ...values, dirty: 0 } },
|
||||
],
|
||||
},
|
||||
keepDirtyOps: { ref, ops: [{ op: 'update', table: 'events', id: local.eventId, values }] },
|
||||
};
|
||||
},
|
||||
|
||||
planPairs() {
|
||||
return [];
|
||||
},
|
||||
planZoneChange() {
|
||||
return null;
|
||||
},
|
||||
planReminderOwnerChange() {
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
/** An Events column list the fake accepts in `where`, for tests that read rows. */
|
||||
export const EVENT_TITLE = Events.TITLE;
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* Packs item op groups into provider batches and applies them
|
||||
* (docs/device-sync.md, "SyncState", "Native module API", "Failure matrix"):
|
||||
*
|
||||
* - a group is never split; `refs` are rebased to the batch and the group's
|
||||
* first op is its yield point, so the provider may commit between groups
|
||||
* but never inside one;
|
||||
* - at most `maxOps` ops and about `maxBytes` of JSON per batch; groups that
|
||||
* carry photo bytes travel alone (Binder budget);
|
||||
* - the SyncState op, when there is one, is the last op of the last batch
|
||||
* and never a yield point: the stored state can't get ahead of its rows.
|
||||
*
|
||||
* A failed batch may have been committed up to a yield point the provider
|
||||
* took, so its groups are retried one by one (their asserts make a retry of
|
||||
* a committed group fail harmlessly); a group that fails an assert is
|
||||
* re-read and re-planned up to `maxReplans` times, then given up. `tooLarge`
|
||||
* splits the batch; `permission` ends the run.
|
||||
*/
|
||||
import type { OpGroup } from '../planner';
|
||||
import type { BatchFailure, OpResult, ProviderOp, ProviderPort } from '../types';
|
||||
import { RunAbort } from './errors';
|
||||
import { hasWrites } from './provider';
|
||||
import type { Tail } from './sync-state';
|
||||
|
||||
export interface Work {
|
||||
group: OpGroup;
|
||||
/** The group applied; `results` are its ops' results (new row ids for inserts). */
|
||||
applied?(results: OpResult[]): void | Promise<void>;
|
||||
/** After an assert failure: read the item again and plan it again; null when nothing is left to write. */
|
||||
replan?(): Promise<Work | null>;
|
||||
/** Given up: the item's rows stay as they are. */
|
||||
failed?(reason: BatchFailure, message: string): void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface BatchWriterOptions {
|
||||
maxOps: number;
|
||||
maxBytes: number;
|
||||
maxReplans: number;
|
||||
}
|
||||
|
||||
function sizeOf(ops: readonly ProviderOp[]): number {
|
||||
return JSON.stringify(ops).length;
|
||||
}
|
||||
|
||||
function carriesBlob(ops: readonly ProviderOp[]): boolean {
|
||||
return ops.some(
|
||||
(op) =>
|
||||
(op.op === 'insert' || op.op === 'update') &&
|
||||
Object.values(op.values).some((v) => v !== null && typeof v === 'object'),
|
||||
);
|
||||
}
|
||||
|
||||
/** A group's ops at `offset` in a batch: refs rebased, the first op a yield point, no other. */
|
||||
function placeGroup(ops: readonly ProviderOp[], offset: number): ProviderOp[] {
|
||||
return ops.map((op, i) => {
|
||||
const copy = { ...op } as ProviderOp & { yieldAllowed?: boolean };
|
||||
if (copy.op === 'insert' && copy.refs) {
|
||||
const refs: Record<string, number> = {};
|
||||
for (const [column, index] of Object.entries(copy.refs)) refs[column] = index + offset;
|
||||
copy.refs = refs;
|
||||
}
|
||||
if (i === 0) copy.yieldAllowed = true;
|
||||
else delete copy.yieldAllowed;
|
||||
return copy;
|
||||
});
|
||||
}
|
||||
|
||||
export function buildBatch(groups: readonly OpGroup[], tailOp?: ProviderOp): ProviderOp[] {
|
||||
const out: ProviderOp[] = [];
|
||||
for (const group of groups) out.push(...placeGroup(group.ops, out.length));
|
||||
if (tailOp) {
|
||||
const { yieldAllowed: _never, ...last } = tailOp as ProviderOp & { yieldAllowed?: boolean };
|
||||
out.push(last as ProviderOp);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Prepends ops to a group (e.g. an engine-side assert), shifting the group's refs. */
|
||||
export function prependOps(group: OpGroup, ops: ProviderOp[]): OpGroup {
|
||||
return { ref: group.ref, ops: [...ops, ...placeGroup(group.ops, ops.length).map(stripYield)] };
|
||||
}
|
||||
|
||||
/** One group running `first` then `second` atomically. */
|
||||
export function concatGroups(ref: string, first: OpGroup, second: OpGroup): OpGroup {
|
||||
return {
|
||||
ref,
|
||||
ops: [...first.ops.map(stripYield), ...placeGroup(second.ops, first.ops.length).map(stripYield)],
|
||||
};
|
||||
}
|
||||
|
||||
function stripYield(op: ProviderOp): ProviderOp {
|
||||
const { yieldAllowed: _y, ...rest } = op as ProviderOp & { yieldAllowed?: boolean };
|
||||
return rest as ProviderOp;
|
||||
}
|
||||
|
||||
export class BatchWriter {
|
||||
/** Batches sent, for tests and logs. */
|
||||
batchesSent = 0;
|
||||
|
||||
constructor(
|
||||
private readonly port: ProviderPort,
|
||||
private readonly options: BatchWriterOptions,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Applies the works in order; `tail` (the SyncState op) goes last, after
|
||||
* every group of `works` was applied or given up.
|
||||
*/
|
||||
async write(works: readonly Work[], tail?: Tail): Promise<void> {
|
||||
const todo = works.filter((w) => hasWrites(w.group.ops));
|
||||
const batches = this.pack(todo);
|
||||
if (batches.length === 0) {
|
||||
if (tail) await this.applyTail(tail);
|
||||
return;
|
||||
}
|
||||
for (let i = 0; i < batches.length; i++) {
|
||||
const last = i === batches.length - 1;
|
||||
const batch = batches[i];
|
||||
// A photo batch is already at the Binder budget: its state op goes alone.
|
||||
const tailHere = last && tail && !(batch.length === 1 && carriesBlob(batch[0].group.ops)) ? tail : undefined;
|
||||
await this.applyBatch(batch, tailHere);
|
||||
if (last && tail && !tailHere) await this.applyTail(tail);
|
||||
}
|
||||
}
|
||||
|
||||
private pack(works: readonly Work[]): Work[][] {
|
||||
const batches: Work[][] = [];
|
||||
let current: Work[] = [];
|
||||
let ops = 0;
|
||||
let bytes = 0;
|
||||
const flush = () => {
|
||||
if (current.length) batches.push(current);
|
||||
current = [];
|
||||
ops = 0;
|
||||
bytes = 0;
|
||||
};
|
||||
for (const work of works) {
|
||||
if (carriesBlob(work.group.ops)) {
|
||||
flush();
|
||||
batches.push([work]);
|
||||
continue;
|
||||
}
|
||||
const n = work.group.ops.length;
|
||||
const b = sizeOf(work.group.ops);
|
||||
if (current.length && (ops + n > this.options.maxOps || bytes + b > this.options.maxBytes)) flush();
|
||||
current.push(work);
|
||||
ops += n;
|
||||
bytes += b;
|
||||
}
|
||||
flush();
|
||||
return batches;
|
||||
}
|
||||
|
||||
private async send(ops: ProviderOp[]) {
|
||||
this.batchesSent++;
|
||||
const result = await this.port.applyBatch(ops);
|
||||
if (!result.ok) {
|
||||
if (result.reason === 'permission') throw new RunAbort('permission', result.message);
|
||||
// The native side refused an op outside our account: an engine bug, never retried.
|
||||
if (result.reason === 'scope') throw new Error(`Provider refused an op: ${result.message}`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private async applyBatch(works: Work[], tail?: Tail): Promise<void> {
|
||||
const tailOp = tail?.op();
|
||||
const result = await this.send(buildBatch(works.map((w) => w.group), tailOp));
|
||||
if (result.ok) {
|
||||
let offset = 0;
|
||||
for (const work of works) {
|
||||
const n = work.group.ops.length;
|
||||
await work.applied?.(result.results.slice(offset, offset + n));
|
||||
offset += n;
|
||||
}
|
||||
tail?.applied();
|
||||
return;
|
||||
}
|
||||
if (result.reason === 'tooLarge' && works.length > 1) {
|
||||
const middle = Math.ceil(works.length / 2);
|
||||
await this.applyBatch(works.slice(0, middle));
|
||||
await this.applyBatch(works.slice(middle), tail);
|
||||
return;
|
||||
}
|
||||
if (works.length === 1 && !tail) {
|
||||
await this.retryAlone(works[0], result.reason, result.message);
|
||||
return;
|
||||
}
|
||||
for (const work of works) await this.applyAlone(work, 0);
|
||||
if (tail) await this.applyTail(tail);
|
||||
}
|
||||
|
||||
private async applyAlone(work: Work, attempt: number): Promise<void> {
|
||||
if (!hasWrites(work.group.ops)) return;
|
||||
const result = await this.send(buildBatch([work.group]));
|
||||
if (result.ok) {
|
||||
await work.applied?.(result.results);
|
||||
return;
|
||||
}
|
||||
await this.retryAlone(work, result.reason, result.message, attempt);
|
||||
}
|
||||
|
||||
private async retryAlone(work: Work, reason: BatchFailure, message: string, attempt = 0): Promise<void> {
|
||||
if (reason === 'assert' && work.replan && attempt < this.options.maxReplans) {
|
||||
const next = await work.replan();
|
||||
if (next) await this.applyAlone(next, attempt + 1);
|
||||
return;
|
||||
}
|
||||
await work.failed?.(reason, message);
|
||||
}
|
||||
|
||||
private async applyTail(tail: Tail): Promise<void> {
|
||||
const result = await this.send([...buildBatch([], tail.op())]);
|
||||
if (!result.ok) throw new Error(`Could not store the sync state: ${result.reason} ${result.message}`);
|
||||
tail.applied();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,663 @@
|
||||
/**
|
||||
* Calendars: server calendars ↔ Calendars rows, events ↔ Events rows with
|
||||
* their exceptions, attendees and reminders (docs/device-sync.md, "Calendar
|
||||
* mapping"). Tasks never sync; a calendar that holds only tasks gets no row.
|
||||
*/
|
||||
import { findTasksOnlyCalendarIds, isTaskLikeObject, SCAN_PROPERTIES, type ScannedCalendarObject } from '../../lib/calendar-component-detection';
|
||||
import { collectionKey, parseCollectionKey, parseObjectRef, pendingUidOf } from '../common/ids';
|
||||
import type { CalendarContext, CalendarEventWire, CalendarLike, CalendarPlanner, LocalCalendar, LocalEvent, OpGroup } from '../planner';
|
||||
import { CALENDAR_AUTHORITY, type ReminderOwner, type Row } from '../types';
|
||||
import { CALENDAR_EVENT_PROPERTIES, CALENDAR_PROPERTIES } from '../wire';
|
||||
import { calendarAddresses, type CalendarAddresses, type CapabilityAccount } from '../jmap/session';
|
||||
import type { Work } from './batch';
|
||||
import type { RunEnv } from './context';
|
||||
import type { SubscriptionCalendar } from './deps';
|
||||
import { ItemSync, type Pending } from './item-sync';
|
||||
import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds';
|
||||
import { poisonFingerprint } from './poison';
|
||||
import { flag, num, str } from './provider';
|
||||
import { isCollectionSelected } from './selection';
|
||||
import { accountOf, type SyncState } from './sync-state';
|
||||
|
||||
const MASTER = 'original_id IS NULL AND original_sync_id IS NULL';
|
||||
const WITH_IDENTITY = "_sync_id IS NOT NULL AND _sync_id NOT LIKE '~pending/%'";
|
||||
/** CalendarProvider leaves DIRTY (and may leave DELETED) NULL on sync-adapter inserts: NULL is clean. */
|
||||
const CLEAN = '(dirty IS NULL OR dirty = 0) AND (deleted IS NULL OR deleted = 0)';
|
||||
|
||||
function onIds(map: Record<string, boolean> | undefined | null): string[] {
|
||||
return Object.entries(map ?? {})
|
||||
.filter(([, on]) => on === true)
|
||||
.map(([id]) => id);
|
||||
}
|
||||
|
||||
function union(a: readonly string[], b: readonly string[]): string[] {
|
||||
return [...new Set([...a, ...b])];
|
||||
}
|
||||
|
||||
export class CalendarSync extends ItemSync {
|
||||
protected readonly itemType = 'CalendarEvent' as const;
|
||||
protected readonly itemProperties = CALENDAR_EVENT_PROPERTIES;
|
||||
protected readonly parentFilter = 'inCalendar' as const;
|
||||
protected readonly parentProperty = 'calendarIds' as const;
|
||||
|
||||
readonly eventKind: Kind<LocalEvent, CalendarEventWire>;
|
||||
private readonly planner: CalendarPlanner;
|
||||
private readonly calendarColumns: string[];
|
||||
private readonly eventColumns: string[];
|
||||
private readonly attendeeColumns: string[];
|
||||
private readonly reminderColumns: string[];
|
||||
|
||||
private readonly calendars = new Map<string, CalendarLike[]>();
|
||||
private readonly syncedKeys = new Set<string>();
|
||||
private readonly readOnlyKeys = new Set<string>();
|
||||
private readonly rowByKey = new Map<string, number>();
|
||||
private readonly keyByRow = new Map<number, string>();
|
||||
/** Calendar rows of deselected calendars: dropped after the upload. */
|
||||
private deleteRows: number[] = [];
|
||||
/** Per account: calendars found to hold only tasks (true) or events again (false). */
|
||||
private readonly taskOnlyNotes = new Map<string, Map<string, boolean>>();
|
||||
private readonly collectionStates = new Map<string, string>();
|
||||
private addresses: CalendarAddresses = { ownerAccount: '', selfAddresses: [] };
|
||||
private subscriptions: SubscriptionCalendar[] = [];
|
||||
private zone = 'UTC';
|
||||
/** Who the Reminders rows of this run are written for (the stored owner until the owner-change pass). */
|
||||
private reminderOwner: ReminderOwner = 'device';
|
||||
|
||||
constructor(env: RunEnv) {
|
||||
super(env);
|
||||
this.planner = env.deps.planners.calendar;
|
||||
this.calendarColumns = union(this.planner.calendarColumns, ['_id']);
|
||||
this.eventColumns = union(this.planner.eventColumns, ['_id', 'calendar_id', '_sync_id', 'original_id', 'original_sync_id']);
|
||||
this.attendeeColumns = union(this.planner.attendeeColumns, ['_id', 'event_id']);
|
||||
this.reminderColumns = union(this.planner.reminderColumns, ['_id', 'event_id']);
|
||||
this.eventKind = this.makeEventKind();
|
||||
}
|
||||
|
||||
// ── Kind ──
|
||||
|
||||
private makeEventKind(): Kind<LocalEvent, CalendarEventWire> {
|
||||
const planner = this.planner;
|
||||
return {
|
||||
name: 'event',
|
||||
table: 'events',
|
||||
identityColumn: '_sync_id',
|
||||
poisonColumn: 'sync_data5',
|
||||
meta: (e) => {
|
||||
const identity = parseObjectRef(e.syncId) ? e.syncId : null;
|
||||
const pendingUid = pendingUidOf(e.syncId);
|
||||
const calendarKey = this.keyByRow.get(e.calendarRowId) ?? null;
|
||||
const acct = accountOfRef(identity) ?? accountOfRef(calendarKey);
|
||||
return {
|
||||
rowId: e.eventId,
|
||||
// A split clone shares its source's _SYNC_ID but uploads as a new event.
|
||||
sourceId: e.split === 'clone' ? null : identity,
|
||||
pending: e.pending ?? (pendingUid && calendarKey ? { uid: pendingUid, target: calendarKey } : null),
|
||||
dirty: e.dirty || e.split !== undefined || e.exceptions.some((x) => x.dirty || x.deleted),
|
||||
deleted: e.deleted,
|
||||
isNew: !identity || e.split === 'clone',
|
||||
poison: e.poison,
|
||||
collections: e.shadow && acct
|
||||
? onIds(e.shadow.calendarIds as Record<string, boolean>).map((id) => collectionKey(acct, id))
|
||||
: calendarKey ? [calendarKey] : [],
|
||||
};
|
||||
},
|
||||
fingerprint: (e) =>
|
||||
poisonFingerprint({
|
||||
deleted: e.deleted,
|
||||
shadow: e.shadow,
|
||||
cells: e.cells,
|
||||
attendees: e.attendees.map((a) => JSON.stringify(a.cells)).sort(),
|
||||
reminders: e.reminders.map((r) => JSON.stringify(r.cells)).sort(),
|
||||
exceptions: e.exceptions
|
||||
.map((x) => JSON.stringify([x.deleted, x.cells, x.attendees.map((a) => a.cells), x.reminders.map((r) => r.cells)]))
|
||||
.sort(),
|
||||
}),
|
||||
loadByRefs: async (refs) => {
|
||||
const out = new Map<string, LocalEvent>();
|
||||
for (const e of await this.loadMastersIn('_sync_id', refs)) {
|
||||
if (e.split !== 'clone' && e.syncId && parseObjectRef(e.syncId)) out.set(e.syncId, e);
|
||||
}
|
||||
return out;
|
||||
},
|
||||
loadByRowIds: (ids) => this.loadMastersIn('_id', ids),
|
||||
loadNew: () => this.loadMastersWhere("_sync_id IS NULL OR _sync_id LIKE '~pending/%'"),
|
||||
planDownload: (event, local, acct) => planner.planDownload(event, local, this.ctx(acct)),
|
||||
planLocalDelete: (e) => planner.planLocalDelete(e),
|
||||
planBaselineHeal: (e) => planner.planBaselineHeal(e),
|
||||
planUpload: (e, acct) => planner.planUpload(e, this.ctx(acct)),
|
||||
planAccepted: (e, event, acct) => planner.planAccepted(e, event, this.ctx(acct)),
|
||||
};
|
||||
}
|
||||
|
||||
protected kinds(): Kind[] {
|
||||
return [this.eventKind];
|
||||
}
|
||||
|
||||
/** Tasks never sync (explicit `@type: "Task"`, or CalDAV VTODOs recognised by their task-only fields). */
|
||||
protected kindOf(object: ServerObject): Kind | null {
|
||||
return isTaskLikeObject(object as ScannedCalendarObject) ? null : this.eventKind;
|
||||
}
|
||||
|
||||
// ── Loading ──
|
||||
|
||||
private async decodeWithExceptions(masters: Row[]): Promise<LocalEvent[]> {
|
||||
if (!masters.length) return [];
|
||||
const byId = new Map<number, Row>();
|
||||
for (const row of masters) byId.set(num(row._id), row);
|
||||
const ids = masters.map((r) => num(r._id));
|
||||
const syncIds = masters.map((r) => str(r._sync_id)).filter((s): s is string => !!s);
|
||||
for (const row of await this.env.reader.rowsIn('events', this.eventColumns, 'original_id', ids)) byId.set(num(row._id), row);
|
||||
if (syncIds.length) {
|
||||
for (const row of await this.env.reader.rowsIn('events', this.eventColumns, 'original_sync_id', syncIds)) byId.set(num(row._id), row);
|
||||
}
|
||||
const all = [...byId.values()];
|
||||
const eventIds = all.map((r) => num(r._id));
|
||||
const attendees = await this.env.reader.rowsIn('attendees', this.attendeeColumns, 'event_id', eventIds);
|
||||
const reminders = await this.env.reader.rowsIn('reminders', this.reminderColumns, 'event_id', eventIds);
|
||||
return this.planner.decodeEvents(all, attendees, reminders);
|
||||
}
|
||||
|
||||
private async loadMastersIn(column: '_id' | '_sync_id', values: ReadonlyArray<string | number>): Promise<LocalEvent[]> {
|
||||
if (!values.length) return [];
|
||||
return this.decodeWithExceptions(await this.env.reader.rowsIn('events', this.eventColumns, column, values, { where: MASTER }));
|
||||
}
|
||||
|
||||
private async loadMastersWhere(where: string, args: Array<string | number> = []): Promise<LocalEvent[]> {
|
||||
return this.decodeWithExceptions(await this.env.reader.rows('events', this.eventColumns, `(${where}) AND ${MASTER}`, args));
|
||||
}
|
||||
|
||||
private async masterIds(where: string, args: Array<string | number> = []): Promise<number[]> {
|
||||
return (await this.env.reader.rows('events', ['_id'], `(${where}) AND ${MASTER}`, args)).map((r) => num(r._id));
|
||||
}
|
||||
|
||||
private async loadCalendars(): Promise<LocalCalendar[]> {
|
||||
return (await this.env.reader.rows('calendars', this.calendarColumns)).map((r) => this.planner.decodeCalendar(r));
|
||||
}
|
||||
|
||||
private async refreshCalendarRows(): Promise<void> {
|
||||
this.rowByKey.clear();
|
||||
this.keyByRow.clear();
|
||||
for (const calendar of await this.loadCalendars()) {
|
||||
if (!calendar.syncId || !parseCollectionKey(calendar.syncId)) continue;
|
||||
this.rowByKey.set(calendar.syncId, calendar.calendarRowId);
|
||||
this.keyByRow.set(calendar.calendarRowId, calendar.syncId);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Collections ──
|
||||
|
||||
private isFeed(account: CapabilityAccount, calendarId: string): boolean {
|
||||
return this.subscriptions.some(
|
||||
(s) => s.calendarId === calendarId && (s.jmapAccountId === account.id || (s.jmapAccountId === null && account.primary)),
|
||||
);
|
||||
}
|
||||
|
||||
private noteTaskOnly(acct: string, calendarId: string, taskOnly: boolean): void {
|
||||
let notes = this.taskOnlyNotes.get(acct);
|
||||
if (!notes) this.taskOnlyNotes.set(acct, (notes = new Map()));
|
||||
notes.set(calendarId, taskOnly);
|
||||
}
|
||||
|
||||
private isTaskOnly(acct: string, calendarId: string): boolean {
|
||||
return this.taskOnlyNotes.get(acct)?.get(calendarId) ?? (this.env.store.account(acct).taskOnly ?? []).includes(calendarId);
|
||||
}
|
||||
|
||||
/** A calendar whose first objects are all tasks (Stalwart offers no per-calendar component set). */
|
||||
private async scanTaskOnly(acct: string, calendarId: string): Promise<boolean> {
|
||||
const ids = await this.env.jmap.queryFirst('CalendarEvent', acct, { inCalendar: calendarId }, this.env.tuning.taskScanLimit);
|
||||
if (!ids.length) return false;
|
||||
const { list } = await this.env.jmap.get<ScannedCalendarObject>('CalendarEvent', acct, ids, SCAN_PROPERTIES);
|
||||
return findTasksOnlyCalendarIds(list.map((o) => ({ ...o, calendarIds: { [calendarId]: true } })), [calendarId]).has(calendarId);
|
||||
}
|
||||
|
||||
protected async collections(write: boolean): Promise<void> {
|
||||
this.zone = this.env.deps.deviceZone();
|
||||
this.reminderOwner = this.env.store.committed.reminderOwner ?? this.env.prefs.reminderOwner;
|
||||
this.addresses = await calendarAddresses(this.env.jmap, this.env.accountName);
|
||||
this.subscriptions = await this.env.deps.subscriptionCalendars();
|
||||
this.syncedKeys.clear();
|
||||
this.readOnlyKeys.clear();
|
||||
const selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }> = [];
|
||||
for (const account of this.env.accounts) {
|
||||
const { list, state } = await this.env.jmap.getContainers<CalendarLike>('Calendar', account.id, CALENDAR_PROPERTIES);
|
||||
this.calendars.set(account.id, list);
|
||||
if (state) {
|
||||
this.collectionStates.set(account.id, state);
|
||||
this.env.recordKnownState(account.id, 'Calendar', state);
|
||||
}
|
||||
const stored = this.env.store.account(account.id);
|
||||
const reconciling = !stored.itemsState || !!stored.reconcile;
|
||||
for (const calendar of list) {
|
||||
if (calendar.myRights?.mayReadItems === false) continue;
|
||||
if (!isCollectionSelected(this.env.prefs.calendarSelection, account, CALENDAR_AUTHORITY, calendar)) continue;
|
||||
const key = collectionKey(account.id, calendar.id);
|
||||
// Checked when a calendar comes to the device, and again on every full reconcile.
|
||||
if (write && (reconciling || (!stored.selected.includes(key) && !this.isTaskOnly(account.id, calendar.id)))) {
|
||||
this.noteTaskOnly(account.id, calendar.id, await this.scanTaskOnly(account.id, calendar.id));
|
||||
}
|
||||
if (this.isTaskOnly(account.id, calendar.id)) continue;
|
||||
const rights = calendar.myRights;
|
||||
const readOnly = this.isFeed(account, calendar.id) || (!!rights && !rights.mayWriteAll && !rights.mayWriteOwn);
|
||||
this.syncedKeys.add(key);
|
||||
if (readOnly) this.readOnlyKeys.add(key);
|
||||
selected.push({ jmapAccountId: account.id, calendar, readOnly, accountName: account.personal ? '' : account.name });
|
||||
}
|
||||
}
|
||||
await this.refreshCalendarRows();
|
||||
this.deleteRows = [];
|
||||
if (!write) return;
|
||||
const plan = this.planner.planCalendars(selected, await this.loadCalendars(), this.calendarCtx());
|
||||
await this.env.writer.write(plan.groups.map((group) => ({ group })));
|
||||
this.deleteRows = plan.deleteCalendarRows;
|
||||
await this.refreshCalendarRows();
|
||||
}
|
||||
|
||||
protected syncedCollections(acct: string): string[] {
|
||||
return (this.calendars.get(acct) ?? [])
|
||||
.filter((c) => this.syncedKeys.has(collectionKey(acct, c.id)))
|
||||
.map((c) => c.id);
|
||||
}
|
||||
|
||||
protected inSelection(acct: string, object: ServerObject): boolean {
|
||||
return onIds(object.calendarIds as Record<string, boolean>).some((id) => this.syncedKeys.has(collectionKey(acct, id)));
|
||||
}
|
||||
|
||||
private findCalendar(calendarId: string): CalendarLike | undefined {
|
||||
for (const account of this.env.accounts) {
|
||||
const found = this.calendars.get(account.id)?.find((c) => c.id === calendarId);
|
||||
if (found) return found;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private calendarCtx(): Omit<CalendarContext, 'jmapAccountId'> {
|
||||
const env = this.env;
|
||||
const primary = this.env.accounts.find((a) => a.primary)?.id ?? this.env.accounts[0]?.id ?? '';
|
||||
return {
|
||||
now: env.now(),
|
||||
mintKey: (taken) => env.mintKey(taken),
|
||||
mintUid: () => env.mintUid(),
|
||||
deviceZone: this.zone,
|
||||
ownerAccount: this.addresses.ownerAccount,
|
||||
selfAddresses: this.addresses.selfAddresses,
|
||||
reminderOwner: this.reminderOwner,
|
||||
calendar: (calendarId) => this.findCalendar(calendarId),
|
||||
calendarRowId: (calendarId) => this.rowByKey.get(collectionKey(primary, calendarId)) ?? null,
|
||||
calendarIdOfRow: (rowId) => {
|
||||
const parsed = parseCollectionKey(this.keyByRow.get(rowId));
|
||||
return parsed ? { jmapAccountId: parsed.accountId, calendarId: parsed.id } : null;
|
||||
},
|
||||
isReadOnly: (calendarId) => [...this.readOnlyKeys].some((k) => parseCollectionKey(k)?.id === calendarId),
|
||||
isSelected: (key) => this.syncedKeys.has(key),
|
||||
};
|
||||
}
|
||||
|
||||
ctx(acct: string): CalendarContext {
|
||||
return {
|
||||
...this.calendarCtx(),
|
||||
jmapAccountId: acct,
|
||||
calendar: (calendarId) => this.calendars.get(acct)?.find((c) => c.id === calendarId),
|
||||
calendarRowId: (calendarId) => this.rowByKey.get(collectionKey(acct, calendarId)) ?? null,
|
||||
isReadOnly: (calendarId) => this.readOnlyKeys.has(collectionKey(acct, calendarId)),
|
||||
};
|
||||
}
|
||||
|
||||
protected decorateState(acct: string, next: SyncState): () => void {
|
||||
const account = accountOf(next, acct);
|
||||
const notes = this.taskOnlyNotes.get(acct);
|
||||
const consumed = new Map(notes ?? []);
|
||||
if (consumed.size) {
|
||||
const taskOnly = new Set(account.taskOnly ?? []);
|
||||
for (const [id, only] of consumed) {
|
||||
if (only) taskOnly.add(id);
|
||||
else taskOnly.delete(id);
|
||||
}
|
||||
account.taskOnly = [...taskOnly];
|
||||
}
|
||||
const collections = this.collectionStates.get(acct);
|
||||
if (collections) account.collectionsState = collections;
|
||||
return () => {
|
||||
for (const [id, only] of consumed) if (notes?.get(id) === only) notes.delete(id);
|
||||
};
|
||||
}
|
||||
|
||||
protected async localIdentities(acct: string): Promise<Map<string, { dirty: boolean; deleted: boolean }>> {
|
||||
const out = new Map<string, { dirty: boolean; deleted: boolean }>();
|
||||
const rows = await this.env.reader.rows('events', ['_id', '_sync_id', 'dirty', 'deleted'], `_sync_id LIKE ? AND ${MASTER}`, [`${acct}/%`]);
|
||||
for (const row of rows) {
|
||||
const id = idInAccount(str(row._sync_id), acct);
|
||||
if (id) out.set(id, { dirty: flag(row.dirty), deleted: flag(row.deleted) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Uploads ──
|
||||
|
||||
/**
|
||||
* Masters with local changes: dirty, deleted or new ones, masters of dirty
|
||||
* or deleted exception rows, and both masters of a CONTENT_EXCEPTION_URI
|
||||
* split (two masters sharing a _SYNC_ID; the older one's rule changed
|
||||
* without DIRTY), loaded together so the planner sees the split.
|
||||
*/
|
||||
protected async loadUploadItems(): Promise<Held[]> {
|
||||
const rows = await this.env.reader.rows(
|
||||
'events',
|
||||
['_id', '_sync_id', 'original_id', 'original_sync_id', 'uid2445'],
|
||||
"dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'",
|
||||
);
|
||||
const ids = new Set<number>();
|
||||
const refs = new Set<string>();
|
||||
const newUids = new Set<string>();
|
||||
for (const row of rows) {
|
||||
if (row.original_id !== null && row.original_id !== undefined) ids.add(num(row.original_id));
|
||||
else if (str(row.original_sync_id)) refs.add(str(row.original_sync_id) as string);
|
||||
else {
|
||||
ids.add(num(row._id));
|
||||
const uid = str(row.uid2445);
|
||||
if (uid && !parseObjectRef(str(row._sync_id))) newUids.add(uid);
|
||||
}
|
||||
}
|
||||
const bySyncId = new Map<string, number[]>();
|
||||
for (const row of await this.env.reader.rows('events', ['_id', '_sync_id'], `_sync_id IS NOT NULL AND ${MASTER}`)) {
|
||||
const syncId = str(row._sync_id) as string;
|
||||
bySyncId.set(syncId, [...(bySyncId.get(syncId) ?? []), num(row._id)]);
|
||||
}
|
||||
for (const [syncId, masters] of bySyncId) {
|
||||
if (masters.length > 1 || refs.has(syncId)) for (const id of masters) ids.add(id);
|
||||
}
|
||||
// A new master may carry a uid another row has (an app copied an event): those rows come along, so
|
||||
// the planner sees the collision and gives the new one a fresh uid instead of adopting the other's object.
|
||||
if (newUids.size) {
|
||||
for (const row of await this.env.reader.rowsIn('events', ['_id'], 'uid2445', [...newUids], { where: MASTER })) ids.add(num(row._id));
|
||||
}
|
||||
const events = await this.loadMastersIn('_id', [...ids]);
|
||||
return events.map((local) => ({ kind: this.eventKind as Kind, local }));
|
||||
}
|
||||
|
||||
/** Event uids are unique per account: only an event in the create's calendar can be the row's own create. */
|
||||
protected adoptionTargetMatches(acct: string, object: ServerObject, target: string): boolean {
|
||||
const parsed = parseCollectionKey(target);
|
||||
return !!parsed && parsed.accountId === acct && (object.calendarIds as Record<string, boolean> | undefined)?.[parsed.id] === true;
|
||||
}
|
||||
|
||||
protected accountOfItem(held: Held): string | null {
|
||||
return super.accountOfItem(held) ?? accountOfRef(this.keyByRow.get((held.local as LocalEvent).calendarRowId) ?? null);
|
||||
}
|
||||
|
||||
protected claimAccount(held: Held): string | null {
|
||||
return accountOfRef(this.keyByRow.get((held.local as LocalEvent).calendarRowId) ?? null);
|
||||
}
|
||||
|
||||
/** A split clone is claimed after its source's rule was planned (see prePhase). */
|
||||
protected claimable(held: Held): boolean {
|
||||
return (held.local as LocalEvent).split !== 'clone';
|
||||
}
|
||||
|
||||
protected async countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set<string> }> {
|
||||
let count = 0;
|
||||
const accounts = new Set<string>();
|
||||
for (const held of items) {
|
||||
const meta = held.kind.meta(held.local);
|
||||
if (!meta.deleted || !meta.sourceId) continue;
|
||||
count++;
|
||||
const acct = accountOfRef(meta.sourceId);
|
||||
if (acct) accounts.add(acct);
|
||||
}
|
||||
const synced = (await this.masterIds(WITH_IDENTITY)).length;
|
||||
return { count, synced: Math.max(synced, count), accounts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Before the creates: a split's source uploads its rule while the split is
|
||||
* still visible, then its clone is claimed as a new event; and deleted +
|
||||
* new masters of the run that are one edit (a move, a series turned
|
||||
* single) upload as patches of the existing object.
|
||||
*/
|
||||
protected async prePhase(acct: string): Promise<void> {
|
||||
let items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct);
|
||||
const sources = items.filter((h) => (h.local as LocalEvent).split === 'source');
|
||||
const clones = items.filter((h) => (h.local as LocalEvent).split === 'clone');
|
||||
if (sources.length || clones.length) {
|
||||
const works: Work[] = [];
|
||||
const ready: Pending[] = [];
|
||||
for (const held of sources) {
|
||||
const pending = await this.planItem(acct, held, works);
|
||||
if (pending) ready.push(pending);
|
||||
}
|
||||
// A clone (of a split, or a new master whose uid another row carries) needs no source:
|
||||
// it claims a fresh uid here and is created with the other new events.
|
||||
for (const held of clones) {
|
||||
const pending = await this.planItem(acct, held, works);
|
||||
if (pending) ready.push(pending);
|
||||
}
|
||||
await this.env.writer.write(works);
|
||||
if (ready.length) await this.send(acct, ready);
|
||||
items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct);
|
||||
}
|
||||
const deleted = items.map((h) => h.local as LocalEvent).filter((e) => {
|
||||
const m = this.eventKind.meta(e);
|
||||
return m.deleted && !!m.sourceId;
|
||||
});
|
||||
const fresh = items.map((h) => h.local as LocalEvent).filter((e) => {
|
||||
const m = this.eventKind.meta(e);
|
||||
return m.isNew && !m.deleted && !!m.pending;
|
||||
});
|
||||
if (!deleted.length || !fresh.length) return;
|
||||
let pairs: ReturnType<CalendarPlanner['planPairs']>;
|
||||
try {
|
||||
pairs = this.planner.planPairs(deleted, fresh, this.ctx(acct));
|
||||
} catch (error) {
|
||||
// Unpaired, they upload as ordinary creates and deletions.
|
||||
this.plannerFailed(acct, `pairs:${acct}`, 'upload', error);
|
||||
return;
|
||||
}
|
||||
if (!pairs.length) return;
|
||||
await this.env.checkpoints.check('upload:pairs');
|
||||
const pending: Pending[] = pairs.map((pair) => {
|
||||
const meta = this.eventKind.meta(pair.deleted);
|
||||
const ownId = idInAccount(meta.sourceId, acct);
|
||||
return {
|
||||
held: { kind: this.eventKind, local: pair.deleted },
|
||||
meta,
|
||||
fingerprint: this.eventKind.fingerprint(pair.deleted),
|
||||
actions: pair.actions as Pending['actions'],
|
||||
accept: async (objects) => {
|
||||
// The new row takes the identity and exceptions of the old one, which is purged.
|
||||
await this.env.writer.write([{ group: pair.ops }]);
|
||||
const [moved] = await this.eventKind.loadByRowIds([pair.fresh.eventId]);
|
||||
const server = ownId ? objects.get(ownId) : undefined;
|
||||
if (!moved || !server) {
|
||||
if (ownId) this.markStale(acct, ownId);
|
||||
return [];
|
||||
}
|
||||
const held: Held = { kind: this.eventKind, local: moved };
|
||||
try {
|
||||
return [this.acceptWork(acct, { held, meta: this.eventKind.meta(moved), fingerprint: '', actions: [] }, server, false)];
|
||||
} catch (error) {
|
||||
this.plannerFailed(acct, refOf(acct, server.id), 'upload', error, server.id);
|
||||
return [];
|
||||
}
|
||||
},
|
||||
};
|
||||
});
|
||||
await this.send(acct, pending);
|
||||
}
|
||||
|
||||
// ── After the upload ──
|
||||
|
||||
/** A master with nothing waiting for an upload. */
|
||||
private isClean(e: LocalEvent): boolean {
|
||||
const m = this.eventKind.meta(e);
|
||||
return !m.dirty && !m.deleted && !m.isNew;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deselected calendars lose their rows (the provider deletes their events)
|
||||
* once nothing in them waits for an upload; until then only their clean
|
||||
* events go. Events that left every synced calendar go when clean.
|
||||
*/
|
||||
protected async dropOutsideSelection(acct: string): Promise<void> {
|
||||
const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id)));
|
||||
const deselected = this.env.store.account(acct).selected.filter((key) => !synced.has(key));
|
||||
const rows = this.deleteRows.filter((row) => accountOfRef(this.keyByRow.get(row) ?? null) === acct);
|
||||
const outside = [...(this.outside.get(acct) ?? [])];
|
||||
if (!deselected.length && !rows.length && !outside.length) return;
|
||||
const kept = new Set<string>();
|
||||
for (const rowId of rows) {
|
||||
await this.env.checkpoints.check('deselect');
|
||||
const key = this.keyByRow.get(rowId) as string;
|
||||
if (await this.dropCalendarRow(rowId)) continue;
|
||||
kept.add(key);
|
||||
}
|
||||
const works: Work[] = [];
|
||||
for (const local of (await this.eventKind.loadByRefs(outside)).values()) {
|
||||
const meta = this.eventKind.meta(local);
|
||||
const id = idInAccount(meta.sourceId, acct);
|
||||
if (!id || !meta.collections.length || meta.collections.some((k) => synced.has(k))) continue;
|
||||
const work = this.removeWork(acct, id, { kind: this.eventKind, local }, 'outside');
|
||||
if (work) works.push(work);
|
||||
}
|
||||
const selected = [...synced, ...kept];
|
||||
await this.env.writer.write(works, this.tail([acct], (next) => {
|
||||
accountOf(next, acct).selected = selected;
|
||||
}));
|
||||
this.outside.delete(acct);
|
||||
}
|
||||
|
||||
/** Deletes a calendar row when none of its events waits for an upload; else only its clean events. True when the row went. */
|
||||
private async dropCalendarRow(rowId: number): Promise<boolean> {
|
||||
const waiting = await this.env.reader.rows(
|
||||
'events',
|
||||
['_id'],
|
||||
"calendar_id = ? AND (dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%')",
|
||||
[rowId],
|
||||
);
|
||||
if (!waiting.length) {
|
||||
await this.env.writer.write([{ group: { ref: `calendar:${rowId}`, ops: [{ op: 'delete', table: 'calendars', id: rowId }] } }]);
|
||||
return true;
|
||||
}
|
||||
const clean = await this.masterIds(`calendar_id = ? AND ${CLEAN} AND ${WITH_IDENTITY}`, [rowId]);
|
||||
for (let i = 0; i < clean.length; i += this.env.tuning.chunkSize) {
|
||||
await this.env.checkpoints.check('deselect');
|
||||
const works: Work[] = [];
|
||||
for (const local of await this.eventKind.loadByRowIds(clean.slice(i, i + this.env.tuning.chunkSize))) {
|
||||
const meta = this.eventKind.meta(local);
|
||||
const acct = accountOfRef(meta.sourceId);
|
||||
const id = acct ? idInAccount(meta.sourceId, acct) : null;
|
||||
if (!acct || !id || !this.isClean(local)) continue;
|
||||
const work = this.removeWork(acct, id, { kind: this.eventKind, local }, 'outside');
|
||||
if (work) works.push(work);
|
||||
}
|
||||
await this.env.writer.write(works);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected async afterSync(): Promise<void> {
|
||||
// Rows of calendars that belong to no account we know (garbage, or a vanished account).
|
||||
for (const rowId of this.deleteRows) {
|
||||
if (this.keyByRow.has(rowId) && accountOfRef(this.keyByRow.get(rowId) ?? null)) continue;
|
||||
await this.dropCalendarRow(rowId);
|
||||
}
|
||||
await this.zonePass();
|
||||
await this.reminderOwnerPass();
|
||||
}
|
||||
|
||||
/** Every clean master, in chunks, through `plan`; `finish` rides with the last chunk's batch. */
|
||||
private async passOverCleanMasters(
|
||||
label: string,
|
||||
plan: (e: LocalEvent, acct: string) => OpGroup | null,
|
||||
finish: (next: SyncState) => void,
|
||||
): Promise<void> {
|
||||
const ids = await this.masterIds(`${CLEAN} AND ${WITH_IDENTITY}`);
|
||||
const size = this.env.tuning.chunkSize;
|
||||
if (!ids.length) {
|
||||
await this.env.writer.write([], this.env.store.tail(finish));
|
||||
return;
|
||||
}
|
||||
for (let i = 0; i < ids.length; i += size) {
|
||||
await this.env.checkpoints.check(label);
|
||||
const works: Work[] = [];
|
||||
for (const local of await this.eventKind.loadByRowIds(ids.slice(i, i + size))) {
|
||||
const sourceId = this.eventKind.meta(local).sourceId;
|
||||
const acct = accountOfRef(sourceId);
|
||||
if (!acct || !this.isClean(local)) continue;
|
||||
try {
|
||||
const group = plan(local, acct);
|
||||
// A group that fails its assert was edited meanwhile: the upload takes it from here.
|
||||
if (group) works.push({ group });
|
||||
} catch (error) {
|
||||
this.plannerFailed(acct, sourceId as string, 'download', error);
|
||||
}
|
||||
}
|
||||
await this.env.writer.write(works, i + size >= ids.length ? this.env.store.tail(finish) : undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The device zone changed: clean floating events are rewritten so they
|
||||
* keep their wall time. `deviceZonePending` marks the pass until its last
|
||||
* chunk stores the new zone.
|
||||
*/
|
||||
private async zonePass(): Promise<void> {
|
||||
const committed = this.env.store.committed;
|
||||
const stored = committed.deviceZone ?? null;
|
||||
if (!stored) {
|
||||
await this.env.writer.write([], this.env.store.tail((next) => {
|
||||
next.deviceZone = this.zone;
|
||||
next.deviceZonePending = null;
|
||||
}));
|
||||
return;
|
||||
}
|
||||
if (stored === this.zone && !committed.deviceZonePending) return;
|
||||
if (committed.deviceZonePending !== this.zone) {
|
||||
await this.env.writer.write([], this.env.store.tail((next) => {
|
||||
next.deviceZonePending = this.zone;
|
||||
}));
|
||||
}
|
||||
await this.passOverCleanMasters(
|
||||
'zone',
|
||||
(e, acct) => this.planner.planZoneChange(e, stored, this.ctx(acct)),
|
||||
(next) => {
|
||||
next.deviceZone = this.zone;
|
||||
next.deviceZonePending = null;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The reminder owner changed (the run's uploads used the old one, so
|
||||
* pending reminder edits went up first): Reminders rows and baselines are
|
||||
* rewritten for the new owner, and the calendars' MAX_REMINDERS follows.
|
||||
*/
|
||||
private async reminderOwnerPass(): Promise<void> {
|
||||
const stored = this.env.store.committed.reminderOwner ?? null;
|
||||
const wanted = this.env.prefs.reminderOwner;
|
||||
if (stored === wanted) return;
|
||||
this.reminderOwner = wanted;
|
||||
if (!stored) {
|
||||
// The first run wrote everything for the wanted owner already.
|
||||
await this.env.writer.write([], this.env.store.tail((next) => {
|
||||
next.reminderOwner = wanted;
|
||||
}));
|
||||
return;
|
||||
}
|
||||
await this.passOverCleanMasters(
|
||||
'reminders',
|
||||
(e, acct) => this.planner.planReminderOwnerChange(e, this.ctx(acct)),
|
||||
(next) => {
|
||||
next.reminderOwner = wanted;
|
||||
},
|
||||
);
|
||||
const selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }> = [];
|
||||
for (const account of this.env.accounts) {
|
||||
for (const calendar of this.calendars.get(account.id) ?? []) {
|
||||
const key = collectionKey(account.id, calendar.id);
|
||||
if (!this.syncedKeys.has(key)) continue;
|
||||
selected.push({ jmapAccountId: account.id, calendar, readOnly: this.readOnlyKeys.has(key), accountName: account.personal ? '' : account.name });
|
||||
}
|
||||
}
|
||||
const plan = this.planner.planCalendars(selected, await this.loadCalendars(), this.calendarCtx());
|
||||
await this.env.writer.write(plan.groups.map((group) => ({ group })));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Checkpoints sit between chunks and phases (docs/device-sync.md, "A sync
|
||||
* run"): the run stops cleanly when the framework cancelled it or its time
|
||||
* is nearly up, keeps SyncManager's traffic monitor happy during long local
|
||||
* phases (it cancels a sync with no network traffic for 60 s), and lets the
|
||||
* JS thread breathe so the UI keeps rendering when it shares the runtime.
|
||||
*/
|
||||
import type { JmapCaller } from '../jmap/caller';
|
||||
import { StopRun } from './errors';
|
||||
|
||||
export interface CheckpointOptions {
|
||||
now(): number;
|
||||
/** Epoch ms by which the run must have finished. */
|
||||
deadline: number;
|
||||
/** Stop starting work this long before the deadline. */
|
||||
marginMs: number;
|
||||
echoAfterMs: number;
|
||||
isCancelled(): Promise<boolean>;
|
||||
yieldThread(): Promise<void>;
|
||||
}
|
||||
|
||||
export class Checkpoints {
|
||||
private jmap: JmapCaller | null = null;
|
||||
/** Test hook: runs at every checkpoint (crash injection). */
|
||||
onCheckpoint?: (label: string) => void | Promise<void>;
|
||||
|
||||
constructor(private readonly options: CheckpointOptions) {}
|
||||
|
||||
attach(jmap: JmapCaller): void {
|
||||
this.jmap = jmap;
|
||||
}
|
||||
|
||||
budgetMs(): number {
|
||||
return this.options.deadline - this.options.now();
|
||||
}
|
||||
|
||||
/** Throws StopRun when the run must end here. */
|
||||
async check(label = ''): Promise<void> {
|
||||
await this.onCheckpoint?.(label);
|
||||
if (this.budgetMs() < this.options.marginMs) throw new StopRun('deadline');
|
||||
if (await this.options.isCancelled()) throw new StopRun('cancelled');
|
||||
if (this.jmap && this.jmap.idleMs() >= this.options.echoAfterMs) await this.jmap.echo();
|
||||
await this.options.yieldThread();
|
||||
}
|
||||
|
||||
/** Throws StopRun('deadline') unless at least `ms` of budget remain (a create batch needs two request timeouts). */
|
||||
requireBudget(ms: number): void {
|
||||
if (this.budgetMs() < ms) throw new StopRun('deadline');
|
||||
}
|
||||
|
||||
/** Keeps traffic flowing during a long local phase without the other checks. */
|
||||
async keepAlive(): Promise<void> {
|
||||
if (this.jmap && this.jmap.idleMs() >= this.options.echoAfterMs) await this.jmap.echo();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,743 @@
|
||||
/**
|
||||
* Contacts: address books ↔ raw contacts and groups of one Android account
|
||||
* (docs/device-sync.md, "Contacts mapping"). Group cards (`kind: "group"`)
|
||||
* are Groups rows and are always written before the contacts of the same
|
||||
* download, so memberships never name a group that does not exist yet.
|
||||
*/
|
||||
import { MimeType } from '../android-columns';
|
||||
import { collectionKey, parseCollectionKey } from '../common/ids';
|
||||
import { parseJsonColumn } from '../common/json';
|
||||
import type {
|
||||
AddressBookLike,
|
||||
ContactCardWire,
|
||||
ContactsContext,
|
||||
ContactsPlanner,
|
||||
LocalContact,
|
||||
LocalGroup,
|
||||
} from '../planner';
|
||||
import { CONTACTS_AUTHORITY, type Row } from '../types';
|
||||
import { ADDRESS_BOOK_PROPERTIES, CONTACT_CARD_PROPERTIES } from '../wire';
|
||||
import type { Work } from './batch';
|
||||
import type { RunEnv } from './context';
|
||||
import { ItemSync, type Pending } from './item-sync';
|
||||
import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds';
|
||||
import { poisonFingerprint } from './poison';
|
||||
import { flag, num, str } from './provider';
|
||||
import { isCollectionSelected } from './selection';
|
||||
import { accountOf, type SyncState } from './sync-state';
|
||||
|
||||
/**
|
||||
* The context the engine hands the contacts planner. `groupsOf` is
|
||||
* not part of `ContactsContext` yet: memberships are stored on the group
|
||||
* cards (`members` holds member uids), so a contact's GroupMembership rows
|
||||
* need the reverse lookup.
|
||||
*/
|
||||
export interface ContactsRunContext extends ContactsContext {
|
||||
/** SOURCE_IDs of the group cards whose `members` include this uid. */
|
||||
groupsOf(uid: string): string[];
|
||||
}
|
||||
|
||||
const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
|
||||
|
||||
export function toBase64(bytes: Uint8Array): string {
|
||||
let out = '';
|
||||
for (let i = 0; i < bytes.length; i += 3) {
|
||||
const a = bytes[i];
|
||||
const b = i + 1 < bytes.length ? bytes[i + 1] : 0;
|
||||
const c = i + 2 < bytes.length ? bytes[i + 2] : 0;
|
||||
const n = (a << 16) | (b << 8) | c;
|
||||
out += BASE64[(n >> 18) & 63] + BASE64[(n >> 12) & 63];
|
||||
out += i + 1 < bytes.length ? BASE64[(n >> 6) & 63] : '=';
|
||||
out += i + 2 < bytes.length ? BASE64[n & 63] : '=';
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Base64 of a `data:` URI's bytes, or null. */
|
||||
export function dataUriBase64(uri: unknown): string | null {
|
||||
if (typeof uri !== 'string' || !/^data:/i.test(uri)) return null;
|
||||
const comma = uri.indexOf(',');
|
||||
if (comma < 0) return null;
|
||||
const header = uri.slice(5, comma);
|
||||
const payload = uri.slice(comma + 1);
|
||||
if (/;base64$/i.test(header)) return payload.replace(/\s+/g, '');
|
||||
const bytes: number[] = [];
|
||||
for (let i = 0; i < payload.length; i++) {
|
||||
if (payload[i] === '%' && /^[0-9a-f]{2}$/i.test(payload.slice(i + 1, i + 3))) {
|
||||
bytes.push(parseInt(payload.slice(i + 1, i + 3), 16));
|
||||
i += 2;
|
||||
} else {
|
||||
bytes.push(payload.charCodeAt(i) & 0xff);
|
||||
}
|
||||
}
|
||||
return toBase64(Uint8Array.from(bytes));
|
||||
}
|
||||
|
||||
/** The name a Relation row shows for a card. */
|
||||
export function displayNameOf(card: Pick<ContactCardWire, 'name'> | null | undefined): string | null {
|
||||
const full = card?.name?.full;
|
||||
if (typeof full === 'string' && full.trim()) return full.trim();
|
||||
const parts = (card?.name?.components ?? [])
|
||||
.filter((c) => c && c.kind !== 'separator' && typeof c.value === 'string' && c.value.trim())
|
||||
.map((c) => (c.value as string).trim());
|
||||
return parts.length ? parts.join(' ') : null;
|
||||
}
|
||||
|
||||
function onIds(map: Record<string, boolean> | undefined | null): string[] {
|
||||
return Object.entries(map ?? {})
|
||||
.filter(([, on]) => on === true)
|
||||
.map(([id]) => id);
|
||||
}
|
||||
|
||||
function union(a: readonly string[], b: readonly string[]): string[] {
|
||||
return [...new Set([...a, ...b])];
|
||||
}
|
||||
|
||||
export class ContactsSync extends ItemSync {
|
||||
protected readonly itemType = 'ContactCard' as const;
|
||||
protected readonly itemProperties = CONTACT_CARD_PROPERTIES;
|
||||
protected readonly parentFilter = 'inAddressBook' as const;
|
||||
protected readonly parentProperty = 'addressBookIds' as const;
|
||||
|
||||
readonly contactKind: Kind<LocalContact, ContactCardWire>;
|
||||
readonly groupKind: Kind<LocalGroup, ContactCardWire>;
|
||||
private readonly planner: ContactsPlanner;
|
||||
private readonly rawColumns: string[];
|
||||
private readonly dataColumns: string[];
|
||||
private readonly groupColumns: string[];
|
||||
|
||||
private readonly books = new Map<string, AddressBookLike[]>();
|
||||
private readonly syncedKeys = new Set<string>();
|
||||
private readonly collectionStates = new Map<string, string>();
|
||||
/** Group SOURCE_ID → Groups row id, and member uid → group SOURCE_IDs; rebuilt after group writes. */
|
||||
private readonly groupRows = new Map<string, number>();
|
||||
private readonly membership = new Map<string, Set<string>>();
|
||||
private groupsStale = true;
|
||||
/** Per account: group refs that appeared (true) or went (false), for `SyncState.groups`. */
|
||||
private readonly groupChanges = new Map<string, Map<string, boolean>>();
|
||||
/** Per account: group refs listed in `groups` whose rows an app hard-deleted. */
|
||||
private absent = new Map<string, string[]>();
|
||||
private readonly blobPhotos = new Map<string, string | null>();
|
||||
private readonly devicePhotos = new Map<number, string | null>();
|
||||
private uidNames: Map<string, string> | null = null;
|
||||
private uidRefs: Map<string, string[]> | null = null;
|
||||
/** Per account: raw contacts the upload phases looked at, for their memberships. */
|
||||
private readonly uploadRows = new Map<string, Set<number>>();
|
||||
|
||||
constructor(env: RunEnv) {
|
||||
super(env);
|
||||
this.planner = env.deps.planners.contacts;
|
||||
this.rawColumns = union(this.planner.rawContactColumns, ['_id']);
|
||||
this.dataColumns = union(this.planner.dataColumns, ['_id', 'raw_contact_id', 'mimetype']);
|
||||
this.groupColumns = union(this.planner.groupColumns, ['_id']);
|
||||
this.contactKind = this.makeContactKind();
|
||||
this.groupKind = this.makeGroupKind();
|
||||
}
|
||||
|
||||
// ── Kinds ──
|
||||
|
||||
private makeContactKind(): Kind<LocalContact, ContactCardWire> {
|
||||
const planner = this.planner;
|
||||
return {
|
||||
name: 'contact',
|
||||
table: 'raw_contacts',
|
||||
identityColumn: 'sourceid',
|
||||
poisonColumn: 'sync4',
|
||||
meta: (c) => ({
|
||||
rowId: c.rawContactId,
|
||||
sourceId: c.sourceId,
|
||||
pending: c.pending,
|
||||
dirty: c.dirty,
|
||||
deleted: c.deleted,
|
||||
isNew: !c.sourceId,
|
||||
poison: c.poison,
|
||||
collections: c.collections,
|
||||
}),
|
||||
fingerprint: (c) =>
|
||||
poisonFingerprint({
|
||||
deleted: c.deleted,
|
||||
shadow: c.shadow,
|
||||
rows: c.rows.map((r) => JSON.stringify([r.mimetype, r.cells])).sort(),
|
||||
}),
|
||||
loadByRefs: async (refs) => {
|
||||
const out = new Map<string, LocalContact>();
|
||||
for (const c of await this.loadContactsIn('sourceid', refs)) if (c.sourceId) out.set(c.sourceId, c);
|
||||
return out;
|
||||
},
|
||||
loadByRowIds: (ids) => this.loadContactsIn('_id', ids),
|
||||
loadNew: () => this.loadContactsWhere('sourceid IS NULL'),
|
||||
planDownload: (card, local, acct) => planner.planDownload(card, local, this.ctx(acct)),
|
||||
planLocalDelete: (c) => planner.planLocalDelete(c),
|
||||
planBaselineHeal: (c) => planner.planBaselineHeal(c),
|
||||
planUpload: (c, acct) => planner.planUpload(c, this.ctx(acct)),
|
||||
planAccepted: (c, card, acct) => planner.planAccepted(c, card, this.ctx(acct)),
|
||||
};
|
||||
}
|
||||
|
||||
private makeGroupKind(): Kind<LocalGroup, ContactCardWire> {
|
||||
const planner = this.planner;
|
||||
return {
|
||||
name: 'group',
|
||||
table: 'groups',
|
||||
identityColumn: 'sourceid',
|
||||
poisonColumn: 'sync4',
|
||||
meta: (g) => ({
|
||||
rowId: g.groupId,
|
||||
sourceId: g.sourceId,
|
||||
pending: g.pending,
|
||||
dirty: g.dirty,
|
||||
deleted: g.deleted,
|
||||
isNew: !g.sourceId,
|
||||
poison: g.poison,
|
||||
collections: this.cardKeys(accountOfRef(g.sourceId), g.shadow),
|
||||
}),
|
||||
fingerprint: (g) => poisonFingerprint({ deleted: g.deleted, title: g.title, shadow: g.shadow }),
|
||||
loadByRefs: async (refs) => {
|
||||
const out = new Map<string, LocalGroup>();
|
||||
for (const g of await this.loadGroupsIn('sourceid', refs)) if (g.sourceId) out.set(g.sourceId, g);
|
||||
return out;
|
||||
},
|
||||
loadByRowIds: (ids) => this.loadGroupsIn('_id', ids),
|
||||
loadNew: () => this.loadGroupsWhere('sourceid IS NULL'),
|
||||
planDownload: (card, local, acct) => planner.planGroupDownload(card, local, this.ctx(acct)),
|
||||
planLocalDelete: (g) => planner.planGroupLocalDelete(g),
|
||||
planBaselineHeal: () => null,
|
||||
planUpload: (g, acct) => planner.planGroupUpload(g, this.ctx(acct)),
|
||||
planAccepted: (g, card, acct) => planner.planGroupAccepted(g, card, this.ctx(acct)),
|
||||
};
|
||||
}
|
||||
|
||||
protected kinds(): Kind[] {
|
||||
return [this.groupKind, this.contactKind];
|
||||
}
|
||||
|
||||
protected kindOf(object: ServerObject): Kind {
|
||||
return object.kind === 'group' ? this.groupKind : this.contactKind;
|
||||
}
|
||||
|
||||
// ── Loading ──
|
||||
|
||||
private async decodeContacts(rawRows: Row[]): Promise<LocalContact[]> {
|
||||
if (!rawRows.length) return [];
|
||||
const data = await this.env.reader.rowsIn('data', this.dataColumns, 'raw_contact_id', rawRows.map((r) => num(r._id)));
|
||||
const byContact = new Map<number, Row[]>();
|
||||
for (const row of data) {
|
||||
const id = num(row.raw_contact_id);
|
||||
let list = byContact.get(id);
|
||||
if (!list) byContact.set(id, (list = []));
|
||||
list.push(row);
|
||||
}
|
||||
return rawRows.map((rc) => this.planner.decodeContact(rc, byContact.get(num(rc._id)) ?? []));
|
||||
}
|
||||
|
||||
private async loadContactsIn(column: '_id' | 'sourceid', values: ReadonlyArray<string | number>): Promise<LocalContact[]> {
|
||||
if (!values.length) return [];
|
||||
return this.decodeContacts(await this.env.reader.rowsIn('raw_contacts', this.rawColumns, column, values));
|
||||
}
|
||||
|
||||
private async loadContactsWhere(where?: string, args?: Array<string | number>): Promise<LocalContact[]> {
|
||||
return this.decodeContacts(await this.env.reader.rows('raw_contacts', this.rawColumns, where, args));
|
||||
}
|
||||
|
||||
private async loadGroupsIn(column: '_id' | 'sourceid', values: ReadonlyArray<string | number>): Promise<LocalGroup[]> {
|
||||
if (!values.length) return [];
|
||||
const rows = await this.env.reader.rowsIn('groups', this.groupColumns, column, values);
|
||||
return rows.map((r) => this.planner.decodeGroup(r));
|
||||
}
|
||||
|
||||
private async loadGroupsWhere(where?: string, args?: Array<string | number>): Promise<LocalGroup[]> {
|
||||
const rows = await this.env.reader.rows('groups', this.groupColumns, where, args);
|
||||
return rows.map((r) => this.planner.decodeGroup(r));
|
||||
}
|
||||
|
||||
/** Group rows by SOURCE_ID and who is a member of which group, from the group cards' shadows. */
|
||||
private async refreshGroups(): Promise<void> {
|
||||
this.groupRows.clear();
|
||||
this.membership.clear();
|
||||
for (const group of await this.loadGroupsWhere()) {
|
||||
if (!group.sourceId || group.deleted) continue;
|
||||
this.groupRows.set(group.sourceId, group.groupId);
|
||||
for (const uid of onIds(group.shadow?.members as Record<string, boolean> | undefined)) {
|
||||
let set = this.membership.get(uid);
|
||||
if (!set) this.membership.set(uid, (set = new Set()));
|
||||
set.add(group.sourceId);
|
||||
}
|
||||
}
|
||||
this.groupsStale = false;
|
||||
}
|
||||
|
||||
// ── Collections ──
|
||||
|
||||
private cardKeys(acct: string | null, card: Pick<ContactCardWire, 'addressBookIds'> | null | undefined): string[] {
|
||||
if (!acct || !card) return [];
|
||||
return onIds(card.addressBookIds).map((id) => collectionKey(acct, id));
|
||||
}
|
||||
|
||||
private book(key: string): AddressBookLike | undefined {
|
||||
const parsed = parseCollectionKey(key);
|
||||
return parsed ? this.books.get(parsed.accountId)?.find((b) => b.id === parsed.id) : undefined;
|
||||
}
|
||||
|
||||
private writable(key: string): boolean {
|
||||
const book = this.book(key);
|
||||
return !!book && book.myRights?.mayWrite !== false;
|
||||
}
|
||||
|
||||
protected async collections(write: boolean): Promise<void> {
|
||||
this.syncedKeys.clear();
|
||||
for (const account of this.env.accounts) {
|
||||
const { list, state } = await this.env.jmap.getContainers<AddressBookLike>('AddressBook', account.id, ADDRESS_BOOK_PROPERTIES);
|
||||
this.books.set(account.id, list);
|
||||
for (const book of list) {
|
||||
if (isCollectionSelected(this.env.prefs.contactsSelection, account, CONTACTS_AUTHORITY, book)) {
|
||||
this.syncedKeys.add(collectionKey(account.id, book.id));
|
||||
}
|
||||
}
|
||||
if (state) {
|
||||
this.collectionStates.set(account.id, state);
|
||||
this.env.recordKnownState(account.id, 'AddressBook', state);
|
||||
}
|
||||
}
|
||||
if (write) await this.ensureSettingsRow();
|
||||
}
|
||||
|
||||
/** One Settings row per account with UNGROUPED_VISIBLE=1: the default 0 hides every contact without a visible group. */
|
||||
private async ensureSettingsRow(): Promise<void> {
|
||||
const rows = await this.env.reader.rows('settings', ['ungrouped_visible', 'should_sync']);
|
||||
const values = { ungrouped_visible: 1, should_sync: 1 };
|
||||
if (!rows.length) {
|
||||
await this.env.writer.write([{ group: { ref: 'settings', ops: [{ op: 'insert', table: 'settings', values }] } }]);
|
||||
} else if (rows.some((r) => num(r.ungrouped_visible) !== 1 || num(r.should_sync) !== 1)) {
|
||||
await this.env.writer.write([{ group: { ref: 'settings', ops: [{ op: 'update', table: 'settings', values }] } }]);
|
||||
}
|
||||
}
|
||||
|
||||
protected syncedCollections(acct: string): string[] {
|
||||
return (this.books.get(acct) ?? [])
|
||||
.filter((b) => this.syncedKeys.has(collectionKey(acct, b.id)))
|
||||
.map((b) => b.id);
|
||||
}
|
||||
|
||||
protected inSelection(acct: string, object: ServerObject): boolean {
|
||||
return this.cardKeys(acct, object as ContactCardWire).some((k) => this.syncedKeys.has(k));
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a contact created on the device goes: the chosen book when it
|
||||
* syncs and is writable, else the personal account's default (or first)
|
||||
* synced writable book; null when there is none.
|
||||
*/
|
||||
createTarget(): string | null {
|
||||
const chosen = this.env.prefs.newContactsAddressBook;
|
||||
if (chosen && this.syncedKeys.has(chosen) && this.writable(chosen)) return chosen;
|
||||
const primary = this.env.accounts.find((a) => a.primary) ?? this.env.accounts[0];
|
||||
if (!primary) return null;
|
||||
const books = [...(this.books.get(primary.id) ?? [])].sort(
|
||||
(a, b) => Number(!!b.isDefault) - Number(!!a.isDefault) || (a.sortOrder ?? 0) - (b.sortOrder ?? 0) || a.id.localeCompare(b.id),
|
||||
);
|
||||
const book = books.find((b) => {
|
||||
const key = collectionKey(primary.id, b.id);
|
||||
return this.syncedKeys.has(key) && this.writable(key);
|
||||
});
|
||||
return book ? collectionKey(primary.id, book.id) : null;
|
||||
}
|
||||
|
||||
private photoBytes(entry: { uri?: string; blobId?: string }): string | null {
|
||||
if (entry.blobId) return this.blobPhotos.get(entry.blobId) ?? null;
|
||||
return dataUriBase64(entry.uri);
|
||||
}
|
||||
|
||||
ctx(acct: string): ContactsRunContext {
|
||||
const env = this.env;
|
||||
return {
|
||||
jmapAccountId: acct,
|
||||
now: env.now(),
|
||||
mintKey: (taken) => env.mintKey(taken),
|
||||
mintUid: () => env.mintUid(),
|
||||
selectedCollections: (card) => this.cardKeys(acct, card).filter((k) => this.syncedKeys.has(k)),
|
||||
isReadOnly: (card) => {
|
||||
const keys = this.cardKeys(acct, card);
|
||||
return keys.length > 0 && keys.every((k) => this.book(k)?.myRights?.mayWrite === false);
|
||||
},
|
||||
isSelected: (key) => this.syncedKeys.has(key),
|
||||
createTarget: () => this.createTarget(),
|
||||
photoBytes: (entry) => this.photoBytes(entry),
|
||||
devicePhoto: (rawContactId) => this.devicePhotos.get(rawContactId) ?? null,
|
||||
nameForUid: (uid) => this.uidNames?.get(uid) ?? null,
|
||||
groupRowIdBySourceId: (sourceId) => this.groupRows.get(sourceId) ?? null,
|
||||
groupsOf: (uid) => [...(this.membership.get(uid) ?? [])],
|
||||
};
|
||||
}
|
||||
|
||||
// ── Downloads ──
|
||||
|
||||
/**
|
||||
* Group cards first: a membership naming an unknown group makes the
|
||||
* provider create an empty one. Memberships live on the group cards, so
|
||||
* when a group's `members` changed, its old and new members come along in
|
||||
* the same download (before its state op): their GroupMembership rows
|
||||
* change although their own cards did not.
|
||||
*/
|
||||
protected async order(acct: string, ids: string[]): Promise<{ first: string[]; rest: string[]; missing: string[] }> {
|
||||
if (!ids.length) return { first: [], rest: [], missing: [] };
|
||||
const { list, notFound } = await this.env.jmap.get<{ id: string; kind?: string; members?: Record<string, boolean> }>(
|
||||
'ContactCard',
|
||||
acct,
|
||||
ids,
|
||||
['id', 'kind', 'members'],
|
||||
);
|
||||
const groups = list.filter((c) => c.kind === 'group');
|
||||
const groupIds = new Set(groups.map((c) => c.id));
|
||||
const found = new Set(list.map((c) => c.id));
|
||||
const rest = ids.filter((id) => found.has(id) && !groupIds.has(id));
|
||||
const members = (await this.membersToRefresh(acct, groups)).filter((id) => !groupIds.has(id));
|
||||
return {
|
||||
first: ids.filter((id) => groupIds.has(id)),
|
||||
rest: members.length ? [...new Set([...rest, ...members])].sort() : rest,
|
||||
missing: notFound,
|
||||
};
|
||||
}
|
||||
|
||||
/** Contacts of `acct` whose membership in one of these group cards flipped since the group's shadow. */
|
||||
private async membersToRefresh(acct: string, groups: Array<{ id: string; members?: Record<string, boolean> }>): Promise<string[]> {
|
||||
if (!groups.length) return [];
|
||||
const locals = await this.groupKind.loadByRefs(groups.map((g) => refOf(acct, g.id)));
|
||||
const flipped = new Set<string>();
|
||||
for (const group of groups) {
|
||||
const before = new Set(onIds(locals.get(refOf(acct, group.id))?.shadow?.members as Record<string, boolean> | undefined));
|
||||
const after = new Set(onIds(group.members));
|
||||
for (const uid of before) if (!after.has(uid)) flipped.add(uid);
|
||||
for (const uid of after) if (!before.has(uid)) flipped.add(uid);
|
||||
}
|
||||
if (!flipped.size) return [];
|
||||
await this.ensureShadowIndex();
|
||||
const out: string[] = [];
|
||||
for (const uid of flipped) {
|
||||
for (const ref of this.uidRefs?.get(uid) ?? []) {
|
||||
const id = idInAccount(ref, acct);
|
||||
if (id) out.push(id);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
protected async beforePlanning(acct: string, objects: ServerObject[]): Promise<void> {
|
||||
const cards = objects as ContactCardWire[];
|
||||
if (this.groupsStale && cards.some((c) => c.kind !== 'group')) await this.refreshGroups();
|
||||
for (const card of cards) {
|
||||
const photo = Object.values(card.media ?? {}).find((m) => m?.kind === 'photo') as
|
||||
| { blobId?: string; mediaType?: string }
|
||||
| undefined;
|
||||
if (!photo?.blobId || this.blobPhotos.has(photo.blobId)) continue;
|
||||
try {
|
||||
this.blobPhotos.set(photo.blobId, toBase64(await this.env.jmap.downloadBlob(acct, photo.blobId, photo.mediaType)));
|
||||
} catch (error) {
|
||||
this.env.log('photo download failed', error);
|
||||
this.blobPhotos.set(photo.blobId, null);
|
||||
}
|
||||
}
|
||||
if (cards.some((c) => Object.keys(c.relatedTo ?? {}).length > 0)) await this.ensureShadowIndex();
|
||||
for (const card of cards) this.indexCard(acct, card);
|
||||
}
|
||||
|
||||
private indexCard(acct: string, card: ContactCardWire & { id?: string }): void {
|
||||
if (!this.uidNames || !this.uidRefs || typeof card.uid !== 'string' || card.kind === 'group') return;
|
||||
const name = displayNameOf(card);
|
||||
if (name) this.uidNames.set(card.uid, name);
|
||||
if (card.id) {
|
||||
const ref = refOf(acct, card.id);
|
||||
const refs = this.uidRefs.get(card.uid) ?? [];
|
||||
if (!refs.includes(ref)) this.uidRefs.set(card.uid, [...refs, ref]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The synced cards by uid (their names for Relation rows, their rows for
|
||||
* membership changes), read from the shadows on first need and kept up to
|
||||
* date with every card downloaded afterwards.
|
||||
*/
|
||||
private async ensureShadowIndex(): Promise<void> {
|
||||
if (this.uidNames && this.uidRefs) return;
|
||||
const names = new Map<string, string>();
|
||||
const refs = new Map<string, string[]>();
|
||||
for (const row of await this.env.reader.rows('raw_contacts', ['sourceid', 'sync2'], 'sourceid IS NOT NULL')) {
|
||||
const shadow = parseJsonColumn<ContactCardWire>(row.sync2);
|
||||
const ref = str(row.sourceid);
|
||||
if (!shadow || typeof shadow.uid !== 'string' || !ref) continue;
|
||||
const name = displayNameOf(shadow);
|
||||
if (name) names.set(shadow.uid, name);
|
||||
refs.set(shadow.uid, [...(refs.get(shadow.uid) ?? []), ref]);
|
||||
}
|
||||
this.uidNames = names;
|
||||
this.uidRefs = refs;
|
||||
}
|
||||
|
||||
protected adoptionTargetMatches(acct: string, object: ServerObject, target: string): boolean {
|
||||
return this.cardKeys(acct, object as ContactCardWire).includes(target);
|
||||
}
|
||||
|
||||
protected lookupScope(_acct: string, collectionId: string): string | null {
|
||||
// Card uids are unique per address book.
|
||||
return collectionId;
|
||||
}
|
||||
|
||||
private groupChange(acct: string, ref: string, present: boolean): void {
|
||||
let changes = this.groupChanges.get(acct);
|
||||
if (!changes) this.groupChanges.set(acct, (changes = new Map()));
|
||||
changes.set(ref, present);
|
||||
this.groupsStale = true;
|
||||
}
|
||||
|
||||
protected onDownloaded(kind: Kind, acct: string, object: ServerObject): void {
|
||||
if (kind === this.groupKind) this.groupChange(acct, refOf(acct, object.id), true);
|
||||
else this.indexCard(acct, object as ContactCardWire);
|
||||
}
|
||||
|
||||
protected onRemoved(kind: Kind, acct: string, id: string): void {
|
||||
if (kind === this.groupKind) this.groupChange(acct, refOf(acct, id), false);
|
||||
}
|
||||
|
||||
protected onAccepted(kind: Kind, acct: string, server: ServerObject): void {
|
||||
if (kind === this.groupKind) this.groupChange(acct, refOf(acct, server.id), true);
|
||||
else this.indexCard(acct, server as ContactCardWire);
|
||||
}
|
||||
|
||||
protected decorateState(acct: string, next: SyncState): () => void {
|
||||
const account = accountOf(next, acct);
|
||||
const changes = this.groupChanges.get(acct);
|
||||
const consumed = new Map(changes ?? []);
|
||||
if (consumed.size || account.groups === undefined) {
|
||||
const groups = new Set(account.groups ?? []);
|
||||
for (const [ref, present] of consumed) {
|
||||
if (present) groups.add(ref);
|
||||
else groups.delete(ref);
|
||||
}
|
||||
account.groups = [...groups];
|
||||
}
|
||||
const collections = this.collectionStates.get(acct);
|
||||
if (collections) account.collectionsState = collections;
|
||||
return () => {
|
||||
for (const [ref, present] of consumed) if (changes?.get(ref) === present) changes.delete(ref);
|
||||
};
|
||||
}
|
||||
|
||||
protected async localIdentities(acct: string): Promise<Map<string, { dirty: boolean; deleted: boolean }>> {
|
||||
const out = new Map<string, { dirty: boolean; deleted: boolean }>();
|
||||
for (const table of ['groups', 'raw_contacts'] as const) {
|
||||
for (const row of await this.env.reader.rows(table, ['_id', 'sourceid', 'dirty', 'deleted'], 'sourceid LIKE ?', [`${acct}/%`])) {
|
||||
const id = idInAccount(str(row.sourceid), acct);
|
||||
if (id) out.set(id, { dirty: flag(row.dirty), deleted: flag(row.deleted) });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Uploads ──
|
||||
|
||||
protected async loadUploadItems(): Promise<Held[]> {
|
||||
const groups = await this.loadGroupsWhere('dirty = 1 OR deleted = 1 OR sourceid IS NULL');
|
||||
const contacts = await this.loadContactsWhere('dirty = 1 OR deleted = 1 OR sourceid IS NULL');
|
||||
return [
|
||||
...groups.map((local) => ({ kind: this.groupKind as Kind, local })),
|
||||
...contacts.map((local) => ({ kind: this.contactKind as Kind, local })),
|
||||
];
|
||||
}
|
||||
|
||||
protected claimAccount(): string | null {
|
||||
const target = this.createTarget();
|
||||
return accountOfRef(target) ?? this.env.accounts.find((a) => a.primary)?.id ?? this.env.accounts[0]?.id ?? null;
|
||||
}
|
||||
|
||||
protected noteUploadItems(acct: string, items: Held[]): void {
|
||||
let rows = this.uploadRows.get(acct);
|
||||
if (!rows) this.uploadRows.set(acct, (rows = new Set()));
|
||||
for (const held of items) if (held.kind === this.contactKind) rows.add(held.kind.meta(held.local).rowId);
|
||||
}
|
||||
|
||||
/** The device photo of every contact about to be planned that has a photo row (JPEG, ≤ 512 px). */
|
||||
protected async beforeUploadPlanning(items: Held[]): Promise<void> {
|
||||
for (const held of items) {
|
||||
if (held.kind !== this.contactKind) continue;
|
||||
const contact = held.local as LocalContact;
|
||||
if (this.devicePhotos.has(contact.rawContactId) || !contact.rows.some((r) => r.mimetype === MimeType.PHOTO)) continue;
|
||||
try {
|
||||
const photo = await this.env.reader.port.readPhoto(contact.rawContactId, 512);
|
||||
this.devicePhotos.set(contact.rawContactId, photo?.jpegBase64 ?? null);
|
||||
} catch (error) {
|
||||
this.env.log('photo read failed', error);
|
||||
this.devicePhotos.set(contact.rawContactId, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Group refs the SyncState lists whose rows are gone: an app hard-deleted them (Fossify does). */
|
||||
private async absentGroups(): Promise<Map<string, string[]>> {
|
||||
const present = new Set<string>();
|
||||
for (const row of await this.env.reader.rows('groups', ['sourceid'], 'sourceid IS NOT NULL')) {
|
||||
const ref = str(row.sourceid);
|
||||
if (ref) present.add(ref);
|
||||
}
|
||||
const out = new Map<string, string[]>();
|
||||
for (const [acct, account] of Object.entries(this.env.store.committed.accounts)) {
|
||||
const changes = this.groupChanges.get(acct);
|
||||
const refs = (account.groups ?? []).filter((ref) => !present.has(ref) && changes?.get(ref) !== false);
|
||||
if (refs.length) out.set(acct, refs);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
protected async countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set<string> }> {
|
||||
let count = 0;
|
||||
const accounts = new Set<string>();
|
||||
for (const held of items) {
|
||||
const meta = held.kind.meta(held.local);
|
||||
if (!meta.deleted || !meta.sourceId) continue;
|
||||
count++;
|
||||
const acct = accountOfRef(meta.sourceId);
|
||||
if (acct) accounts.add(acct);
|
||||
}
|
||||
this.absent = await this.absentGroups();
|
||||
for (const [acct, refs] of this.absent) {
|
||||
count += refs.length;
|
||||
accounts.add(acct);
|
||||
}
|
||||
const synced =
|
||||
(await this.env.reader.rows('raw_contacts', ['_id'], 'sourceid IS NOT NULL')).length +
|
||||
(await this.env.reader.rows('groups', ['_id'], 'sourceid IS NOT NULL')).length;
|
||||
return { count, synced: Math.max(synced, count), accounts };
|
||||
}
|
||||
|
||||
protected onDiscard(next: SyncState, accounts: Set<string>): void {
|
||||
for (const acct of accounts) {
|
||||
const gone = new Set(this.absent.get(acct) ?? []);
|
||||
if (!gone.size) continue;
|
||||
const account = accountOf(next, acct);
|
||||
account.groups = (account.groups ?? []).filter((ref) => !gone.has(ref));
|
||||
}
|
||||
this.absent = new Map();
|
||||
}
|
||||
|
||||
protected async extraPhase(acct: string, deletionsAllowed: boolean): Promise<void> {
|
||||
if (deletionsAllowed) await this.deleteAbsentGroups(acct);
|
||||
await this.uploadMemberships(acct);
|
||||
}
|
||||
|
||||
/**
|
||||
* A group hard-deleted on the device: destroyed on the server, or, when
|
||||
* it is also in address books that do not sync, removed from the synced
|
||||
* ones only.
|
||||
*/
|
||||
private async deleteAbsentGroups(acct: string): Promise<void> {
|
||||
const refs = (this.absent.get(acct) ?? []).filter((ref) => this.groupChanges.get(acct)?.get(ref) !== false);
|
||||
if (!refs.length) return;
|
||||
await this.env.checkpoints.check('upload:groups');
|
||||
const ids = refs.map((ref) => idInAccount(ref, acct)).filter((id): id is string => !!id);
|
||||
const { list, notFound } = await this.env.jmap.get<ContactCardWire & { id: string }>('ContactCard', acct, ids, ['id', 'addressBookIds']);
|
||||
for (const id of notFound) this.groupChange(acct, refOf(acct, id), false);
|
||||
const update: Record<string, Record<string, unknown>> = {};
|
||||
const destroy: string[] = [];
|
||||
for (const card of list) {
|
||||
const books = onIds(card.addressBookIds);
|
||||
const synced = books.filter((id) => this.syncedKeys.has(collectionKey(acct, id)));
|
||||
if (synced.length && synced.length < books.length) {
|
||||
update[card.id] = Object.fromEntries(synced.map((id) => [`addressBookIds/${id}`, null]));
|
||||
} else {
|
||||
destroy.push(card.id);
|
||||
}
|
||||
}
|
||||
if (!destroy.length && !Object.keys(update).length) return;
|
||||
const { response } = await this.setChecked(acct, { update, destroy });
|
||||
const done = [...(response.destroyed ?? []), ...Object.keys(response.updated ?? {})];
|
||||
for (const id of done) {
|
||||
this.groupChange(acct, refOf(acct, id), false);
|
||||
this.env.report.stats.uploaded.deleted++;
|
||||
}
|
||||
for (const [id, error] of [...Object.entries(response.notDestroyed ?? {}), ...Object.entries(response.notUpdated ?? {})]) {
|
||||
if (error?.type === 'notFound') this.groupChange(acct, refOf(acct, id), false);
|
||||
else this.env.report.itemError({ ref: refOf(acct, id), side: 'upload', type: error?.type ?? 'serverFail', description: error?.description });
|
||||
}
|
||||
}
|
||||
|
||||
/** Memberships edited on the device become patches of the group cards' `members`. */
|
||||
private async uploadMemberships(acct: string): Promise<void> {
|
||||
const rows = [...(this.uploadRows.get(acct) ?? [])];
|
||||
if (!rows.length) return;
|
||||
await this.env.checkpoints.check('upload:memberships');
|
||||
const contacts = (await this.contactKind.loadByRowIds(rows)).filter((c) => !c.deleted);
|
||||
const groups = (await this.loadGroupsWhere('sourceid LIKE ?', [`${acct}/%`])).filter(
|
||||
(g) => accountOfRef(g.sourceId) === acct && !g.deleted,
|
||||
);
|
||||
const actions = this.planner.planMembershipUploads(contacts, groups, this.ctx(acct));
|
||||
if (!actions.length) return;
|
||||
const byGroup = new Map<string, Pending>();
|
||||
for (const action of actions) {
|
||||
const id = action.kind === 'create' ? null : action.id;
|
||||
const group = id ? groups.find((g) => g.sourceId === refOf(acct, id)) : undefined;
|
||||
if (!group) {
|
||||
this.env.report.itemError({ ref: id ? refOf(acct, id) : 'membership', side: 'upload', type: 'unknownGroup' });
|
||||
continue;
|
||||
}
|
||||
let pending = byGroup.get(group.sourceId as string);
|
||||
if (!pending) {
|
||||
const held: Held = { kind: this.groupKind, local: group };
|
||||
pending = { held, meta: this.groupKind.meta(group), fingerprint: this.groupKind.fingerprint(group), actions: [] };
|
||||
byGroup.set(group.sourceId as string, pending);
|
||||
}
|
||||
pending.actions.push(action as Pending['actions'][number]);
|
||||
}
|
||||
if (byGroup.size) await this.send(acct, [...byGroup.values()]);
|
||||
}
|
||||
|
||||
// ── Deselection ──
|
||||
|
||||
/**
|
||||
* After the upload: rows of contacts in no synced address book go, when
|
||||
* clean (a deselected book, or a card moved away on the server). Dirty
|
||||
* ones stay until their upload succeeds, and keep their book selected.
|
||||
*/
|
||||
protected async dropOutsideSelection(acct: string): Promise<void> {
|
||||
const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id)));
|
||||
const committed = this.env.store.account(acct);
|
||||
const deselected = committed.selected.filter((key) => !synced.has(key));
|
||||
const outside = [...(this.outside.get(acct) ?? [])];
|
||||
if (!deselected.length && !outside.length) return;
|
||||
|
||||
const kept = new Set<string>();
|
||||
const consider = (held: Held, works: Work[]) => {
|
||||
const meta = held.kind.meta(held.local);
|
||||
const id = idInAccount(meta.sourceId, acct);
|
||||
// Rows whose collections are unknown are never dropped.
|
||||
if (!id || !meta.collections.length || meta.collections.some((k) => synced.has(k))) return;
|
||||
const work = this.removeWork(acct, id, held, 'outside');
|
||||
if (work) works.push(work);
|
||||
else for (const key of meta.collections) if (deselected.includes(key)) kept.add(key);
|
||||
};
|
||||
if (deselected.length) {
|
||||
const groupWorks: Work[] = [];
|
||||
for (const group of await this.loadGroupsWhere('sourceid LIKE ?', [`${acct}/%`])) consider({ kind: this.groupKind, local: group }, groupWorks);
|
||||
await this.env.writer.write(groupWorks);
|
||||
const rows = await this.env.reader.rows('raw_contacts', ['_id', 'sourceid'], 'sourceid LIKE ?', [`${acct}/%`]);
|
||||
const ids = rows.filter((r) => accountOfRef(str(r.sourceid)) === acct).map((r) => num(r._id));
|
||||
for (let i = 0; i < ids.length; i += this.env.tuning.chunkSize) {
|
||||
await this.env.checkpoints.check('deselect');
|
||||
const works: Work[] = [];
|
||||
for (const contact of await this.contactKind.loadByRowIds(ids.slice(i, i + this.env.tuning.chunkSize))) {
|
||||
consider({ kind: this.contactKind, local: contact }, works);
|
||||
}
|
||||
await this.env.writer.write(works);
|
||||
}
|
||||
const selected = [...synced, ...kept];
|
||||
// The book leaves `selected` with its last dropped row; one still holding a dirty item stays.
|
||||
await this.env.writer.write([], this.tail([acct], (next) => {
|
||||
accountOf(next, acct).selected = selected;
|
||||
}));
|
||||
} else {
|
||||
const works: Work[] = [];
|
||||
for (const kind of this.kinds()) {
|
||||
for (const local of (await kind.loadByRefs(outside)).values()) consider({ kind, local }, works);
|
||||
}
|
||||
await this.env.writer.write(works);
|
||||
}
|
||||
this.outside.delete(acct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* What one run (or teardown) of one Android account and authority carries
|
||||
* through its phases.
|
||||
*/
|
||||
import type { AccountSyncPrefs, Authority, RunExtras } from '../types';
|
||||
import type { JmapCaller } from '../jmap/caller';
|
||||
import type { CapabilityAccount } from '../jmap/session';
|
||||
import type { BatchWriter } from './batch';
|
||||
import type { Checkpoints } from './checkpoint';
|
||||
import type { EngineDeps, Tuning } from './deps';
|
||||
import type { ProviderReader } from './provider';
|
||||
import type { ReportBuilder } from './report';
|
||||
import type { StateStore } from './sync-state';
|
||||
|
||||
export interface RunEnv {
|
||||
readonly deps: EngineDeps;
|
||||
readonly tuning: Tuning;
|
||||
/** A teardown only uploads (and never writes calendar rows). */
|
||||
readonly mode: 'sync' | 'teardown';
|
||||
readonly registryId: string;
|
||||
readonly accountName: string;
|
||||
readonly authority: Authority;
|
||||
readonly extras: RunExtras;
|
||||
readonly reader: ProviderReader;
|
||||
readonly writer: BatchWriter;
|
||||
readonly report: ReportBuilder;
|
||||
readonly checkpoints: Checkpoints;
|
||||
readonly store: StateStore;
|
||||
readonly jmap: JmapCaller;
|
||||
/** Accounts with the authority's capability, the primary first. */
|
||||
readonly accounts: CapabilityAccount[];
|
||||
readonly prefs: AccountSyncPrefs;
|
||||
now(): number;
|
||||
mintKey(taken?: Iterable<string>): string;
|
||||
mintUid(): string;
|
||||
sleep(ms: number): Promise<void>;
|
||||
log(message: string, detail?: unknown): void;
|
||||
recordKnownState(jmapAccountId: string, type: string, state: string): void;
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Everything the sync engine reaches outside itself. The engine is pure
|
||||
* TypeScript: src/device-sync/task.ts wires these to the native module, a
|
||||
* detached JMAPClient and the app's stores; the tests wire them to the fake
|
||||
* provider, the fake JMAP server and test planners.
|
||||
*/
|
||||
import type { CalendarPlanner, ContactsPlanner } from '../planner';
|
||||
import type { RandomSource } from '../common/ids';
|
||||
import type { AccountSyncPrefs, Authority, ProviderPort, RunStatus } from '../types';
|
||||
import type { JmapConnection } from '../jmap/connection';
|
||||
|
||||
export type { JmapConnection };
|
||||
|
||||
/** A calendar that mirrors an iCal feed (read-only on the device). */
|
||||
export interface SubscriptionCalendar {
|
||||
/** JMAP account of the mirror calendar; null for subscriptions saved before accounts were recorded (the primary). */
|
||||
jmapAccountId: string | null;
|
||||
calendarId: string;
|
||||
}
|
||||
|
||||
export interface EngineDeps {
|
||||
/** The provider of one Android account and authority. */
|
||||
provider(accountName: string, authority: Authority): ProviderPort;
|
||||
/** A JMAP connection for the registry account (rebuilt once on an auth failure); throws the client's errors. */
|
||||
jmap(registryId: string): Promise<JmapConnection>;
|
||||
/** The account's device-sync preferences (the store is hydrated first). */
|
||||
prefs(registryId: string): Promise<AccountSyncPrefs>;
|
||||
planners: { contacts: ContactsPlanner; calendar: CalendarPlanner };
|
||||
/** Epoch milliseconds. */
|
||||
now(): number;
|
||||
/** Whether the framework cancelled the run, or nobody waits for it any more. */
|
||||
isCancelled(runId: string): Promise<boolean>;
|
||||
random: RandomSource;
|
||||
/** Feed-subscription calendars (read-only on the device). */
|
||||
subscriptionCalendars(): Promise<SubscriptionCalendar[]>;
|
||||
/** Android's `getSyncAutomatically` for the account and authority. */
|
||||
isSyncEnabled(accountName: string, authority: Authority): Promise<boolean>;
|
||||
/** The device's IANA time zone (floating events are written in it). */
|
||||
deviceZone(): string;
|
||||
/** The last run's status for the settings UI. */
|
||||
recordStatus(registryId: string, authority: Authority, status: RunStatus): void | Promise<void>;
|
||||
/** The latest state the engine synced for a JMAP account and type, so the app skips the echo of our own writes. */
|
||||
recordKnownState(registryId: string, jmapAccountId: string, type: string, state: string): void | Promise<void>;
|
||||
/** One notification that deep-links to sign-in, after an auth failure the rebuild did not fix. */
|
||||
notifyAuthProblem(registryId: string, accountName: string, authority: Authority): void | Promise<void>;
|
||||
/** Lets the JS thread render between chunks (default: `setTimeout(0)`). */
|
||||
yieldThread?(): Promise<void>;
|
||||
/** Waits before a retry, e.g. the uid lookup's back-off (default: `setTimeout`). */
|
||||
sleep?(ms: number): Promise<void>;
|
||||
log?(message: string, detail?: unknown): void;
|
||||
/** Overrides for tests. */
|
||||
tuning?: Partial<Tuning>;
|
||||
}
|
||||
|
||||
/** Sizes and limits from docs/device-sync.md ("Limits and performance", "Deletion threshold"). */
|
||||
export interface Tuning {
|
||||
/** Raw contacts or events planned per chunk. */
|
||||
chunkSize: number;
|
||||
/** Provider ops per applyBatch, with a yield point at every item group. */
|
||||
maxBatchOps: number;
|
||||
/** Serialized bytes per applyBatch (the Binder budget is 1 MB for the whole transaction). */
|
||||
maxBatchBytes: number;
|
||||
/** Re-reads and re-plans of an item whose group failed an assert. */
|
||||
maxReplans: number;
|
||||
/** Creates per `/set` (a create batch is not interruptible). */
|
||||
maxCreatesPerSet: number;
|
||||
/** Serialized bytes per `/set` call (Stalwart splits bigger calls into several commits). */
|
||||
maxSetBytes: number;
|
||||
/** `stateMismatch` retries of an account's upload. */
|
||||
maxStateMismatchRetries: number;
|
||||
/** Stop starting work this long before the deadline. */
|
||||
deadlineMarginMs: number;
|
||||
/** A create batch starts only with this much budget left (two request timeouts). */
|
||||
createBudgetMs: number;
|
||||
/** A `Core/echo` when a local phase has gone this long without a JMAP request. */
|
||||
echoAfterMs: number;
|
||||
/** Deletions above this count … */
|
||||
deletionThresholdCount: number;
|
||||
/** … and above this share of the authority's synced objects set `tooManyDeletions`. */
|
||||
deletionThresholdRatio: number;
|
||||
/** A full reconcile that finds nothing while the device holds more than this aborts. */
|
||||
safetyAbortRows: number;
|
||||
/** Back-off of a poisoned item: first delay, doubling up to the cap. */
|
||||
poisonBackoffMs: number;
|
||||
poisonBackoffMaxMs: number;
|
||||
/** Retries of a uid lookup after a duplicate-uid error, with these delays. */
|
||||
uidLookupDelaysMs: number[];
|
||||
/** Objects scanned to tell a task-only calendar. */
|
||||
taskScanLimit: number;
|
||||
/** Item errors kept in a report. */
|
||||
maxItemErrors: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_TUNING: Tuning = {
|
||||
chunkSize: 50,
|
||||
maxBatchOps: 400,
|
||||
maxBatchBytes: 300_000,
|
||||
maxReplans: 3,
|
||||
maxCreatesPerSet: 50,
|
||||
maxSetBytes: 1_000_000,
|
||||
maxStateMismatchRetries: 3,
|
||||
deadlineMarginMs: 30_000,
|
||||
createBudgetMs: 60_000,
|
||||
echoAfterMs: 40_000,
|
||||
deletionThresholdCount: 50,
|
||||
deletionThresholdRatio: 0.2,
|
||||
safetyAbortRows: 10,
|
||||
poisonBackoffMs: 60 * 60_000,
|
||||
poisonBackoffMaxMs: 24 * 60 * 60_000,
|
||||
uidLookupDelaysMs: [500, 1500, 4000],
|
||||
taskScanLimit: 50,
|
||||
maxItemErrors: 50,
|
||||
};
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* How a run stops early, and what each failure means for the report
|
||||
* (docs/device-sync.md, "Headless payload, report and SyncResult" and
|
||||
* "Failure matrix").
|
||||
*/
|
||||
import type { RunOutcome } from '../types';
|
||||
import { isMethodError, isRequestLimitError, retryAfterMs, transportFailure } from '../jmap/errors';
|
||||
|
||||
/** Out of time, or cancelled: stop cleanly between chunks, ask for another sync. */
|
||||
export class StopRun extends Error {
|
||||
constructor(readonly reason: 'cancelled' | 'deadline') {
|
||||
super(reason === 'deadline' ? 'Out of time; the rest follows in the next sync' : 'Cancelled');
|
||||
this.name = 'StopRun';
|
||||
}
|
||||
}
|
||||
|
||||
/** A run ends with this outcome. `noProgress` zeroes the counts so SyncManager backs off instead of retrying at once. */
|
||||
export class RunAbort extends Error {
|
||||
constructor(
|
||||
readonly outcome: RunOutcome,
|
||||
message: string,
|
||||
readonly options: { noProgress?: boolean; delayUntil?: number } = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'RunAbort';
|
||||
}
|
||||
}
|
||||
|
||||
/** The server changed since the download an upload was planned from. */
|
||||
export class StateMismatch extends Error {
|
||||
constructor(readonly jmapAccountId: string) {
|
||||
super(`stateMismatch in ${jmapAccountId}`);
|
||||
this.name = 'StateMismatch';
|
||||
}
|
||||
}
|
||||
|
||||
export interface ClassifiedFailure {
|
||||
outcome: RunOutcome;
|
||||
message: string;
|
||||
delayUntil?: number;
|
||||
moreRecordsToGet?: boolean;
|
||||
noProgress?: boolean;
|
||||
/** Tell the user to sign in again. */
|
||||
authProblem?: boolean;
|
||||
}
|
||||
|
||||
function messageOf(error: unknown): string {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message.length > 300 ? `${message.slice(0, 300)}…` : message;
|
||||
}
|
||||
|
||||
/** A provider refusal for a missing runtime permission (the native module rejects with it). */
|
||||
export function isPermissionFailure(error: unknown): boolean {
|
||||
const e = error as { code?: unknown; message?: unknown } | null;
|
||||
return e?.code === 'permission' || (typeof e?.message === 'string' && /\bpermission\b|SecurityException/i.test(e.message));
|
||||
}
|
||||
|
||||
export function classifyFailure(error: unknown, now: number): ClassifiedFailure {
|
||||
if (error instanceof StopRun) {
|
||||
return { outcome: 'cancelled', message: error.message, moreRecordsToGet: true };
|
||||
}
|
||||
if (error instanceof RunAbort) {
|
||||
return { outcome: error.outcome, message: error.message, noProgress: error.options.noProgress, delayUntil: error.options.delayUntil };
|
||||
}
|
||||
if (error instanceof StateMismatch) {
|
||||
return { outcome: 'io', message: 'The server kept changing during the upload', noProgress: true };
|
||||
}
|
||||
switch (transportFailure(error)) {
|
||||
case 'auth':
|
||||
return { outcome: 'auth', message: messageOf(error), authProblem: true };
|
||||
case 'network':
|
||||
case 'timeout':
|
||||
return { outcome: 'io', message: messageOf(error) };
|
||||
case 'rateLimit': {
|
||||
const wait = retryAfterMs(error) ?? 60_000;
|
||||
return { outcome: 'io', message: messageOf(error), delayUntil: Math.ceil((now + wait) / 1000) };
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (isPermissionFailure(error)) return { outcome: 'permission', message: messageOf(error) };
|
||||
if (isRequestLimitError(error)) return { outcome: 'internal', message: messageOf(error) };
|
||||
if (isMethodError(error)) return { outcome: 'io', message: `${error.type}: ${messageOf(error)}` };
|
||||
if (error instanceof Error && /^(JMAP request failed|Invalid JSON response)/.test(error.message)) {
|
||||
return { outcome: 'io', message: messageOf(error) };
|
||||
}
|
||||
return { outcome: 'internal', message: messageOf(error) };
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/**
|
||||
* The device sync engine (docs/device-sync.md). Pure TypeScript: every
|
||||
* dependency comes in through `EngineDeps`; src/device-sync/task.ts wires
|
||||
* the real ones.
|
||||
*/
|
||||
export { runDeviceSync, hasLocalWork } from './run';
|
||||
export { teardownAuthority, countPending, type TeardownOptions } from './teardown';
|
||||
export { DEFAULT_TUNING, type EngineDeps, type JmapConnection, type SubscriptionCalendar, type Tuning } from './deps';
|
||||
export { parseSyncState, serializeSyncState, type SyncState, type AccountState } from './sync-state';
|
||||
export { lockKey, isLocked } from './mutex';
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* The engine's view of one kind of local item (a contact, a group, an event
|
||||
* master with its exceptions): what it needs to know about an item and the
|
||||
* planner calls it makes for it. The contacts and calendar sync plug their
|
||||
* planners in through this, so downloads, uploads, retries and the guards
|
||||
* are written once (engine/item-sync.ts).
|
||||
*/
|
||||
import type { AcceptedPlan, DownloadPlan, OpGroup, PendingCreate, PoisonMarker, UploadPlan } from '../planner';
|
||||
import type { ProviderOp } from '../types';
|
||||
|
||||
export type KindName = 'contact' | 'group' | 'event';
|
||||
|
||||
/** A server object as `/get` returns it. */
|
||||
export interface ServerObject {
|
||||
id: string;
|
||||
uid?: string;
|
||||
[property: string]: unknown;
|
||||
}
|
||||
|
||||
export interface ItemMeta {
|
||||
rowId: number;
|
||||
/** `<jmapAccountId>/<id>` of the server object; null until created (events: a pending marker is no identity). */
|
||||
sourceId: string | null;
|
||||
pending: PendingCreate | null;
|
||||
/** Local changes wait for an upload (events: the master or one of its exceptions). */
|
||||
dirty: boolean;
|
||||
deleted: boolean;
|
||||
/** Never reached the server: no identity (docs/device-sync.md, "Identity"). */
|
||||
isNew: boolean;
|
||||
poison: PoisonMarker | null;
|
||||
/** Collection keys the server object was in when last seen. */
|
||||
collections: string[];
|
||||
}
|
||||
|
||||
/** A decoded local item and its kind. */
|
||||
export interface Held {
|
||||
kind: Kind;
|
||||
local: unknown;
|
||||
}
|
||||
|
||||
export interface Kind<L = any, O extends ServerObject = any> {
|
||||
readonly name: KindName;
|
||||
readonly table: 'raw_contacts' | 'groups' | 'events';
|
||||
/** The column holding the identity, for the "not inserted yet" guard of a download insert. */
|
||||
readonly identityColumn: 'sourceid' | '_sync_id';
|
||||
readonly poisonColumn: 'sync4' | 'sync_data5';
|
||||
meta(local: L): ItemMeta;
|
||||
/** What an upload is computed from, for poison markers. */
|
||||
fingerprint(local: L): string;
|
||||
/** Items by identity, one per ref. */
|
||||
loadByRefs(refs: readonly string[]): Promise<Map<string, L>>;
|
||||
loadByRowIds(rowIds: readonly number[]): Promise<L[]>;
|
||||
/** Items without identity (new ones), for adoption by pending uid. */
|
||||
loadNew(): Promise<L[]>;
|
||||
planDownload(object: O, local: L | null, jmapAccountId: string): DownloadPlan;
|
||||
planLocalDelete(local: L): OpGroup;
|
||||
planBaselineHeal(local: L): OpGroup | null;
|
||||
planUpload(local: L, jmapAccountId: string): UploadPlan<O>;
|
||||
planAccepted(local: L, server: O, jmapAccountId: string): AcceptedPlan;
|
||||
}
|
||||
|
||||
export function refOf(jmapAccountId: string, id: string): string {
|
||||
return `${jmapAccountId}/${id}`;
|
||||
}
|
||||
|
||||
/** The id part of a `<jmapAccountId>/<id>` ref of `jmapAccountId`, else null. */
|
||||
export function idInAccount(ref: string | null, jmapAccountId: string): string | null {
|
||||
if (!ref) return null;
|
||||
const prefix = `${jmapAccountId}/`;
|
||||
return ref.startsWith(prefix) && !ref.includes('#') ? ref.slice(prefix.length) : null;
|
||||
}
|
||||
|
||||
export function accountOfRef(ref: string | null): string | null {
|
||||
if (!ref) return null;
|
||||
const slash = ref.indexOf('/');
|
||||
return slash > 0 ? ref.slice(0, slash) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A download insert first asserts that no row has the identity yet: when a
|
||||
* failed batch was committed up to a yield point, the retry must not insert
|
||||
* the item twice.
|
||||
*/
|
||||
export function insertGuard(kind: Kind, ref: string): ProviderOp {
|
||||
return { op: 'assert', table: kind.table, where: `${kind.identityColumn} = ?`, args: [ref], expectCount: 0 };
|
||||
}
|
||||
|
||||
export function poisonOp(kind: Kind, rowId: number, marker: PoisonMarker | null): ProviderOp {
|
||||
return {
|
||||
op: 'update',
|
||||
table: kind.table,
|
||||
id: rowId,
|
||||
values: { [kind.poisonColumn]: marker ? JSON.stringify(marker) : null },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* One sync run or app-side operation (turning sync off, sign-out) at a time
|
||||
* per registry account and authority. Module level on purpose: a headless
|
||||
* run and the UI share one JS runtime (docs/device-sync.md, "Architecture"),
|
||||
* so a module-level lock covers both.
|
||||
*/
|
||||
import type { Authority } from '../types';
|
||||
|
||||
const tails = new Map<string, Promise<void>>();
|
||||
const stopRequests = new Map<string, number>();
|
||||
|
||||
export function lockKey(registryId: string, authority: Authority): string {
|
||||
return `${registryId}\u0000${authority}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for the lock; resolves with its release function, or null when
|
||||
* `waitMs` passed first (the place in the queue is then handed on).
|
||||
*/
|
||||
export function acquireLock(key: string, waitMs = Infinity): Promise<(() => void) | null> {
|
||||
const previous = tails.get(key) ?? Promise.resolve();
|
||||
let pass!: () => void;
|
||||
const mine = new Promise<void>((resolve) => {
|
||||
pass = resolve;
|
||||
});
|
||||
const tail = previous.then(() => mine);
|
||||
tails.set(key, tail);
|
||||
void tail.then(() => {
|
||||
if (tails.get(key) === tail) tails.delete(key);
|
||||
});
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const timer = Number.isFinite(waitMs)
|
||||
? setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(null);
|
||||
void previous.then(pass);
|
||||
}, Math.max(0, waitMs))
|
||||
: null;
|
||||
void previous.then(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
let released = false;
|
||||
resolve(() => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
pass();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function isLocked(key: string): boolean {
|
||||
return tails.has(key);
|
||||
}
|
||||
|
||||
/** Asks the run holding the lock to stop at its next checkpoint (it reports `cancelled`). */
|
||||
export function requestStop(key: string): void {
|
||||
stopRequests.set(key, (stopRequests.get(key) ?? 0) + 1);
|
||||
}
|
||||
|
||||
export function clearStopRequest(key: string): void {
|
||||
stopRequests.delete(key);
|
||||
}
|
||||
|
||||
export function stopRequested(key: string): boolean {
|
||||
return (stopRequests.get(key) ?? 0) > 0;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* Poison markers (docs/device-sync.md, "Uploads", SetErrors): an item the
|
||||
* server refused (`invalidProperties`, `invalidPatch`, `tooLarge`,
|
||||
* `overQuota`, …) is not retried until its back-off ends (1 h, doubling to
|
||||
* a day) or it changes. The engine writes the marker (SYNC4 / SYNC_DATA5)
|
||||
* and checks it before planning an upload; the fingerprint covers what the
|
||||
* item would upload from: its mapped rows, its flags and its shadow.
|
||||
*/
|
||||
import { jsonHash } from '../common/json';
|
||||
import type { PoisonMarker } from '../planner';
|
||||
|
||||
export function poisonFingerprint(content: unknown): string {
|
||||
return jsonHash(content);
|
||||
}
|
||||
|
||||
/** True while the marker holds: same fingerprint and the back-off not over. */
|
||||
export function isBackedOff(marker: PoisonMarker | null, fingerprint: string, now: number): boolean {
|
||||
return !!marker && marker.fp === fingerprint && marker.until > now;
|
||||
}
|
||||
|
||||
/** The next marker after a refusal: attempts count up while the item stays the same. */
|
||||
export function nextMarker(
|
||||
previous: PoisonMarker | null,
|
||||
fingerprint: string,
|
||||
error: { type: string; description?: string },
|
||||
now: number,
|
||||
backoff: { firstMs: number; maxMs: number },
|
||||
): PoisonMarker {
|
||||
const n = previous && previous.fp === fingerprint ? previous.n + 1 : 1;
|
||||
const delay = Math.min(backoff.firstMs * 2 ** (n - 1), backoff.maxMs);
|
||||
const marker: PoisonMarker = { fp: fingerprint, type: error.type, n, until: now + delay };
|
||||
if (error.description) marker.description = error.description.slice(0, 500);
|
||||
return marker;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Reads through the provider port as row objects. Every query is scoped to
|
||||
* our account by the native side; the SQL stays within the subset the fake
|
||||
* provider evaluates (src/device-sync/__tests__/fakes/fake-sql.ts).
|
||||
*/
|
||||
import type { ProviderOp, ProviderPort, ProviderTable, Row } from '../types';
|
||||
import { RunAbort, isPermissionFailure } from './errors';
|
||||
|
||||
/** Host parameters per `IN (…)`: well under SQLite's 999. */
|
||||
const IN_CHUNK = 400;
|
||||
|
||||
export class ProviderReader {
|
||||
constructor(readonly port: ProviderPort) {}
|
||||
|
||||
async rows(table: ProviderTable, columns: readonly string[], where?: string, args?: Array<string | number>): Promise<Row[]> {
|
||||
let result;
|
||||
try {
|
||||
result = await this.port.query({ table, columns: [...columns], ...(where ? { where, args: args ?? [] } : {}) });
|
||||
} catch (error) {
|
||||
if (isPermissionFailure(error)) throw new RunAbort('permission', (error as Error).message);
|
||||
throw error;
|
||||
}
|
||||
const { columns: names, rows } = result;
|
||||
return rows.map((cells) => {
|
||||
const row: Row = {};
|
||||
names.forEach((name, i) => {
|
||||
row[name] = cells[i] ?? null;
|
||||
});
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
/** Rows whose `column` is one of `values` (in chunks), optionally narrowed by `extra`. */
|
||||
async rowsIn(
|
||||
table: ProviderTable,
|
||||
columns: readonly string[],
|
||||
column: string,
|
||||
values: ReadonlyArray<string | number>,
|
||||
extra?: { where: string; args?: Array<string | number> },
|
||||
): Promise<Row[]> {
|
||||
const out: Row[] = [];
|
||||
const unique = [...new Set(values)];
|
||||
for (let i = 0; i < unique.length; i += IN_CHUNK) {
|
||||
const part = unique.slice(i, i + IN_CHUNK);
|
||||
const inClause = `${column} IN (${part.map(() => '?').join(', ')})`;
|
||||
const where = extra ? `(${inClause}) AND (${extra.where})` : inClause;
|
||||
out.push(...(await this.rows(table, columns, where, [...part, ...(extra?.args ?? [])])));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async readSyncState(): Promise<string | null> {
|
||||
try {
|
||||
return await this.port.readSyncState();
|
||||
} catch (error) {
|
||||
if (isPermissionFailure(error)) throw new RunAbort('permission', (error as Error).message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function num(value: unknown): number {
|
||||
return typeof value === 'number' ? value : Number(value ?? 0);
|
||||
}
|
||||
|
||||
export function flag(value: unknown): boolean {
|
||||
return value !== null && value !== undefined && value !== '' && Number(value) !== 0;
|
||||
}
|
||||
|
||||
export function str(value: unknown): string | null {
|
||||
return typeof value === 'string' && value ? value : value === null || value === undefined || value === '' ? null : String(value);
|
||||
}
|
||||
|
||||
/** True when an op group writes something (asserts alone change nothing). */
|
||||
export function hasWrites(ops: readonly ProviderOp[]): boolean {
|
||||
return ops.some((op) => op.op !== 'assert');
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* The run report (`RunReport`, docs/device-sync.md "Headless payload, report
|
||||
* and SyncResult"). Counts are objects actually written or sent, so a run
|
||||
* that repeated a page without writing anything is no progress to
|
||||
* SyncManager.
|
||||
*/
|
||||
import type { Authority, ItemError, RunOutcome, RunReport, RunStats, RunStatus } from '../types';
|
||||
|
||||
export function emptyStats(): RunStats {
|
||||
return {
|
||||
downloaded: { created: 0, updated: 0, deleted: 0 },
|
||||
uploaded: { created: 0, updated: 0, deleted: 0 },
|
||||
entries: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export class ReportBuilder {
|
||||
readonly stats = emptyStats();
|
||||
conflicts = 0;
|
||||
private readonly errors: ItemError[] = [];
|
||||
private errorCount = 0;
|
||||
private readonly notes = new Set<string>();
|
||||
tooManyDeletions?: { count: number; threshold: number };
|
||||
|
||||
constructor(
|
||||
readonly runId: string,
|
||||
readonly authority: Authority,
|
||||
readonly startedAt: number,
|
||||
private readonly maxItemErrors = 50,
|
||||
) {}
|
||||
|
||||
itemError(error: ItemError): void {
|
||||
this.errorCount++;
|
||||
// One entry per item and side: a retried item is reported once, with its latest error.
|
||||
const existing = this.errors.findIndex((e) => e.ref === error.ref && e.side === error.side);
|
||||
if (existing >= 0) {
|
||||
this.errors[existing] = error;
|
||||
this.errorCount--;
|
||||
return;
|
||||
}
|
||||
if (this.errors.length < this.maxItemErrors) this.errors.push(error);
|
||||
}
|
||||
|
||||
/** A note for the report's message ("invitations not sent", …). */
|
||||
note(text: string): void {
|
||||
this.notes.add(text);
|
||||
}
|
||||
|
||||
get itemErrorCount(): number {
|
||||
return this.errorCount;
|
||||
}
|
||||
|
||||
build(
|
||||
outcome: RunOutcome,
|
||||
now: number,
|
||||
extra: { message?: string; delayUntil?: number; moreRecordsToGet?: boolean; noProgress?: boolean } = {},
|
||||
): RunReport {
|
||||
const stats: RunStats = JSON.parse(JSON.stringify(this.stats));
|
||||
if (extra.noProgress) {
|
||||
stats.downloaded = { created: 0, updated: 0, deleted: 0 };
|
||||
stats.uploaded = { created: 0, updated: 0, deleted: 0 };
|
||||
}
|
||||
const message = [extra.message, ...this.notes].filter(Boolean).join('; ');
|
||||
const report: RunReport = {
|
||||
v: 1,
|
||||
runId: this.runId,
|
||||
authority: this.authority,
|
||||
outcome,
|
||||
startedAt: this.startedAt,
|
||||
durationMs: Math.max(0, now - this.startedAt),
|
||||
stats,
|
||||
conflicts: this.conflicts,
|
||||
itemErrors: this.errors.slice(),
|
||||
};
|
||||
if (message) report.message = message;
|
||||
if (outcome === 'tooManyDeletions' && this.tooManyDeletions) report.tooManyDeletions = this.tooManyDeletions;
|
||||
if (extra.delayUntil !== undefined) report.delayUntil = extra.delayUntil;
|
||||
if (extra.moreRecordsToGet) report.moreRecordsToGet = true;
|
||||
return report;
|
||||
}
|
||||
}
|
||||
|
||||
export function statusOf(report: RunReport, itemErrorCount = report.itemErrors.length): RunStatus {
|
||||
const status: RunStatus = {
|
||||
at: report.startedAt + report.durationMs,
|
||||
outcome: report.outcome,
|
||||
durationMs: report.durationMs,
|
||||
conflicts: report.conflicts,
|
||||
itemErrors: itemErrorCount,
|
||||
stats: report.stats,
|
||||
};
|
||||
if (report.message) status.message = report.message;
|
||||
return status;
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* One sync run of one Android account and authority (docs/device-sync.md,
|
||||
* "A sync run"): preflight, then the authority's phases, then the report.
|
||||
* Never throws: every failure becomes an outcome.
|
||||
*/
|
||||
import { makeKeyMinter, uuidFrom } from '../common/ids';
|
||||
import { JmapCaller } from '../jmap/caller';
|
||||
import type { JmapConnection } from '../jmap/connection';
|
||||
import { accountsWithCapability } from '../jmap/session';
|
||||
import {
|
||||
CALENDAR_AUTHORITY,
|
||||
CONTACTS_AUTHORITY,
|
||||
JMAP_CALENDARS,
|
||||
JMAP_CONTACTS,
|
||||
type Authority,
|
||||
type RunExtras,
|
||||
type RunPayload,
|
||||
type RunReport,
|
||||
} from '../types';
|
||||
import { BatchWriter } from './batch';
|
||||
import { CalendarSync } from './calendar-sync';
|
||||
import { Checkpoints } from './checkpoint';
|
||||
import { ContactsSync } from './contacts-sync';
|
||||
import type { RunEnv } from './context';
|
||||
import { DEFAULT_TUNING, type EngineDeps, type Tuning } from './deps';
|
||||
import { classifyFailure, RunAbort } from './errors';
|
||||
import { acquireLock, lockKey, stopRequested } from './mutex';
|
||||
import { ProviderReader, str } from './provider';
|
||||
import { ReportBuilder, statusOf } from './report';
|
||||
import { emptySyncState, parseSyncState, StateStore, type SyncState } from './sync-state';
|
||||
import type { ItemSync } from './item-sync';
|
||||
|
||||
export function tuningOf(deps: EngineDeps): Tuning {
|
||||
return { ...DEFAULT_TUNING, ...deps.tuning };
|
||||
}
|
||||
|
||||
const defaultYield = () => new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function capabilityOf(authority: Authority): string {
|
||||
return authority === CONTACTS_AUTHORITY ? JMAP_CONTACTS : JMAP_CALENDARS;
|
||||
}
|
||||
|
||||
function itemTypeOf(authority: Authority): string {
|
||||
return authority === CONTACTS_AUTHORITY ? 'ContactCard' : 'CalendarEvent';
|
||||
}
|
||||
|
||||
/** Rows waiting for an upload (dirty, deleted, new), or state work a run must do whatever its extras. */
|
||||
export async function hasLocalWork(reader: ProviderReader, authority: Authority, state: SyncState): Promise<boolean> {
|
||||
for (const account of Object.values(state.accounts)) {
|
||||
if (account.reconcile || account.stale.length) return true;
|
||||
}
|
||||
if (authority === CALENDAR_AUTHORITY) {
|
||||
const rows = await reader.rows('events', ['_id'], "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'");
|
||||
return rows.length > 0;
|
||||
}
|
||||
const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL');
|
||||
if (contacts.length) return true;
|
||||
const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']);
|
||||
const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean));
|
||||
if (groups.some((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1)) return true;
|
||||
// A group an app hard-deleted is only found by its absence.
|
||||
return Object.values(state.accounts).some((a) => (a.groups ?? []).some((ref) => !present.has(ref)));
|
||||
}
|
||||
|
||||
export interface EnvOptions {
|
||||
deps: EngineDeps;
|
||||
tuning: Tuning;
|
||||
mode: 'sync' | 'teardown';
|
||||
registryId: string;
|
||||
accountName: string;
|
||||
authority: Authority;
|
||||
extras: RunExtras;
|
||||
deadline: number;
|
||||
isCancelled(): Promise<boolean>;
|
||||
reader: ProviderReader;
|
||||
report: ReportBuilder;
|
||||
stored: SyncState;
|
||||
connection: JmapConnection;
|
||||
}
|
||||
|
||||
/**
|
||||
* The run's environment once the preflight passed: the JMAP connection is
|
||||
* open and its accounts known. Throws RunAbort('unsupported') when no
|
||||
* account offers the authority's capability.
|
||||
*/
|
||||
export async function createEnv(options: EnvOptions): Promise<{ env: RunEnv; sync: ItemSync }> {
|
||||
const { deps, tuning, registryId, authority, connection } = options;
|
||||
const jmap = new JmapCaller(connection.port, () => deps.now());
|
||||
const accounts = accountsWithCapability(jmap.session(), capabilityOf(authority));
|
||||
if (!accounts.length) {
|
||||
throw new RunAbort('unsupported', `The server offers no ${authority === CONTACTS_AUTHORITY ? 'contacts' : 'calendars'} for this account`);
|
||||
}
|
||||
const prefs = await deps.prefs(registryId);
|
||||
const recordKnownState = (jmapAccountId: string, type: string, state: string) => {
|
||||
try {
|
||||
void Promise.resolve(deps.recordKnownState(registryId, jmapAccountId, type, state)).catch(() => undefined);
|
||||
} catch {
|
||||
// Only an echo hint for the app.
|
||||
}
|
||||
};
|
||||
const store = new StateStore(options.stored, { registryId, origin: connection.origin }, (before, after) => {
|
||||
for (const [acct, account] of Object.entries(after.accounts)) {
|
||||
const state = account.itemsState;
|
||||
if (state && state !== before.accounts[acct]?.itemsState) recordKnownState(acct, itemTypeOf(authority), state);
|
||||
}
|
||||
});
|
||||
const checkpoints = new Checkpoints({
|
||||
now: () => deps.now(),
|
||||
deadline: options.deadline,
|
||||
marginMs: tuning.deadlineMarginMs,
|
||||
echoAfterMs: tuning.echoAfterMs,
|
||||
isCancelled: options.isCancelled,
|
||||
yieldThread: deps.yieldThread ?? defaultYield,
|
||||
});
|
||||
checkpoints.attach(jmap);
|
||||
const mintKey = makeKeyMinter(deps.random);
|
||||
const env: RunEnv = {
|
||||
deps,
|
||||
tuning,
|
||||
mode: options.mode,
|
||||
registryId,
|
||||
accountName: options.accountName,
|
||||
authority,
|
||||
extras: options.extras,
|
||||
reader: options.reader,
|
||||
writer: new BatchWriter(options.reader.port, {
|
||||
maxOps: tuning.maxBatchOps,
|
||||
maxBytes: tuning.maxBatchBytes,
|
||||
maxReplans: tuning.maxReplans,
|
||||
}),
|
||||
report: options.report,
|
||||
checkpoints,
|
||||
store,
|
||||
jmap,
|
||||
accounts,
|
||||
prefs,
|
||||
now: () => deps.now(),
|
||||
mintKey: (taken) => mintKey(taken),
|
||||
mintUid: () => uuidFrom(deps.random),
|
||||
sleep: deps.sleep ?? defaultSleep,
|
||||
log: (message, detail) => deps.log?.(message, detail),
|
||||
recordKnownState,
|
||||
};
|
||||
const sync = authority === CONTACTS_AUTHORITY ? new ContactsSync(env) : new CalendarSync(env);
|
||||
return { env, sync };
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one sync and returns its report. Holds the (registry account,
|
||||
* authority) lock for the whole run; a run that cannot get it before its
|
||||
* deadline reports `cancelled` and asks for another sync.
|
||||
*/
|
||||
export async function runDeviceSync(payload: RunPayload, deps: EngineDeps): Promise<RunReport> {
|
||||
const tuning = tuningOf(deps);
|
||||
const report = new ReportBuilder(payload.runId, payload.authority, deps.now(), tuning.maxItemErrors);
|
||||
const key = lockKey(payload.registryId, payload.authority);
|
||||
const release = await acquireLock(key, Math.max(0, payload.deadline - deps.now() - tuning.deadlineMarginMs));
|
||||
let result: RunReport;
|
||||
if (!release) {
|
||||
result = report.build('cancelled', deps.now(), { message: 'Another sync of this account is still running', moreRecordsToGet: true });
|
||||
} else {
|
||||
try {
|
||||
result = await runLocked(payload, deps, tuning, report, key);
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
try {
|
||||
await deps.recordStatus(payload.registryId, payload.authority, statusOf(result, report.itemErrorCount));
|
||||
} catch (error) {
|
||||
deps.log?.('recording the run status failed', error);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runLocked(payload: RunPayload, deps: EngineDeps, tuning: Tuning, report: ReportBuilder, key: string): Promise<RunReport> {
|
||||
const finish = (outcome: RunReport['outcome'], extra: Parameters<ReportBuilder['build']>[2] = {}) =>
|
||||
report.build(outcome, deps.now(), extra);
|
||||
const extras = payload.extras ?? {};
|
||||
try {
|
||||
if (!(await deps.isSyncEnabled(payload.accountName, payload.authority))) {
|
||||
return finish('disabled', { message: 'Sync is off for this account' });
|
||||
}
|
||||
const reader = new ProviderReader(deps.provider(payload.accountName, payload.authority));
|
||||
const { state: stored, readable } = parseSyncState(await reader.readSyncState());
|
||||
if (!readable) deps.log?.('unreadable SyncState: starting over with a full reconcile');
|
||||
if (stored.owner && stored.owner.registryId !== payload.registryId) {
|
||||
return finish('internal', { message: 'The rows on this device were written for another account' });
|
||||
}
|
||||
// Providers schedule an upload sync for every account 30 s after any app write.
|
||||
if (extras.upload && !(await hasLocalWork(reader, payload.authority, stored))) return finish('ok');
|
||||
|
||||
const connection = await deps.jmap(payload.registryId);
|
||||
if (stored.owner && stored.owner.origin !== connection.origin) {
|
||||
return finish('internal', { message: 'The rows on this device were written for another server' });
|
||||
}
|
||||
const { sync } = await createEnv({
|
||||
deps,
|
||||
tuning,
|
||||
mode: 'sync',
|
||||
registryId: payload.registryId,
|
||||
accountName: payload.accountName,
|
||||
authority: payload.authority,
|
||||
extras,
|
||||
deadline: payload.deadline,
|
||||
isCancelled: async () => stopRequested(key) || (await deps.isCancelled(payload.runId)),
|
||||
reader,
|
||||
report,
|
||||
stored: readable ? stored : emptySyncState(),
|
||||
connection,
|
||||
});
|
||||
await sync.sync();
|
||||
return finish(report.tooManyDeletions ? 'tooManyDeletions' : 'ok');
|
||||
} catch (error) {
|
||||
const failure = classifyFailure(error, deps.now());
|
||||
if (failure.authProblem) {
|
||||
try {
|
||||
await deps.notifyAuthProblem(payload.registryId, payload.accountName, payload.authority);
|
||||
} catch (notifyError) {
|
||||
deps.log?.('auth notification failed', notifyError);
|
||||
}
|
||||
}
|
||||
if (failure.outcome === 'internal') deps.log?.('device sync failed', error);
|
||||
return finish(failure.outcome, {
|
||||
message: failure.message,
|
||||
delayUntil: failure.delayUntil,
|
||||
moreRecordsToGet: failure.moreRecordsToGet,
|
||||
noProgress: failure.noProgress,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Which collections sync. The store keeps explicit choices only; a
|
||||
* collection the user never touched follows the default: the personal
|
||||
* account's collections on, shared accounts' off (docs/device-sync.md,
|
||||
* "App integration"), except the webmail's allow-list address book.
|
||||
*/
|
||||
import { collectionKey } from '../common/ids';
|
||||
import { CONTACTS_AUTHORITY, type Authority } from '../types';
|
||||
|
||||
/**
|
||||
* The address book the webmail keeps its allow-listed senders in. Mirrors
|
||||
* TRUSTED_SENDERS_BOOK_NAME in src/stores/contacts-store.ts, which the pure
|
||||
* engine cannot import.
|
||||
*/
|
||||
export const TRUSTED_SENDERS_BOOK_NAME = 'Trusted Senders';
|
||||
|
||||
/**
|
||||
* Whether a collection the user never chose syncs: those of the personal
|
||||
* account do, shared ones don't; nor does the "Trusted Senders" address
|
||||
* book, a mail filter rather than people to call.
|
||||
*/
|
||||
export function collectionDefaultOn(personal: boolean, authority: Authority, name: string | null | undefined): boolean {
|
||||
if (!personal) return false;
|
||||
return !(authority === CONTACTS_AUTHORITY && name === TRUSTED_SENDERS_BOOK_NAME);
|
||||
}
|
||||
|
||||
/** An explicit choice wins; otherwise the default. */
|
||||
export function isCollectionSelected(
|
||||
explicit: Readonly<Record<string, boolean>>,
|
||||
account: { id: string; personal: boolean },
|
||||
authority: Authority,
|
||||
collection: { id: string; name?: string | null },
|
||||
): boolean {
|
||||
const choice = explicit[collectionKey(account.id, collection.id)];
|
||||
return typeof choice === 'boolean' ? choice : collectionDefaultOn(account.personal, authority, collection.name);
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* The SyncState blob (docs/device-sync.md, "SyncState"): one per Android
|
||||
* account and authority, versioned JSON the provider stores next to the
|
||||
* rows. It is only ever written as the last op of a batch whose rows it
|
||||
* describes, so a crash never leaves it ahead of them.
|
||||
*/
|
||||
import { clone } from '../common/json';
|
||||
import type { ProviderOp, ReminderOwner } from '../types';
|
||||
|
||||
export const SYNC_STATE_VERSION = 1;
|
||||
|
||||
/** A full reconcile in progress. */
|
||||
export interface ReconcileMarker {
|
||||
/** The item state taken before listing anything; null when a reconcile was asked for and has not started. */
|
||||
from: string | null;
|
||||
phase: 'ids' | 'objects';
|
||||
/** Objects processed so far, in the reconcile's order. */
|
||||
position: number;
|
||||
/** The last id processed in that (sorted) order: a resumed reconcile continues after it. */
|
||||
after: string | null;
|
||||
}
|
||||
|
||||
export interface AccountState {
|
||||
/** AddressBook / Calendar state of the last collections read. */
|
||||
collectionsState: string | null;
|
||||
/** ContactCard / CalendarEvent state the rows describe. */
|
||||
itemsState: string | null;
|
||||
/** Collection keys the rows were written for. */
|
||||
selected: string[];
|
||||
/** Object ids whose rows could not be written; fetched again at the start of every run, nothing uploads for them meanwhile. */
|
||||
stale: string[];
|
||||
reconcile: ReconcileMarker | null;
|
||||
/** Contacts: refs of the group cards present on the device (a hard-deleted group is found by its absence). */
|
||||
groups?: string[];
|
||||
/** Calendar: ids of calendars found to hold only tasks. */
|
||||
taskOnly?: string[];
|
||||
}
|
||||
|
||||
export interface SyncState {
|
||||
v: typeof SYNC_STATE_VERSION;
|
||||
/** The registry account and server origin the rows belong to. */
|
||||
owner: { registryId: string; origin: string } | null;
|
||||
accounts: Record<string, AccountState>;
|
||||
/** Calendar: the zone floating events were written in. */
|
||||
deviceZone?: string | null;
|
||||
/** Calendar: the zone a zone-change pass is moving floating events to. */
|
||||
deviceZonePending?: string | null;
|
||||
/** Calendar: who the Reminders rows were written for (a change rewrites them). */
|
||||
reminderOwner?: ReminderOwner | null;
|
||||
}
|
||||
|
||||
export function emptySyncState(): SyncState {
|
||||
return { v: SYNC_STATE_VERSION, owner: null, accounts: {} };
|
||||
}
|
||||
|
||||
export function emptyAccountState(): AccountState {
|
||||
return { collectionsState: null, itemsState: null, selected: [], stale: [], reconcile: null };
|
||||
}
|
||||
|
||||
const strings = (value: unknown): string[] =>
|
||||
Array.isArray(value) ? value.filter((v): v is string => typeof v === 'string') : [];
|
||||
const stringOrNull = (value: unknown): string | null => (typeof value === 'string' && value ? value : null);
|
||||
|
||||
function parseReconcile(value: unknown): ReconcileMarker | null {
|
||||
if (!value || typeof value !== 'object') return null;
|
||||
const r = value as Record<string, unknown>;
|
||||
return {
|
||||
from: stringOrNull(r.from),
|
||||
phase: r.phase === 'objects' ? 'objects' : 'ids',
|
||||
position: typeof r.position === 'number' && r.position >= 0 ? r.position : 0,
|
||||
after: stringOrNull(r.after),
|
||||
};
|
||||
}
|
||||
|
||||
function parseAccount(value: unknown): AccountState {
|
||||
const a = (value && typeof value === 'object' ? value : {}) as Record<string, unknown>;
|
||||
const out: AccountState = {
|
||||
collectionsState: stringOrNull(a.collectionsState),
|
||||
itemsState: stringOrNull(a.itemsState),
|
||||
selected: strings(a.selected),
|
||||
stale: strings(a.stale),
|
||||
reconcile: parseReconcile(a.reconcile),
|
||||
};
|
||||
if (a.groups !== undefined) out.groups = strings(a.groups);
|
||||
if (a.taskOnly !== undefined) out.taskOnly = strings(a.taskOnly);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* The stored blob. `readable` is false when there was one that could not be
|
||||
* understood (another version, garbage): the rows are then treated as ours
|
||||
* and every account starts over with a full reconcile.
|
||||
*/
|
||||
export function parseSyncState(text: string | null): { state: SyncState; readable: boolean } {
|
||||
if (!text) return { state: emptySyncState(), readable: true };
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = JSON.parse(text);
|
||||
} catch {
|
||||
return { state: emptySyncState(), readable: false };
|
||||
}
|
||||
const r = (raw && typeof raw === 'object' ? raw : null) as Record<string, unknown> | null;
|
||||
if (!r || r.v !== SYNC_STATE_VERSION) return { state: emptySyncState(), readable: false };
|
||||
const owner = r.owner as Record<string, unknown> | null | undefined;
|
||||
const state: SyncState = {
|
||||
v: SYNC_STATE_VERSION,
|
||||
owner:
|
||||
owner && typeof owner.registryId === 'string' && typeof owner.origin === 'string'
|
||||
? { registryId: owner.registryId, origin: owner.origin }
|
||||
: null,
|
||||
accounts: {},
|
||||
};
|
||||
if (r.accounts && typeof r.accounts === 'object') {
|
||||
for (const [id, account] of Object.entries(r.accounts as Record<string, unknown>)) {
|
||||
state.accounts[id] = parseAccount(account);
|
||||
}
|
||||
}
|
||||
if ('deviceZone' in r) state.deviceZone = stringOrNull(r.deviceZone);
|
||||
if ('deviceZonePending' in r) state.deviceZonePending = stringOrNull(r.deviceZonePending);
|
||||
if (r.reminderOwner === 'device' || r.reminderOwner === 'bulwark') state.reminderOwner = r.reminderOwner;
|
||||
return { state, readable: true };
|
||||
}
|
||||
|
||||
export function serializeSyncState(state: SyncState): string {
|
||||
return JSON.stringify(state);
|
||||
}
|
||||
|
||||
/** The op that ends a batch and moves the stored state along with its rows. */
|
||||
export interface Tail {
|
||||
op(): ProviderOp;
|
||||
/** Called once the batch carrying `op()` applied. */
|
||||
applied(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* The committed SyncState of a run. Changes are proposed as a tail: the
|
||||
* batch writer calls `op()` right before sending the last batch (so the
|
||||
* change sees what earlier batches decided, e.g. items put on `stale`), and
|
||||
* the state is only adopted after that batch applied.
|
||||
*/
|
||||
export class StateStore {
|
||||
private state: SyncState;
|
||||
|
||||
constructor(
|
||||
initial: SyncState,
|
||||
private readonly owner: { registryId: string; origin: string } | null,
|
||||
private readonly onCommit?: (before: SyncState, after: SyncState) => void,
|
||||
) {
|
||||
this.state = clone(initial);
|
||||
}
|
||||
|
||||
get committed(): Readonly<SyncState> {
|
||||
return this.state;
|
||||
}
|
||||
|
||||
account(jmapAccountId: string): Readonly<AccountState> {
|
||||
return this.state.accounts[jmapAccountId] ?? emptyAccountState();
|
||||
}
|
||||
|
||||
/** A tail applying `mutate` to a copy of the committed state (plus the owner). */
|
||||
tail(mutate: (next: SyncState) => void): Tail {
|
||||
let next: SyncState | null = null;
|
||||
return {
|
||||
op: () => {
|
||||
next = clone(this.state);
|
||||
if (this.owner) next.owner = { ...this.owner };
|
||||
mutate(next);
|
||||
return { op: 'syncState', value: serializeSyncState(next) };
|
||||
},
|
||||
applied: () => {
|
||||
if (!next) return;
|
||||
const before = this.state;
|
||||
this.state = next;
|
||||
this.onCommit?.(before, next);
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** The account's entry in a state being built, created when missing. */
|
||||
export function accountOf(state: SyncState, jmapAccountId: string): AccountState {
|
||||
let account = state.accounts[jmapAccountId];
|
||||
if (!account) {
|
||||
account = emptyAccountState();
|
||||
state.accounts[jmapAccountId] = account;
|
||||
}
|
||||
return account;
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Turning device sync off for an authority, or signing out
|
||||
* (docs/device-sync.md, "App integration", Lifecycle): under the run lock,
|
||||
* the device's dirty, deleted and new items are uploaded first (bounded in
|
||||
* time, failures ignored), then every row of the authority and its SyncState
|
||||
* go. When changes are still waiting, nothing is deleted unless the user
|
||||
* confirmed losing them (`force`).
|
||||
*/
|
||||
import { CALENDAR_AUTHORITY, type Authority, type ProviderOp } from '../types';
|
||||
import { BatchWriter } from './batch';
|
||||
import type { EngineDeps } from './deps';
|
||||
import { acquireLock, clearStopRequest, lockKey, requestStop } from './mutex';
|
||||
import { ProviderReader, str } from './provider';
|
||||
import { ReportBuilder } from './report';
|
||||
import { createEnv, tuningOf } from './run';
|
||||
import { parseSyncState } from './sync-state';
|
||||
|
||||
export interface TeardownOptions {
|
||||
/** Upload waiting changes first (default true). */
|
||||
uploadFirst?: boolean;
|
||||
/** Time for the upload (default 30 s). */
|
||||
timeoutMs?: number;
|
||||
/** Delete even when changes could not be uploaded (the user confirmed). */
|
||||
force?: boolean;
|
||||
}
|
||||
|
||||
/** Items still waiting for an upload: dirty, deleted, new, and groups deleted by absence. */
|
||||
export async function countPending(reader: ProviderReader, authority: Authority): Promise<number> {
|
||||
const { state } = parseSyncState(await reader.readSyncState());
|
||||
if (authority === CALENDAR_AUTHORITY) {
|
||||
const rows = await reader.rows(
|
||||
'events',
|
||||
['_id', 'original_id', 'original_sync_id'],
|
||||
"dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'",
|
||||
);
|
||||
const masters = new Set(rows.map((r) => str(r.original_id) ?? str(r.original_sync_id) ?? `row:${String(r._id)}`));
|
||||
return masters.size;
|
||||
}
|
||||
const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL');
|
||||
const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']);
|
||||
const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean));
|
||||
const waitingGroups = groups.filter((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1).length;
|
||||
const absent = Object.values(state.accounts).reduce((n, a) => n + (a.groups ?? []).filter((ref) => !present.has(ref)).length, 0);
|
||||
return contacts.length + waitingGroups + absent;
|
||||
}
|
||||
|
||||
function deleteEverything(authority: Authority): ProviderOp[] {
|
||||
// Sync-adapter deletes: raw contacts take their data rows along, calendar rows their events.
|
||||
return authority === CALENDAR_AUTHORITY
|
||||
? [{ op: 'delete', table: 'calendars' }]
|
||||
: [
|
||||
{ op: 'delete', table: 'raw_contacts' },
|
||||
{ op: 'delete', table: 'groups' },
|
||||
{ op: 'delete', table: 'settings' },
|
||||
];
|
||||
}
|
||||
|
||||
export async function teardownAuthority(
|
||||
deps: EngineDeps,
|
||||
registryId: string,
|
||||
accountName: string,
|
||||
authority: Authority,
|
||||
options: TeardownOptions = {},
|
||||
): Promise<{ pending: number }> {
|
||||
const tuning = tuningOf(deps);
|
||||
const key = lockKey(registryId, authority);
|
||||
// A sync holding the lock stops at its next checkpoint instead of running to its deadline.
|
||||
requestStop(key);
|
||||
const release = await acquireLock(key);
|
||||
clearStopRequest(key);
|
||||
try {
|
||||
const reader = new ProviderReader(deps.provider(accountName, authority));
|
||||
if (options.uploadFirst !== false) {
|
||||
const started = deps.now();
|
||||
try {
|
||||
const { state: stored, readable } = parseSyncState(await reader.readSyncState());
|
||||
// Rows written for another registry account are never uploaded to this one.
|
||||
if (!stored.owner || stored.owner.registryId === registryId) {
|
||||
const connection = await deps.jmap(registryId);
|
||||
if (!stored.owner || stored.owner.origin === connection.origin) {
|
||||
const report = new ReportBuilder('teardown', authority, started, tuning.maxItemErrors);
|
||||
const { sync } = await createEnv({
|
||||
deps,
|
||||
tuning: { ...tuning, deadlineMarginMs: 0, createBudgetMs: 0 },
|
||||
mode: 'teardown',
|
||||
registryId,
|
||||
accountName,
|
||||
authority,
|
||||
extras: {},
|
||||
deadline: started + (options.timeoutMs ?? 30_000),
|
||||
isCancelled: async () => false,
|
||||
reader,
|
||||
report,
|
||||
stored: readable ? stored : parseSyncState(null).state,
|
||||
connection,
|
||||
});
|
||||
await sync.uploadOnly();
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Offline, signed out, out of time: whatever did not go up is counted below.
|
||||
deps.log?.('upload before teardown stopped', error);
|
||||
}
|
||||
}
|
||||
const pending = await countPending(reader, authority);
|
||||
if (pending > 0 && !options.force) return { pending };
|
||||
const writer = new BatchWriter(reader.port, {
|
||||
maxOps: tuning.maxBatchOps,
|
||||
maxBytes: tuning.maxBatchBytes,
|
||||
maxReplans: 0,
|
||||
});
|
||||
await writer.write(
|
||||
[
|
||||
{
|
||||
group: { ref: 'teardown', ops: deleteEverything(authority) },
|
||||
failed: (reason, message) => {
|
||||
throw new Error(`Could not remove the synced rows: ${reason} ${message}`);
|
||||
},
|
||||
},
|
||||
],
|
||||
{ op: () => ({ op: 'syncState', value: '' }), applied: () => undefined },
|
||||
);
|
||||
return { pending: 0 };
|
||||
} finally {
|
||||
release?.();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
/**
|
||||
* The engine's JMAP calls on one registry account's `JmapPort`: one request
|
||||
* in flight at a time, the time of the last request (for the keep-alive
|
||||
* echo), and the method helpers device sync needs, each sized by the
|
||||
* session's limits (docs/device-sync.md, "A sync run" and "Limits").
|
||||
*
|
||||
* Pure: the port is a detached JMAPClient on the device and the fake server
|
||||
* in tests.
|
||||
*/
|
||||
import { requireMethodResult } from '../../api/jmap-result';
|
||||
import type { JMAPResponseBody } from '../../api/types';
|
||||
import {
|
||||
JMAP_CALENDARS,
|
||||
JMAP_CONTACTS,
|
||||
JMAP_CORE,
|
||||
type JmapCoreLimits,
|
||||
type JmapInvocation,
|
||||
type JmapPort,
|
||||
type JmapResponse,
|
||||
type JmapSessionView,
|
||||
} from '../types';
|
||||
import type { PatchObject } from '../common/patch';
|
||||
|
||||
export type ItemType = 'ContactCard' | 'CalendarEvent';
|
||||
export type ContainerType = 'AddressBook' | 'Calendar';
|
||||
type DataType = ItemType | ContainerType;
|
||||
|
||||
/** `/changes` pages: Stalwart caps at 5000, a small page keeps every page a quick checkpoint. */
|
||||
export const MAX_CHANGES = 256;
|
||||
/** `/query` pages (Stalwart's `query_max_results`). */
|
||||
export const QUERY_PAGE = 5000;
|
||||
/** Ids re-read at the start of the next `/query` page, so a deletion that shifts the list skips nothing. */
|
||||
const QUERY_OVERLAP = 100;
|
||||
|
||||
export function capabilityOf(type: DataType): string {
|
||||
return type === 'ContactCard' || type === 'AddressBook' ? JMAP_CONTACTS : JMAP_CALENDARS;
|
||||
}
|
||||
|
||||
function using(type: DataType): string[] {
|
||||
return [JMAP_CORE, capabilityOf(type)];
|
||||
}
|
||||
|
||||
export interface ChangesPage {
|
||||
oldState: string;
|
||||
newState: string;
|
||||
hasMoreChanges: boolean;
|
||||
created: string[];
|
||||
updated: string[];
|
||||
destroyed: string[];
|
||||
}
|
||||
|
||||
export interface SetErrorShape {
|
||||
type: string;
|
||||
description?: string;
|
||||
properties?: string[];
|
||||
}
|
||||
|
||||
export interface SetResponse {
|
||||
oldState?: string;
|
||||
newState: string;
|
||||
created?: Record<string, { id: string } & Record<string, unknown>>;
|
||||
updated?: Record<string, unknown>;
|
||||
destroyed?: string[];
|
||||
notCreated?: Record<string, SetErrorShape>;
|
||||
notUpdated?: Record<string, SetErrorShape>;
|
||||
notDestroyed?: Record<string, SetErrorShape>;
|
||||
}
|
||||
|
||||
export interface SetArgs {
|
||||
create?: Record<string, Record<string, unknown>>;
|
||||
update?: Record<string, PatchObject>;
|
||||
destroy?: string[];
|
||||
sendSchedulingMessages?: boolean;
|
||||
}
|
||||
|
||||
export class JmapCaller {
|
||||
private chain: Promise<unknown> = Promise.resolve();
|
||||
private lastRequestAt: number;
|
||||
/** Requests sent, for tests and logs. */
|
||||
requestCount = 0;
|
||||
|
||||
constructor(
|
||||
readonly port: JmapPort,
|
||||
private readonly now: () => number,
|
||||
) {
|
||||
this.lastRequestAt = now();
|
||||
}
|
||||
|
||||
session(): JmapSessionView {
|
||||
return this.port.session();
|
||||
}
|
||||
|
||||
limits(): JmapCoreLimits {
|
||||
return this.port.limits();
|
||||
}
|
||||
|
||||
/** Milliseconds since the last request started or ended. */
|
||||
idleMs(): number {
|
||||
return this.now() - this.lastRequestAt;
|
||||
}
|
||||
|
||||
/**
|
||||
* One request at a time: the server's `maxConcurrentRequests` is a per-user
|
||||
* budget the UI shares, and a sync run has no reason to spend more than one.
|
||||
*/
|
||||
request(calls: JmapInvocation[], usingCaps: string[]): Promise<JmapResponse> {
|
||||
const run = async () => {
|
||||
this.lastRequestAt = this.now();
|
||||
this.requestCount++;
|
||||
try {
|
||||
return await this.port.request(calls, usingCaps);
|
||||
} finally {
|
||||
this.lastRequestAt = this.now();
|
||||
}
|
||||
};
|
||||
const result = this.chain.then(run, run);
|
||||
this.chain = result.catch(() => undefined);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** One method call; a method-level error throws `JMAPMethodError` with its `type`. */
|
||||
async call<T>(method: string, args: Record<string, unknown>, usingCaps: string[]): Promise<T> {
|
||||
const response = await this.request([[method, args, '0']], usingCaps);
|
||||
return requireMethodResult<T>(response as unknown as JMAPResponseBody, '0', method);
|
||||
}
|
||||
|
||||
/** Network traffic for SyncManager's "no traffic for 60 s" monitor during long local phases. */
|
||||
async echo(): Promise<void> {
|
||||
await this.request([['Core/echo', { ping: this.now() }, '0']], [JMAP_CORE]);
|
||||
}
|
||||
|
||||
/** Bytes of a blob (a blob-backed photo), in turn with the requests. */
|
||||
downloadBlob(accountId: string, blobId: string, type?: string): Promise<Uint8Array> {
|
||||
const run = async () => {
|
||||
this.lastRequestAt = this.now();
|
||||
try {
|
||||
return await this.port.downloadBlob(accountId, blobId, type);
|
||||
} finally {
|
||||
this.lastRequestAt = this.now();
|
||||
}
|
||||
};
|
||||
const result = this.chain.then(run, run);
|
||||
this.chain = result.catch(() => undefined);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Reads ──
|
||||
|
||||
/**
|
||||
* Objects by id with explicit properties, in calls of at most
|
||||
* `maxObjectsInGet` ids (never `ids: null`, which Stalwart silently caps).
|
||||
*/
|
||||
async get<T>(
|
||||
type: DataType,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
properties: readonly string[],
|
||||
): Promise<{ list: T[]; notFound: string[]; state: string | null }> {
|
||||
const list: T[] = [];
|
||||
const notFound: string[] = [];
|
||||
let state: string | null = null;
|
||||
const size = Math.max(1, this.limits().maxObjectsInGet);
|
||||
for (let i = 0; i < ids.length; i += size) {
|
||||
const body = await this.call<{ list?: T[]; notFound?: string[]; state?: string }>(
|
||||
`${type}/get`,
|
||||
{ accountId, ids: ids.slice(i, i + size), properties: [...properties] },
|
||||
using(type),
|
||||
);
|
||||
list.push(...(body.list ?? []));
|
||||
notFound.push(...(body.notFound ?? []));
|
||||
state = body.state ?? state;
|
||||
}
|
||||
return { list, notFound, state };
|
||||
}
|
||||
|
||||
/** Every container of the account (there are few; Stalwart allows 250 per account). */
|
||||
async getContainers<T>(type: ContainerType, accountId: string, properties: readonly string[]): Promise<{ list: T[]; state: string | null }> {
|
||||
const body = await this.call<{ list?: T[]; state?: string }>(
|
||||
`${type}/get`,
|
||||
{ accountId, ids: null, properties: [...properties] },
|
||||
using(type),
|
||||
);
|
||||
return { list: body.list ?? [], state: body.state ?? null };
|
||||
}
|
||||
|
||||
/** The current state of a type, taken before a full reconcile lists anything. */
|
||||
async takeState(type: ItemType, accountId: string): Promise<string> {
|
||||
const body = await this.call<{ state?: string }>(`${type}/get`, { accountId, ids: [], properties: ['id'] }, using(type));
|
||||
return body.state ?? '';
|
||||
}
|
||||
|
||||
async changes(type: ItemType, accountId: string, sinceState: string, maxChanges = MAX_CHANGES): Promise<ChangesPage> {
|
||||
const body = await this.call<Partial<ChangesPage>>(
|
||||
`${type}/changes`,
|
||||
{ accountId, sinceState, maxChanges },
|
||||
using(type),
|
||||
);
|
||||
return {
|
||||
oldState: body.oldState ?? sinceState,
|
||||
newState: body.newState ?? sinceState,
|
||||
hasMoreChanges: Boolean(body.hasMoreChanges),
|
||||
created: body.created ?? [],
|
||||
updated: body.updated ?? [],
|
||||
destroyed: body.destroyed ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Every id matching `filter`, paged by `position` in pages of at most 5000.
|
||||
* Pages overlap a little: a deletion between two pages shifts the list and
|
||||
* would otherwise skip an object.
|
||||
*/
|
||||
async queryAll(type: ItemType, accountId: string, filter: Record<string, unknown>, pageSize = QUERY_PAGE): Promise<string[]> {
|
||||
const seen = new Set<string>();
|
||||
let position = 0;
|
||||
for (let round = 0; round < 1000; round++) {
|
||||
const body = await this.call<{ ids?: string[]; total?: number; position?: number }>(
|
||||
`${type}/query`,
|
||||
{ accountId, filter, position, limit: pageSize, calculateTotal: true },
|
||||
using(type),
|
||||
);
|
||||
const ids = body.ids ?? [];
|
||||
for (const id of ids) seen.add(id);
|
||||
const total = typeof body.total === 'number' ? body.total : null;
|
||||
const end = position + ids.length;
|
||||
if (ids.length === 0) break;
|
||||
// A server may cap the page below `pageSize`; the total says whether more follow.
|
||||
if (total !== null ? end >= total : ids.length < pageSize) break;
|
||||
position = end - Math.min(QUERY_OVERLAP, Math.floor(ids.length / 4));
|
||||
}
|
||||
return [...seen];
|
||||
}
|
||||
|
||||
/** The first `limit` ids matching `filter` (task-only calendar detection). */
|
||||
async queryFirst(type: ItemType, accountId: string, filter: Record<string, unknown>, limit: number): Promise<string[]> {
|
||||
const body = await this.call<{ ids?: string[] }>(`${type}/query`, { accountId, filter, limit }, using(type));
|
||||
return body.ids ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Ids of the objects carrying each uid, in `collectionId` when given
|
||||
* (contact uids are unique per address book, event uids per account). The
|
||||
* uid filter reads Stalwart's asynchronous search index, so a just-created
|
||||
* object can be missing.
|
||||
*/
|
||||
async lookupUids(
|
||||
type: ItemType,
|
||||
accountId: string,
|
||||
uids: readonly string[],
|
||||
collectionId: string | null,
|
||||
): Promise<Map<string, string[]>> {
|
||||
const found = new Map<string, string[]>();
|
||||
if (uids.length === 0) return found;
|
||||
const byUid: Record<string, unknown> =
|
||||
uids.length === 1 ? { uid: uids[0] } : { operator: 'OR', conditions: uids.map((uid) => ({ uid })) };
|
||||
const parentFilter = type === 'ContactCard' ? 'inAddressBook' : 'inCalendar';
|
||||
const filter = collectionId === null
|
||||
? byUid
|
||||
: { operator: 'AND', conditions: [{ [parentFilter]: collectionId }, byUid] };
|
||||
const ids = await this.queryAll(type, accountId, filter);
|
||||
if (ids.length === 0) return found;
|
||||
const { list } = await this.get<{ id: string; uid?: string }>(type, accountId, ids, ['id', 'uid']);
|
||||
for (const object of list) {
|
||||
if (typeof object.uid !== 'string' || !uids.includes(object.uid)) continue;
|
||||
found.set(object.uid, [...(found.get(object.uid) ?? []), object.id]);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
// ── Writes ──
|
||||
|
||||
/**
|
||||
* One `/set` call, guarded by `ifInState` when there is one; with
|
||||
* `thenGet`, a `/get` of those ids in the same request (updates only:
|
||||
* Stalwart resolves no result reference to a `/set`, and a create's id is
|
||||
* only known from the response). A method-level error (`stateMismatch`, …)
|
||||
* throws `JMAPMethodError`.
|
||||
*/
|
||||
async set<T>(
|
||||
type: ItemType,
|
||||
accountId: string,
|
||||
args: SetArgs,
|
||||
ifInState: string | null,
|
||||
thenGet?: { ids: string[]; properties: readonly string[] },
|
||||
): Promise<{ response: SetResponse; got: T[] | null }> {
|
||||
const setArgs: Record<string, unknown> = { accountId };
|
||||
if (ifInState) setArgs.ifInState = ifInState;
|
||||
if (args.create && Object.keys(args.create).length) setArgs.create = args.create;
|
||||
if (args.update && Object.keys(args.update).length) setArgs.update = args.update;
|
||||
if (args.destroy?.length) setArgs.destroy = args.destroy;
|
||||
if (args.sendSchedulingMessages) setArgs.sendSchedulingMessages = true;
|
||||
const calls: JmapInvocation[] = [[`${type}/set`, setArgs, 's']];
|
||||
const withGet = !!thenGet && thenGet.ids.length > 0 && thenGet.ids.length <= this.limits().maxObjectsInGet;
|
||||
if (withGet) {
|
||||
calls.push([`${type}/get`, { accountId, ids: thenGet!.ids, properties: [...thenGet!.properties] }, 'g']);
|
||||
}
|
||||
const res = (await this.request(calls, using(type))) as unknown as JMAPResponseBody;
|
||||
const response = requireMethodResult<SetResponse>(res, 's', `${type}/set`);
|
||||
let got: T[] | null = null;
|
||||
if (withGet) {
|
||||
got = requireMethodResult<{ list?: T[] }>(res, 'g', `${type}/get`).list ?? [];
|
||||
} else if (thenGet && thenGet.ids.length > 0) {
|
||||
got = (await this.get<T>(type, accountId, thenGet.ids, thenGet.properties)).list;
|
||||
}
|
||||
return { response, got };
|
||||
}
|
||||
}
|
||||
|
||||
// ── SetErrors ──
|
||||
|
||||
/**
|
||||
* What a SetError means for the item (docs/device-sync.md, "Uploads"):
|
||||
* - `notFound`: gone on the server (an update: a remote delete; a destroy: done);
|
||||
* - `forbidden`: not allowed (retried without scheduling messages, else reverted);
|
||||
* - `uidExists`: `invalidProperties` on `uid`: the object exists already (a
|
||||
* lost response, or the client's automatic retry of a create);
|
||||
* - `poison`: the object itself is refused; back off until it changes.
|
||||
*/
|
||||
export type SetErrorClass = 'notFound' | 'forbidden' | 'uidExists' | 'poison';
|
||||
|
||||
export function classifySetError(error: SetErrorShape | undefined, op: 'create' | 'update' | 'destroy'): SetErrorClass {
|
||||
const type = error?.type;
|
||||
if (type === 'notFound') return 'notFound';
|
||||
if (type === 'forbidden') return 'forbidden';
|
||||
if (op === 'create' && type === 'invalidProperties' && error?.properties?.includes('uid')) return 'uidExists';
|
||||
return 'poison';
|
||||
}
|
||||
|
||||
/** The existing card Stalwart names in a duplicate-uid error ("… already exists with id c5."). */
|
||||
export function existingIdFromUidError(error: SetErrorShape | undefined): string | null {
|
||||
const m = /already exists with id ([A-Za-z0-9_-]+)/.exec(error?.description ?? '');
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* The JMAP side of a device sync run on the device: a `JmapPort` over a
|
||||
* detached `JMAPClient` of one registry account. Never the UI's `jmapClient`
|
||||
* singleton: a run can share the JS runtime with the UI (the headless task
|
||||
* starts with `allowedInForeground`), and re-binding the singleton would send
|
||||
* the UI's requests with another account's credentials (the widgets and the
|
||||
* push task avoid it for the same reason).
|
||||
*
|
||||
* This file is the React Native boundary of `src/device-sync/jmap`; the rest
|
||||
* of the folder is pure and runs against the fake server in tests.
|
||||
*/
|
||||
import { JMAPClient } from '../../api/jmap-client';
|
||||
import { JMAP_CORE, type JmapCoreLimits, type JmapPort, type JmapResponse, type JmapSessionView } from '../types';
|
||||
import { openWithAuthRebuild, type JmapConnection } from './connection';
|
||||
|
||||
/** RFC 8620 fallbacks when the server advertises no core limit. */
|
||||
const FALLBACK_LIMITS: JmapCoreLimits = {
|
||||
maxObjectsInGet: 500,
|
||||
maxObjectsInSet: 500,
|
||||
maxCallsInRequest: 16,
|
||||
maxConcurrentRequests: 4,
|
||||
maxSizeRequest: 10_000_000,
|
||||
};
|
||||
|
||||
function positive(value: unknown, fallback: number): number {
|
||||
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
export function coreLimits(capabilities: Record<string, unknown> | undefined): JmapCoreLimits {
|
||||
const core = (capabilities?.[JMAP_CORE] ?? {}) as Partial<Record<keyof JmapCoreLimits, unknown>>;
|
||||
return {
|
||||
maxObjectsInGet: positive(core.maxObjectsInGet, FALLBACK_LIMITS.maxObjectsInGet),
|
||||
maxObjectsInSet: positive(core.maxObjectsInSet, FALLBACK_LIMITS.maxObjectsInSet),
|
||||
maxCallsInRequest: positive(core.maxCallsInRequest, FALLBACK_LIMITS.maxCallsInRequest),
|
||||
maxConcurrentRequests: positive(core.maxConcurrentRequests, FALLBACK_LIMITS.maxConcurrentRequests),
|
||||
maxSizeRequest: positive(core.maxSizeRequest, FALLBACK_LIMITS.maxSizeRequest),
|
||||
};
|
||||
}
|
||||
|
||||
function originOf(url: string | null): string {
|
||||
const m = /^(https?:\/\/[^/?#]+)/i.exec(url ?? '');
|
||||
return m ? m[1].toLowerCase() : (url ?? '');
|
||||
}
|
||||
|
||||
function sessionView(client: JMAPClient): JmapSessionView {
|
||||
const session = client.currentSession;
|
||||
if (!session) throw new Error('JMAP session not loaded');
|
||||
const accounts: JmapSessionView['accounts'] = {};
|
||||
for (const [id, account] of Object.entries(session.accounts ?? {})) {
|
||||
accounts[id] = {
|
||||
name: account.name,
|
||||
isPersonal: account.isPersonal,
|
||||
isReadOnly: account.isReadOnly,
|
||||
accountCapabilities: account.accountCapabilities ?? {},
|
||||
};
|
||||
}
|
||||
return {
|
||||
username: session.username || client.username || '',
|
||||
accounts,
|
||||
primaryAccounts: session.primaryAccounts ?? {},
|
||||
capabilities: session.capabilities ?? {},
|
||||
};
|
||||
}
|
||||
|
||||
function clientPort(client: JMAPClient): JmapPort {
|
||||
return {
|
||||
session: () => sessionView(client),
|
||||
limits: () => coreLimits(client.currentSession?.capabilities),
|
||||
request: async (calls, using) => (await client.request(calls, using)) as unknown as JmapResponse,
|
||||
downloadBlob: async (accountId, blobId, type) =>
|
||||
new Uint8Array(await client.fetchBlobArrayBuffer(blobId, undefined, type, accountId)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A port for the registry account `registryId` (`AccountEntry.id`). No
|
||||
* stored credentials or rejected ones are retried once with a fresh client,
|
||||
* then fail with an AuthenticationError; an unreachable server throws a
|
||||
* NetworkError and touches nothing.
|
||||
*/
|
||||
export function openJmapPort(registryId: string): Promise<JmapConnection> {
|
||||
return openWithAuthRebuild(async () => {
|
||||
const client = new JMAPClient();
|
||||
if (!(await client.loadAccount(registryId))) return null;
|
||||
return { port: clientPort(client), origin: originOf(client.serverUrl) };
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* A registry account's JMAP connection for one sync run, with the one
|
||||
* rebuild the design allows when the server rejects the credentials: the UI
|
||||
* (or another run) may have rotated the OAuth refresh token this client
|
||||
* still holds, and a fresh client reads the rotated one from secure storage
|
||||
* (docs/device-sync.md, "A sync run", preflight).
|
||||
*/
|
||||
import type { JmapPort } from '../types';
|
||||
import { authError, transportFailure } from './errors';
|
||||
|
||||
export interface JmapConnection {
|
||||
port: JmapPort;
|
||||
/** Origin of the server (`https://host[:port]`), recorded as the SyncState owner. */
|
||||
origin: string;
|
||||
}
|
||||
|
||||
/** Loads a client; null when the account has no usable stored credentials. */
|
||||
export type ConnectionLoader = () => Promise<JmapConnection | null>;
|
||||
|
||||
async function loadOnce(load: ConnectionLoader): Promise<JmapConnection | null> {
|
||||
try {
|
||||
return await load();
|
||||
} catch (error) {
|
||||
if (transportFailure(error) === 'auth') return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the connection: a missing session or an AuthenticationError is
|
||||
* retried once with a fresh client; still failing, it throws an
|
||||
* AuthenticationError. Network trouble propagates as it came. Later
|
||||
* requests rebuild once more the same way when the server answers 401.
|
||||
*/
|
||||
export async function openWithAuthRebuild(load: ConnectionLoader): Promise<JmapConnection> {
|
||||
const first = await loadOnce(load);
|
||||
const opened = first ?? (await load());
|
||||
if (!opened) throw authError('No usable credentials for this account');
|
||||
|
||||
let current = opened.port;
|
||||
let rebuilt = !first;
|
||||
const retryOnAuth = async <T>(send: (port: JmapPort) => Promise<T>): Promise<T> => {
|
||||
try {
|
||||
return await send(current);
|
||||
} catch (error) {
|
||||
// A 401 comes back before the server ran anything, so replaying is safe.
|
||||
if (transportFailure(error) !== 'auth' || rebuilt) throw error;
|
||||
rebuilt = true;
|
||||
const next = await load();
|
||||
if (!next) throw error;
|
||||
current = next.port;
|
||||
return send(current);
|
||||
}
|
||||
};
|
||||
const port: JmapPort = {
|
||||
session: () => current.session(),
|
||||
limits: () => current.limits(),
|
||||
request: (calls, using) => retryOnAuth((p) => p.request(calls, using)),
|
||||
downloadBlob: (accountId, blobId, type) => retryOnAuth((p) => p.downloadBlob(accountId, blobId, type)),
|
||||
};
|
||||
return { port, origin: opened.origin };
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Transport failures of the JMAP side, recognised by `error.name` as
|
||||
* jmap-client.ts sets it (AuthenticationError, NetworkError,
|
||||
* RequestTimeoutError, RateLimitError), so the engine never imports the
|
||||
* client: it stays pure and runs against the fake server in tests.
|
||||
*/
|
||||
import { JMAPMethodError } from '../../api/jmap-result';
|
||||
|
||||
export { JMAPMethodError };
|
||||
|
||||
/**
|
||||
* - `auth`: the server rejected the credentials (a 401 even after a token refresh);
|
||||
* - `network` / `timeout`: no response; the server may have applied the request;
|
||||
* - `rateLimit`: a 429, with `retryAfterMs`;
|
||||
* - `other`: anything else (a non-OK HTTP status, a body that is not JSON, a bug).
|
||||
*/
|
||||
export type TransportFailure = 'auth' | 'network' | 'timeout' | 'rateLimit' | 'other';
|
||||
|
||||
export function transportFailure(error: unknown): TransportFailure {
|
||||
switch ((error as { name?: unknown } | null)?.name) {
|
||||
case 'AuthenticationError':
|
||||
case 'TotpRequiredError':
|
||||
return 'auth';
|
||||
case 'NetworkError':
|
||||
return 'network';
|
||||
case 'RequestTimeoutError':
|
||||
return 'timeout';
|
||||
case 'RateLimitError':
|
||||
return 'rateLimit';
|
||||
default:
|
||||
return 'other';
|
||||
}
|
||||
}
|
||||
|
||||
/** Milliseconds a RateLimitError asks us to wait, or null. */
|
||||
export function retryAfterMs(error: unknown): number | null {
|
||||
const ms = (error as { retryAfterMs?: unknown } | null)?.retryAfterMs;
|
||||
return typeof ms === 'number' && Number.isFinite(ms) && ms >= 0 ? ms : null;
|
||||
}
|
||||
|
||||
/** A method-level error (`['error', {type}]`) of the given type, e.g. `cannotCalculateChanges`. */
|
||||
export function isMethodError(error: unknown, type?: string): error is JMAPMethodError {
|
||||
return error instanceof JMAPMethodError && (type === undefined || error.type === type);
|
||||
}
|
||||
|
||||
/**
|
||||
* jmap-client.ts turns Stalwart's HTTP 400 request-level limit error into a
|
||||
* plain `JMAP request failed: 400 - {…limit…}`. The engine sizes every
|
||||
* request by the session's limits, so one of those is a bug, not an outage.
|
||||
*/
|
||||
export function isRequestLimitError(error: unknown): boolean {
|
||||
const message = (error as { message?: unknown } | null)?.message;
|
||||
return typeof message === 'string'
|
||||
&& /JMAP request failed: 400/.test(message)
|
||||
&& message.includes('urn:ietf:params:jmap:error:limit');
|
||||
}
|
||||
|
||||
export function authError(message: string): Error {
|
||||
const error = new Error(message);
|
||||
error.name = 'AuthenticationError';
|
||||
return error;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* What device sync reads from a JMAP session: the accounts that hold address
|
||||
* books or calendars, and the user's own calendar addresses.
|
||||
*/
|
||||
import { JMAP_CORE, type JmapSessionView } from '../types';
|
||||
import type { JmapCaller } from './caller';
|
||||
|
||||
const JMAP_SUBMISSION = 'urn:ietf:params:jmap:submission';
|
||||
|
||||
export interface CapabilityAccount {
|
||||
id: string;
|
||||
name: string;
|
||||
/** The capability's primary account (the user's own). */
|
||||
primary: boolean;
|
||||
/** Collections of personal accounts are selected by default, shared ones are not. */
|
||||
personal: boolean;
|
||||
readOnly: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* The primary account of the capability (`primaryAccounts[capability]`)
|
||||
* first, then every other account that advertises it (shared accounts).
|
||||
*/
|
||||
export function accountsWithCapability(session: JmapSessionView, capability: string): CapabilityAccount[] {
|
||||
const out: CapabilityAccount[] = [];
|
||||
const primary = session.primaryAccounts?.[capability];
|
||||
const add = (id: string, isPrimary: boolean) => {
|
||||
const account = session.accounts?.[id];
|
||||
if (!account || out.some((a) => a.id === id)) return;
|
||||
out.push({
|
||||
id,
|
||||
name: account.name || id,
|
||||
primary: isPrimary,
|
||||
personal: isPrimary || account.isPersonal,
|
||||
readOnly: !!account.isReadOnly,
|
||||
});
|
||||
};
|
||||
if (primary) add(primary, true);
|
||||
for (const [id, account] of Object.entries(session.accounts ?? {})) {
|
||||
if (id !== primary && account.accountCapabilities && capability in account.accountCapabilities) add(id, false);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface CalendarAddresses {
|
||||
/** `OWNER_ACCOUNT` of our calendars: never empty (Etar crashes on a null owner). */
|
||||
ownerAccount: string;
|
||||
/** Every address of the user, lowercase, without `mailto:`. */
|
||||
selfAddresses: string[];
|
||||
}
|
||||
|
||||
function normalizeAddress(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
const address = value.trim().replace(/^mailto:/i, '').toLowerCase();
|
||||
return address.includes('@') ? address : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The login when it is an email address, else the first identity's address;
|
||||
* aliases from identities count as "me" too (docs/device-sync.md,
|
||||
* "Decisions"). Identities are read only when the session offers
|
||||
* submission; a server without `Identity/get` leaves the login.
|
||||
*/
|
||||
export async function calendarAddresses(caller: JmapCaller, fallbackName: string): Promise<CalendarAddresses> {
|
||||
const session = caller.session();
|
||||
const login = normalizeAddress(session.username);
|
||||
const identities: string[] = [];
|
||||
const submissionAccount = session.primaryAccounts?.[JMAP_SUBMISSION];
|
||||
if (submissionAccount) {
|
||||
try {
|
||||
const body = await caller.call<{ list?: Array<{ email?: string }> }>(
|
||||
'Identity/get',
|
||||
{ accountId: submissionAccount, properties: ['email'] },
|
||||
[JMAP_CORE, JMAP_SUBMISSION],
|
||||
);
|
||||
for (const identity of body.list ?? []) {
|
||||
const address = normalizeAddress(identity.email);
|
||||
if (address) identities.push(address);
|
||||
}
|
||||
} catch {
|
||||
// Without identities the login is the only address we know.
|
||||
}
|
||||
}
|
||||
const selfAddresses = [...new Set([...(login ? [login] : []), ...identities])];
|
||||
const ownerAccount = login ?? identities[0] ?? (session.username || fallbackName).toLowerCase();
|
||||
return { ownerAccount, selfAddresses: selfAddresses.length ? selfAddresses : [ownerAccount] };
|
||||
}
|
||||
@@ -182,6 +182,13 @@ export interface ContactsContext extends MapContext {
|
||||
nameForUid(uid: string): string | null;
|
||||
/** Group rows by SOURCE_ID, for memberships. */
|
||||
groupRowIdBySourceId(sourceId: string): number | null;
|
||||
/**
|
||||
* SOURCE_IDs of the synced group cards whose `members` list this uid: the
|
||||
* server keeps memberships on the group cards, so a contact's
|
||||
* GroupMembership rows need the reverse lookup. The engine always provides
|
||||
* it; without it the planner leaves memberships as they are.
|
||||
*/
|
||||
groupsOf?(uid: string): string[];
|
||||
}
|
||||
|
||||
export interface ContactsPlanner {
|
||||
@@ -332,7 +339,11 @@ export interface CalendarPlanner {
|
||||
/** Deletes the master and its exception rows (a sync-adapter delete does not cascade). */
|
||||
planLocalDelete(local: LocalEvent): OpGroup;
|
||||
planBaselineHeal(local: LocalEvent): OpGroup | null;
|
||||
/** May return several actions: a "this and following" split is a rule patch plus a create. */
|
||||
/**
|
||||
* May return several actions. For a "this and following" split, the
|
||||
* source's plan is the rule patch plus its pruned overrides; the new series
|
||||
* is a row of its own and goes through claim and create by itself.
|
||||
*/
|
||||
planUpload(local: LocalEvent, ctx: CalendarContext): UploadPlan<CalendarEventWire>;
|
||||
planAccepted(local: LocalEvent, server: CalendarEventWire, ctx: CalendarContext): AcceptedPlan;
|
||||
/**
|
||||
|
||||
+172
-23
@@ -1,41 +1,190 @@
|
||||
/**
|
||||
* Entry of the BulwarkDeviceSync headless task, which the contacts and
|
||||
* calendar sync adapters start for every sync (index.ts registers it). See
|
||||
* docs/device-sync.md.
|
||||
* calendar sync adapters start for every sync (index.ts registers it), and
|
||||
* the teardown the app calls when sync is turned off or an account signs
|
||||
* out. See docs/device-sync.md.
|
||||
*
|
||||
* The run must always end with `finishRun`, called before the task's promise
|
||||
* This is the React Native boundary of the engine: it wires the native
|
||||
* module, a detached JMAPClient per registry account (never the UI's
|
||||
* singleton) and the device-sync store into the pure engine.
|
||||
*
|
||||
* A run always ends with `finishRun`, called before the task's promise
|
||||
* settles: JS timers stop once the task is over, and a sync adapter waiting
|
||||
* for a report that never comes holds its sync until the deadline.
|
||||
*
|
||||
* The sync engine is not wired in yet, so a run only reports that device
|
||||
* sync is off; nothing in the app can turn it on at this point.
|
||||
* for a report that never comes holds its sync until the deadline. Nothing
|
||||
* here throws: every failure becomes a report.
|
||||
*/
|
||||
import { NativeModules } from 'react-native';
|
||||
import type { DeviceSyncNativeModule, RunPayload, RunReport } from './types';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
import { detectDeviceLocale, isSupportedLocale, translate, type LocaleCode } from '../i18n';
|
||||
import { randomBytes } from '../lib/random';
|
||||
import { accountSyncPrefs, useDeviceSyncStore, waitForDeviceSyncHydration } from '../stores/device-sync-store';
|
||||
import type { RandomSource } from './common/ids';
|
||||
import {
|
||||
runDeviceSync,
|
||||
teardownAuthority as teardownWithDeps,
|
||||
type EngineDeps,
|
||||
type SubscriptionCalendar,
|
||||
type TeardownOptions,
|
||||
} from './engine';
|
||||
import { emptyStats } from './engine/report';
|
||||
import { openJmapPort } from './jmap/client-port';
|
||||
import { createProviderPort, finishRun, getSyncSettings, isRunCancelled, showSyncProblem } from './native';
|
||||
import type { CalendarPlanner, ContactsPlanner } from './planner';
|
||||
import type { Authority, RunPayload, RunReport } from './types';
|
||||
|
||||
export async function runDeviceSyncTask(payload: RunPayload): Promise<void> {
|
||||
const native = NativeModules.BulwarkDeviceSync as DeviceSyncNativeModule | undefined;
|
||||
if (!native || !payload?.runId) return;
|
||||
const report: RunReport = {
|
||||
// Mirrors the persist names of the locale and feed-subscription stores; read
|
||||
// straight from storage so a headless run loads neither store.
|
||||
const LOCALE_STORAGE_KEY = 'webmail:locale:v1';
|
||||
const SUBSCRIPTIONS_STORAGE_KEY = 'calendar-subscriptions';
|
||||
/** Where "sign in again" leads (the accounts pane), as for a registry eviction. */
|
||||
const SIGN_IN_URI = 'bulwarkmobile://settings/account';
|
||||
|
||||
// The contacts and calendar planners land in the next
|
||||
// commits. Until then a run fails before it touches a row.
|
||||
function notWired(): never {
|
||||
throw new Error('Device sync planners are not wired into this build yet');
|
||||
}
|
||||
const contactsPlanner = new Proxy({}, { get: notWired }) as ContactsPlanner;
|
||||
const calendarPlanner = new Proxy({}, { get: notWired }) as CalendarPlanner;
|
||||
|
||||
/** Random numbers from the platform CSPRNG, buffered (uids and map keys of device-made items). */
|
||||
function secureRandom(): RandomSource {
|
||||
let buffer: Uint8Array = new Uint8Array(0);
|
||||
let offset = 0;
|
||||
return () => {
|
||||
if (offset + 4 > buffer.length) {
|
||||
buffer = randomBytes(256);
|
||||
offset = 0;
|
||||
}
|
||||
const n = ((buffer[offset] << 24) >>> 0) + (buffer[offset + 1] << 16) + (buffer[offset + 2] << 8) + buffer[offset + 3];
|
||||
offset += 4;
|
||||
return n / 2 ** 32;
|
||||
};
|
||||
}
|
||||
|
||||
/** Feed-subscription calendars (read-only on the device), from the subscriptions store's storage. */
|
||||
async function subscriptionCalendars(): Promise<SubscriptionCalendar[]> {
|
||||
try {
|
||||
const raw = await AsyncStorage.getItem(SUBSCRIPTIONS_STORAGE_KEY);
|
||||
const subscriptions = raw ? (JSON.parse(raw) as { state?: { subscriptions?: unknown } }).state?.subscriptions : null;
|
||||
if (!Array.isArray(subscriptions)) return [];
|
||||
return subscriptions
|
||||
.filter((s): s is { calendarId: string; accountId?: unknown } => typeof s?.calendarId === 'string')
|
||||
.map((s) => ({ calendarId: s.calendarId, jmapAccountId: typeof s.accountId === 'string' ? s.accountId : null }));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** The app's language: the in-app override, else the device's. */
|
||||
async function appLocale(): Promise<LocaleCode> {
|
||||
try {
|
||||
const raw = await AsyncStorage.getItem(LOCALE_STORAGE_KEY);
|
||||
const override = raw ? (JSON.parse(raw) as { override?: unknown }).override : null;
|
||||
if (typeof override === 'string' && isSupportedLocale(override)) return override;
|
||||
} catch {
|
||||
// The device language it is.
|
||||
}
|
||||
return detectDeviceLocale();
|
||||
}
|
||||
|
||||
function deviceZone(): string {
|
||||
try {
|
||||
return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
|
||||
} catch {
|
||||
return 'UTC';
|
||||
}
|
||||
}
|
||||
|
||||
async function notifyAuthProblem(_registryId: string, accountName: string): Promise<void> {
|
||||
const locale = await appLocale();
|
||||
await showSyncProblem(
|
||||
accountName,
|
||||
translate(locale, 'device_sync.auth_problem_title', 'Sign in again to keep syncing'),
|
||||
translate(
|
||||
locale,
|
||||
'device_sync.auth_problem_text',
|
||||
'The server no longer accepts the sign-in of {account}, so its contacts and calendars on this device stopped syncing. Sign in again to resume. Changes made on this device are kept.',
|
||||
{ account: accountName },
|
||||
),
|
||||
SIGN_IN_URI,
|
||||
translate(locale, 'device_sync.problem_channel', 'Sync problems'),
|
||||
);
|
||||
}
|
||||
|
||||
export function createTaskDeps(): EngineDeps {
|
||||
return {
|
||||
provider: createProviderPort,
|
||||
jmap: openJmapPort,
|
||||
prefs: async (registryId) => {
|
||||
await waitForDeviceSyncHydration();
|
||||
return accountSyncPrefs(registryId);
|
||||
},
|
||||
planners: { contacts: contactsPlanner, calendar: calendarPlanner },
|
||||
now: () => Date.now(),
|
||||
isCancelled: isRunCancelled,
|
||||
random: secureRandom(),
|
||||
subscriptionCalendars,
|
||||
isSyncEnabled: async (accountName, authority) => (await getSyncSettings(accountName)).authorities?.[authority]?.automatic === true,
|
||||
deviceZone,
|
||||
// zustand 5 persists on every set(), hydrated or not: wait, or the stored preferences are overwritten.
|
||||
recordStatus: async (registryId, authority, status) => {
|
||||
await waitForDeviceSyncHydration();
|
||||
useDeviceSyncStore.getState().recordRunStatus(registryId, authority, status);
|
||||
},
|
||||
recordKnownState: async (registryId, jmapAccountId, type, state) => {
|
||||
await waitForDeviceSyncHydration();
|
||||
useDeviceSyncStore.getState().recordKnownState(registryId, jmapAccountId, type, state);
|
||||
},
|
||||
notifyAuthProblem,
|
||||
log: (message, detail) => {
|
||||
console.warn(`[device-sync] ${message}`, detail instanceof Error ? detail.message : (detail ?? ''));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function failureReport(payload: RunPayload, error: unknown): RunReport {
|
||||
return {
|
||||
v: 1,
|
||||
runId: payload.runId,
|
||||
authority: payload.authority,
|
||||
outcome: 'disabled',
|
||||
message: 'Device sync is not available in this build',
|
||||
outcome: 'internal',
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
startedAt: Date.now(),
|
||||
durationMs: 0,
|
||||
stats: {
|
||||
downloaded: { created: 0, updated: 0, deleted: 0 },
|
||||
uploaded: { created: 0, updated: 0, deleted: 0 },
|
||||
entries: 0,
|
||||
skipped: 0,
|
||||
},
|
||||
stats: emptyStats(),
|
||||
conflicts: 0,
|
||||
itemErrors: [],
|
||||
};
|
||||
}
|
||||
|
||||
export async function runDeviceSyncTask(payload: RunPayload): Promise<void> {
|
||||
if (!payload?.runId) return;
|
||||
let report: RunReport;
|
||||
try {
|
||||
await native.finishRun(payload.runId, JSON.stringify(report));
|
||||
} catch {
|
||||
report = await runDeviceSync(payload, createTaskDeps());
|
||||
} catch (error) {
|
||||
report = failureReport(payload, error);
|
||||
}
|
||||
try {
|
||||
await finishRun(payload.runId, report);
|
||||
} catch (error) {
|
||||
// The adapter ends the run on its own when the task finishes.
|
||||
console.warn('[device-sync] finishRun failed', error instanceof Error ? error.message : error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Turning sync off for an authority (or signing out): uploads what the
|
||||
* device changed (bounded by `timeoutMs`, default 30 s), then removes the
|
||||
* authority's rows and SyncState. When changes are still waiting
|
||||
* (`pending > 0`) nothing is deleted unless `force` is set, once the user
|
||||
* confirmed losing them. Runs under the same lock as sync runs.
|
||||
*/
|
||||
export function teardownAuthority(
|
||||
registryId: string,
|
||||
accountName: string,
|
||||
authority: Authority,
|
||||
options?: TeardownOptions,
|
||||
): Promise<{ pending: number }> {
|
||||
return teardownWithDeps(createTaskDeps(), registryId, accountName, authority, options);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user