mirror of
https://github.com/bulwarkmail/webmail.git
synced 2026-11-04 08:15:41 +00:00
chore: merge contacts and files into the settings branch
This commit is contained in:
Linus Rath
+35
-8
@@ -21,6 +21,7 @@ import { isTaskLikeObject, type ScannedCalendarObject } from "@/lib/calendar-com
|
||||
import { DEFAULT_CALENDAR_COMPONENTS, mkCalendarCollection, newCalendarCollectionName } from "@/lib/webdav/calendar-collection";
|
||||
import { sanitizeDisplayName, splitMailbox } from "@/lib/rfc5322-mailbox";
|
||||
import { decodeFileNodeName } from "./filenode-name";
|
||||
import { PartialAccountsError } from "./partial-accounts";
|
||||
import { fileNodeKind, fromDraftFileNode, toDraftFileNodeCreate } from "./filenode-kind";
|
||||
import { fileNodeDepth } from "./filenode-depth";
|
||||
import { contactFromWire, contactToWire } from "./contact-wire";
|
||||
@@ -7419,10 +7420,16 @@ export class JMAPClient implements IJMAPClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws PartialAccountsError when an account couldn't answer, with what the
|
||||
* others gave, so a failure doesn't read as that account having no books.
|
||||
*/
|
||||
async getAllAddressBooks(): Promise<AddressBook[]> {
|
||||
const failed: string[] = [];
|
||||
let failure: unknown;
|
||||
const allBooks: AddressBook[] = [];
|
||||
const primaryId = this.getContactsAccountId();
|
||||
try {
|
||||
const allBooks: AddressBook[] = [];
|
||||
const primaryId = this.getContactsAccountId();
|
||||
const accountIds = this.getContactCapableAccountIds();
|
||||
|
||||
for (const accountId of accountIds) {
|
||||
@@ -7445,17 +7452,23 @@ export class JMAPClient implements IJMAPClient {
|
||||
isShared: !isPrimary,
|
||||
}));
|
||||
allBooks.push(...books);
|
||||
} else if (response.methodResponses?.[0]?.[0] === "error") {
|
||||
const refused = response.methodResponses[0][1];
|
||||
throw new Error(refused?.description || refused?.type || "AddressBook/get failed");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch address books for account ${accountId}:`, error);
|
||||
failed.push(accountId);
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
|
||||
return allBooks;
|
||||
if (failed.length === 0) return allBooks;
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch all address books:', error);
|
||||
return this.getAddressBooks();
|
||||
}
|
||||
throw new PartialAccountsError(allBooks, failed, primaryId, failure);
|
||||
}
|
||||
|
||||
async createAddressBook(name: string): Promise<AddressBook> {
|
||||
@@ -8322,6 +8335,8 @@ export class JMAPClient implements IJMAPClient {
|
||||
], this.contactUsing());
|
||||
|
||||
const queryResult = response.methodResponses?.[0];
|
||||
// A refused call isn't an empty address book: the caller keeps what it has.
|
||||
if (queryResult?.[0] === "error") throw new Error(queryResult[1]?.description || queryResult[1]?.type || "ContactCard/query failed");
|
||||
if (queryResult?.[0] !== "ContactCard/query") break;
|
||||
|
||||
const ids: string[] = queryResult[1].ids || [];
|
||||
@@ -8344,8 +8359,10 @@ export class JMAPClient implements IJMAPClient {
|
||||
["ContactCard/get", { accountId, ids: chunk }, "g"],
|
||||
], this.contactUsing());
|
||||
|
||||
if (response.methodResponses?.[0]?.[0] === "ContactCard/get") {
|
||||
const list = (response.methodResponses[0][1].list || []) as ContactCard[];
|
||||
const getResult = response.methodResponses?.[0];
|
||||
if (getResult?.[0] === "error") throw new Error(getResult[1]?.description || getResult[1]?.type || "ContactCard/get failed");
|
||||
if (getResult?.[0] === "ContactCard/get") {
|
||||
const list = (getResult[1].list || []) as ContactCard[];
|
||||
allContacts.push(...list.map(contactFromWire));
|
||||
}
|
||||
}
|
||||
@@ -8365,10 +8382,17 @@ export class JMAPClient implements IJMAPClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws PartialAccountsError when an account couldn't answer, with what the
|
||||
* others gave, so a failure doesn't read as that account having no cards.
|
||||
*/
|
||||
async getAllContacts(): Promise<ContactCard[]> {
|
||||
const failed: string[] = [];
|
||||
let failure: unknown;
|
||||
let allContacts: ContactCard[] = [];
|
||||
const primaryId = this.getContactsAccountId();
|
||||
try {
|
||||
const allContacts: ContactCard[] = [];
|
||||
const primaryId = this.getContactsAccountId();
|
||||
allContacts = [];
|
||||
const accountIds = this.getContactCapableAccountIds();
|
||||
|
||||
for (const accountId of accountIds) {
|
||||
@@ -8391,14 +8415,17 @@ export class JMAPClient implements IJMAPClient {
|
||||
allContacts.push(...contacts);
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch contacts for account ${accountId}:`, error);
|
||||
failed.push(accountId);
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
|
||||
return allContacts;
|
||||
if (failed.length === 0) return allContacts;
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch all contacts:', error);
|
||||
return this.getContacts();
|
||||
}
|
||||
throw new PartialAccountsError(allContacts, failed, primaryId, failure);
|
||||
}
|
||||
|
||||
async getContact(contactId: string, accountId?: string): Promise<ContactCard | null> {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* A listing across JMAP accounts where at least one account couldn't answer.
|
||||
* It carries what the others gave. An account that failed isn't an account
|
||||
* with nothing in it, so the caller keeps what it had for that one.
|
||||
*/
|
||||
export class PartialAccountsError<T> extends Error {
|
||||
constructor(
|
||||
readonly items: T[],
|
||||
readonly failedAccountIds: string[],
|
||||
readonly primaryAccountId: string,
|
||||
cause: unknown,
|
||||
) {
|
||||
super(cause instanceof Error ? cause.message : 'An account could not answer');
|
||||
this.name = 'PartialAccountsError';
|
||||
}
|
||||
}
|
||||
|
||||
/** By name: `instanceof` fails across Next's bundle copies. */
|
||||
export function partialAccounts<T>(error: unknown): PartialAccountsError<T> | null {
|
||||
return error instanceof Error && error.name === 'PartialAccountsError' ? error as PartialAccountsError<T> : null;
|
||||
}
|
||||
|
||||
/** What to show after a partial listing: the answers, and what's held for the accounts that failed. */
|
||||
export function withFailedAccountsKept<T extends { accountId?: string }>(partial: PartialAccountsError<T>, held: readonly T[]): T[] {
|
||||
const failed = new Set(partial.failedAccountIds);
|
||||
return [...partial.items, ...held.filter((item) => failed.has(item.accountId ?? partial.primaryAccountId))];
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
// @vitest-environment node
|
||||
/**
|
||||
* The real JMAPClient reading the store's answers: what the client's own code
|
||||
* makes of them is the contract, not the store's idea of the shape.
|
||||
*/
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { JMAPClient } from '@/lib/jmap/client';
|
||||
import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks';
|
||||
import { partialAccounts } from '@/lib/jmap/partial-accounts';
|
||||
import type { AddressBook, ContactCard } from '@/lib/jmap/types';
|
||||
import { DEFAULT_SCOPE } from '../area';
|
||||
import { contactsArea } from '../contacts';
|
||||
import { AccountEngine } from '../engine/account-engine';
|
||||
import { JMAP_ACCOUNT } from './helpers/fake-jmap';
|
||||
import { CONTACTS_SESSION, FakeContacts, SHARED_ACCOUNT } from './helpers/fake-contacts';
|
||||
import { openMemoryStore } from './helpers/node-adapter';
|
||||
|
||||
afterEach(() => setOfflineHooks(null));
|
||||
|
||||
async function offlineClient(server: FakeContacts): Promise<{ client: JMAPClient; engine: AccountEngine }> {
|
||||
const engine = new AccountEngine({
|
||||
account: 'a@fake.test',
|
||||
db: openMemoryStore(),
|
||||
session: CONTACTS_SESSION,
|
||||
transport: server,
|
||||
areas: [contactsArea],
|
||||
scope: DEFAULT_SCOPE,
|
||||
sleep: async () => {},
|
||||
});
|
||||
await engine.requestSync();
|
||||
engine.setOnline(false);
|
||||
const hooks: Partial<OfflineHooks> = {
|
||||
skipNetwork: () => true,
|
||||
answer: (_client, calls) => engine.answer(calls),
|
||||
record: () => {},
|
||||
noteNetwork: () => {},
|
||||
};
|
||||
setOfflineHooks(hooks as OfflineHooks);
|
||||
const client = JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', {
|
||||
...CONTACTS_SESSION,
|
||||
capabilities: { ...CONTACTS_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 2 } },
|
||||
});
|
||||
return { client, engine };
|
||||
}
|
||||
|
||||
function seeded(): FakeContacts {
|
||||
const server = new FakeContacts();
|
||||
for (const full of ['Anna Bauer', 'Bernd Graf', 'Carla Müller']) server.addCard({ name: { full } });
|
||||
server.addBook({ id: 'teambook', name: 'Team' }, SHARED_ACCOUNT);
|
||||
server.addCard({ id: 'c1', name: { full: 'Dora Team' }, addressBookIds: { teambook: true } }, SHARED_ACCOUNT);
|
||||
return server;
|
||||
}
|
||||
|
||||
describe('the client over the store', () => {
|
||||
it('reports a store that has nothing yet as a failure, not as no contacts', async () => {
|
||||
const engine = new AccountEngine({
|
||||
account: 'a@fake.test', db: openMemoryStore(), session: CONTACTS_SESSION, transport: new FakeContacts(),
|
||||
areas: [contactsArea], scope: DEFAULT_SCOPE, sleep: async () => {},
|
||||
});
|
||||
// Answered per call, with one more call the store can answer: the error isn't hidden by a null answer.
|
||||
const perCall: Partial<OfflineHooks> = {
|
||||
skipNetwork: () => true,
|
||||
answer: async (_client, calls) => (await engine.answer(calls)) ?? {
|
||||
methodResponses: calls.map(([, , id]) => ['error', { type: 'serverUnavailable' }, id]),
|
||||
},
|
||||
record: () => {},
|
||||
noteNetwork: () => {},
|
||||
};
|
||||
setOfflineHooks(perCall as OfflineHooks);
|
||||
const client = JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', CONTACTS_SESSION);
|
||||
await expect(client.getAllContacts()).rejects.toThrow();
|
||||
await expect(client.getAllAddressBooks()).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('lists all contacts and books across accounts, paging in its own batch size', async () => {
|
||||
const { client } = await offlineClient(seeded());
|
||||
const contacts = await client.getAllContacts();
|
||||
expect(contacts.map((c) => c.name?.full).sort()).toEqual(['Anna Bauer', 'Bernd Graf', 'Carla Müller', 'Dora Team']);
|
||||
const shared = contacts.find((c) => c.name?.full === 'Dora Team');
|
||||
expect(shared).toMatchObject({ id: `${SHARED_ACCOUNT}:c1`, originalId: 'c1', isShared: true });
|
||||
const books = await client.getAllAddressBooks();
|
||||
expect(books.map((b) => b.id).sort()).toEqual(['book1', `${SHARED_ACCOUNT}:teambook`]);
|
||||
});
|
||||
|
||||
it('says which account failed, with what the others gave', async () => {
|
||||
const { client, engine } = await offlineClient(seeded());
|
||||
const failPrimary: Partial<OfflineHooks> = {
|
||||
skipNetwork: () => true,
|
||||
answer: async (_client, calls) => (calls[0][1].accountId === JMAP_ACCOUNT
|
||||
? { methodResponses: calls.map(([, , id]) => ['error', { type: 'serverUnavailable' }, id]) }
|
||||
: engine.answer(calls)),
|
||||
record: () => {},
|
||||
noteNetwork: () => {},
|
||||
};
|
||||
setOfflineHooks(failPrimary as OfflineHooks);
|
||||
const contacts = await client.getAllContacts().catch((error: unknown) => error);
|
||||
const partial = partialAccounts<ContactCard>(contacts);
|
||||
expect(partial?.failedAccountIds).toEqual([JMAP_ACCOUNT]);
|
||||
expect(partial?.primaryAccountId).toBe(JMAP_ACCOUNT);
|
||||
expect(partial?.items.map((c) => c.name?.full)).toEqual(['Dora Team']);
|
||||
const books = partialAccounts<AddressBook>(await client.getAllAddressBooks().catch((error: unknown) => error));
|
||||
expect(books?.failedAccountIds).toEqual([JMAP_ACCOUNT]);
|
||||
expect(books?.items.map((b) => b.originalId)).toEqual(['teambook']);
|
||||
});
|
||||
|
||||
it('searches with its own filter', async () => {
|
||||
const { client } = await offlineClient(seeded());
|
||||
expect((await client.searchContacts('müll')).map((c) => c.name?.full)).toEqual(['Carla Müller']);
|
||||
expect((await client.searchContacts('team')).map((c) => c.originalId)).toEqual(['c1']);
|
||||
});
|
||||
|
||||
it('creates, edits and deletes through its own methods with the stand-in id', async () => {
|
||||
const { client, engine } = await offlineClient(seeded());
|
||||
const created = await client.createContact({ name: { full: 'Neue Person' }, emails: { e: { address: 'neu@example.org' } } });
|
||||
expect(created.id).toBeTruthy();
|
||||
expect(created.addressBookIds).toEqual({ book1: true });
|
||||
await client.updateContact(created.id, { name: { full: 'Neue Person B' } });
|
||||
expect((await client.getContact(created.id))?.name?.full).toBe('Neue Person B');
|
||||
expect((await client.searchContacts('person b')).map((c) => c.id)).toEqual([created.id]);
|
||||
await client.deleteContact(created.id);
|
||||
expect(await client.getContact(created.id)).toBeNull();
|
||||
expect(await engine.areaCall('contacts.pending', undefined)).toBe(0);
|
||||
});
|
||||
|
||||
it('throws the server-style error for an edit to a card it does not have', async () => {
|
||||
const { client } = await offlineClient(seeded());
|
||||
await expect(client.updateContact('nope', { name: { full: 'X' } })).rejects.toThrow();
|
||||
await expect(client.deleteContact('nope')).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('reads an account the store has as the primary contacts account', async () => {
|
||||
const { client } = await offlineClient(seeded());
|
||||
expect(client.getContactsAccountId()).toBe(JMAP_ACCOUNT);
|
||||
expect((await client.getContacts('book1')).length).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
// @vitest-environment node
|
||||
/**
|
||||
* The real JMAPClient listing files from the store, in each way a server says
|
||||
* what a node is, plus what it does with a write.
|
||||
*/
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { JMAPClient } from '@/lib/jmap/client';
|
||||
import { fileNodeKind } from '@/lib/jmap/filenode-kind';
|
||||
import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks';
|
||||
import { DEFAULT_SCOPE } from '../area';
|
||||
import { AccountEngine } from '../engine/account-engine';
|
||||
import { filesArea } from '../files';
|
||||
import { FakeFiles, FILES, FILES_SESSION, SHARED_FILES_ACCOUNT, type FakeFilesOptions } from './helpers/fake-files';
|
||||
import { openMemoryStore } from './helpers/node-adapter';
|
||||
|
||||
afterEach(() => setOfflineHooks(null));
|
||||
|
||||
async function offlineClient(server: FakeFiles): Promise<JMAPClient> {
|
||||
const engine = new AccountEngine({
|
||||
account: 'a@fake.test',
|
||||
db: openMemoryStore(),
|
||||
session: FILES_SESSION,
|
||||
transport: server,
|
||||
areas: [filesArea],
|
||||
scope: DEFAULT_SCOPE,
|
||||
sleep: async () => {},
|
||||
});
|
||||
await engine.requestSync();
|
||||
engine.setOnline(false);
|
||||
setOfflineHooks({
|
||||
skipNetwork: () => true,
|
||||
answer: (_client, calls) => engine.answer(calls),
|
||||
record: () => {},
|
||||
noteNetwork: () => {},
|
||||
} as Partial<OfflineHooks> as OfflineHooks);
|
||||
return JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', {
|
||||
...FILES_SESSION,
|
||||
capabilities: { ...FILES_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500 } },
|
||||
});
|
||||
}
|
||||
|
||||
describe('file listing offline', () => {
|
||||
it.each<[string, FakeFilesOptions]>([
|
||||
['nodeType (0.16.6 and later, 1.0 as tested)', { style: 'nodeType' }],
|
||||
['blobId null only (0.16.0-5)', { style: 'blob' }],
|
||||
['@type and mediaType (1.0.0)', { style: 'draft' }],
|
||||
])('tells folders from files: %s', async (_label, options) => {
|
||||
const server = new FakeFiles(options);
|
||||
const folder = server.addFolder('Docs');
|
||||
server.addFile('note.txt', folder.id);
|
||||
const client = await offlineClient(server);
|
||||
expect(client.supportsFiles()).toBe(true);
|
||||
|
||||
const nodes = await client.listAllFileNodes();
|
||||
const byName = new Map(nodes.map((n) => [n.name, n]));
|
||||
expect(fileNodeKind(byName.get('Docs') as never)).toBe('directory');
|
||||
expect(fileNodeKind(byName.get('note.txt') as never)).toBe('file');
|
||||
expect(byName.get('note.txt')).toMatchObject({ parentId: folder.id, type: 'text/plain', size: 10 });
|
||||
expect((await client.listFileNodes(folder.id)).map((n) => n.name)).toEqual(['note.txt']);
|
||||
expect((await client.listFileNodes(null)).map((n) => n.name)).toEqual(['Docs']);
|
||||
});
|
||||
|
||||
it('lists an account larger than one /get, ending its paging cleanly', async () => {
|
||||
const server = new FakeFiles({ getAll: 'refuse' });
|
||||
const folder = server.addFolder('Docs');
|
||||
for (let i = 0; i < 700; i += 1) server.addFile(`f${i}`, folder.id);
|
||||
const client = await offlineClient(server);
|
||||
expect(await client.listAllFileNodes()).toHaveLength(701);
|
||||
});
|
||||
|
||||
it('lists shared accounts with namespaced ids', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFolder('Mine');
|
||||
const shared = server.addFolder('Theirs', null, undefined, SHARED_FILES_ACCOUNT);
|
||||
const client = await offlineClient(server);
|
||||
const nodes = await client.listAllFileNodesAcrossAccounts();
|
||||
expect(nodes.find((n) => n.name === 'Theirs')).toMatchObject({ id: `${SHARED_FILES_ACCOUNT}:${shared.id}`, isShared: true });
|
||||
expect(nodes.find((n) => n.name === 'Mine')).toMatchObject({ isShared: false });
|
||||
});
|
||||
|
||||
it('fails a write with an error rather than losing it', async () => {
|
||||
const server = new FakeFiles();
|
||||
const client = await offlineClient(server);
|
||||
await expect(client.createFileDirectory('New', null)).rejects.toThrow();
|
||||
expect(FILES in FILES_SESSION.capabilities).toBe(true);
|
||||
expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,535 @@
|
||||
// @vitest-environment node
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { DEFAULT_SCOPE, type StoreEvent, type StoreScope } from '../area';
|
||||
import { AccountEngine } from '../engine/account-engine';
|
||||
import { TransportError } from '../engine/transport';
|
||||
import { fileBlob, FILE_CACHE_CAP_BYTES, filesArea } from '../files';
|
||||
import type { StorageAdapter } from '../storage';
|
||||
import { JMAP_ACCOUNT } from './helpers/fake-jmap';
|
||||
import { FakeFiles, FILES_SESSION, SHARED_FILES_ACCOUNT } from './helpers/fake-files';
|
||||
import { openMemoryStore } from './helpers/node-adapter';
|
||||
|
||||
const NOW = Date.UTC(2026, 10, 3);
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any -- answers are shaped like the server's */
|
||||
|
||||
interface Rig {
|
||||
db: StorageAdapter;
|
||||
engine: AccountEngine;
|
||||
clock: { at: number };
|
||||
events: StoreEvent[];
|
||||
}
|
||||
|
||||
function rigFor(server: FakeFiles, scope: Partial<StoreScope> = {}): Rig {
|
||||
const db = openMemoryStore();
|
||||
const clock = { at: NOW };
|
||||
const events: StoreEvent[] = [];
|
||||
const engine = new AccountEngine({
|
||||
account: 'a@fake.test',
|
||||
db,
|
||||
session: FILES_SESSION,
|
||||
transport: server,
|
||||
areas: [filesArea],
|
||||
scope: { ...DEFAULT_SCOPE, ...scope },
|
||||
emit: (event) => events.push(event),
|
||||
now: () => clock.at,
|
||||
sleep: async () => {},
|
||||
});
|
||||
return { db, engine, clock, events };
|
||||
}
|
||||
|
||||
async function synced(server: FakeFiles, scope: Partial<StoreScope> = {}): Promise<Rig> {
|
||||
const rig = rigFor(server, scope);
|
||||
await rig.engine.requestSync();
|
||||
return rig;
|
||||
}
|
||||
|
||||
async function call(engine: AccountEngine, name: string, args: Record<string, unknown> = {}): Promise<any> {
|
||||
const response = await engine.answer([[name, { accountId: JMAP_ACCOUNT, ...args }, '0']]);
|
||||
if (!response) throw new Error('nothing answered');
|
||||
const [answered, result] = response.methodResponses[0];
|
||||
if (answered === 'error') throw new Error(result.type);
|
||||
return result;
|
||||
}
|
||||
|
||||
const ref = (id: string, jmap = JMAP_ACCOUNT) => ({ jmap, id });
|
||||
|
||||
function names(rows: Array<{ name?: unknown }>): unknown[] {
|
||||
return rows.map((row) => row.name).sort();
|
||||
}
|
||||
|
||||
describe('file metadata sync', () => {
|
||||
it('stores small accounts from one FileNode/get', async () => {
|
||||
const server = new FakeFiles();
|
||||
const folder = server.addFolder('Docs');
|
||||
server.addFile('a.txt', folder.id);
|
||||
const { engine } = await synced(server);
|
||||
expect(server.calls.filter(([n]) => n === 'FileNode/query')).toHaveLength(0);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['Docs', 'a.txt']);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['0.16 cuts the list at 500', { getAll: 'cut' as const }],
|
||||
['1.0 refuses a list over 500', { getAll: 'refuse' as const }],
|
||||
])('pages past the /get limit: %s', async (_label, options) => {
|
||||
const server = new FakeFiles(options);
|
||||
const folder = server.addFolder('Docs');
|
||||
for (let i = 0; i < 1234; i += 1) server.addFile(`f${i}.txt`, folder.id);
|
||||
const { db } = await synced(server);
|
||||
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(1235);
|
||||
for (const [name, args] of server.calls) {
|
||||
if (name === 'FileNode/get' && Array.isArray(args.ids)) expect(args.ids.length).toBeLessThanOrEqual(500);
|
||||
}
|
||||
});
|
||||
|
||||
it('finds folders the query leaves out on old servers, by the parents of what it did find', async () => {
|
||||
const server = new FakeFiles({ queryOmitsFolders: true, style: 'blob' });
|
||||
for (let i = 0; i < 600; i += 1) server.addFile(`early${i}.txt`);
|
||||
const top = server.addFolder('Top');
|
||||
const inner = server.addFolder('Inner', top.id);
|
||||
server.addFile('deep.txt', inner.id);
|
||||
const { db } = await synced(server);
|
||||
const kinds = db.all<{ name: string; kind: string }>("SELECT name, kind FROM file_node WHERE name IN ('Top', 'Inner', 'deep.txt') ORDER BY name");
|
||||
expect(kinds).toEqual([{ name: 'Inner', kind: 'directory' }, { name: 'Top', kind: 'directory' }, { name: 'deep.txt', kind: 'file' }]);
|
||||
});
|
||||
|
||||
it('continues a stopped full sync from its cursor', async () => {
|
||||
const server = new FakeFiles({ getAll: 'refuse' });
|
||||
for (let i = 0; i < 6000; i += 1) server.addFile(`f${i}`);
|
||||
const rig = rigFor(server);
|
||||
let gets = 0;
|
||||
server.beforeRequest = () => {
|
||||
gets += 1;
|
||||
// State capture, the refused list, a query page of 5000 and two requests of gets: then the network goes.
|
||||
if (gets === 6) server.offline = true;
|
||||
};
|
||||
await rig.engine.requestSync();
|
||||
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(5000);
|
||||
server.offline = false;
|
||||
server.beforeRequest = null;
|
||||
server.calls.length = 0;
|
||||
await rig.engine.requestSync();
|
||||
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(6000);
|
||||
expect(server.calls.find(([n]) => n === 'FileNode/query')?.[1].position).toBe(5000);
|
||||
expect(server.calls.filter(([n, a]) => n === 'FileNode/get' && a.ids === null && a.accountId === JMAP_ACCOUNT)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('follows changes through FileNode/changes', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a.txt');
|
||||
const b = server.addFile('b.txt');
|
||||
const { engine } = await synced(server);
|
||||
server.calls.length = 0;
|
||||
server.renameNode(a.id, 'renamed.txt');
|
||||
server.removeNode(b.id);
|
||||
server.addFolder('New');
|
||||
await engine.requestSync();
|
||||
expect(server.calls.some(([n]) => n === 'FileNode/changes')).toBe(true);
|
||||
expect(server.calls.some(([n, args]) => n === 'FileNode/get' && args.ids === null)).toBe(false);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['New', 'renamed.txt']);
|
||||
});
|
||||
|
||||
it('starts over when the server can no longer give changes, and keeps a pin on a file still there', async () => {
|
||||
const server = new FakeFiles();
|
||||
const gone = server.addFile('gone.txt');
|
||||
const kept = server.addFile('kept.txt');
|
||||
const { db, engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(kept.id));
|
||||
await engine.requestSync();
|
||||
server.removeNode(gone.id);
|
||||
server.account().coll.truncatedBelow = server.account().coll.log.length;
|
||||
await engine.requestSync();
|
||||
expect(db.all('SELECT id FROM file_node').map((r) => r.id)).toEqual([kept.id]);
|
||||
expect(db.get<{ pinned: number }>('SELECT pinned FROM file_node WHERE id = ?', [kept.id])?.pinned).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps accounts apart, even with the same ids', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('own.txt', null, 5, { id: 'same' });
|
||||
server.addFile('team.txt', null, 5, { id: 'same', blobId: 'team-blob' }, SHARED_FILES_ACCOUNT);
|
||||
const { engine } = await synced(server);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: ['same'] })).list)).toEqual(['own.txt']);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: ['same'], accountId: SHARED_FILES_ACCOUNT })).list)).toEqual(['team.txt']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('file answers', () => {
|
||||
it('says so instead of an empty list when nothing was read yet', async () => {
|
||||
const { engine } = rigFor(new FakeFiles());
|
||||
expect(await engine.answer([['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '0']])).toBeNull();
|
||||
});
|
||||
|
||||
it('gives the last server state and only asked-for properties', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a.txt');
|
||||
const { engine } = await synced(server);
|
||||
const got = await call(engine, 'FileNode/get', { ids: [file.id, 'nope'], properties: ['name', 'parentId'] });
|
||||
expect(got.state).toBe(server.account().coll.state());
|
||||
expect(got.list).toEqual([{ id: file.id, name: 'a.txt', parentId: null }]);
|
||||
expect(got.notFound).toEqual(['nope']);
|
||||
});
|
||||
|
||||
it('answers a query by parent, also at the root, which the real server refuses to take', async () => {
|
||||
const server = new FakeFiles();
|
||||
const folder = server.addFolder('Docs');
|
||||
const inside = server.addFile('in.txt', folder.id);
|
||||
const root = server.addFile('root.txt');
|
||||
const { engine } = await synced(server);
|
||||
expect((await call(engine, 'FileNode/query', { filter: { parentId: folder.id } })).ids).toEqual([inside.id]);
|
||||
expect((await call(engine, 'FileNode/query', { filter: { parentId: null } })).ids).toEqual([folder.id, root.id]);
|
||||
});
|
||||
|
||||
it('pages and sorts a query', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('b', null, 30);
|
||||
server.addFile('a', null, 10);
|
||||
server.addFile('c', null, 20);
|
||||
const { engine } = await synced(server);
|
||||
const page = await call(engine, 'FileNode/query', { filter: {}, position: 1, limit: 1, calculateTotal: true });
|
||||
expect(page).toMatchObject({ position: 1, total: 3, ids: [expect.any(String)] });
|
||||
const bySize = await call(engine, 'FileNode/query', { filter: {}, sort: [{ property: 'size', isAscending: false }] });
|
||||
const list = (await call(engine, 'FileNode/get', { ids: bySize.ids })).list;
|
||||
expect(list.map((n: { name: string }) => n.name)).toEqual(['b', 'c', 'a']);
|
||||
});
|
||||
|
||||
it('refuses what it cannot read, for that call only', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
const response = await engine.answer([
|
||||
['FileNode/query', { accountId: JMAP_ACCOUNT, filter: { hasFoo: true } }, '0'],
|
||||
['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'],
|
||||
]);
|
||||
expect(response?.methodResponses[0][1].type).toBe('unsupportedFilter');
|
||||
expect(response?.methodResponses[1][0]).toBe('FileNode/get');
|
||||
});
|
||||
|
||||
it('answers FileNode/set with a clear per-call error, not silence', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
const response = await engine.answer([
|
||||
['FileNode/set', { accountId: JMAP_ACCOUNT, create: { n0: { name: 'x' } } }, '0'],
|
||||
['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'],
|
||||
]);
|
||||
expect(response?.methodResponses[0]).toEqual(['error', { type: 'serverUnavailable', description: 'FileNode/set needs the server' }, '0']);
|
||||
expect(response?.methodResponses[1][0]).toBe('FileNode/get');
|
||||
expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pinned files', () => {
|
||||
it('fetches a pinned file and gives it back byte for byte, past the chunk size', async () => {
|
||||
const server = new FakeFiles();
|
||||
const content = new Uint8Array(3 * 1024 * 1024 + 17).map((_, i) => i % 251);
|
||||
const file = server.addFile('big.bin', null, content);
|
||||
const { db, engine } = await synced(server);
|
||||
expect(server.downloads).toEqual([]);
|
||||
|
||||
expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: false });
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toEqual([{ jmap: JMAP_ACCOUNT, blobId: file.blobId }]);
|
||||
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(4);
|
||||
|
||||
const read = await engine.areaCall('files.read', ref(file.id)) as any;
|
||||
expect(read.name).toBe('big.bin');
|
||||
expect(read.type).toBe('text/plain');
|
||||
expect(Buffer.compare(read.data, content)).toBe(0);
|
||||
expect(Buffer.compare(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)?.data as Uint8Array, content)).toBe(0);
|
||||
expect(await engine.areaCall('files.blob', { jmap: JMAP_ACCOUNT, blobId: file.blobId })).toMatchObject({ size: content.length });
|
||||
// Already kept: not fetched again.
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: true });
|
||||
});
|
||||
|
||||
it('names a file with a %-encoded name from 0.16 WebDAV the way the client shows it', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('My%20Report.pdf');
|
||||
const { engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(file.id));
|
||||
await engine.requestSync();
|
||||
expect((await engine.areaCall('files.read', ref(file.id)) as any).name).toBe('My Report.pdf');
|
||||
expect((await engine.areaCall('files.kept', undefined) as any).files[0].name).toBe('My Report.pdf');
|
||||
// The stored node stays as the server sent it. The client decodes it once.
|
||||
expect((await call(engine, 'FileNode/get', { ids: [file.id] })).list[0].name).toBe('My%20Report.pdf');
|
||||
});
|
||||
|
||||
it('gives nothing for a file not kept', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a.txt');
|
||||
const { db, engine } = await synced(server);
|
||||
expect(await engine.areaCall('files.read', ref(file.id))).toBeNull();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)).toBeNull();
|
||||
});
|
||||
|
||||
it('refuses to pin a folder or an unknown id', async () => {
|
||||
const server = new FakeFiles();
|
||||
const folder = server.addFolder('Docs');
|
||||
const { engine } = await synced(server);
|
||||
await expect(engine.areaCall('files.pin', ref(folder.id))).rejects.toThrow('Only files');
|
||||
await expect(engine.areaCall('files.pin', ref('nope'))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('downloads one at a time', async () => {
|
||||
const server = new FakeFiles();
|
||||
const files = [server.addFile('a'), server.addFile('b'), server.addFile('c')];
|
||||
const { engine } = await synced(server);
|
||||
for (const file of files) await engine.areaCall('files.pin', ref(file.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(3);
|
||||
expect(server.mostAtOnce).toBe(1);
|
||||
});
|
||||
|
||||
it('fetches a pinned file over the one-file limit, and skips an opened one', async () => {
|
||||
const server = new FakeFiles();
|
||||
const pinned = server.addFile('pinned.bin', null, 100);
|
||||
const opened = server.addFile('opened.bin', null, 100);
|
||||
const { engine } = await synced(server, { maxAttachmentBytes: 50 });
|
||||
await engine.areaCall('files.pin', ref(pinned.id));
|
||||
await engine.areaCall('files.noteOpened', ref(opened.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads.map((d) => d.blobId)).toEqual([pinned.blobId]);
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
const row = (name: string) => status.files.find((f: { name: string }) => f.name === name);
|
||||
expect(row('pinned.bin')).toMatchObject({ pinned: true, kept: true, error: null });
|
||||
expect(row('opened.bin')).toMatchObject({ pinned: false, kept: false, error: 'tooLarge' });
|
||||
});
|
||||
|
||||
it('does not take pins past the total limit, and says so', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a', null, 60);
|
||||
const b = server.addFile('b', null, 60);
|
||||
const { engine, clock } = await synced(server, { maxTotalBytes: 100 });
|
||||
await engine.areaCall('files.pin', ref(a.id));
|
||||
clock.at += 1;
|
||||
await engine.areaCall('files.pin', ref(b.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
expect(status.files.filter((f: { kept: boolean }) => f.kept)).toHaveLength(1);
|
||||
expect(status.files.filter((f: { error: string | null }) => f.error === 'overLimit')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('keeps nothing when the scope forbids it, and drops what was kept', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const rig = await synced(server);
|
||||
await rig.engine.areaCall('files.pin', ref(file.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
|
||||
rig.engine.setScope({ ...DEFAULT_SCOPE, attachments: false });
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).toBeNull();
|
||||
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ pinned: true, kept: false, error: 'notAllowed' });
|
||||
|
||||
rig.engine.setScope(DEFAULT_SCOPE);
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
});
|
||||
|
||||
it('goes by the real size when the metadata has it wrong', async () => {
|
||||
const server = new FakeFiles();
|
||||
const opened = server.addFile('opened', null, new Uint8Array(100), { size: 1 });
|
||||
const { engine } = await synced(server, { maxAttachmentBytes: 50 });
|
||||
await engine.areaCall('files.noteOpened', ref(opened.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
expect(await engine.areaCall('files.read', ref(opened.id))).toBeNull();
|
||||
expect((await engine.areaCall('files.kept', undefined) as any).files[0].error).toBe('tooLarge');
|
||||
});
|
||||
|
||||
it('does not store a file unpinned while it was downloading', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(file.id));
|
||||
const original = server.download.bind(server);
|
||||
server.download = async (jmap, blobId) => {
|
||||
await engine.areaCall('files.unpin', ref(file.id));
|
||||
return original(jmap, blobId);
|
||||
};
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
expect(await engine.areaCall('files.read', ref(file.id))).toBeNull();
|
||||
});
|
||||
|
||||
it('lets a never-opened file go on unpin, and keeps an opened one as recent', async () => {
|
||||
const server = new FakeFiles();
|
||||
const never = server.addFile('never');
|
||||
const opened = server.addFile('opened');
|
||||
const { engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(never.id));
|
||||
await engine.areaCall('files.pin', ref(opened.id));
|
||||
await engine.areaCall('files.noteOpened', ref(opened.id));
|
||||
await engine.requestSync();
|
||||
await engine.areaCall('files.unpin', ref(never.id));
|
||||
await engine.areaCall('files.unpin', ref(opened.id));
|
||||
expect(await engine.areaCall('files.read', ref(never.id))).toBeNull();
|
||||
expect(await engine.areaCall('files.read', ref(opened.id))).not.toBeNull();
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
expect(status.files.map((f: { name: string; pinned: boolean }) => [f.name, f.pinned])).toEqual([['opened', false]]);
|
||||
});
|
||||
|
||||
it('drops the old content and fetches the new one when a pinned file changes', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a.txt', null, new Uint8Array([1, 2, 3]));
|
||||
const { db, engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(file.id));
|
||||
await engine.requestSync();
|
||||
const old = file.blobId as string;
|
||||
const fresh = server.replaceContent(file.id, new Uint8Array([9, 9]));
|
||||
await engine.requestSync();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, old)).toBeNull();
|
||||
expect([...(fileBlob({ db }, JMAP_ACCOUNT, fresh)?.data ?? [])]).toEqual([9, 9]);
|
||||
expect(Array.from((await engine.areaCall('files.read', ref(file.id)) as any).data)).toEqual([9, 9]);
|
||||
});
|
||||
|
||||
it('drops the content when the file is removed on the server, unless another node shares it', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a', null, 5, { blobId: 'shared-blob' });
|
||||
const b = server.addFile('b', null, 5, { blobId: 'shared-blob' });
|
||||
const { db, engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(a.id));
|
||||
await engine.areaCall('files.pin', ref(b.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
server.removeNode(a.id);
|
||||
await engine.requestSync();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).not.toBeNull();
|
||||
server.removeNode(b.id);
|
||||
await engine.requestSync();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).toBeNull();
|
||||
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0);
|
||||
});
|
||||
|
||||
it('carries on past one bad file, and hands a network failure to the engine', async () => {
|
||||
const server = new FakeFiles();
|
||||
const bad = server.addFile('bad');
|
||||
const good = server.addFile('good');
|
||||
server.blobs.delete(bad.blobId as string);
|
||||
const { engine, clock } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(bad.id));
|
||||
clock.at += 1;
|
||||
await engine.areaCall('files.pin', ref(good.id));
|
||||
await engine.requestSync();
|
||||
expect(await engine.areaCall('files.read', ref(good.id))).not.toBeNull();
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
expect(status.files.find((f: { name: string }) => f.name === 'bad').error).toBe('download');
|
||||
|
||||
});
|
||||
|
||||
it('hands a network failure to the engine, which ends the pass, and goes on next time', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const rig = await synced(server);
|
||||
await rig.engine.areaCall('files.pin', ref(file.id));
|
||||
const original = server.download.bind(server);
|
||||
server.download = async () => {
|
||||
throw new TransportError('network', 'Failed to fetch');
|
||||
};
|
||||
const syncedBefore = rig.events.filter((e) => e.type === 'synced').length;
|
||||
await rig.engine.requestSync();
|
||||
expect(rig.events.some((e) => e.type === 'error')).toBe(false);
|
||||
expect(rig.events.filter((e) => e.type === 'synced').length).toBe(syncedBefore);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ kept: false, error: null });
|
||||
|
||||
server.download = original;
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('recently opened files', () => {
|
||||
it('keeps an opened file without a pin', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: false });
|
||||
await engine.requestSync();
|
||||
expect(await engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: true });
|
||||
});
|
||||
|
||||
it('pushes out the oldest unpinned first, and never a pinned one', async () => {
|
||||
const server = new FakeFiles();
|
||||
const files = Object.fromEntries(['old', 'pinnedOldest', 'mid', 'new', 'newest'].map((name) => [name, server.addFile(name, null, 10)]));
|
||||
const rig = await synced(server, { maxTotalBytes: 30 });
|
||||
const open = async (name: string) => {
|
||||
rig.clock.at += 1000;
|
||||
await rig.engine.areaCall('files.noteOpened', ref(files[name].id));
|
||||
await rig.engine.requestSync();
|
||||
};
|
||||
await open('pinnedOldest');
|
||||
await rig.engine.areaCall('files.pin', ref(files.pinnedOldest.id));
|
||||
await open('old');
|
||||
await open('mid');
|
||||
const kept = async () => (await rig.engine.areaCall('files.kept', undefined) as any).files
|
||||
.filter((f: { kept: boolean }) => f.kept).map((f: { name: string }) => f.name).sort();
|
||||
expect(await kept()).toEqual(['mid', 'old', 'pinnedOldest']);
|
||||
|
||||
await open('new');
|
||||
expect(await kept()).toEqual(['mid', 'new', 'pinnedOldest']);
|
||||
await open('mid');
|
||||
await open('newest');
|
||||
expect(await kept()).toEqual(['mid', 'newest', 'pinnedOldest']);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).bytes).toBe(30);
|
||||
|
||||
// A pinned file was never pushed out and had to come back.
|
||||
expect(server.downloads.filter((d) => d.blobId === files.pinnedOldest.blobId)).toHaveLength(1);
|
||||
// Pushed out means forgotten: the next pass doesn't fetch it straight back.
|
||||
expect(server.downloads.filter((d) => d.blobId === files.old.blobId)).toHaveLength(1);
|
||||
const count = server.downloads.length;
|
||||
await rig.engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(count);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).files.map((f: { name: string }) => f.name)).not.toContain('old');
|
||||
});
|
||||
|
||||
it('pushes out recent files for a pin, and keeps a pin past the cap', async () => {
|
||||
const server = new FakeFiles();
|
||||
const recent = server.addFile('recent', null, 20);
|
||||
const pin = server.addFile('pin', null, 20);
|
||||
const rig = await synced(server, { maxTotalBytes: 25 });
|
||||
await rig.engine.areaCall('files.noteOpened', ref(recent.id));
|
||||
await rig.engine.requestSync();
|
||||
rig.clock.at += 1000;
|
||||
await rig.engine.areaCall('files.pin', ref(pin.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(recent.id))).toBeNull();
|
||||
expect(await rig.engine.areaCall('files.read', ref(pin.id))).not.toBeNull();
|
||||
});
|
||||
|
||||
it('keeps recently opened files under 500 MB whatever the total limit is', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('new', null, 10);
|
||||
const rig = await synced(server, { maxTotalBytes: 2 * FILE_CACHE_CAP_BYTES });
|
||||
// As if an older file already took all of it: only the row counts here.
|
||||
rig.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [JMAP_ACCOUNT, 'huge', 'x', FILE_CACHE_CAP_BYTES, NOW - 1000]);
|
||||
rig.db.run("INSERT INTO file_node (jmap, id, name, kind, blob_id, size, json, opened_at) VALUES (?, 'hugenode', 'huge', 'file', 'huge', ?, '{}', ?)", [JMAP_ACCOUNT, FILE_CACHE_CAP_BYTES, NOW - 1000]);
|
||||
await rig.engine.areaCall('files.noteOpened', ref(file.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(fileBlob({ db: rig.db }, JMAP_ACCOUNT, 'huge')).toBeNull();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).capBytes).toBe(FILE_CACHE_CAP_BYTES);
|
||||
});
|
||||
|
||||
it('drops an old opened file that does not fit rather than a newer one', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a', null, 20);
|
||||
const b = server.addFile('b', null, 20);
|
||||
const rig = await synced(server, { maxTotalBytes: 25 });
|
||||
await rig.engine.areaCall('files.noteOpened', ref(a.id));
|
||||
rig.clock.at += 1000;
|
||||
await rig.engine.areaCall('files.noteOpened', ref(b.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(b.id))).not.toBeNull();
|
||||
expect(await rig.engine.areaCall('files.read', ref(a.id))).toBeNull();
|
||||
expect(server.downloads.map((d) => d.blobId)).toEqual([b.blobId, a.blobId]);
|
||||
const count = server.downloads.length;
|
||||
await rig.engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(count);
|
||||
});
|
||||
});
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
@@ -0,0 +1,268 @@
|
||||
/**
|
||||
* FakeJmap with contacts, as measured on both Stalwarts (2026-11-03): uid is a
|
||||
* query filter, a second card with a uid is refused with invalidProperties,
|
||||
* unknown ids give per-record notFound, AddressBook state doesn't move when
|
||||
* cards change, `ContactCard/get` with ids null stops at 500.
|
||||
*/
|
||||
|
||||
import { TransportError, type JmapResponse, type MethodCall, type StoreSession } from '../../engine/transport';
|
||||
import { Collection, FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './fake-jmap';
|
||||
|
||||
export const CONTACTS = 'urn:ietf:params:jmap:contacts';
|
||||
export const SHARED_ACCOUNT = 'shared1';
|
||||
export const MAIL_ONLY_ACCOUNT = 'mailonly';
|
||||
|
||||
export const CONTACTS_SESSION: StoreSession = {
|
||||
...FAKE_SESSION,
|
||||
primaryAccounts: { ...FAKE_SESSION.primaryAccounts, [CONTACTS]: JMAP_ACCOUNT },
|
||||
accounts: {
|
||||
[JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [CONTACTS]: {} } },
|
||||
[SHARED_ACCOUNT]: { name: 'team@example.org', isPersonal: false, accountCapabilities: { [CONTACTS]: {} } },
|
||||
[MAIL_ONLY_ACCOUNT]: { name: 'list@example.org', isPersonal: false, accountCapabilities: { 'urn:ietf:params:jmap:mail': {} } },
|
||||
},
|
||||
capabilities: { [CONTACTS]: {} },
|
||||
};
|
||||
|
||||
type Obj = Record<string, unknown> & { id: string };
|
||||
|
||||
/** The /changes rules of the base fake, for another collection. */
|
||||
export function changesFor(coll: Collection, args: Record<string, unknown>): Record<string, unknown> {
|
||||
const since = String(args.sinceState);
|
||||
if (!/^s\d+$/.test(since)) throw Object.assign(new Error('bad state'), { jmapError: 'invalidArguments' });
|
||||
const from = Number(since.slice(1));
|
||||
if (from < coll.truncatedBelow || from > coll.log.length) {
|
||||
throw Object.assign(new Error('log truncated'), { jmapError: 'cannotCalculateChanges' });
|
||||
}
|
||||
const maxChanges = Math.min(Number(args.maxChanges ?? 5000) || 5000, 5000);
|
||||
return { accountId: args.accountId, oldState: since, ...coll.changes(since, maxChanges) };
|
||||
}
|
||||
|
||||
function storedMedia(card: Obj): void {
|
||||
const media = card.media as Record<string, Record<string, unknown>> | undefined;
|
||||
for (const [key, entry] of Object.entries(media ?? {})) {
|
||||
if (typeof entry.uri !== 'string' || !entry.uri.startsWith('data:')) continue;
|
||||
const { uri, ...rest } = entry;
|
||||
(media as Record<string, unknown>)[key] = { ...rest, blobId: `stored-${uri.length}-${key}` };
|
||||
}
|
||||
}
|
||||
|
||||
function applyServerPatch(card: Obj, patch: Record<string, unknown>): void {
|
||||
for (const [path, value] of Object.entries(patch)) {
|
||||
const steps = path.split('/');
|
||||
let node = card as Record<string, unknown>;
|
||||
for (const step of steps.slice(0, -1)) node = node[step] as Record<string, unknown>;
|
||||
const last = steps[steps.length - 1];
|
||||
if (value === null) delete node[last];
|
||||
else node[last] = value;
|
||||
}
|
||||
}
|
||||
|
||||
class ContactAccount {
|
||||
readonly books = new Map<string, Obj>();
|
||||
readonly cards = new Map<string, Obj>();
|
||||
readonly coll = { AddressBook: new Collection(), ContactCard: new Collection() };
|
||||
}
|
||||
|
||||
export class FakeContacts extends FakeJmap {
|
||||
private readonly contactAccounts = new Map<string, ContactAccount>();
|
||||
private nextCard = 1;
|
||||
/** Set to apply the next ContactCard/set and then lose its response. */
|
||||
loseNextSet = false;
|
||||
/** Card ids (or creation ids) whose next set is refused with this error. */
|
||||
readonly refuse = new Map<string, { type: string; description?: string; properties?: string[] }>();
|
||||
/** A server that lets two cards have one uid, as nothing in the RFC forbids it. */
|
||||
allowDuplicateUids = false;
|
||||
/** Stalwart 1.0 stores a data: URI in `media` as a blob: `uri` goes, `blobId` comes (measured). */
|
||||
mediaAsBlobs = false;
|
||||
/** Set to apply the next ContactCard/set, answer it, and then lose the network. */
|
||||
offlineAfterSet = false;
|
||||
/** Set to lose the network before the next ContactCard/set reaches the server. */
|
||||
offlineBeforeSet = false;
|
||||
/** The most ids ContactCard/get and calls a request take, as a session can say. */
|
||||
maxObjectsInGet = 500;
|
||||
/** Answers a ContactCard/set call as a whole, instead of handling it. */
|
||||
failSet: ((args: Record<string, unknown>) => { jmapError: string } | { http: number } | null) | null = null;
|
||||
|
||||
constructor(books: string[] = ['book1']) {
|
||||
super();
|
||||
for (const id of books) this.addBook({ id, name: id, isDefault: id === books[0] });
|
||||
}
|
||||
|
||||
account(id: string = JMAP_ACCOUNT): ContactAccount {
|
||||
let found = this.contactAccounts.get(id);
|
||||
if (!found) {
|
||||
found = new ContactAccount();
|
||||
this.contactAccounts.set(id, found);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
addBook(book: Obj, accountId: string = JMAP_ACCOUNT): void {
|
||||
const account = this.account(accountId);
|
||||
account.books.set(book.id, book);
|
||||
account.coll.AddressBook.push('created', book.id);
|
||||
}
|
||||
|
||||
addCard(partial: Partial<Obj> = {}, accountId: string = JMAP_ACCOUNT): Obj {
|
||||
const account = this.account(accountId);
|
||||
const n = this.nextCard++;
|
||||
const card: Obj = {
|
||||
'@type': 'Card',
|
||||
version: '1.0',
|
||||
id: `c${n}`,
|
||||
uid: `urn:uuid:card-${n}`,
|
||||
addressBookIds: { [[...account.books.keys()][0] ?? 'book1']: true },
|
||||
kind: 'individual',
|
||||
name: { full: `Person ${n}` },
|
||||
...partial,
|
||||
};
|
||||
account.cards.set(card.id, card);
|
||||
account.coll.ContactCard.push('created', card.id);
|
||||
return card;
|
||||
}
|
||||
|
||||
editCard(id: string, patch: Record<string, unknown>, accountId: string = JMAP_ACCOUNT): void {
|
||||
const account = this.account(accountId);
|
||||
applyServerPatch(account.cards.get(id) as Obj, patch);
|
||||
account.coll.ContactCard.push('updated', id);
|
||||
}
|
||||
|
||||
removeCard(id: string, accountId: string = JMAP_ACCOUNT): void {
|
||||
const account = this.account(accountId);
|
||||
account.cards.delete(id);
|
||||
account.coll.ContactCard.push('destroyed', id);
|
||||
}
|
||||
|
||||
private contactSet(account: ContactAccount, args: Record<string, unknown>): Record<string, unknown> {
|
||||
const result: Record<string, unknown> = { accountId: args.accountId, oldState: account.coll.ContactCard.state() };
|
||||
const created: Record<string, unknown> = {};
|
||||
const notCreated: Record<string, unknown> = {};
|
||||
const updated: Record<string, unknown> = {};
|
||||
const notUpdated: Record<string, unknown> = {};
|
||||
const destroyed: string[] = [];
|
||||
const notDestroyed: Record<string, unknown> = {};
|
||||
for (const [cid, raw] of Object.entries((args.create ?? {}) as Record<string, Record<string, unknown>>)) {
|
||||
const refused = this.refuse.get(cid) ?? this.refuse.get(String(raw.uid));
|
||||
const clash = [...account.cards.values()].find((card) => card.uid === raw.uid);
|
||||
if (refused) {
|
||||
notCreated[cid] = refused;
|
||||
} else if (clash && !this.allowDuplicateUids) {
|
||||
notCreated[cid] = { type: 'invalidProperties', description: `Contact with UID ${String(raw.uid)} already exists with id ${clash.id}.`, properties: ['uid'] };
|
||||
} else {
|
||||
const id = `c${this.nextCard++}`;
|
||||
const stored: Obj = { '@type': 'Card', version: '1.0', ...structuredClone(raw), id };
|
||||
if (this.mediaAsBlobs) storedMedia(stored);
|
||||
account.cards.set(id, stored);
|
||||
account.coll.ContactCard.push('created', id);
|
||||
created[cid] = { id };
|
||||
}
|
||||
}
|
||||
for (const [id, patch] of Object.entries((args.update ?? {}) as Record<string, Record<string, unknown>>)) {
|
||||
const card = account.cards.get(id);
|
||||
const refused = this.refuse.get(id);
|
||||
if (refused) notUpdated[id] = refused;
|
||||
else if (!card) notUpdated[id] = { type: 'notFound' };
|
||||
else {
|
||||
applyServerPatch(card, structuredClone(patch));
|
||||
if (this.mediaAsBlobs) storedMedia(card);
|
||||
account.coll.ContactCard.push('updated', id);
|
||||
updated[id] = null;
|
||||
}
|
||||
}
|
||||
for (const id of (args.destroy ?? []) as string[]) {
|
||||
const refused = this.refuse.get(id);
|
||||
if (refused) notDestroyed[id] = refused;
|
||||
else if (!account.cards.has(id)) notDestroyed[id] = { type: 'notFound' };
|
||||
else {
|
||||
account.cards.delete(id);
|
||||
account.coll.ContactCard.push('destroyed', id);
|
||||
destroyed.push(id);
|
||||
}
|
||||
}
|
||||
for (const [key, value] of Object.entries({ created, notCreated, updated, notUpdated, notDestroyed })) {
|
||||
if (Object.keys(value).length > 0) result[key] = value;
|
||||
}
|
||||
if (destroyed.length > 0) result.destroyed = destroyed;
|
||||
result.newState = account.coll.ContactCard.state();
|
||||
return result;
|
||||
}
|
||||
|
||||
protected override handle(name: string, args: Record<string, unknown>): Record<string, unknown> {
|
||||
const account = this.account(String(args.accountId));
|
||||
switch (name) {
|
||||
case 'AddressBook/get': {
|
||||
const ids = args.ids as string[] | null | undefined;
|
||||
const all = [...account.books.values()];
|
||||
return {
|
||||
accountId: args.accountId,
|
||||
state: account.coll.AddressBook.state(),
|
||||
list: ids ? all.filter((b) => ids.includes(b.id)) : all,
|
||||
notFound: ids ? ids.filter((id) => !account.books.has(id)) : [],
|
||||
};
|
||||
}
|
||||
case 'AddressBook/changes':
|
||||
return changesFor(account.coll.AddressBook, args);
|
||||
case 'ContactCard/changes':
|
||||
return changesFor(account.coll.ContactCard, args);
|
||||
case 'ContactCard/query': {
|
||||
const filter = (args.filter ?? {}) as { uid?: string; inAddressBook?: string };
|
||||
let list = [...account.cards.values()];
|
||||
if (filter.uid) list = list.filter((c) => c.uid === filter.uid);
|
||||
if (filter.inAddressBook) list = list.filter((c) => (c.addressBookIds as Record<string, boolean>)[filter.inAddressBook as string]);
|
||||
let position = Number(args.position ?? 0);
|
||||
if (args.anchor !== undefined) {
|
||||
const at = list.findIndex((c) => c.id === args.anchor);
|
||||
if (at < 0) throw Object.assign(new Error('anchorNotFound'), { jmapError: 'anchorNotFound' });
|
||||
position = at + Number(args.anchorOffset ?? 0);
|
||||
}
|
||||
const limit = Number(args.limit ?? 5000) || 5000;
|
||||
return {
|
||||
accountId: args.accountId,
|
||||
queryState: account.coll.ContactCard.state(),
|
||||
canCalculateChanges: true,
|
||||
position,
|
||||
ids: list.slice(position, position + Math.min(limit, 5000)).map((c) => c.id),
|
||||
...(args.calculateTotal ? { total: list.length } : {}),
|
||||
};
|
||||
}
|
||||
case 'ContactCard/get': {
|
||||
const ids = args.ids as string[] | null | undefined;
|
||||
if (ids && ids.length > this.maxObjectsInGet) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' });
|
||||
const chosen = ids ?? [...account.cards.keys()].slice(0, this.maxObjectsInGet);
|
||||
return {
|
||||
accountId: args.accountId,
|
||||
state: account.coll.ContactCard.state(),
|
||||
list: chosen.filter((id) => account.cards.has(id)).map((id) => structuredClone(account.cards.get(id))),
|
||||
notFound: chosen.filter((id) => !account.cards.has(id)),
|
||||
};
|
||||
}
|
||||
case 'ContactCard/set': {
|
||||
const failure = this.failSet?.(args);
|
||||
if (failure && 'http' in failure) throw new TransportError('http', `JMAP request failed: ${failure.http}`, failure.http, 'urn:ietf:params:jmap:error:limit');
|
||||
if (failure) throw Object.assign(new Error(failure.jmapError), failure);
|
||||
return this.contactSet(account, args);
|
||||
}
|
||||
default:
|
||||
return super.handle(name, args);
|
||||
}
|
||||
}
|
||||
|
||||
override async request(calls: MethodCall[], using?: string[]): Promise<JmapResponse> {
|
||||
// The real server fails the whole request for a capability it doesn't list.
|
||||
void using;
|
||||
if (this.offlineBeforeSet && calls.some(([name]) => name === 'ContactCard/set')) {
|
||||
this.offlineBeforeSet = false;
|
||||
throw new TransportError('network', 'Failed to fetch');
|
||||
}
|
||||
const response = await super.request(calls);
|
||||
if (this.offlineAfterSet && calls.some(([name]) => name === 'ContactCard/set')) {
|
||||
this.offlineAfterSet = false;
|
||||
this.offline = true;
|
||||
}
|
||||
if (this.loseNextSet && calls.some(([name]) => name === 'ContactCard/set')) {
|
||||
this.loseNextSet = false;
|
||||
throw new TransportError('network', 'Connection dropped after the server took the request');
|
||||
}
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* FakeJmap with files, as measured on both Stalwarts (2026-11-03) and as the
|
||||
* client's comments record: `filter: {parentId: null}` is an HTTP 400 that
|
||||
* fails the whole request, a folder has null blobId, type and size. What differs
|
||||
* between versions is set per server:
|
||||
* - `style` is how a node says what it is: `nodeType` (0.16.6 and later, 1.0 as
|
||||
* tested), `blob` (0.16.0-5: nothing but blobId null), `draft` (1.0.0: `@type`
|
||||
* and `mediaType`).
|
||||
* - `getAll`: ids null past 500 is silently cut on 0.16 and refused on 1.0.
|
||||
* - `queryOmitsFolders`: FileNode/query before 0.16.6.
|
||||
*/
|
||||
|
||||
import { TransportError, type StoreSession } from '../../engine/transport';
|
||||
import { changesFor } from './fake-contacts';
|
||||
import { Collection, FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './fake-jmap';
|
||||
|
||||
export const FILES = 'urn:ietf:params:jmap:filenode';
|
||||
export const SHARED_FILES_ACCOUNT = 'shared1';
|
||||
|
||||
export const FILES_SESSION: StoreSession = {
|
||||
...FAKE_SESSION,
|
||||
primaryAccounts: { ...FAKE_SESSION.primaryAccounts, [FILES]: JMAP_ACCOUNT },
|
||||
accounts: {
|
||||
[JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [FILES]: {} } },
|
||||
[SHARED_FILES_ACCOUNT]: { name: 'team@example.org', isPersonal: false, accountCapabilities: { [FILES]: {} } },
|
||||
},
|
||||
capabilities: { [FILES]: {} },
|
||||
};
|
||||
|
||||
export interface FakeNode {
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
name: string;
|
||||
blobId: string | null;
|
||||
type: string | null;
|
||||
size: number | null;
|
||||
modified?: string;
|
||||
}
|
||||
|
||||
export type FolderStyle = 'nodeType' | 'blob' | 'draft';
|
||||
|
||||
export interface FakeFilesOptions {
|
||||
style?: FolderStyle;
|
||||
getAll?: 'cut' | 'refuse';
|
||||
queryOmitsFolders?: boolean;
|
||||
}
|
||||
|
||||
class FilesAccount {
|
||||
readonly nodes = new Map<string, FakeNode>();
|
||||
readonly coll = new Collection();
|
||||
}
|
||||
|
||||
export class FakeFiles extends FakeJmap {
|
||||
private readonly fileAccounts = new Map<string, FilesAccount>();
|
||||
private readonly options: Required<FakeFilesOptions>;
|
||||
private nextNode = 1;
|
||||
readonly blobs = new Map<string, Uint8Array>();
|
||||
/** In order, with the account. */
|
||||
readonly downloads: Array<{ jmap: string; blobId: string }> = [];
|
||||
downloading = 0;
|
||||
mostAtOnce = 0;
|
||||
|
||||
constructor(options: FakeFilesOptions = {}) {
|
||||
super();
|
||||
this.options = { style: 'nodeType', getAll: 'cut', queryOmitsFolders: false, ...options };
|
||||
}
|
||||
|
||||
account(id: string = JMAP_ACCOUNT): FilesAccount {
|
||||
let found = this.fileAccounts.get(id);
|
||||
if (!found) {
|
||||
found = new FilesAccount();
|
||||
this.fileAccounts.set(id, found);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
addFolder(name: string, parentId: string | null = null, id?: string, accountId: string = JMAP_ACCOUNT): FakeNode {
|
||||
const account = this.account(accountId);
|
||||
const node: FakeNode = { id: id ?? `n${this.nextNode++}`, parentId, name, blobId: null, type: null, size: null };
|
||||
account.nodes.set(node.id, node);
|
||||
account.coll.push('created', node.id);
|
||||
return node;
|
||||
}
|
||||
|
||||
addFile(name: string, parentId: string | null = null, content: Uint8Array | number = 10, extra: Partial<FakeNode> = {}, accountId: string = JMAP_ACCOUNT): FakeNode {
|
||||
const account = this.account(accountId);
|
||||
const id = extra.id ?? `n${this.nextNode++}`;
|
||||
const blobId = extra.blobId ?? `blob-${id}`;
|
||||
const data = typeof content === 'number' ? new Uint8Array(content).fill(id.length) : content;
|
||||
this.blobs.set(blobId, data);
|
||||
const node: FakeNode = { id, parentId, name, blobId, type: 'text/plain', size: data.length, ...extra };
|
||||
account.nodes.set(id, node);
|
||||
account.coll.push('created', id);
|
||||
return node;
|
||||
}
|
||||
|
||||
/** A new version of the file: new blob, same node. */
|
||||
replaceContent(id: string, content: Uint8Array, accountId: string = JMAP_ACCOUNT): string {
|
||||
const account = this.account(accountId);
|
||||
const node = account.nodes.get(id) as FakeNode;
|
||||
const blobId = `${node.blobId as string}-v`;
|
||||
this.blobs.set(blobId, content);
|
||||
node.blobId = blobId;
|
||||
node.size = content.length;
|
||||
account.coll.push('updated', id);
|
||||
return blobId;
|
||||
}
|
||||
|
||||
renameNode(id: string, name: string, accountId: string = JMAP_ACCOUNT): void {
|
||||
const account = this.account(accountId);
|
||||
(account.nodes.get(id) as FakeNode).name = name;
|
||||
account.coll.push('updated', id);
|
||||
}
|
||||
|
||||
removeNode(id: string, accountId: string = JMAP_ACCOUNT): void {
|
||||
const account = this.account(accountId);
|
||||
account.nodes.delete(id);
|
||||
account.coll.push('destroyed', id);
|
||||
}
|
||||
|
||||
private wire(node: FakeNode, properties: string[] | undefined): Record<string, unknown> {
|
||||
const isFolder = node.blobId === null;
|
||||
const full: Record<string, unknown> = { id: node.id, parentId: node.parentId, name: node.name, blobId: node.blobId, size: node.size, created: '2026-10-01T00:00:00Z', modified: node.modified ?? '2026-10-01T00:00:00Z' };
|
||||
if (this.options.style === 'draft') {
|
||||
full['@type'] = isFolder ? 'Directory' : 'File';
|
||||
full.mediaType = node.type;
|
||||
} else {
|
||||
full.type = node.type;
|
||||
if (this.options.style === 'nodeType') full.nodeType = isFolder ? 'directory' : 'file';
|
||||
}
|
||||
if (!properties) return full;
|
||||
const out: Record<string, unknown> = { id: node.id };
|
||||
for (const p of properties) if (p in full) out[p] = full[p];
|
||||
return out;
|
||||
}
|
||||
|
||||
protected override handle(name: string, args: Record<string, unknown>): Record<string, unknown> {
|
||||
if (!name.startsWith('FileNode/')) return super.handle(name, args);
|
||||
const account = this.account(String(args.accountId));
|
||||
switch (name) {
|
||||
case 'FileNode/changes':
|
||||
return changesFor(account.coll, args);
|
||||
case 'FileNode/get': {
|
||||
const ids = args.ids as string[] | null;
|
||||
if (ids && ids.length > 500) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' });
|
||||
if (!ids && account.nodes.size > 500 && this.options.getAll === 'refuse') {
|
||||
throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' });
|
||||
}
|
||||
const chosen = ids ?? [...account.nodes.keys()].slice(0, 500);
|
||||
const properties = args.properties as string[] | undefined;
|
||||
return {
|
||||
accountId: args.accountId,
|
||||
state: account.coll.state(),
|
||||
list: chosen.filter((id) => account.nodes.has(id)).map((id) => this.wire(account.nodes.get(id) as FakeNode, properties)),
|
||||
notFound: chosen.filter((id) => !account.nodes.has(id)),
|
||||
};
|
||||
}
|
||||
case 'FileNode/query': {
|
||||
const filter = (args.filter ?? {}) as { parentId?: string | null };
|
||||
// Measured on both servers: this fails the whole request.
|
||||
if (filter.parentId === null) throw new TransportError('http', 'JMAP request failed: 400', 400, 'urn:ietf:params:jmap:error:notRequest');
|
||||
let list = [...account.nodes.values()];
|
||||
if (this.options.queryOmitsFolders) list = list.filter((n) => n.blobId !== null);
|
||||
if (filter.parentId) list = list.filter((n) => n.parentId === filter.parentId);
|
||||
const position = Number(args.position ?? 0);
|
||||
const limit = Math.min(Number(args.limit ?? 5000) || 5000, 5000);
|
||||
return {
|
||||
accountId: args.accountId,
|
||||
queryState: account.coll.state(),
|
||||
canCalculateChanges: true,
|
||||
position,
|
||||
ids: list.slice(position, position + limit).map((n) => n.id),
|
||||
...(args.calculateTotal ? { total: list.length } : {}),
|
||||
};
|
||||
}
|
||||
default:
|
||||
throw Object.assign(new Error(name), { jmapError: 'unknownMethod' });
|
||||
}
|
||||
}
|
||||
|
||||
override async download(jmap: string, blobId: string): Promise<Uint8Array> {
|
||||
if (this.offline) throw new TransportError('network', 'Failed to fetch');
|
||||
this.downloads.push({ jmap, blobId });
|
||||
this.downloading += 1;
|
||||
this.mostAtOnce = Math.max(this.mostAtOnce, this.downloading);
|
||||
await Promise.resolve();
|
||||
this.downloading -= 1;
|
||||
const data = this.blobs.get(blobId);
|
||||
if (!data) throw new TransportError('http', 'Download failed: 404', 404);
|
||||
return data;
|
||||
}
|
||||
}
|
||||
@@ -286,7 +286,7 @@ export class FakeJmap implements JmapTransport {
|
||||
return { methodResponses: done };
|
||||
}
|
||||
|
||||
async download(): Promise<Uint8Array> {
|
||||
async download(_jmap: string, _blobId: string, _type: string, _name: string): Promise<Uint8Array> {
|
||||
return new Uint8Array();
|
||||
}
|
||||
|
||||
|
||||
@@ -80,6 +80,8 @@ export interface StoreContext {
|
||||
overlay<T extends { id: string }>(jmap: string, type: string, records: T[]): T[];
|
||||
/** Attachments and files the other areas keep: `scope.maxTotalBytes` is for all of them together. */
|
||||
keptByOthers(area: string): number;
|
||||
/** Starts a sync pass, or one more after the running pass. Resolves when they're done. */
|
||||
requestSync(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+3
-1
@@ -2,7 +2,9 @@
|
||||
|
||||
import type { StoreArea } from './area';
|
||||
import { calendarArea } from './calendar';
|
||||
import { contactsArea } from './contacts';
|
||||
import { filesArea } from './files';
|
||||
import { mailArea } from './mail';
|
||||
import { outboxArea } from './outbox';
|
||||
|
||||
export const STORE_AREAS: readonly StoreArea[] = [mailArea, outboxArea, calendarArea];
|
||||
export const STORE_AREAS: readonly StoreArea[] = [mailArea, outboxArea, calendarArea, contactsArea, filesArea];
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { StoreArea } from '../area';
|
||||
import { CONTACT_METHODS } from './methods';
|
||||
import { listConflicts, pendingCount } from './pending';
|
||||
import { CONTACT_MIGRATIONS } from './schema';
|
||||
import { contactsSyncer } from './sync';
|
||||
|
||||
export const contactsArea: StoreArea = {
|
||||
name: 'contacts',
|
||||
migrations: CONTACT_MIGRATIONS,
|
||||
methods: CONTACT_METHODS,
|
||||
syncers: [contactsSyncer],
|
||||
calls: {
|
||||
/** Sends what's queued, in the next sync pass, and says what's left. */
|
||||
'contacts.flush': async (ctx) => {
|
||||
await ctx.requestSync();
|
||||
return { pending: pendingCount(ctx.db), conflicts: listConflicts(ctx.db).length };
|
||||
},
|
||||
/** How many cards have edits waiting to go out. */
|
||||
'contacts.pending': (ctx) => pendingCount(ctx.db),
|
||||
/** Edits the server refused or that met a change there. The server's copy is the one kept. */
|
||||
'contacts.conflicts': (ctx) => listConflicts(ctx.db),
|
||||
/** The user has seen this one. */
|
||||
'contacts.dismissConflict': (ctx, params: { id: number }) => {
|
||||
ctx.db.run('DELETE FROM contact_conflict WHERE id = ?', [params.id]);
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,213 @@
|
||||
/** Contact reads and queued edits, answered from the store. */
|
||||
|
||||
import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area';
|
||||
import { listObjects } from '../engine/core-schema';
|
||||
import type { SqlValue } from '../storage';
|
||||
import { queueCreate, queueDestroy, queueUpdate, resolveCardId, type SetError } from './pending';
|
||||
import { CONTACTS_CAPABILITY, foldText } from './records';
|
||||
import type { JsonObject, Patch } from './patch';
|
||||
|
||||
type Filter = Record<string, unknown>;
|
||||
|
||||
interface Sql { sql: string; params: SqlValue[] }
|
||||
|
||||
function jmapOf(ctx: MethodContext, args: Record<string, unknown>): string {
|
||||
return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[CONTACTS_CAPABILITY] ?? ctx.defaultJmap;
|
||||
}
|
||||
|
||||
/**
|
||||
* The last state from the server. Throws until a first full pass finished, so
|
||||
* an answer never passes off part of a list as the whole. A later resync
|
||||
* (phase `list`) goes on answering from the copy it's refreshing.
|
||||
*/
|
||||
function syncedState(ctx: MethodContext, jmap: string, type: string): string | null {
|
||||
const row = ctx.db.get<{ state: string | null; phase: string | null }>('SELECT state, phase FROM sync_state WHERE jmap = ? AND type = ?', [jmap, type]);
|
||||
if (!row || (!row.state && row.phase !== 'list')) throw new MethodError('serverUnavailable', 'Contacts not kept yet');
|
||||
return row.state;
|
||||
}
|
||||
|
||||
function pick(object: Record<string, unknown>, properties: unknown): Record<string, unknown> {
|
||||
if (!Array.isArray(properties)) return object;
|
||||
const out: Record<string, unknown> = { id: object.id };
|
||||
for (const property of properties as string[]) if (property in object) out[property] = object[property];
|
||||
return out;
|
||||
}
|
||||
|
||||
function quote(term: string): string {
|
||||
return `"${term.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
function terms(text: string): string[] {
|
||||
return foldText(text).split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* A `text` condition, one part of the client's textFilter (search-utils.ts):
|
||||
* words, `word*` or a "quoted phrase". Every word has to match somewhere. The
|
||||
* trigram index finds it anywhere in a name, which takes three characters.
|
||||
* Shorter ones match the start of a word.
|
||||
*/
|
||||
function textCondition(value: string, rank: string[]): Sql {
|
||||
const phrase = /^"(.+)"$/.exec(value.trim());
|
||||
const words = terms(phrase ? phrase[1] : value);
|
||||
if (words.length === 0) return { sql: '0', params: [] };
|
||||
if (phrase) {
|
||||
rank.push(quote(words.join(' ')));
|
||||
return { sql: 'c.rowid IN (SELECT rowid FROM contact_fts_word WHERE contact_fts_word MATCH ?)', params: [quote(words.join(' '))] };
|
||||
}
|
||||
const parts: string[] = [];
|
||||
const params: SqlValue[] = [];
|
||||
for (const word of words) {
|
||||
rank.push(`${quote(word)}*`);
|
||||
if ([...word].length >= 3) {
|
||||
parts.push('c.rowid IN (SELECT rowid FROM contact_fts_tri WHERE contact_fts_tri MATCH ?)');
|
||||
params.push(quote(word));
|
||||
} else {
|
||||
parts.push('c.rowid IN (SELECT rowid FROM contact_fts_word WHERE contact_fts_word MATCH ?)');
|
||||
params.push(`${quote(word)}*`);
|
||||
}
|
||||
}
|
||||
return { sql: parts.join(' AND '), params };
|
||||
}
|
||||
|
||||
function filterSql(filter: Filter, rank: string[]): Sql {
|
||||
if (typeof filter.operator === 'string') {
|
||||
const parts = (Array.isArray(filter.conditions) ? filter.conditions as Filter[] : []).map((c) => filterSql(c, rank));
|
||||
const params = parts.flatMap((p) => p.params);
|
||||
const joined = (joiner: string) => parts.map((p) => `(${p.sql})`).join(joiner);
|
||||
if (filter.operator === 'AND') return { sql: parts.length ? joined(' AND ') : '1', params };
|
||||
if (filter.operator === 'OR') return { sql: parts.length ? joined(' OR ') : '0', params };
|
||||
if (filter.operator === 'NOT') return { sql: parts.length ? `NOT (${joined(' OR ')})` : '1', params };
|
||||
throw new MethodError('unsupportedFilter', `Unknown operator ${String(filter.operator)}`);
|
||||
}
|
||||
const conditions: string[] = [];
|
||||
const params: SqlValue[] = [];
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
if (key === 'text') {
|
||||
const text = textCondition(String(value), rank);
|
||||
conditions.push(text.sql);
|
||||
params.push(...text.params);
|
||||
} else if (key === 'inAddressBook') {
|
||||
conditions.push('EXISTS (SELECT 1 FROM contact_card_book b WHERE b.card_rowid = c.rowid AND b.book_id = ?)');
|
||||
params.push(String(value));
|
||||
} else {
|
||||
throw new MethodError('unsupportedFilter', `${key} isn't available offline`);
|
||||
}
|
||||
}
|
||||
return { sql: conditions.length ? conditions.join(' AND ') : '1', params };
|
||||
}
|
||||
|
||||
const addressBookGet: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
const state = syncedState(ctx, jmap, 'AddressBook');
|
||||
const books = listObjects<Record<string, unknown>>(ctx.db, jmap, 'AddressBook');
|
||||
const ids = Array.isArray(args.ids) ? args.ids as string[] : null;
|
||||
const chosen = ids ? books.filter((book) => ids.includes(book.id as string)) : books;
|
||||
return {
|
||||
accountId: jmap,
|
||||
...(state ? { state } : {}),
|
||||
list: chosen.map((book) => pick(book, args.properties)),
|
||||
notFound: ids ? ids.filter((id) => !books.some((book) => book.id === id)) : [],
|
||||
};
|
||||
};
|
||||
|
||||
const contactQuery: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
syncedState(ctx, jmap, 'ContactCard');
|
||||
if (args.anchor !== undefined) throw new MethodError('unsupportedFilter', 'anchor needs the server');
|
||||
if (Array.isArray(args.sort) && args.sort.length > 0) throw new MethodError('unsupportedSort', 'Sorted lists need the server');
|
||||
const rank: string[] = [];
|
||||
const where = args.filter ? filterSql(args.filter as Filter, rank) : { sql: '1', params: [] };
|
||||
const position = Math.max(0, Number(args.position ?? 0));
|
||||
// limit 0 means no limit on Stalwart, and the client can't tell a local answer apart.
|
||||
const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1;
|
||||
// Names starting with what was typed come before ones merely containing it.
|
||||
const first = rank.length > 0 ? 'c.rowid IN (SELECT rowid FROM contact_fts_word WHERE contact_fts_word MATCH ?) DESC, ' : '';
|
||||
const ids = ctx.db.all<{ id: string }>(
|
||||
`SELECT c.id FROM contact_card c WHERE c.jmap = ? AND (${where.sql}) ORDER BY ${first}c.sort_key, c.id LIMIT ? OFFSET ?`,
|
||||
[jmap, ...where.params, ...(rank.length > 0 ? [rank.join(' AND ')] : []), limit, position],
|
||||
).map((row) => String(row.id));
|
||||
const result: MethodResult = {
|
||||
accountId: jmap,
|
||||
// Local answers aren't a position in the server's log, so this is a token only.
|
||||
queryState: 'local',
|
||||
canCalculateChanges: false,
|
||||
position,
|
||||
ids,
|
||||
};
|
||||
if (args.calculateTotal) {
|
||||
result.total = Number(ctx.db.get<{ n: number }>(
|
||||
`SELECT COUNT(*) AS n FROM contact_card c WHERE c.jmap = ? AND (${where.sql})`,
|
||||
[jmap, ...where.params],
|
||||
)?.n ?? 0);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
const contactGet: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
const state = syncedState(ctx, jmap, 'ContactCard');
|
||||
const list: Array<Record<string, unknown>> = [];
|
||||
const notFound: string[] = [];
|
||||
const rows = Array.isArray(args.ids)
|
||||
? (args.ids as string[]).map((id) => ({ id, row: ctx.db.get<{ json: string }>('SELECT json FROM contact_card WHERE jmap = ? AND id = ?', [jmap, resolveCardId(ctx.db, jmap, id)]) }))
|
||||
: ctx.db.all<{ json: string; id: string }>('SELECT json, id FROM contact_card WHERE jmap = ? ORDER BY sort_key, id', [jmap]).map((row) => ({ id: row.id, row }));
|
||||
for (const { id, row } of rows) {
|
||||
if (row) list.push(pick(JSON.parse(row.json) as Record<string, unknown>, args.properties));
|
||||
else notFound.push(id);
|
||||
}
|
||||
return { accountId: jmap, ...(state ? { state } : {}), list, notFound };
|
||||
};
|
||||
|
||||
/**
|
||||
* Queues the edits. The answer leaves out oldState and newState: nothing
|
||||
* reached the server, so there's no real state to name, and the client reads
|
||||
* neither from ContactCard/set.
|
||||
*/
|
||||
const contactSet: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
syncedState(ctx, jmap, 'ContactCard');
|
||||
const created: Record<string, { id: string; uid: string }> = {};
|
||||
const notCreated: Record<string, SetError> = {};
|
||||
const updated: Record<string, null> = {};
|
||||
const notUpdated: Record<string, SetError> = {};
|
||||
const destroyed: string[] = [];
|
||||
const notDestroyed: Record<string, SetError> = {};
|
||||
ctx.db.transaction(() => {
|
||||
for (const [cid, card] of Object.entries((args.create ?? {}) as Record<string, JsonObject>)) {
|
||||
const outcome = queueCreate(ctx, jmap, card);
|
||||
if ('id' in outcome) created[cid] = outcome;
|
||||
else notCreated[cid] = outcome;
|
||||
}
|
||||
for (const [id, patch] of Object.entries((args.update ?? {}) as Record<string, Patch>)) {
|
||||
const refused = queueUpdate(ctx, jmap, id, patch);
|
||||
if (refused) notUpdated[id] = refused;
|
||||
else updated[id] = null;
|
||||
}
|
||||
for (const id of (args.destroy ?? []) as string[]) {
|
||||
const refused = queueDestroy(ctx, jmap, id);
|
||||
if (refused) notDestroyed[id] = refused;
|
||||
else destroyed.push(id);
|
||||
}
|
||||
});
|
||||
if (Object.keys(created).length + Object.keys(updated).length + destroyed.length > 0) {
|
||||
ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] });
|
||||
}
|
||||
const some = (name: string, value: object) => (Object.keys(value).length > 0 ? { [name]: value } : {});
|
||||
return {
|
||||
accountId: jmap,
|
||||
...some('created', created),
|
||||
...some('notCreated', notCreated),
|
||||
...some('updated', updated),
|
||||
...some('notUpdated', notUpdated),
|
||||
...some('destroyed', destroyed),
|
||||
...some('notDestroyed', notDestroyed),
|
||||
};
|
||||
};
|
||||
|
||||
export const CONTACT_METHODS: Readonly<Record<string, MethodHandler>> = {
|
||||
'AddressBook/get': addressBookGet,
|
||||
'ContactCard/query': contactQuery,
|
||||
'ContactCard/get': contactGet,
|
||||
'ContactCard/set': contactSet,
|
||||
};
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* RFC 8620 §5.3 patches: keys are JSON pointers without the leading slash, a
|
||||
* null value removes the property. Written so the common outbox can use the
|
||||
* same rules for its per-record merge.
|
||||
*/
|
||||
|
||||
export type JsonObject = Record<string, unknown>;
|
||||
export type Patch = Record<string, unknown>;
|
||||
|
||||
export class InvalidPatchError extends Error {
|
||||
constructor(path: string) {
|
||||
super(`Can't apply a patch at ${path}`);
|
||||
this.name = 'InvalidPatchError';
|
||||
}
|
||||
}
|
||||
|
||||
function steps(path: string): string[] {
|
||||
return path.split('/').map((step) => step.replace(/~1/g, '/').replace(/~0/g, '~'));
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is JsonObject {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** A copy of the object with the patch applied. Throws when a path leads nowhere. */
|
||||
export function applyPatch<T extends JsonObject>(object: T, patch: Patch): T {
|
||||
const root: JsonObject = { ...object };
|
||||
for (const [path, value] of Object.entries(patch)) {
|
||||
const parts = steps(path);
|
||||
let node = root;
|
||||
for (const step of parts.slice(0, -1)) {
|
||||
const next = node[step];
|
||||
if (!isObject(next)) throw new InvalidPatchError(path);
|
||||
const copy = { ...next };
|
||||
node[step] = copy;
|
||||
node = copy;
|
||||
}
|
||||
const last = parts[parts.length - 1];
|
||||
if (value === null) delete node[last];
|
||||
else node[last] = value;
|
||||
}
|
||||
return root as T;
|
||||
}
|
||||
|
||||
/** One patch that does what the older one and then the newer one would do. */
|
||||
export function mergePatches(older: Patch, newer: Patch): Patch {
|
||||
const merged: Patch = { ...older };
|
||||
for (const [path, value] of Object.entries(newer)) {
|
||||
// A newer write to a whole property replaces older writes below it.
|
||||
for (const key of Object.keys(merged)) {
|
||||
if (key.startsWith(`${path}/`)) delete merged[key];
|
||||
}
|
||||
// A newer write below an older whole-property write goes into that value.
|
||||
const parent = Object.keys(merged).find((key) => path.startsWith(`${key}/`));
|
||||
if (parent && isObject(merged[parent])) {
|
||||
const holder = applyPatch({ value: merged[parent] }, { [`value/${path.slice(parent.length + 1)}`]: value });
|
||||
merged[parent] = holder.value;
|
||||
} else {
|
||||
merged[path] = value;
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** The top-level properties a patch writes to. */
|
||||
export function touchedProperties(patch: Patch): string[] {
|
||||
return [...new Set(Object.keys(patch).map((path) => steps(path)[0]))];
|
||||
}
|
||||
|
||||
function isNothing(value: unknown): boolean {
|
||||
return value == null || (isObject(value) && Object.keys(value).length === 0);
|
||||
}
|
||||
|
||||
/** Deep equality that ignores key order. Absent, null and an empty map count as the same. */
|
||||
export function sameJson(a: unknown, b: unknown): boolean {
|
||||
if (a === b || (isNothing(a) && isNothing(b))) return true;
|
||||
if (Array.isArray(a) && Array.isArray(b)) {
|
||||
return a.length === b.length && a.every((item, i) => sameJson(item, b[i]));
|
||||
}
|
||||
if (isObject(a) && isObject(b)) {
|
||||
return [...new Set([...Object.keys(a), ...Object.keys(b)])].every((key) => sameJson(a[key], b[key]));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Stalwart 1.0 keeps a data: URI in `media` as a blob: `uri` goes and `blobId` comes (measured). */
|
||||
function inlineKeptAsBlob(wanted: JsonObject, actual: JsonObject): boolean {
|
||||
return typeof wanted.uri === 'string' && wanted.uri.startsWith('data:') && actual.uri == null && typeof actual.blobId === 'string';
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `actual` is `wanted` as far as a server can tell. A key only the
|
||||
* server has is its own, unless `base` shows it was there before the edit:
|
||||
* then the edit meant to remove it, and it hasn't happened yet.
|
||||
*/
|
||||
export function coversJson(wanted: unknown, actual: unknown, base?: unknown): boolean {
|
||||
if (!isObject(wanted) || !isObject(actual)) return sameJson(wanted, actual);
|
||||
const before = isObject(base) ? base : {};
|
||||
const inline = inlineKeptAsBlob(wanted, actual);
|
||||
for (const key of Object.keys(wanted)) {
|
||||
if (inline && key === 'uri') continue;
|
||||
if (!coversJson(wanted[key], actual[key], before[key])) return false;
|
||||
}
|
||||
for (const key of Object.keys(actual)) {
|
||||
if (key in wanted || (inline && key === 'blobId')) continue;
|
||||
if (before[key] != null) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
/**
|
||||
* Contact edits made while the server can't be reached, and sending them.
|
||||
*
|
||||
* One row per card, whatever number of edits it had. The visible card in
|
||||
* contact_card already has the edits on top, and sync keeps it so. Sending
|
||||
* runs at the start of each sync pass, so it never overlaps with itself.
|
||||
*
|
||||
* What a row becomes on its way out:
|
||||
* - Update: the server's card is read first. Nothing to do if it already has
|
||||
* the values (a lost response). A conflict if the server changed one of the
|
||||
* properties the edit touches. Otherwise the patch goes out.
|
||||
* - Destroy: done if the card is gone, a conflict if it changed since.
|
||||
* - Create: a request that may have reached the server looks for the card's
|
||||
* uid first, so a lost response doesn't make a second card. Stalwart also
|
||||
* refuses a second card with the same uid. Either way the card found is only
|
||||
* ours if it holds what was sent, or an offline card with someone else's uid
|
||||
* would silently turn into theirs.
|
||||
*
|
||||
* The local copy after sending is always the server's own read-back, never what
|
||||
* was sent: Stalwart 1.0 stores a data: URI as a blob. A base built from what
|
||||
* was sent would read that as someone else's change. If the network goes before
|
||||
* the read-back, that counts as a lost response: the row stays, and the next
|
||||
* pass finds its edit already there. `sent` is what the last request held, so
|
||||
* the next pass can tell its own edit from someone else's.
|
||||
*/
|
||||
|
||||
import type { StoreContext, SyncContext } from '../area';
|
||||
import { requestLimits, type RequestLimits } from '../engine/limits';
|
||||
import { JmapMethodError, responseFor, TransportError, type MethodCall } from '../engine/transport';
|
||||
import type { StorageAdapter } from '../storage';
|
||||
import { applyPatch, coversJson, InvalidPatchError, mergePatches, sameJson, touchedProperties, type JsonObject, type Patch } from './patch';
|
||||
import { cardDisplayName, CONTACTS_USING, readCard, removeCard, writeCard, type Card } from './records';
|
||||
|
||||
export const STAND_IN_PREFIX = 'bulwark-local-card-';
|
||||
/** What a held upload's stand-in blob id starts with (lib/offline/queue.ts). */
|
||||
const LOCAL_BLOB_MARK = '"blobId":"bulwark-local-';
|
||||
|
||||
const SET_BATCH = 100;
|
||||
|
||||
/** Of the session's request limit. A bigger card would never go out. */
|
||||
function maxRecordBytes(ctx: StoreContext): number {
|
||||
return Math.floor(requestLimits(ctx.session).maxSizeRequest * 0.8);
|
||||
}
|
||||
|
||||
/** Set errors that say nothing about the edit itself: it goes out again next time. */
|
||||
const TRANSIENT_ERRORS = new Set(['serverFail', 'serverUnavailable', 'rateLimit']);
|
||||
/** Server-kept timestamps: their changing doesn't mean someone edited the card. */
|
||||
const SERVER_STAMPS = ['updated', 'created'];
|
||||
|
||||
export interface PendingRow {
|
||||
seq: number;
|
||||
jmap: string;
|
||||
id: string;
|
||||
op: 'create' | 'update' | 'destroy';
|
||||
uid: string | null;
|
||||
payload: string | null;
|
||||
base: string | null;
|
||||
attempted: number;
|
||||
sent: string | null;
|
||||
queued_at: number;
|
||||
}
|
||||
|
||||
export interface SetError { type: string; description?: string; properties?: string[] }
|
||||
|
||||
export interface ContactConflict {
|
||||
id: number;
|
||||
jmap: string;
|
||||
recordId: string;
|
||||
kind: 'refused' | 'changed' | 'gone';
|
||||
summary: string;
|
||||
local: { op: string; payload: unknown };
|
||||
server: unknown;
|
||||
error: string | null;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
export function pendingFor(db: StorageAdapter, jmap: string, id: string): PendingRow | undefined {
|
||||
return db.get<PendingRow>('SELECT * FROM contact_pending WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
}
|
||||
|
||||
function dropPending(db: StorageAdapter, jmap: string, id: string): void {
|
||||
db.run('DELETE FROM contact_pending WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
}
|
||||
|
||||
function savePending(db: StorageAdapter, ctx: { now(): number }, row: { jmap: string; id: string; op: PendingRow['op']; uid?: string | null; payload: string | null; base: string | null }): void {
|
||||
db.run(
|
||||
`INSERT INTO contact_pending (jmap, id, op, uid, payload, base, queued_at) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (jmap, id) DO UPDATE SET op = excluded.op, uid = excluded.uid, payload = excluded.payload, base = excluded.base`,
|
||||
[row.jmap, row.id, row.op, row.uid ?? null, row.payload, row.base, ctx.now()],
|
||||
);
|
||||
}
|
||||
|
||||
/** A stand-in id that already went through becomes the real one. */
|
||||
export function resolveCardId(db: StorageAdapter, jmap: string, id: string): string {
|
||||
return db.get<{ real_id: string }>('SELECT real_id FROM contact_id_map WHERE jmap = ? AND local_id = ?', [jmap, id])?.real_id ?? id;
|
||||
}
|
||||
|
||||
/** A held upload can't be named in a card that goes out later. The client falls back to a data: URI. */
|
||||
function holdsLocalBlob(value: unknown): boolean {
|
||||
return JSON.stringify(value).includes(LOCAL_BLOB_MARK);
|
||||
}
|
||||
|
||||
const TOO_LARGE_ERROR: SetError = { type: 'tooLarge', description: 'This contact is too large to save' };
|
||||
|
||||
const LOCAL_BLOB_ERROR: SetError = {
|
||||
type: 'invalidProperties',
|
||||
description: 'A photo that is not on the server yet cannot be used here',
|
||||
properties: ['media'],
|
||||
};
|
||||
|
||||
export function queueCreate(ctx: StoreContext, jmap: string, card: JsonObject): { id: string; uid: string } | SetError {
|
||||
if (holdsLocalBlob(card)) return LOCAL_BLOB_ERROR;
|
||||
if (Object.keys((card.addressBookIds ?? {}) as object).length === 0) {
|
||||
// The client picks the default book. None known means there's nowhere to put it.
|
||||
return { type: 'invalidProperties', description: 'A contact needs an address book', properties: ['addressBookIds'] };
|
||||
}
|
||||
const id = `${STAND_IN_PREFIX}${crypto.randomUUID()}`;
|
||||
const uid = typeof card.uid === 'string' && card.uid ? card.uid : `urn:uuid:${crypto.randomUUID()}`;
|
||||
const payload = JSON.stringify({ ...card, uid });
|
||||
if (payload.length > maxRecordBytes(ctx)) return TOO_LARGE_ERROR;
|
||||
savePending(ctx.db, ctx, { jmap, id, op: 'create', uid, payload, base: null });
|
||||
writeCard(ctx.db, jmap, { '@type': 'Card', version: '1.0', ...card, uid, id });
|
||||
return { id, uid };
|
||||
}
|
||||
|
||||
export function queueUpdate(ctx: StoreContext, jmap: string, rawId: string, patch: Patch): SetError | null {
|
||||
const id = resolveCardId(ctx.db, jmap, rawId);
|
||||
const card = readCard(ctx.db, jmap, id);
|
||||
if (!card) return { type: 'notFound' };
|
||||
if ('id' in patch) return { type: 'invalidProperties', description: 'The id property is immutable.', properties: ['id'] };
|
||||
if (holdsLocalBlob(patch)) return LOCAL_BLOB_ERROR;
|
||||
const pending = pendingFor(ctx.db, jmap, id);
|
||||
let visible: Card;
|
||||
try {
|
||||
visible = applyPatch(card, patch);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidPatchError) return { type: 'invalidPatch', description: error.message };
|
||||
throw error;
|
||||
}
|
||||
if (pending?.op === 'create') {
|
||||
const payload = JSON.stringify(applyPatch(JSON.parse(pending.payload as string) as JsonObject, patch));
|
||||
if (payload.length > maxRecordBytes(ctx)) return TOO_LARGE_ERROR;
|
||||
savePending(ctx.db, ctx, { jmap, id, op: 'create', uid: pending.uid, payload, base: null });
|
||||
} else {
|
||||
const merged = JSON.stringify(pending ? mergePatches(JSON.parse(pending.payload as string) as Patch, patch) : patch);
|
||||
if (merged.length > maxRecordBytes(ctx)) return TOO_LARGE_ERROR;
|
||||
savePending(ctx.db, ctx, { jmap, id, op: 'update', payload: merged, base: pending ? pending.base : JSON.stringify(card) });
|
||||
}
|
||||
writeCard(ctx.db, jmap, visible);
|
||||
return null;
|
||||
}
|
||||
|
||||
export function queueDestroy(ctx: StoreContext, jmap: string, rawId: string): SetError | null {
|
||||
const id = resolveCardId(ctx.db, jmap, rawId);
|
||||
const card = readCard(ctx.db, jmap, id);
|
||||
if (!card) return { type: 'notFound' };
|
||||
const pending = pendingFor(ctx.db, jmap, id);
|
||||
if (pending?.op === 'create' && !pending.attempted) {
|
||||
// Never reached the server, so it never has to.
|
||||
dropPending(ctx.db, jmap, id);
|
||||
} else if (pending?.op === 'create') {
|
||||
// It may have reached the server. Sending decides, then destroys it.
|
||||
savePending(ctx.db, ctx, { jmap, id, op: 'destroy', uid: pending.uid, payload: pending.payload, base: null });
|
||||
} else {
|
||||
savePending(ctx.db, ctx, { jmap, id, op: 'destroy', payload: null, base: pending ? pending.base : JSON.stringify(card) });
|
||||
}
|
||||
removeCard(ctx.db, jmap, id);
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- What sync does around queued edits ----
|
||||
|
||||
/** Stores a card the server sent, with whatever is queued for it on top. */
|
||||
export function storeServerCard(db: StorageAdapter, jmap: string, card: Card, gen?: number): void {
|
||||
const pending = pendingFor(db, jmap, card.id);
|
||||
if (pending?.op === 'destroy') return;
|
||||
let visible = card;
|
||||
if (pending?.op === 'update') {
|
||||
try {
|
||||
visible = applyPatch(card, JSON.parse(pending.payload as string) as Patch);
|
||||
} catch (error) {
|
||||
// The patch no longer fits the card. Sending turns this into a conflict.
|
||||
if (!(error instanceof InvalidPatchError)) throw error;
|
||||
}
|
||||
}
|
||||
writeCard(db, jmap, visible, gen);
|
||||
}
|
||||
|
||||
/** The server no longer has this card. */
|
||||
export function serverDestroyed(ctx: StoreContext, jmap: string, id: string): void {
|
||||
const pending = pendingFor(ctx.db, jmap, id);
|
||||
if (pending?.op === 'update') recordConflict(ctx, pending, 'gone', null, null);
|
||||
if (pending && pending.op !== 'create') dropPending(ctx.db, jmap, id);
|
||||
removeCard(ctx.db, jmap, id);
|
||||
}
|
||||
|
||||
function recordConflict(ctx: StoreContext, row: PendingRow, kind: ContactConflict['kind'], server: Card | null, error: SetError | null): void {
|
||||
const local = row.payload ? JSON.parse(row.payload) as JsonObject : null;
|
||||
const known = server ?? readCard(ctx.db, row.jmap, row.id) ?? (row.base ? JSON.parse(row.base) as JsonObject : local);
|
||||
ctx.db.run(
|
||||
'INSERT INTO contact_conflict (jmap, record_id, kind, summary, local, server, error, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
|
||||
[
|
||||
row.jmap,
|
||||
row.id,
|
||||
kind,
|
||||
cardDisplayName(known ?? {}),
|
||||
JSON.stringify({ op: row.op, payload: local }),
|
||||
server ? JSON.stringify(server) : null,
|
||||
error ? [error.type, error.description].filter(Boolean).join(': ') : null,
|
||||
ctx.now(),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
export function listConflicts(db: StorageAdapter): ContactConflict[] {
|
||||
return db.all<{ id: number; jmap: string; record_id: string; kind: ContactConflict['kind']; summary: string; local: string; server: string | null; error: string | null; created_at: number }>(
|
||||
'SELECT * FROM contact_conflict ORDER BY id',
|
||||
).map((row) => ({
|
||||
id: Number(row.id),
|
||||
jmap: row.jmap,
|
||||
recordId: row.record_id,
|
||||
kind: row.kind,
|
||||
summary: row.summary,
|
||||
local: JSON.parse(row.local) as ContactConflict['local'],
|
||||
server: row.server ? JSON.parse(row.server) : null,
|
||||
error: row.error,
|
||||
createdAt: Number(row.created_at),
|
||||
}));
|
||||
}
|
||||
|
||||
export function pendingCount(db: StorageAdapter): number {
|
||||
return Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM contact_pending')?.n ?? 0);
|
||||
}
|
||||
|
||||
// ---- Sending ----
|
||||
|
||||
function withoutStamps(card: JsonObject): JsonObject {
|
||||
const copy = { ...card };
|
||||
for (const key of SERVER_STAMPS) delete copy[key];
|
||||
return copy;
|
||||
}
|
||||
|
||||
async function readServerCards(ctx: SyncContext, jmap: string, ids: string[], limits: RequestLimits): Promise<Map<string, Card>> {
|
||||
const cards = new Map<string, Card>();
|
||||
for (let i = 0; i < ids.length; i += limits.maxObjectsInGet) {
|
||||
const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + limits.maxObjectsInGet) }, 'g']], CONTACTS_USING);
|
||||
for (const card of responseFor(response, 'g')[1].list as Card[]) cards.set(card.id, card);
|
||||
}
|
||||
return cards;
|
||||
}
|
||||
|
||||
/** The server's cards with these uids. */
|
||||
async function cardsByUid(ctx: SyncContext, jmap: string, uids: string[], limits: RequestLimits): Promise<Map<string, Card>> {
|
||||
const ids = new Map<string, string>();
|
||||
for (let i = 0; i < uids.length; i += limits.maxCallsInRequest) {
|
||||
const group = uids.slice(i, i + limits.maxCallsInRequest);
|
||||
const calls: MethodCall[] = group.map((uid, n) => ['ContactCard/query', { accountId: jmap, filter: { uid }, limit: 1 }, `q${n}`]);
|
||||
const response = await ctx.transport.request(calls, CONTACTS_USING);
|
||||
group.forEach((uid, n) => {
|
||||
const found = responseFor(response, `q${n}`)[1].ids as string[];
|
||||
if (found.length > 0) ids.set(uid, found[0]);
|
||||
});
|
||||
}
|
||||
const cards = await readServerCards(ctx, jmap, [...ids.values()], limits);
|
||||
const byUid = new Map<string, Card>();
|
||||
for (const [uid, id] of ids) {
|
||||
const card = cards.get(id);
|
||||
if (card) byUid.set(uid, card);
|
||||
}
|
||||
return byUid;
|
||||
}
|
||||
|
||||
/** Whether a card the server has under our uid is the one we sent. */
|
||||
function holdsSent(sentPayload: string, card: Card): boolean {
|
||||
const sent = JSON.parse(sentPayload) as JsonObject;
|
||||
return Object.keys(sent).every((key) => coversJson(sent[key], card[key]));
|
||||
}
|
||||
|
||||
const UID_TAKEN: SetError = { type: 'invalidProperties', description: 'Another contact already has this uid', properties: ['uid'] };
|
||||
|
||||
/** The create went through, or had already. The stand-in row becomes the server's card. */
|
||||
function settleCreated(ctx: SyncContext, sent: PendingRow, server: Card): void {
|
||||
const { db } = ctx;
|
||||
const { jmap, id: standIn } = sent;
|
||||
const now = pendingFor(db, jmap, standIn);
|
||||
db.run('INSERT OR REPLACE INTO contact_id_map (jmap, local_id, real_id) VALUES (?, ?, ?)', [jmap, standIn, server.id]);
|
||||
dropPending(db, jmap, standIn);
|
||||
removeCard(db, jmap, standIn);
|
||||
if (now?.op === 'destroy' || !now) {
|
||||
// Deleted while the create was on its way: delete the real one next.
|
||||
savePending(db, ctx, { jmap, id: server.id, op: 'destroy', payload: null, base: JSON.stringify(server) });
|
||||
return;
|
||||
}
|
||||
if (now.payload !== sent.payload) {
|
||||
// Edited while the create was on its way: what differs goes out as an edit.
|
||||
const before = JSON.parse(sent.payload as string) as JsonObject;
|
||||
const after = JSON.parse(now.payload as string) as JsonObject;
|
||||
const patch: Patch = {};
|
||||
for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) {
|
||||
if (!sameJson(before[key], after[key])) patch[key.replace(/~/g, '~0').replace(/\//g, '~1')] = after[key] ?? null;
|
||||
}
|
||||
savePending(db, ctx, { jmap, id: server.id, op: 'update', payload: JSON.stringify(patch), base: JSON.stringify(server) });
|
||||
writeCard(db, jmap, applyPatch(server, patch));
|
||||
return;
|
||||
}
|
||||
writeCard(db, jmap, server);
|
||||
}
|
||||
|
||||
interface Decision { create: Map<string, PendingRow>; update: Map<string, PendingRow>; destroy: Map<string, PendingRow> }
|
||||
|
||||
/**
|
||||
* Whether someone else moved one of these properties since the edit was queued.
|
||||
* The server holding what our last request sent is our own doing, not a change:
|
||||
* its response may have been lost.
|
||||
*/
|
||||
function movedOnServer(row: PendingRow, server: Card, keys: readonly string[]): boolean {
|
||||
if (!row.base) return false;
|
||||
const base = JSON.parse(row.base) as JsonObject;
|
||||
let own: JsonObject | null = null;
|
||||
if (row.sent) {
|
||||
try {
|
||||
own = applyPatch(base, JSON.parse(row.sent) as Patch);
|
||||
} catch (error) {
|
||||
if (!(error instanceof InvalidPatchError)) throw error;
|
||||
}
|
||||
}
|
||||
return keys.some((key) => !sameJson(base[key], server[key]) && !(own && coversJson(own[key], server[key], base[key])));
|
||||
}
|
||||
|
||||
/** What to do with one row now that the server's copy is known. Conflicts are settled here. */
|
||||
function decide(ctx: SyncContext, row: PendingRow, server: Card | undefined, serverByUid: Map<string, Card>, out: Decision): void {
|
||||
const { db } = ctx;
|
||||
const { jmap, id } = row;
|
||||
const uidHit = row.uid ? serverByUid.get(row.uid) : undefined;
|
||||
if (row.op === 'create') {
|
||||
if (!uidHit) {
|
||||
out.create.set(id, row);
|
||||
} else if (holdsSent(row.sent ?? (row.payload as string), uidHit)) {
|
||||
settleCreated(ctx, { ...row, payload: row.sent ?? row.payload }, uidHit);
|
||||
} else {
|
||||
recordConflict(ctx, row, 'refused', null, UID_TAKEN);
|
||||
dropPending(db, jmap, id);
|
||||
removeCard(db, jmap, id);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (row.op === 'destroy' && id.startsWith(STAND_IN_PREFIX)) {
|
||||
// A create that may have gone out, deleted since. Only a card holding what was sent is ours to delete.
|
||||
dropPending(db, jmap, id);
|
||||
const sent = row.sent ?? row.payload;
|
||||
if (uidHit && sent && holdsSent(sent, uidHit)) {
|
||||
savePending(db, ctx, { jmap, id: uidHit.id, op: 'destroy', payload: null, base: JSON.stringify(uidHit) });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (row.op === 'destroy') {
|
||||
if (!server) {
|
||||
dropPending(db, jmap, id);
|
||||
return;
|
||||
}
|
||||
const stable = withoutStamps(server);
|
||||
const base = row.base ? withoutStamps(JSON.parse(row.base) as JsonObject) : {};
|
||||
if (movedOnServer(row, server, [...new Set([...Object.keys(stable), ...Object.keys(base)])])) {
|
||||
recordConflict(ctx, row, 'changed', server, null);
|
||||
dropPending(db, jmap, id);
|
||||
writeCard(db, jmap, server);
|
||||
} else {
|
||||
out.destroy.set(id, row);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!server) {
|
||||
recordConflict(ctx, row, 'gone', null, null);
|
||||
dropPending(db, jmap, id);
|
||||
removeCard(db, jmap, id);
|
||||
return;
|
||||
}
|
||||
const patch = JSON.parse(row.payload as string) as Patch;
|
||||
const base = row.base ? JSON.parse(row.base) as JsonObject : {};
|
||||
let applied: Card | null = null;
|
||||
try {
|
||||
applied = applyPatch(server, patch);
|
||||
} catch (error) {
|
||||
if (!(error instanceof InvalidPatchError)) throw error;
|
||||
}
|
||||
const touched = touchedProperties(patch);
|
||||
if (applied && touched.every((key) => coversJson((applied as Card)[key], server[key], base[key]))) {
|
||||
dropPending(db, jmap, id);
|
||||
writeCard(db, jmap, server);
|
||||
return;
|
||||
}
|
||||
if (!applied || movedOnServer(row, server, touched)) {
|
||||
recordConflict(ctx, row, 'changed', server, null);
|
||||
dropPending(db, jmap, id);
|
||||
writeCard(db, jmap, server);
|
||||
return;
|
||||
}
|
||||
out.update.set(id, row);
|
||||
}
|
||||
|
||||
function settleUpdated(ctx: SyncContext, sent: PendingRow, server: Card | undefined): void {
|
||||
const { db } = ctx;
|
||||
const now = pendingFor(db, sent.jmap, sent.id);
|
||||
if (!now) return;
|
||||
if (now.op === 'update' && now.payload === sent.payload) {
|
||||
dropPending(db, sent.jmap, sent.id);
|
||||
if (server) writeCard(db, sent.jmap, server);
|
||||
return;
|
||||
}
|
||||
// Edited again while this was on its way: that edit now builds on what the server holds.
|
||||
if (!server) return;
|
||||
db.run('UPDATE contact_pending SET base = ?, sent = NULL, attempted = 0 WHERE seq = ?', [JSON.stringify(server), now.seq]);
|
||||
if (now.op === 'update') storeServerCard(db, sent.jmap, server);
|
||||
}
|
||||
|
||||
function settleRefused(ctx: SyncContext, sent: PendingRow, error: SetError, server: Card | undefined): void {
|
||||
const { db } = ctx;
|
||||
if (TRANSIENT_ERRORS.has(error.type)) return;
|
||||
const now = pendingFor(db, sent.jmap, sent.id);
|
||||
if (now && (now.op !== sent.op || now.payload !== sent.payload)) {
|
||||
// Edited again while this was on its way, so the refused edit is still in
|
||||
// the row. It goes out once more, and a second refusal is the conflict.
|
||||
db.run('UPDATE contact_pending SET sent = NULL, attempted = 0 WHERE seq = ?', [now.seq]);
|
||||
return;
|
||||
}
|
||||
if (sent.op === 'destroy' && error.type === 'notFound') {
|
||||
dropPending(db, sent.jmap, sent.id);
|
||||
return;
|
||||
}
|
||||
const kind = error.type === 'notFound' ? 'gone' : 'refused';
|
||||
recordConflict(ctx, sent, kind, server ?? null, error);
|
||||
dropPending(db, sent.jmap, sent.id);
|
||||
const restore = server ?? (sent.base ? JSON.parse(sent.base) as Card : null);
|
||||
if (restore && kind === 'refused') writeCard(db, sent.jmap, restore);
|
||||
else removeCard(db, sent.jmap, sent.id);
|
||||
}
|
||||
|
||||
/** Takes rows up to about the size of one request. */
|
||||
function nextBatch(db: StorageAdapter, jmap: string, afterSeq: number, limits: RequestLimits): PendingRow[] {
|
||||
const rows = db.all<PendingRow>('SELECT * FROM contact_pending WHERE jmap = ? AND seq > ? ORDER BY seq LIMIT ?', [jmap, afterSeq, Math.min(SET_BATCH, limits.maxObjectsInSet)]);
|
||||
const room = limits.maxSizeRequest / 2;
|
||||
let bytes = 0;
|
||||
const batch: PendingRow[] = [];
|
||||
for (const row of rows) {
|
||||
bytes += row.payload?.length ?? 0;
|
||||
if (batch.length > 0 && bytes > room) break;
|
||||
batch.push(row);
|
||||
}
|
||||
return batch;
|
||||
}
|
||||
|
||||
/** One exchange with the server for these rows: read, decide, send, read back, settle. */
|
||||
async function exchange(ctx: SyncContext, jmap: string, rows: readonly PendingRow[], limits: RequestLimits): Promise<void> {
|
||||
const { db } = ctx;
|
||||
const current = await readServerCards(ctx, jmap, rows.filter((row) => row.op !== 'create' && !row.id.startsWith(STAND_IN_PREFIX)).map((row) => row.id), limits);
|
||||
const serverByUid = await cardsByUid(ctx, jmap, rows.filter((row) => row.uid && row.attempted).map((row) => row.uid as string), limits);
|
||||
|
||||
const out: Decision = { create: new Map(), update: new Map(), destroy: new Map() };
|
||||
db.transaction(() => {
|
||||
for (const { seq, id } of rows) {
|
||||
// The page may have edited it while the reads were on their way.
|
||||
const row = pendingFor(db, jmap, id);
|
||||
if (row && row.seq === seq) decide(ctx, row, current.get(id), serverByUid, out);
|
||||
}
|
||||
for (const row of [...out.create.values(), ...out.update.values()]) {
|
||||
db.run('UPDATE contact_pending SET attempted = 1, sent = payload WHERE seq = ?', [row.seq]);
|
||||
}
|
||||
});
|
||||
if (out.create.size + out.update.size + out.destroy.size === 0) return;
|
||||
|
||||
const creationIds = new Map([...out.create.values()].map((row) => [`c${row.seq}`, row]));
|
||||
const response = await ctx.transport.request([['ContactCard/set', {
|
||||
accountId: jmap,
|
||||
...(creationIds.size > 0 ? { create: Object.fromEntries([...creationIds].map(([cid, row]) => [cid, JSON.parse(row.payload as string)])) } : {}),
|
||||
...(out.update.size > 0 ? { update: Object.fromEntries([...out.update.values()].map((row) => [row.id, JSON.parse(row.payload as string)])) } : {}),
|
||||
...(out.destroy.size > 0 ? { destroy: [...out.destroy.keys()] } : {}),
|
||||
}, 's']], CONTACTS_USING);
|
||||
const [, result] = responseFor(response, 's');
|
||||
|
||||
// What the server now holds. A network failure here is a lost response: the rows stay.
|
||||
const createdIds = new Map<string, string>();
|
||||
for (const cid of creationIds.keys()) {
|
||||
const created = result.created?.[cid] as { id?: unknown } | undefined;
|
||||
if (created?.id) createdIds.set(cid, String(created.id));
|
||||
}
|
||||
const updatedIds = [...out.update.keys()].filter((id) => result.updated && id in result.updated);
|
||||
const readBack = await readServerCards(ctx, jmap, [...createdIds.values(), ...updatedIds], limits);
|
||||
// A card already has this uid: maybe this one, sent before.
|
||||
const uidRefused = [...creationIds].filter(([cid]) => (result.notCreated?.[cid] as SetError | undefined)?.properties?.includes('uid'));
|
||||
const taken = await cardsByUid(ctx, jmap, uidRefused.map(([, row]) => row.uid as string), limits);
|
||||
|
||||
db.transaction(() => {
|
||||
for (const [cid, row] of creationIds) {
|
||||
const realId = createdIds.get(cid);
|
||||
const refused = result.notCreated?.[cid] as SetError | undefined;
|
||||
if (realId) {
|
||||
const card = readBack.get(realId);
|
||||
if (card) {
|
||||
settleCreated(ctx, row, card);
|
||||
} else {
|
||||
// Created and deleted again already. The delta will say the same.
|
||||
dropPending(db, jmap, row.id);
|
||||
removeCard(db, jmap, row.id);
|
||||
}
|
||||
} else if (refused?.properties?.includes('uid')) {
|
||||
const card = taken.get(row.uid as string);
|
||||
if (card && holdsSent(row.payload as string, card)) settleCreated(ctx, row, card);
|
||||
else settleRefused(ctx, row, refused, undefined);
|
||||
} else if (refused) {
|
||||
settleRefused(ctx, row, refused, undefined);
|
||||
}
|
||||
}
|
||||
for (const [id, row] of out.update) {
|
||||
const refused = result.notUpdated?.[id] as SetError | undefined;
|
||||
if (refused) settleRefused(ctx, row, refused, current.get(id));
|
||||
else if (updatedIds.includes(id)) settleUpdated(ctx, row, readBack.get(id));
|
||||
}
|
||||
for (const [id, row] of out.destroy) {
|
||||
const refused = result.notDestroyed?.[id] as SetError | undefined;
|
||||
if (refused) settleRefused(ctx, row, refused, current.get(id));
|
||||
else if ((result.destroyed as string[] | undefined)?.includes(id)) dropPending(db, jmap, id);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** The server said no to the request itself, for a reason that won't go away by itself. */
|
||||
function requestRefusal(error: unknown): SetError | null {
|
||||
if (error instanceof JmapMethodError) return TRANSIENT_ERRORS.has(error.type) ? null : { type: error.type, description: error.message };
|
||||
if (error instanceof TransportError && error.kind === 'http' && error.status !== undefined && error.status >= 400 && error.status < 500) {
|
||||
return { type: 'requestRefused', description: error.message };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** One card may be what makes a request fail: the others go one by one, and that one becomes a conflict. */
|
||||
async function sendRows(ctx: SyncContext, jmap: string, rows: readonly PendingRow[], limits: RequestLimits): Promise<void> {
|
||||
try {
|
||||
await exchange(ctx, jmap, rows, limits);
|
||||
} catch (error) {
|
||||
const refusal = requestRefusal(error);
|
||||
if (!refusal) throw error;
|
||||
if (rows.length > 1) {
|
||||
for (const row of rows) {
|
||||
if (ctx.stopped()) return;
|
||||
await sendRows(ctx, jmap, [row], limits);
|
||||
}
|
||||
return;
|
||||
}
|
||||
ctx.db.transaction(() => {
|
||||
const row = pendingFor(ctx.db, jmap, rows[0].id);
|
||||
if (row) settleRefused(ctx, row, refusal, undefined);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends what's queued for one JMAP account. Network failures reach the engine and the rows stay. */
|
||||
export async function flushPending(ctx: SyncContext, jmap: string): Promise<void> {
|
||||
const limits = requestLimits(ctx.session);
|
||||
let afterSeq = 0;
|
||||
let settled = false;
|
||||
try {
|
||||
for (;;) {
|
||||
if (ctx.stopped()) break;
|
||||
const batch = nextBatch(ctx.db, jmap, afterSeq, limits);
|
||||
if (batch.length === 0) break;
|
||||
afterSeq = batch[batch.length - 1].seq;
|
||||
settled = true;
|
||||
await sendRows(ctx, jmap, batch, limits);
|
||||
}
|
||||
} finally {
|
||||
if (settled) ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/** How a ContactCard becomes rows, and how the search index reads it. */
|
||||
|
||||
import type { SqlValue, StorageAdapter } from '../storage';
|
||||
|
||||
export const CONTACTS_CAPABILITY = 'urn:ietf:params:jmap:contacts';
|
||||
export const CONTACTS_USING = ['urn:ietf:params:jmap:core', CONTACTS_CAPABILITY];
|
||||
|
||||
/** As the client asks for them. Without them Stalwart may leave out shareWith and myRights. */
|
||||
export const ADDRESS_BOOK_PROPERTIES = ['id', 'name', 'description', 'sortOrder', 'isDefault', 'isSubscribed', 'shareWith', 'myRights'];
|
||||
|
||||
export type Card = Record<string, unknown> & { id: string };
|
||||
|
||||
function values(map: unknown): Array<Record<string, unknown>> {
|
||||
if (typeof map !== 'object' || map === null) return [];
|
||||
return Object.values(map).filter((v): v is Record<string, unknown> => typeof v === 'object' && v !== null);
|
||||
}
|
||||
|
||||
function strings(items: unknown[]): string[] {
|
||||
return items.filter((item): item is string => typeof item === 'string' && item.length > 0);
|
||||
}
|
||||
|
||||
/** Lower case, no accents: what both the index and the search terms go through. */
|
||||
export function foldText(text: string): string {
|
||||
return text
|
||||
.normalize('NFKD')
|
||||
.replace(/\p{M}+/gu, '')
|
||||
.toLowerCase()
|
||||
.replace(/ß/g, 'ss')
|
||||
.replace(/æ/g, 'ae')
|
||||
.replace(/œ/g, 'oe')
|
||||
.replace(/[øđł]/g, (c) => ({ ø: 'o', đ: 'd', ł: 'l' })[c] as string);
|
||||
}
|
||||
|
||||
/**
|
||||
* The text to index. A German name is indexed both ways, since people type
|
||||
* "Mueller" for "Müller" as often as "Muller".
|
||||
*/
|
||||
function indexText(text: string): string {
|
||||
const folded = foldText(text);
|
||||
const german = foldText(
|
||||
text.normalize('NFC').replace(/[äÄ]/g, 'ae').replace(/[öÖ]/g, 'oe').replace(/[üÜ]/g, 'ue'),
|
||||
);
|
||||
return german === folded ? folded : `${folded} ${german}`;
|
||||
}
|
||||
|
||||
export function cardDisplayName(card: Record<string, unknown>): string {
|
||||
const name = card.name as { full?: unknown; components?: unknown } | undefined;
|
||||
if (typeof name?.full === 'string' && name.full) return name.full;
|
||||
const parts = Array.isArray(name?.components) ? strings((name.components as Array<{ value?: unknown }>).map((c) => c?.value)) : [];
|
||||
if (parts.length > 0) return parts.join(' ');
|
||||
return strings(values(card.emails).map((e) => e.address))[0] ?? '';
|
||||
}
|
||||
|
||||
function cardIndex(card: Record<string, unknown>): { name: string; email: string; org: string } {
|
||||
const name = card.name as { full?: unknown; components?: unknown } | undefined;
|
||||
const components = Array.isArray(name?.components) ? (name.components as Array<{ value?: unknown }>).map((c) => c?.value) : [];
|
||||
const nicknames = values(card.nicknames).map((n) => n.name);
|
||||
const emails = values(card.emails).map((e) => e.address);
|
||||
const organizations = values(card.organizations).flatMap((o) => [
|
||||
o.name,
|
||||
...(Array.isArray(o.units) ? (o.units as Array<{ name?: unknown }>).map((u) => u?.name) : []),
|
||||
]);
|
||||
return {
|
||||
name: indexText(strings([name?.full, ...components, ...nicknames]).join(' ')),
|
||||
email: indexText(strings(emails).join(' ')),
|
||||
org: indexText(strings(organizations).join(' ')),
|
||||
};
|
||||
}
|
||||
|
||||
function books(card: Record<string, unknown>): string[] {
|
||||
const ids = card.addressBookIds;
|
||||
if (typeof ids !== 'object' || ids === null) return [];
|
||||
return Object.entries(ids).filter(([, on]) => on).map(([id]) => id);
|
||||
}
|
||||
|
||||
/** Writes one card and its index rows. Without a gen, the row keeps its own. */
|
||||
export function writeCard(db: StorageAdapter, jmap: string, card: Card, gen?: number): void {
|
||||
const existing = db.get<{ rowid: number; gen: number }>('SELECT rowid, gen FROM contact_card WHERE jmap = ? AND id = ?', [jmap, card.id]);
|
||||
const row: SqlValue[] = [
|
||||
typeof card.uid === 'string' ? card.uid : null,
|
||||
foldText(cardDisplayName(card)),
|
||||
JSON.stringify(card),
|
||||
gen ?? Number(existing?.gen ?? 0),
|
||||
];
|
||||
let rowid: number;
|
||||
if (existing) {
|
||||
rowid = Number(existing.rowid);
|
||||
db.run('UPDATE contact_card SET uid = ?, sort_key = ?, json = ?, gen = ? WHERE rowid = ?', [...row, rowid]);
|
||||
dropIndex(db, rowid);
|
||||
} else {
|
||||
rowid = db.run('INSERT INTO contact_card (uid, sort_key, json, gen, jmap, id) VALUES (?, ?, ?, ?, ?, ?)', [...row, jmap, card.id]).lastInsertRowid;
|
||||
}
|
||||
const text = cardIndex(card);
|
||||
for (const table of ['contact_fts_word', 'contact_fts_tri']) {
|
||||
db.run(`INSERT INTO ${table} (rowid, name, email, org) VALUES (?, ?, ?, ?)`, [rowid, text.name, text.email, text.org]);
|
||||
}
|
||||
for (const book of books(card)) db.run('INSERT OR IGNORE INTO contact_card_book (card_rowid, book_id) VALUES (?, ?)', [rowid, book]);
|
||||
}
|
||||
|
||||
function dropIndex(db: StorageAdapter, rowid: number): void {
|
||||
db.run('DELETE FROM contact_fts_word WHERE rowid = ?', [rowid]);
|
||||
db.run('DELETE FROM contact_fts_tri WHERE rowid = ?', [rowid]);
|
||||
db.run('DELETE FROM contact_card_book WHERE card_rowid = ?', [rowid]);
|
||||
}
|
||||
|
||||
export function removeCard(db: StorageAdapter, jmap: string, id: string): void {
|
||||
const existing = db.get<{ rowid: number }>('SELECT rowid FROM contact_card WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
if (!existing) return;
|
||||
dropIndex(db, Number(existing.rowid));
|
||||
db.run('DELETE FROM contact_card WHERE rowid = ?', [existing.rowid]);
|
||||
}
|
||||
|
||||
export function readCard(db: StorageAdapter, jmap: string, id: string): Card | null {
|
||||
const row = db.get<{ json: string }>('SELECT json FROM contact_card WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
return row ? JSON.parse(row.json) as Card : null;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { Migration } from '../storage';
|
||||
|
||||
export const CONTACT_MIGRATIONS: readonly Migration[] = [
|
||||
{
|
||||
id: 600,
|
||||
name: 'contacts',
|
||||
sql: `
|
||||
CREATE TABLE contact_card (
|
||||
rowid INTEGER PRIMARY KEY,
|
||||
jmap TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
uid TEXT,
|
||||
sort_key TEXT NOT NULL DEFAULT '',
|
||||
-- What the UI sees: the server's card with queued edits on top.
|
||||
json TEXT NOT NULL,
|
||||
gen INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE (jmap, id)
|
||||
);
|
||||
CREATE INDEX contact_card_sort ON contact_card (jmap, sort_key, id);
|
||||
CREATE TABLE contact_card_book (
|
||||
card_rowid INTEGER NOT NULL,
|
||||
book_id TEXT NOT NULL,
|
||||
PRIMARY KEY (card_rowid, book_id)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX contact_card_book_lookup ON contact_card_book (book_id, card_rowid);
|
||||
-- Text goes in through foldText (records.ts), which also takes care of accents.
|
||||
CREATE VIRTUAL TABLE contact_fts_word USING fts5(
|
||||
name, email, org,
|
||||
content = '', contentless_delete = 1,
|
||||
tokenize = 'unicode61 remove_diacritics 0',
|
||||
prefix = '2 3'
|
||||
);
|
||||
CREATE VIRTUAL TABLE contact_fts_tri USING fts5(
|
||||
name, email, org,
|
||||
content = '', contentless_delete = 1,
|
||||
tokenize = 'trigram'
|
||||
);
|
||||
-- One row per card with something to send. Edits to a card merge here.
|
||||
CREATE TABLE contact_pending (
|
||||
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
jmap TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
op TEXT NOT NULL,
|
||||
uid TEXT,
|
||||
-- create: the whole card. update: the merged patch. destroy: nothing.
|
||||
payload TEXT,
|
||||
-- The server's card when the first edit was queued, to see what changed there since.
|
||||
base TEXT,
|
||||
-- A request may have reached the server, so a create looks for its uid first.
|
||||
attempted INTEGER NOT NULL DEFAULT 0,
|
||||
-- The payload that request held: the server may have this, not the newer payload.
|
||||
sent TEXT,
|
||||
queued_at INTEGER NOT NULL,
|
||||
UNIQUE (jmap, id)
|
||||
);
|
||||
-- Stand-in ids the page may still hold after a create went through.
|
||||
CREATE TABLE contact_id_map (
|
||||
jmap TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
real_id TEXT NOT NULL,
|
||||
PRIMARY KEY (jmap, local_id)
|
||||
) WITHOUT ROWID;
|
||||
CREATE TABLE contact_conflict (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
jmap TEXT NOT NULL,
|
||||
record_id TEXT NOT NULL,
|
||||
-- refused: the server said no. changed: edited there too. gone: deleted there.
|
||||
kind TEXT NOT NULL,
|
||||
summary TEXT NOT NULL,
|
||||
local TEXT NOT NULL,
|
||||
server TEXT,
|
||||
error TEXT,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
`,
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* Contact sync for one JMAP account: first what's queued goes out, then the
|
||||
* address books, then the cards. A full pass over the cards captures the
|
||||
* state first and pages, with its cursor in sync_state, so a reload continues
|
||||
* where it stopped. Cards the pass didn't see are gone from the server, so a
|
||||
* page must not skip any: it goes from the last card read (`anchor`), since a
|
||||
* position moves when an earlier card is deleted. Both Stalwarts take that.
|
||||
*/
|
||||
|
||||
import type { SyncContext, Syncer } from '../area';
|
||||
import { runChangesLoop } from '../engine/changes';
|
||||
import { deleteObject, putObject } from '../engine/core-schema';
|
||||
import { requestLimits } from '../engine/limits';
|
||||
import { JmapMethodError, responseFor, TransportError, type MethodCall } from '../engine/transport';
|
||||
import { flushPending, serverDestroyed, storeServerCard } from './pending';
|
||||
import { ADDRESS_BOOK_PROPERTIES, CONTACTS_CAPABILITY, CONTACTS_USING, type Card } from './records';
|
||||
|
||||
const MOST_PAIRS_PER_REQUEST = 4;
|
||||
|
||||
function pageSize(ctx: SyncContext): number {
|
||||
return requestLimits(ctx.session).maxObjectsInGet;
|
||||
}
|
||||
|
||||
// ---- Address books ----
|
||||
|
||||
async function fullBooks(ctx: SyncContext, jmap: string): Promise<void> {
|
||||
const response = await ctx.transport.request([['AddressBook/get', { accountId: jmap, properties: ADDRESS_BOOK_PROPERTIES }, 'b']], CONTACTS_USING);
|
||||
const [, result] = responseFor(response, 'b');
|
||||
ctx.db.transaction(() => {
|
||||
ctx.db.run("DELETE FROM object WHERE jmap = ? AND type = 'AddressBook'", [jmap]);
|
||||
for (const book of result.list as Array<{ id: string }>) putObject(ctx.db, jmap, 'AddressBook', book.id, book);
|
||||
ctx.setSyncState(jmap, 'AddressBook', { state: String(result.state), phase: null, cursor: null });
|
||||
});
|
||||
ctx.emit({ type: 'changed', account: ctx.account, collections: ['AddressBook'] });
|
||||
}
|
||||
|
||||
async function syncBooks(ctx: SyncContext, jmap: string): Promise<void> {
|
||||
const { state, target } = ctx.syncState(jmap, 'AddressBook');
|
||||
if (!state) return fullBooks(ctx, jmap);
|
||||
const outcome = await runChangesLoop({
|
||||
transport: ctx.transport,
|
||||
type: 'AddressBook',
|
||||
jmap,
|
||||
using: CONTACTS_USING,
|
||||
since: state,
|
||||
target,
|
||||
stopped: ctx.stopped,
|
||||
apply: async (page) => {
|
||||
const ids = [...page.created, ...page.updated];
|
||||
const list: Array<{ id: string }> = [];
|
||||
const size = pageSize(ctx);
|
||||
for (let i = 0; i < ids.length; i += size) {
|
||||
const response = await ctx.transport.request([['AddressBook/get', { accountId: jmap, ids: ids.slice(i, i + size), properties: ADDRESS_BOOK_PROPERTIES }, 'b']], CONTACTS_USING);
|
||||
list.push(...responseFor(response, 'b')[1].list);
|
||||
}
|
||||
ctx.db.transaction(() => {
|
||||
for (const book of list) putObject(ctx.db, jmap, 'AddressBook', book.id, book);
|
||||
for (const id of page.destroyed) deleteObject(ctx.db, jmap, 'AddressBook', id);
|
||||
ctx.setSyncState(jmap, 'AddressBook', { state: page.newState });
|
||||
});
|
||||
if (ids.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['AddressBook'] });
|
||||
},
|
||||
});
|
||||
if (outcome.outcome === 'resync') await fullBooks(ctx, jmap);
|
||||
}
|
||||
|
||||
// ---- Cards ----
|
||||
|
||||
interface ListCursor { position: number; anchor: string | null; gen: number; captured: string }
|
||||
|
||||
async function startFullList(ctx: SyncContext, jmap: string): Promise<ListCursor> {
|
||||
const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: [] }, 's']], CONTACTS_USING);
|
||||
const captured = String(responseFor(response, 's')[1].state);
|
||||
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM contact_card WHERE jmap = ?', [jmap])?.gen ?? 0) + 1;
|
||||
const cursor: ListCursor = { position: 0, anchor: null, gen, captured };
|
||||
// Answers wait for the end of a first pass (phase `first`). A resync has an older copy to give.
|
||||
const phase = ctx.syncState(jmap, 'ContactCard').state ? 'list' : 'first';
|
||||
ctx.db.transaction(() => ctx.setSyncState(jmap, 'ContactCard', { state: null, phase, cursor: { ...cursor } }));
|
||||
return cursor;
|
||||
}
|
||||
|
||||
/** Cards from before this pass that it didn't find, leaving alone creates still on this device. */
|
||||
function sweep(ctx: SyncContext, jmap: string, gen: number): void {
|
||||
const stale = ctx.db.all<{ id: string }>(
|
||||
`SELECT c.id FROM contact_card c WHERE c.jmap = ? AND c.gen < ?
|
||||
AND NOT EXISTS (SELECT 1 FROM contact_pending p WHERE p.jmap = c.jmap AND p.id = c.id AND p.op = 'create')`,
|
||||
[jmap, gen],
|
||||
);
|
||||
for (const row of stale) serverDestroyed(ctx, jmap, String(row.id));
|
||||
}
|
||||
|
||||
async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise<boolean> {
|
||||
const cursor = { ...start };
|
||||
const limits = requestLimits(ctx.session);
|
||||
const page = limits.maxObjectsInGet;
|
||||
const pairs = Math.max(1, Math.min(MOST_PAIRS_PER_REQUEST, Math.floor(limits.maxCallsInRequest / 2)));
|
||||
for (;;) {
|
||||
if (ctx.stopped()) return false;
|
||||
const calls: MethodCall[] = [];
|
||||
for (let i = 0; i < pairs; i += 1) {
|
||||
const from = cursor.anchor === null ? { position: i * page } : { anchor: cursor.anchor, anchorOffset: 1 + i * page };
|
||||
calls.push(['ContactCard/query', { accountId: jmap, ...from, limit: page, calculateTotal: i === 0 }, `q${i}`]);
|
||||
calls.push(['ContactCard/get', { accountId: jmap, '#ids': { resultOf: `q${i}`, name: 'ContactCard/query', path: '/ids' } }, `g${i}`]);
|
||||
}
|
||||
const response = await ctx.transport.request(calls, CONTACTS_USING);
|
||||
let total: number | null = null;
|
||||
let got = 0;
|
||||
let reached = 0;
|
||||
let short = false;
|
||||
let last: string | null = null;
|
||||
const cards: Card[] = [];
|
||||
try {
|
||||
for (let i = 0; i < pairs && !short; i += 1) {
|
||||
const [, query] = responseFor(response, `q${i}`);
|
||||
const ids = query.ids as string[];
|
||||
if (i === 0 && typeof query.total === 'number') total = query.total;
|
||||
cards.push(...responseFor(response, `g${i}`)[1].list as Card[]);
|
||||
got += ids.length;
|
||||
if (ids.length > 0) last = ids[ids.length - 1];
|
||||
reached = Number(query.position ?? 0) + ids.length;
|
||||
short = ids.length < page;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!(error instanceof JmapMethodError && error.type === 'anchorNotFound')) throw error;
|
||||
// The card it went from is gone. Going through again costs requests, not cards.
|
||||
cursor.anchor = null;
|
||||
cursor.position = 0;
|
||||
ctx.db.transaction(() => ctx.setSyncState(jmap, 'ContactCard', { cursor: { ...cursor } }));
|
||||
continue;
|
||||
}
|
||||
if (last !== null) cursor.anchor = last;
|
||||
cursor.position = reached;
|
||||
// The server may clamp a page, so a short one only ends this if there's no total to go by.
|
||||
const done = got === 0 || (total === null ? short : reached >= total);
|
||||
ctx.db.transaction(() => {
|
||||
for (const card of cards) storeServerCard(ctx.db, jmap, card, cursor.gen);
|
||||
if (done) {
|
||||
sweep(ctx, jmap, cursor.gen);
|
||||
ctx.setSyncState(jmap, 'ContactCard', { state: cursor.captured, phase: null, cursor: null });
|
||||
} else {
|
||||
ctx.setSyncState(jmap, 'ContactCard', { cursor: { ...cursor } });
|
||||
}
|
||||
});
|
||||
ctx.emit({ type: 'progress', account: ctx.account, area: 'contacts', phase: 'list', done: cursor.position, total });
|
||||
if (done) {
|
||||
ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] });
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: string | null): Promise<'done' | 'resync' | 'stopped'> {
|
||||
const outcome = await runChangesLoop({
|
||||
transport: ctx.transport,
|
||||
type: 'ContactCard',
|
||||
jmap,
|
||||
using: CONTACTS_USING,
|
||||
since,
|
||||
target,
|
||||
stopped: ctx.stopped,
|
||||
apply: async (page) => {
|
||||
const ids = [...page.created, ...page.updated];
|
||||
const fetched: Card[] = [];
|
||||
const size = pageSize(ctx);
|
||||
for (let i = 0; i < ids.length; i += size) {
|
||||
const response = await ctx.transport.request([['ContactCard/get', { accountId: jmap, ids: ids.slice(i, i + size) }, 'g']], CONTACTS_USING);
|
||||
fetched.push(...responseFor(response, 'g')[1].list as Card[]);
|
||||
}
|
||||
ctx.db.transaction(() => {
|
||||
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM contact_card WHERE jmap = ?', [jmap])?.gen ?? 0);
|
||||
for (const card of fetched) storeServerCard(ctx.db, jmap, card, gen);
|
||||
for (const id of page.destroyed) serverDestroyed(ctx, jmap, id);
|
||||
ctx.setSyncState(jmap, 'ContactCard', { state: page.newState });
|
||||
});
|
||||
if (ids.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['ContactCard'] });
|
||||
},
|
||||
});
|
||||
if (outcome.outcome === 'resync') return 'resync';
|
||||
return outcome.outcome === 'stopped' ? 'stopped' : 'done';
|
||||
}
|
||||
|
||||
async function syncCards(ctx: SyncContext, jmap: string): Promise<void> {
|
||||
const row = ctx.syncState(jmap, 'ContactCard');
|
||||
let cursor = row.phase === 'list' || row.phase === 'first' ? row.cursor as unknown as ListCursor : null;
|
||||
if (!cursor && !row.state) cursor = await startFullList(ctx, jmap);
|
||||
if (!cursor && row.state) {
|
||||
const outcome = await syncDelta(ctx, jmap, row.state, row.target);
|
||||
if (outcome === 'stopped') return;
|
||||
if (outcome === 'resync') cursor = await startFullList(ctx, jmap);
|
||||
}
|
||||
if (cursor) {
|
||||
if (!(await runFullList(ctx, jmap, cursor))) return;
|
||||
// Catch what changed while paging.
|
||||
const after = ctx.syncState(jmap, 'ContactCard');
|
||||
if (after.state) await syncDelta(ctx, jmap, after.state, after.target);
|
||||
}
|
||||
}
|
||||
|
||||
export const contactsSyncer: Syncer = {
|
||||
name: 'contacts',
|
||||
capability: CONTACTS_CAPABILITY,
|
||||
async run(ctx, jmap) {
|
||||
try {
|
||||
await flushPending(ctx, jmap);
|
||||
} catch (error) {
|
||||
// No network, no credentials or a busy server is the engine's to handle.
|
||||
// Anything else leaves the edits queued and mustn't stop contacts syncing.
|
||||
if (error instanceof TransportError && error.kind !== 'http') throw error;
|
||||
ctx.emit({ type: 'error', account: ctx.account, area: 'contacts', message: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
if (ctx.stopped()) return;
|
||||
await syncBooks(ctx, jmap);
|
||||
if (ctx.stopped()) return;
|
||||
await syncCards(ctx, jmap);
|
||||
},
|
||||
};
|
||||
@@ -304,6 +304,7 @@ export class AccountEngine {
|
||||
now: this.now,
|
||||
emit: this.emitter,
|
||||
stopped: () => this.closed || !this.online || this.scopeChanged,
|
||||
requestSync: () => this.requestSync(),
|
||||
syncState: (jmap, type) => this.syncState(jmap, type),
|
||||
setSyncState: (jmap, type, row) => this.setSyncState(jmap, type, row),
|
||||
overlay: (jmap, type, records) => this.overlay(jmap, type, records),
|
||||
@@ -321,6 +322,7 @@ export class AccountEngine {
|
||||
emit: this.emitter,
|
||||
overlay: (jmap, type, records) => this.overlay(jmap, type, records),
|
||||
keptByOthers: (area) => this.keptByOthers(area),
|
||||
requestSync: () => this.requestSync(),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/** What one request may hold: the session's limits, never past the ones the engine was built for. */
|
||||
|
||||
import type { StoreSession } from './transport';
|
||||
|
||||
export interface RequestLimits {
|
||||
maxObjectsInGet: number;
|
||||
maxObjectsInSet: number;
|
||||
maxCallsInRequest: number;
|
||||
maxSizeRequest: number;
|
||||
}
|
||||
|
||||
const DEFAULTS: RequestLimits = {
|
||||
maxObjectsInGet: 500,
|
||||
maxObjectsInSet: 500,
|
||||
maxCallsInRequest: 16,
|
||||
maxSizeRequest: 10 * 1024 * 1024,
|
||||
};
|
||||
|
||||
export function requestLimits(session: StoreSession): RequestLimits {
|
||||
const core = (session.capabilities['urn:ietf:params:jmap:core'] ?? {}) as Partial<RequestLimits>;
|
||||
const limit = (key: keyof RequestLimits) => {
|
||||
const value = core[key];
|
||||
return typeof value === 'number' && value > 0 ? Math.min(value, DEFAULTS[key]) : DEFAULTS[key];
|
||||
};
|
||||
return {
|
||||
maxObjectsInGet: limit('maxObjectsInGet'),
|
||||
maxObjectsInSet: limit('maxObjectsInSet'),
|
||||
maxCallsInRequest: limit('maxCallsInRequest'),
|
||||
maxSizeRequest: limit('maxSizeRequest'),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { StoreArea } from '../area';
|
||||
import { fileBlob, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep';
|
||||
import { FILE_METHODS } from './methods';
|
||||
import { FILE_MIGRATIONS } from './schema';
|
||||
import { filesSyncer } from './sync';
|
||||
|
||||
export { fileBlob, FILE_CACHE_CAP_BYTES } from './keep';
|
||||
|
||||
export const filesArea: StoreArea = {
|
||||
name: 'files',
|
||||
migrations: FILE_MIGRATIONS,
|
||||
methods: FILE_METHODS,
|
||||
syncers: [filesSyncer],
|
||||
calls: {
|
||||
/** Keep this file's content on the device until it's unpinned. */
|
||||
'files.pin': (ctx, params: NodeRef) => pinFile(ctx, params),
|
||||
'files.unpin': (ctx, params: NodeRef) => unpinFile(ctx, params),
|
||||
/** The page opened this file: keep its content for a while. */
|
||||
'files.noteOpened': (ctx, params: NodeRef) => noteOpened(ctx, params),
|
||||
/** A kept file's content and name, or null. */
|
||||
'files.read': (ctx, params: NodeRef) => readFile(ctx, params),
|
||||
/** A kept content by blob id, or null. */
|
||||
'files.blob': (ctx, params: { jmap: string; blobId: string }) => fileBlob(ctx, params.jmap, params.blobId),
|
||||
/** Pinned and recently opened files, for a settings page. */
|
||||
'files.kept': (ctx) => keptStatus(ctx),
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* File contents kept on the device: pinned files, and recently opened ones
|
||||
* within a cap. Metadata for every file is in file_node regardless.
|
||||
*
|
||||
* What gets kept:
|
||||
* - A pinned file stays until it's unpinned or removed on the server. The cap
|
||||
* doesn't push it out and the size limit for one file doesn't apply. The
|
||||
* account's total limit (scope.maxTotalBytes, which a policy can lower) does.
|
||||
* - A recently opened file is kept if it's within scope.maxAttachmentBytes, and
|
||||
* pushed out oldest first when the cap is reached. Its opened_at goes with
|
||||
* it, so it isn't fetched straight back.
|
||||
* - With scope.attachments off, nothing is kept.
|
||||
*/
|
||||
|
||||
import { TransportError } from '../engine/transport';
|
||||
import { MethodError, type StoreContext, type SyncContext } from '../area';
|
||||
import type { StorageAdapter } from '../storage';
|
||||
import { nodeMediaType, type WireNode } from './records';
|
||||
|
||||
export const FILE_CACHE_CAP_BYTES = 500 * 1024 * 1024;
|
||||
const CHUNK_BYTES = 1024 * 1024;
|
||||
|
||||
export interface KeptFile { data: Uint8Array; type: string; size: number }
|
||||
|
||||
export interface NodeRef { jmap: string; id: string }
|
||||
|
||||
interface NodeRow {
|
||||
jmap: string;
|
||||
id: string;
|
||||
name: string;
|
||||
kind: string;
|
||||
blob_id: string | null;
|
||||
size: number | null;
|
||||
json: string;
|
||||
pinned: number;
|
||||
opened_at: number | null;
|
||||
keep_error: string | null;
|
||||
}
|
||||
|
||||
interface BlobRow {
|
||||
jmap: string;
|
||||
blob_id: string;
|
||||
size: number;
|
||||
kept_at: number;
|
||||
pinned: number;
|
||||
opened_at: number | null;
|
||||
}
|
||||
|
||||
function capBytes(ctx: Pick<StoreContext, 'scope'>): number {
|
||||
return Math.min(FILE_CACHE_CAP_BYTES, ctx.scope.maxTotalBytes);
|
||||
}
|
||||
|
||||
/** A kept file's content, for the client's blob lookup. */
|
||||
export function fileBlob(ctx: Pick<StoreContext, 'db'>, jmap: string, blobId: string): KeptFile | null {
|
||||
const row = ctx.db.get<{ type: string; size: number }>('SELECT type, size FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
|
||||
if (!row) return null;
|
||||
const data = new Uint8Array(Number(row.size));
|
||||
let at = 0;
|
||||
for (const chunk of ctx.db.all<{ data: Uint8Array }>('SELECT data FROM file_blob_chunk WHERE jmap = ? AND blob_id = ? ORDER BY seq', [jmap, blobId])) {
|
||||
data.set(chunk.data, at);
|
||||
at += chunk.data.length;
|
||||
}
|
||||
return { data, type: row.type, size: Number(row.size) };
|
||||
}
|
||||
|
||||
function hasBlob(db: StorageAdapter, jmap: string, blobId: string): boolean {
|
||||
return Boolean(db.get('SELECT 1 AS x FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]));
|
||||
}
|
||||
|
||||
function dropBlob(db: StorageAdapter, jmap: string, blobId: string): void {
|
||||
db.run('DELETE FROM file_blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
|
||||
db.run('DELETE FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
|
||||
}
|
||||
|
||||
function storeBlob(ctx: StoreContext, jmap: string, blobId: string, type: string, data: Uint8Array): void {
|
||||
dropBlob(ctx.db, jmap, blobId);
|
||||
ctx.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [jmap, blobId, type, data.length, ctx.now()]);
|
||||
for (let seq = 0, at = 0; at < data.length; seq += 1, at += CHUNK_BYTES) {
|
||||
ctx.db.run('INSERT INTO file_blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, data.subarray(at, at + CHUNK_BYTES)]);
|
||||
}
|
||||
}
|
||||
|
||||
/** All kept contents in the store, with what the nodes pointing at each say about it. */
|
||||
function keptBlobs(db: StorageAdapter): BlobRow[] {
|
||||
return db.all<BlobRow>(
|
||||
`SELECT b.jmap, b.blob_id, b.size, b.kept_at, COALESCE(MAX(n.pinned), 0) AS pinned, MAX(n.opened_at) AS opened_at
|
||||
FROM file_blob b LEFT JOIN file_node n ON n.jmap = b.jmap AND n.blob_id = b.blob_id
|
||||
GROUP BY b.jmap, b.blob_id`,
|
||||
);
|
||||
}
|
||||
|
||||
function ageOf(blob: BlobRow): number {
|
||||
return Number(blob.opened_at ?? blob.kept_at);
|
||||
}
|
||||
|
||||
function pushOut(db: StorageAdapter, blob: BlobRow): void {
|
||||
dropBlob(db, blob.jmap, blob.blob_id);
|
||||
db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pushes out unpinned contents, oldest first, so `size` more fits under the
|
||||
* cap. A recently opened file only pushes out older ones and says false when
|
||||
* that isn't enough. A pinned one always gets in.
|
||||
*/
|
||||
function makeRoom(ctx: StoreContext, size: number, incoming: { pinned: boolean; openedAt: number }): boolean {
|
||||
const cap = capBytes(ctx);
|
||||
const blobs = keptBlobs(ctx.db);
|
||||
let used = blobs.reduce((sum, blob) => sum + Number(blob.size), 0);
|
||||
const candidates = blobs
|
||||
.filter((blob) => !blob.pinned && (incoming.pinned || ageOf(blob) < incoming.openedAt))
|
||||
.sort((a, b) => ageOf(a) - ageOf(b));
|
||||
const chosen: BlobRow[] = [];
|
||||
for (const blob of candidates) {
|
||||
if (used + size <= cap) break;
|
||||
used -= Number(blob.size);
|
||||
chosen.push(blob);
|
||||
}
|
||||
if (used + size > cap && !incoming.pinned) return false;
|
||||
for (const blob of chosen) pushOut(ctx.db, blob);
|
||||
return true;
|
||||
}
|
||||
|
||||
function pinnedBytes(db: StorageAdapter): number {
|
||||
return keptBlobs(db).filter((blob) => blob.pinned).reduce((sum, blob) => sum + Number(blob.size), 0);
|
||||
}
|
||||
|
||||
function setError(db: StorageAdapter, jmap: string, blobId: string, error: string | null): void {
|
||||
db.run('UPDATE file_node SET keep_error = ? WHERE jmap = ? AND blob_id = ?', [error, jmap, blobId]);
|
||||
}
|
||||
|
||||
/** Contents no node points at any more: the file was removed or has a new version. */
|
||||
export function dropOrphanBlobs(db: StorageAdapter): void {
|
||||
for (const blob of keptBlobs(db)) {
|
||||
const used = db.get('SELECT 1 AS x FROM file_node WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]);
|
||||
if (!used) dropBlob(db, blob.jmap, blob.blob_id);
|
||||
}
|
||||
}
|
||||
|
||||
interface Wanted { blob_id: string; id: string; name: string; size: number | null; json: string; pinned: number; opened_at: number | null }
|
||||
|
||||
function wantedRow(db: StorageAdapter, jmap: string, blobId: string): Wanted | undefined {
|
||||
return db.get<Wanted>(
|
||||
`SELECT blob_id, id, name, size, json, pinned, opened_at FROM file_node
|
||||
WHERE jmap = ? AND blob_id = ? AND (pinned = 1 OR opened_at IS NOT NULL)
|
||||
ORDER BY pinned DESC, opened_at DESC LIMIT 1`,
|
||||
[jmap, blobId],
|
||||
);
|
||||
}
|
||||
|
||||
/** Fetches what should be kept and isn't yet, one file at a time, pinned files first. */
|
||||
export async function fetchKeptFiles(ctx: SyncContext, jmap: string): Promise<void> {
|
||||
const { db } = ctx;
|
||||
if (!ctx.scope.attachments) {
|
||||
db.transaction(() => {
|
||||
for (const blob of keptBlobs(db)) dropBlob(db, blob.jmap, blob.blob_id);
|
||||
db.run("UPDATE file_node SET keep_error = 'notAllowed' WHERE jmap = ? AND (pinned = 1 OR opened_at IS NOT NULL)", [jmap]);
|
||||
});
|
||||
return;
|
||||
}
|
||||
db.transaction(() => {
|
||||
db.run("UPDATE file_node SET keep_error = NULL WHERE jmap = ? AND keep_error = 'notAllowed'", [jmap]);
|
||||
// The limit may have gone down since last time.
|
||||
makeRoom(ctx, 0, { pinned: true, openedAt: 0 });
|
||||
});
|
||||
const todo = db.all<{ blob_id: string }>(
|
||||
`SELECT n.blob_id FROM file_node n
|
||||
WHERE n.jmap = ? AND n.blob_id IS NOT NULL AND (n.pinned = 1 OR n.opened_at IS NOT NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM file_blob b WHERE b.jmap = n.jmap AND b.blob_id = n.blob_id)
|
||||
GROUP BY n.blob_id ORDER BY MAX(n.pinned) DESC, MAX(n.opened_at) DESC`,
|
||||
[jmap],
|
||||
).map((row) => String(row.blob_id));
|
||||
let done = 0;
|
||||
for (const blobId of todo) {
|
||||
if (ctx.stopped()) return;
|
||||
// The page may have changed its mind during the last download.
|
||||
const want = wantedRow(db, jmap, blobId);
|
||||
if (!want) continue;
|
||||
const known = Number(want.size ?? 0);
|
||||
if (!want.pinned && known > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) {
|
||||
setError(db, jmap, blobId, 'tooLarge');
|
||||
continue;
|
||||
}
|
||||
if (want.pinned && pinnedBytes(db) + known > ctx.scope.maxTotalBytes) {
|
||||
setError(db, jmap, blobId, 'overLimit');
|
||||
continue;
|
||||
}
|
||||
let data: Uint8Array;
|
||||
const type = nodeMediaType(JSON.parse(want.json) as WireNode);
|
||||
try {
|
||||
data = await ctx.transport.download(jmap, blobId, type, want.name);
|
||||
} catch (error) {
|
||||
// No network or no credentials reach the engine. One bad file doesn't stop the others.
|
||||
if (error instanceof TransportError && error.kind === 'http') {
|
||||
setError(db, jmap, blobId, 'download');
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
db.transaction(() => {
|
||||
const now = wantedRow(db, jmap, blobId);
|
||||
if (!now) return;
|
||||
if (now.pinned ? pinnedBytes(db) + data.length > ctx.scope.maxTotalBytes : data.length > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) {
|
||||
// The size in the metadata was wrong or missing. With the real one
|
||||
// noted, the next pass sees this without downloading it again.
|
||||
db.run('UPDATE file_node SET size = ? WHERE jmap = ? AND blob_id = ?', [data.length, jmap, blobId]);
|
||||
setError(db, jmap, blobId, now.pinned ? 'overLimit' : 'tooLarge');
|
||||
return;
|
||||
}
|
||||
if (!makeRoom(ctx, data.length, { pinned: Boolean(now.pinned), openedAt: Number(now.opened_at ?? 0) })) {
|
||||
db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ? AND pinned = 0', [jmap, blobId]);
|
||||
return;
|
||||
}
|
||||
storeBlob(ctx, jmap, blobId, type, data);
|
||||
setError(db, jmap, blobId, null);
|
||||
});
|
||||
done += 1;
|
||||
ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'contents', done, total: todo.length });
|
||||
}
|
||||
if (done > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] });
|
||||
}
|
||||
|
||||
// ---- Calls from the page ----
|
||||
|
||||
function fileRow(db: StorageAdapter, { jmap, id }: NodeRef): NodeRow {
|
||||
const row = db.get<NodeRow>('SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
if (!row) throw new MethodError('notFound', `No file ${id}`);
|
||||
if (!row.blob_id || row.kind !== 'file') throw new MethodError('invalidArguments', 'Only files can be kept');
|
||||
return row;
|
||||
}
|
||||
|
||||
export function pinFile(ctx: StoreContext, ref: NodeRef): { kept: boolean } {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
ctx.db.run('UPDATE file_node SET pinned = 1, keep_error = NULL WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]);
|
||||
const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string);
|
||||
if (!kept) void ctx.requestSync();
|
||||
return { kept };
|
||||
}
|
||||
|
||||
/** Back to being a recently opened file, or let go of it if it never was one. */
|
||||
export function unpinFile(ctx: StoreContext, ref: NodeRef): void {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
ctx.db.transaction(() => {
|
||||
ctx.db.run('UPDATE file_node SET pinned = 0 WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]);
|
||||
if (!wantedRow(ctx.db, ref.jmap, row.blob_id as string)) dropBlob(ctx.db, ref.jmap, row.blob_id as string);
|
||||
});
|
||||
}
|
||||
|
||||
export function noteOpened(ctx: StoreContext, ref: NodeRef): { kept: boolean } {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
ctx.db.run('UPDATE file_node SET opened_at = ?, keep_error = NULL WHERE jmap = ? AND id = ?', [ctx.now(), ref.jmap, ref.id]);
|
||||
const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string);
|
||||
if (!kept) void ctx.requestSync();
|
||||
return { kept };
|
||||
}
|
||||
|
||||
export function readFile(ctx: StoreContext, ref: NodeRef): (KeptFile & { name: string }) | null {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
const kept = fileBlob(ctx, ref.jmap, row.blob_id as string);
|
||||
return kept ? { ...kept, name: row.name } : null;
|
||||
}
|
||||
|
||||
export interface KeptFileStatus {
|
||||
jmap: string;
|
||||
id: string;
|
||||
name: string;
|
||||
size: number | null;
|
||||
pinned: boolean;
|
||||
openedAt: number | null;
|
||||
kept: boolean;
|
||||
/** Why it isn't kept: tooLarge, overLimit, notAllowed or download. */
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** Pinned and recently opened files, and how much room their contents take. */
|
||||
export function keptStatus(ctx: StoreContext): { files: KeptFileStatus[]; bytes: number; capBytes: number } {
|
||||
const rows = ctx.db.all<NodeRow>(
|
||||
`SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node
|
||||
WHERE pinned = 1 OR opened_at IS NOT NULL ORDER BY pinned DESC, opened_at DESC, name`,
|
||||
);
|
||||
return {
|
||||
files: rows.map((row) => ({
|
||||
jmap: row.jmap,
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
size: row.size === null ? null : Number(row.size),
|
||||
pinned: Boolean(row.pinned),
|
||||
openedAt: row.opened_at === null ? null : Number(row.opened_at),
|
||||
kept: row.blob_id !== null && hasBlob(ctx.db, row.jmap, row.blob_id),
|
||||
error: row.keep_error,
|
||||
})),
|
||||
bytes: keptBlobs(ctx.db).reduce((sum, blob) => sum + Number(blob.size), 0),
|
||||
capBytes: capBytes(ctx),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* FileNode reads answered from the store, so listings work offline. Nothing
|
||||
* here answers FileNode/set: the responder gives it a per-call
|
||||
* serverUnavailable, since the design has no offline file writes.
|
||||
*/
|
||||
|
||||
import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area';
|
||||
import type { SqlValue } from '../storage';
|
||||
import { FILES_CAPABILITY } from './records';
|
||||
|
||||
function jmapOf(ctx: MethodContext, args: Record<string, unknown>): string {
|
||||
return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[FILES_CAPABILITY] ?? ctx.defaultJmap;
|
||||
}
|
||||
|
||||
/** The last state from the server. Throws when the list was never read, so an empty answer always means empty. */
|
||||
function syncedState(ctx: MethodContext, jmap: string): string | null {
|
||||
const row = ctx.db.get<{ state: string | null; phase: string | null }>("SELECT state, phase FROM sync_state WHERE jmap = ? AND type = 'FileNode'", [jmap]);
|
||||
if (!row || (!row.state && !row.phase)) throw new MethodError('serverUnavailable', 'Files not kept yet');
|
||||
return row.state;
|
||||
}
|
||||
|
||||
function pick(node: Record<string, unknown>, properties: unknown): Record<string, unknown> {
|
||||
if (!Array.isArray(properties)) return node;
|
||||
const out: Record<string, unknown> = { id: node.id };
|
||||
for (const property of properties as string[]) if (property in node) out[property] = node[property];
|
||||
return out;
|
||||
}
|
||||
|
||||
const SORTS: Record<string, string> = {
|
||||
name: 'name COLLATE NOCASE',
|
||||
size: 'size',
|
||||
created: "json_extract(json, '$.created')",
|
||||
modified: "json_extract(json, '$.modified')",
|
||||
};
|
||||
|
||||
function orderSql(sort: unknown): string {
|
||||
const parts: string[] = [];
|
||||
for (const level of (Array.isArray(sort) ? sort : []) as Array<{ property?: string; isAscending?: boolean }>) {
|
||||
const column = SORTS[String(level.property)];
|
||||
if (!column) throw new MethodError('unsupportedSort', `${String(level.property)} isn't available offline`);
|
||||
parts.push(`${column} ${level.isAscending === false ? 'DESC' : 'ASC'}`);
|
||||
}
|
||||
parts.push('name COLLATE NOCASE ASC', 'id ASC');
|
||||
return parts.join(', ');
|
||||
}
|
||||
|
||||
const fileNodeGet: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
const state = syncedState(ctx, jmap);
|
||||
const list: Array<Record<string, unknown>> = [];
|
||||
const notFound: string[] = [];
|
||||
if (Array.isArray(args.ids)) {
|
||||
for (const id of args.ids as string[]) {
|
||||
const row = ctx.db.get<{ json: string }>('SELECT json FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
if (row) list.push(pick(JSON.parse(row.json) as Record<string, unknown>, args.properties));
|
||||
else notFound.push(id);
|
||||
}
|
||||
} else {
|
||||
// The whole tree at once. The client reads a long list as possibly cut,
|
||||
// pages FileNode/query to check, and finds nothing missing.
|
||||
for (const row of ctx.db.all<{ json: string }>('SELECT json FROM file_node WHERE jmap = ?', [jmap])) {
|
||||
list.push(pick(JSON.parse(row.json) as Record<string, unknown>, args.properties));
|
||||
}
|
||||
}
|
||||
return { accountId: jmap, ...(state ? { state } : {}), list, notFound };
|
||||
};
|
||||
|
||||
const fileNodeQuery: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
syncedState(ctx, jmap);
|
||||
if (args.anchor !== undefined) throw new MethodError('unsupportedFilter', 'anchor needs the server');
|
||||
const filter = (args.filter ?? {}) as Record<string, unknown>;
|
||||
const conditions = ['jmap = ?'];
|
||||
const params: SqlValue[] = [jmap];
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
if (key !== 'parentId') throw new MethodError('unsupportedFilter', `${key} isn't available offline`);
|
||||
if (value === null) {
|
||||
conditions.push('parent_id IS NULL');
|
||||
} else {
|
||||
conditions.push('parent_id = ?');
|
||||
params.push(String(value));
|
||||
}
|
||||
}
|
||||
const where = conditions.join(' AND ');
|
||||
const position = Math.max(0, Number(args.position ?? 0));
|
||||
const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1;
|
||||
const ids = ctx.db.all<{ id: string }>(
|
||||
`SELECT id FROM file_node WHERE ${where} ORDER BY ${orderSql(args.sort)} LIMIT ? OFFSET ?`,
|
||||
[...params, limit, position],
|
||||
).map((row) => String(row.id));
|
||||
const result: MethodResult = {
|
||||
accountId: jmap,
|
||||
// Local answers aren't a position in the server's log, so this is a token only.
|
||||
queryState: 'local',
|
||||
canCalculateChanges: false,
|
||||
position,
|
||||
ids,
|
||||
};
|
||||
if (args.calculateTotal) {
|
||||
result.total = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM file_node WHERE ${where}`, params)?.n ?? 0);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
export const FILE_METHODS: Readonly<Record<string, MethodHandler>> = {
|
||||
'FileNode/get': fileNodeGet,
|
||||
'FileNode/query': fileNodeQuery,
|
||||
};
|
||||
@@ -0,0 +1,50 @@
|
||||
/** How a FileNode becomes a row. The server's own copy stays in `json`, in whichever draft it speaks. */
|
||||
|
||||
import { fileNodeKind, fromDraftFileNode } from '@/lib/jmap/filenode-kind';
|
||||
import { decodeFileNodeName } from '@/lib/jmap/filenode-name';
|
||||
import type { FileNode } from '@/lib/jmap/types';
|
||||
import type { StorageAdapter } from '../storage';
|
||||
|
||||
export const FILES_CAPABILITY = 'urn:ietf:params:jmap:filenode';
|
||||
export const FILES_USING = ['urn:ietf:params:jmap:core', FILES_CAPABILITY];
|
||||
|
||||
/** The client's list (lib/jmap/client.ts). Stalwart leaves out shareWith and myRights unless asked. */
|
||||
export const FILE_NODE_PROPERTIES = [
|
||||
'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified',
|
||||
'nodeType', '@type', 'mediaType', 'shareWith', 'myRights',
|
||||
];
|
||||
|
||||
export type WireNode = Record<string, unknown> & { id: string };
|
||||
|
||||
/** The node's media type, whichever draft named it. */
|
||||
export function nodeMediaType(node: WireNode): string {
|
||||
const { type } = fromDraftFileNode(node as unknown as Partial<FileNode> & { '@type'?: unknown; mediaType?: unknown });
|
||||
return typeof type === 'string' && type ? type : 'application/octet-stream';
|
||||
}
|
||||
|
||||
export function writeNode(db: StorageAdapter, jmap: string, node: WireNode, gen: number): void {
|
||||
const normal = fromDraftFileNode(node as unknown as FileNode & { '@type'?: unknown; mediaType?: unknown });
|
||||
db.run(
|
||||
`INSERT INTO file_node (jmap, id, parent_id, name, kind, blob_id, size, json, gen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (jmap, id) DO UPDATE SET parent_id = excluded.parent_id, name = excluded.name, kind = excluded.kind,
|
||||
size = excluded.size, json = excluded.json, gen = excluded.gen,
|
||||
keep_error = CASE WHEN blob_id IS excluded.blob_id THEN keep_error END,
|
||||
blob_id = excluded.blob_id`,
|
||||
[
|
||||
jmap,
|
||||
node.id,
|
||||
typeof node.parentId === 'string' ? node.parentId : null,
|
||||
// The client decodes names itself from `json`. This column is for the store's own lists.
|
||||
typeof node.name === 'string' ? decodeFileNodeName(node.name) : '',
|
||||
fileNodeKind(normal),
|
||||
typeof node.blobId === 'string' ? node.blobId : null,
|
||||
typeof node.size === 'number' ? node.size : null,
|
||||
JSON.stringify(node),
|
||||
gen,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
export function removeNode(db: StorageAdapter, jmap: string, id: string): void {
|
||||
db.run('DELETE FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { Migration } from '../storage';
|
||||
|
||||
export const FILE_MIGRATIONS: readonly Migration[] = [
|
||||
{
|
||||
id: 700,
|
||||
name: 'files',
|
||||
sql: `
|
||||
CREATE TABLE file_node (
|
||||
jmap TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
parent_id TEXT,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
-- file, directory or symlink, whichever way the server says so.
|
||||
kind TEXT NOT NULL,
|
||||
blob_id TEXT,
|
||||
size INTEGER,
|
||||
json TEXT NOT NULL,
|
||||
gen INTEGER NOT NULL DEFAULT 0,
|
||||
pinned INTEGER NOT NULL DEFAULT 0,
|
||||
-- When the page last opened it. Cleared when its content is pushed out.
|
||||
opened_at INTEGER,
|
||||
-- Why the content isn't kept: tooLarge, overLimit, notAllowed or download.
|
||||
keep_error TEXT,
|
||||
PRIMARY KEY (jmap, id)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX file_node_parent ON file_node (jmap, parent_id);
|
||||
CREATE INDEX file_node_blob ON file_node (jmap, blob_id);
|
||||
CREATE TABLE file_blob (
|
||||
jmap TEXT NOT NULL,
|
||||
blob_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
kept_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (jmap, blob_id)
|
||||
) WITHOUT ROWID;
|
||||
-- In pieces: a whole file bound at once would grow the WASM heap by its
|
||||
-- size, and that memory never comes back.
|
||||
CREATE TABLE file_blob_chunk (
|
||||
jmap TEXT NOT NULL,
|
||||
blob_id TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
PRIMARY KEY (jmap, blob_id, seq)
|
||||
);
|
||||
`,
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* FileNode metadata for one JMAP account, then the contents worth keeping.
|
||||
*
|
||||
* A full pass reads the tree the way the client does (fetchAllFileNodes,
|
||||
* lib/jmap/client.ts). `FileNode/get` with ids null returns everything, but
|
||||
* 0.16 silently stops at maxObjectsInGet and 1.0 refuses a larger account with
|
||||
* requestTooLarge. Past that, ids come from paging FileNode/query. Before
|
||||
* 0.16.6 that query leaves out folders, so parents still unknown afterwards are
|
||||
* read by id. The cursor is in sync_state, so a reload continues.
|
||||
*/
|
||||
|
||||
import type { SyncContext, Syncer } from '../area';
|
||||
import { runChangesLoop } from '../engine/changes';
|
||||
import { JmapMethodError, responseFor, type MethodCall } from '../engine/transport';
|
||||
import { dropOrphanBlobs, fetchKeptFiles } from './keep';
|
||||
import { FILE_NODE_PROPERTIES, FILES_CAPABILITY, FILES_USING, removeNode, writeNode, type WireNode } from './records';
|
||||
|
||||
const MAX_GET = 500;
|
||||
const GETS_PER_REQUEST = 8;
|
||||
const QUERY_PAGE = 5000;
|
||||
|
||||
function getBatch(ctx: SyncContext): number {
|
||||
const core = ctx.session.capabilities['urn:ietf:params:jmap:core'] as { maxObjectsInGet?: number } | undefined;
|
||||
return Math.min(MAX_GET, core?.maxObjectsInGet || MAX_GET);
|
||||
}
|
||||
|
||||
async function fetchNodes(ctx: SyncContext, jmap: string, ids: string[]): Promise<WireNode[]> {
|
||||
const size = getBatch(ctx);
|
||||
const nodes: WireNode[] = [];
|
||||
for (let at = 0; at < ids.length;) {
|
||||
const calls: MethodCall[] = [];
|
||||
for (let i = 0; i < GETS_PER_REQUEST && at < ids.length; i += 1, at += size) {
|
||||
calls.push(['FileNode/get', { accountId: jmap, ids: ids.slice(at, at + size), properties: FILE_NODE_PROPERTIES }, `g${i}`]);
|
||||
}
|
||||
const response = await ctx.transport.request(calls, FILES_USING);
|
||||
calls.forEach(([, , callId]) => nodes.push(...responseFor(response, callId)[1].list as WireNode[]));
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
interface ListCursor { step: 'all' | 'query' | 'parents'; position: number; gen: number; captured: string }
|
||||
|
||||
async function startFullList(ctx: SyncContext, jmap: string): Promise<ListCursor> {
|
||||
const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: [], properties: ['id'] }, 's']], FILES_USING);
|
||||
const captured = String(responseFor(response, 's')[1].state);
|
||||
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0) + 1;
|
||||
const cursor: ListCursor = { step: 'all', position: 0, gen, captured };
|
||||
ctx.db.transaction(() => ctx.setSyncState(jmap, 'FileNode', { state: null, phase: 'list', cursor: { ...cursor } }));
|
||||
return cursor;
|
||||
}
|
||||
|
||||
function saveStep(ctx: SyncContext, jmap: string, nodes: WireNode[], cursor: ListCursor): void {
|
||||
ctx.db.transaction(() => {
|
||||
for (const node of nodes) writeNode(ctx.db, jmap, node, cursor.gen);
|
||||
ctx.setSyncState(jmap, 'FileNode', { cursor: { ...cursor } });
|
||||
});
|
||||
}
|
||||
|
||||
async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise<boolean> {
|
||||
const cursor = { ...start };
|
||||
|
||||
if (cursor.step === 'all') {
|
||||
let nodes: WireNode[] = [];
|
||||
let complete = false;
|
||||
try {
|
||||
const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: null, properties: FILE_NODE_PROPERTIES }, 'a']], FILES_USING);
|
||||
nodes = responseFor(response, 'a')[1].list as WireNode[];
|
||||
complete = nodes.length < getBatch(ctx);
|
||||
} catch (error) {
|
||||
if (!(error instanceof JmapMethodError && error.type === 'requestTooLarge')) throw error;
|
||||
}
|
||||
if (!complete) cursor.step = 'query';
|
||||
else cursor.step = 'parents';
|
||||
saveStep(ctx, jmap, nodes, cursor);
|
||||
}
|
||||
|
||||
while (cursor.step === 'query') {
|
||||
if (ctx.stopped()) return false;
|
||||
const response = await ctx.transport.request([['FileNode/query', { accountId: jmap, filter: {}, position: cursor.position, limit: QUERY_PAGE, calculateTotal: true }, 'q']], FILES_USING);
|
||||
const [, query] = responseFor(response, 'q');
|
||||
const ids = query.ids as string[];
|
||||
const missing = ids.filter((id) => Number(ctx.db.get<{ gen: number }>('SELECT gen FROM file_node WHERE jmap = ? AND id = ?', [jmap, id])?.gen ?? -1) < cursor.gen);
|
||||
const nodes = await fetchNodes(ctx, jmap, missing);
|
||||
cursor.position += ids.length;
|
||||
// The server may clamp the limit, so only an empty page or the total ends this.
|
||||
if (ids.length === 0 || (typeof query.total === 'number' && cursor.position >= query.total)) cursor.step = 'parents';
|
||||
saveStep(ctx, jmap, nodes, cursor);
|
||||
ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'list', done: cursor.position, total: typeof query.total === 'number' ? query.total : null });
|
||||
}
|
||||
|
||||
const asked = new Set<string>();
|
||||
for (;;) {
|
||||
if (ctx.stopped()) return false;
|
||||
const parents = ctx.db.all<{ id: string }>(
|
||||
`SELECT DISTINCT n.parent_id AS id FROM file_node n
|
||||
WHERE n.jmap = ? AND n.gen = ? AND n.parent_id IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM file_node p WHERE p.jmap = n.jmap AND p.id = n.parent_id AND p.gen = ?)`,
|
||||
[jmap, cursor.gen, cursor.gen],
|
||||
).map((row) => String(row.id)).filter((id) => !asked.has(id));
|
||||
if (parents.length === 0) break;
|
||||
for (const id of parents) asked.add(id);
|
||||
saveStep(ctx, jmap, await fetchNodes(ctx, jmap, parents), cursor);
|
||||
}
|
||||
|
||||
ctx.db.transaction(() => {
|
||||
// Whatever this pass didn't see is gone from the server.
|
||||
ctx.db.run('DELETE FROM file_node WHERE jmap = ? AND gen < ?', [jmap, cursor.gen]);
|
||||
ctx.setSyncState(jmap, 'FileNode', { state: cursor.captured, phase: null, cursor: null });
|
||||
});
|
||||
ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: string | null): Promise<'done' | 'resync' | 'stopped'> {
|
||||
const outcome = await runChangesLoop({
|
||||
transport: ctx.transport,
|
||||
type: 'FileNode',
|
||||
jmap,
|
||||
using: FILES_USING,
|
||||
since,
|
||||
target,
|
||||
stopped: ctx.stopped,
|
||||
apply: async (page) => {
|
||||
const nodes = await fetchNodes(ctx, jmap, [...page.created, ...page.updated]);
|
||||
ctx.db.transaction(() => {
|
||||
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0);
|
||||
for (const node of nodes) writeNode(ctx.db, jmap, node, gen);
|
||||
for (const id of page.destroyed) removeNode(ctx.db, jmap, id);
|
||||
ctx.setSyncState(jmap, 'FileNode', { state: page.newState });
|
||||
});
|
||||
if (nodes.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] });
|
||||
},
|
||||
});
|
||||
if (outcome.outcome === 'resync') return 'resync';
|
||||
return outcome.outcome === 'stopped' ? 'stopped' : 'done';
|
||||
}
|
||||
|
||||
export const filesSyncer: Syncer = {
|
||||
name: 'files',
|
||||
capability: FILES_CAPABILITY,
|
||||
async run(ctx, jmap) {
|
||||
const row = ctx.syncState(jmap, 'FileNode');
|
||||
let cursor = row.phase === 'list' ? row.cursor as unknown as ListCursor : null;
|
||||
if (!cursor && !row.state) cursor = await startFullList(ctx, jmap);
|
||||
if (!cursor && row.state) {
|
||||
const outcome = await syncDelta(ctx, jmap, row.state, row.target);
|
||||
if (outcome === 'stopped') return;
|
||||
if (outcome === 'resync') cursor = await startFullList(ctx, jmap);
|
||||
}
|
||||
if (cursor) {
|
||||
if (!(await runFullList(ctx, jmap, cursor))) return;
|
||||
// Catch what changed while paging.
|
||||
const after = ctx.syncState(jmap, 'FileNode');
|
||||
if (after.state && (await syncDelta(ctx, jmap, after.state, after.target)) === 'stopped') return;
|
||||
}
|
||||
if (ctx.stopped()) return;
|
||||
ctx.db.transaction(() => dropOrphanBlobs(ctx.db));
|
||||
await fetchKeptFiles(ctx, jmap);
|
||||
},
|
||||
};
|
||||
@@ -9,10 +9,11 @@ import {
|
||||
normalizeContactPhotoUri,
|
||||
TRUSTED_SENDERS_BOOK_NAME,
|
||||
} from '../contact-store';
|
||||
import type { ContactCard, ContactMedia } from '@/lib/jmap/types';
|
||||
import type { AddressBook, ContactCard, ContactMedia } from '@/lib/jmap/types';
|
||||
import type { IJMAPClient } from '@/lib/jmap/client-interface';
|
||||
import { clearContactPhotoCache } from '@/lib/contact-photo-cache';
|
||||
import { PrincipalsRequestError } from '@/lib/jmap/principals-error';
|
||||
import { PartialAccountsError } from '@/lib/jmap/partial-accounts';
|
||||
import { setClientLookup } from '../client-registry';
|
||||
|
||||
vi.stubGlobal('crypto', { randomUUID: () => '00000000-0000-0000-0000-000000000000' });
|
||||
@@ -1364,6 +1365,78 @@ describe('contact-store', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('an account that cannot answer', () => {
|
||||
it('keeps its contacts and books while the other accounts refresh', async () => {
|
||||
const failing = {
|
||||
getAllContacts: vi.fn().mockRejectedValue(new Error('offline')),
|
||||
getAllAddressBooks: vi.fn().mockRejectedValue(new Error('offline')),
|
||||
} as unknown as IJMAPClient;
|
||||
const working = {
|
||||
getAllContacts: vi.fn().mockResolvedValue([makeContact({ id: 'b1' })]),
|
||||
getAllAddressBooks: vi.fn().mockResolvedValue([{ id: 'bookb', name: 'B' }]),
|
||||
} as unknown as IJMAPClient;
|
||||
const kept = makeContact({ id: 'acc-a::a1', localAccountId: 'acc-a' });
|
||||
const keptBook = { id: 'acc-a::booka', name: 'A', localAccountId: 'acc-a' } as AddressBook;
|
||||
useContactStore.setState({ contacts: [kept], addressBooks: [keptBook] });
|
||||
const accounts = [{ client: failing, localAccountId: 'acc-a' }, { client: working, localAccountId: 'acc-b' }];
|
||||
|
||||
await useContactStore.getState().fetchAllAccountsContacts(accounts);
|
||||
await useContactStore.getState().fetchAllAccountsAddressBooks(accounts);
|
||||
|
||||
const state = useContactStore.getState();
|
||||
expect(state.contacts.map((c) => c.id)).toEqual(['acc-a::a1', 'acc-b::b1']);
|
||||
expect(state.addressBooks.map((b) => b.id)).toEqual(['acc-a::booka', 'acc-b::bookb']);
|
||||
});
|
||||
|
||||
it('keeps the primary account\'s cards when only a shared account answered', async () => {
|
||||
const refusal = new PartialAccountsError<ContactCard>([makeContact({ id: 'shared:s2', accountId: 'shared', isShared: true })], ['me'], 'me', new Error('offline'));
|
||||
const failing = { getAllContacts: vi.fn().mockRejectedValue(refusal) } as unknown as IJMAPClient;
|
||||
useContactStore.setState({
|
||||
contacts: [
|
||||
makeContact({ id: 'p1', accountId: 'me' }),
|
||||
// A card just created has no accountId yet: it's the primary account's.
|
||||
makeContact({ id: 'p2' }),
|
||||
makeContact({ id: 'shared:s1', accountId: 'shared', isShared: true }),
|
||||
],
|
||||
});
|
||||
await useContactStore.getState().fetchContacts(failing);
|
||||
const state = useContactStore.getState();
|
||||
expect(state.contacts.map((c) => c.id).sort()).toEqual(['p1', 'p2', 'shared:s2']);
|
||||
expect(state.error).toBe('Failed to fetch contacts');
|
||||
expect(state.isLoading).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps the primary account\'s books when only a shared account answered', async () => {
|
||||
const refusal = new PartialAccountsError<AddressBook>([{ id: 'shared:b', name: 'B', accountId: 'shared' } as AddressBook], ['me'], 'me', new Error('offline'));
|
||||
const failing = { getAllAddressBooks: vi.fn().mockRejectedValue(refusal) } as unknown as IJMAPClient;
|
||||
useContactStore.setState({ addressBooks: [{ id: 'mine', name: 'Mine', accountId: 'me' } as AddressBook] });
|
||||
await useContactStore.getState().fetchAddressBooks(failing);
|
||||
expect(useContactStore.getState().addressBooks.map((b) => b.id).sort()).toEqual(['mine', 'shared:b']);
|
||||
});
|
||||
|
||||
it('keeps a failed JMAP account of one login while its others refresh', async () => {
|
||||
const refusal = new PartialAccountsError<ContactCard>([makeContact({ id: 's2', accountId: 'shared' })], ['me'], 'me', new Error('offline'));
|
||||
const failing = { getAllContacts: vi.fn().mockRejectedValue(refusal) } as unknown as IJMAPClient;
|
||||
useContactStore.setState({
|
||||
contacts: [
|
||||
makeContact({ id: 'acc-a::p1', accountId: 'me', localAccountId: 'acc-a' }),
|
||||
makeContact({ id: 'acc-a::s1', accountId: 'shared', localAccountId: 'acc-a' }),
|
||||
makeContact({ id: 'acc-b::x', accountId: 'me', localAccountId: 'acc-b' }),
|
||||
],
|
||||
});
|
||||
await useContactStore.getState().fetchAllAccountsContacts([{ client: failing, localAccountId: 'acc-a' }]);
|
||||
expect(useContactStore.getState().contacts.map((c) => c.id).sort()).toEqual(['acc-a::p1', 'acc-a::s2']);
|
||||
});
|
||||
|
||||
it('keeps the list when the only account fails', async () => {
|
||||
const failing = { getAllContacts: vi.fn().mockRejectedValue(new Error('offline')) } as unknown as IJMAPClient;
|
||||
useContactStore.setState({ contacts: [makeContact()] });
|
||||
await useContactStore.getState().fetchContacts(failing);
|
||||
expect(useContactStore.getState().contacts).toHaveLength(1);
|
||||
expect(useContactStore.getState().error).toBe('Failed to fetch contacts');
|
||||
});
|
||||
});
|
||||
|
||||
describe('persistence/partialize', () => {
|
||||
it('should persist contacts when supportsSync is false', () => {
|
||||
const { partialize } = (useContactStore as unknown as { persist: { getOptions: () => { partialize: (state: Record<string, unknown>) => Record<string, unknown> } } }).persist.getOptions();
|
||||
|
||||
@@ -120,6 +120,22 @@ describe('file-store hierarchy (issue #379)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps the listing when refreshing the same folder fails, and starts empty in a new one', async () => {
|
||||
const client = makeMockClient([dir('stuff', 'Stuff', null), file('notes', 'Notes.md', null), file('inner', 'Inner.md', 'stuff')]);
|
||||
useFileStore.getState().initClient(client);
|
||||
await useFileStore.getState().navigate(null);
|
||||
expect(useFileStore.getState().resources.map(r => r.name).sort()).toEqual(['Notes.md', 'Stuff']);
|
||||
|
||||
client.listAllFileNodes = async () => { throw new Error('offline'); };
|
||||
await useFileStore.getState().navigate(null);
|
||||
expect(useFileStore.getState().error).toBe('offline');
|
||||
expect(useFileStore.getState().resources.map(r => r.name).sort()).toEqual(['Notes.md', 'Stuff']);
|
||||
|
||||
await useFileStore.getState().navigate('stuff', 'Stuff');
|
||||
expect(useFileStore.getState().error).toBe('offline');
|
||||
expect(useFileStore.getState().resources).toEqual([]);
|
||||
});
|
||||
|
||||
it('lists only root-level nodes at the root, not the whole tree flattened', async () => {
|
||||
// Stuff > Nonsense > Notes.md / Other.md (the exact shape from the issue)
|
||||
const client = makeMockClient([
|
||||
|
||||
+25
-4
@@ -13,6 +13,7 @@ import { sanitizeDisplayName, splitMailbox } from '@/lib/rfc5322-mailbox';
|
||||
import { inlineContactMedia, writeContactWithNewPhoto, type ContactSaveOptions } from '@/lib/contact-media';
|
||||
import { contactToView } from '@/lib/plugin-projection';
|
||||
import { PrincipalsRequestError } from '@/lib/jmap/principals-error';
|
||||
import { PartialAccountsError, partialAccounts, withFailedAccountsKept } from '@/lib/jmap/partial-accounts';
|
||||
import type { Principal } from '@/lib/jmap/types';
|
||||
import { currentStoreEpoch } from '@/lib/store-epoch';
|
||||
|
||||
@@ -614,7 +615,14 @@ export const useContactStore = create<ContactStore>()(
|
||||
} catch (error) {
|
||||
if (currentStoreEpoch() !== epoch) return;
|
||||
console.error('Failed to fetch contacts:', error);
|
||||
set({ error: 'Failed to fetch contacts', isLoading: false, contactsLoaded: true });
|
||||
// An account that couldn't answer keeps its cards; the others are fresh.
|
||||
const partial = partialAccounts<ContactCard>(error);
|
||||
set((state) => ({
|
||||
error: 'Failed to fetch contacts',
|
||||
isLoading: false,
|
||||
contactsLoaded: true,
|
||||
contacts: partial ? withFailedAccountsKept(partial, state.contacts) : state.contacts,
|
||||
}));
|
||||
}
|
||||
},
|
||||
|
||||
@@ -656,7 +664,11 @@ export const useContactStore = create<ContactStore>()(
|
||||
} catch (error) {
|
||||
if (currentStoreEpoch() !== epoch) return;
|
||||
console.error('Failed to fetch address books:', error);
|
||||
set({ error: 'Failed to fetch address books' });
|
||||
const partial = partialAccounts<AddressBook>(error);
|
||||
set((state) => ({
|
||||
error: 'Failed to fetch address books',
|
||||
addressBooks: partial ? withFailedAccountsKept(partial, state.addressBooks) : state.addressBooks,
|
||||
}));
|
||||
}
|
||||
},
|
||||
|
||||
@@ -671,7 +683,12 @@ export const useContactStore = create<ContactStore>()(
|
||||
return prefixContactsWithLocalAccount(list, localAccountId);
|
||||
} catch (error) {
|
||||
debug.error(`Failed to fetch contacts for account ${localAccountId}:`, error);
|
||||
return [] as ContactCard[];
|
||||
// Keep what this login's cards were: it just couldn't answer.
|
||||
const held = get().contacts.filter((contact) => contact.localAccountId === localAccountId);
|
||||
const partial = partialAccounts<ContactCard>(error);
|
||||
if (!partial) return held;
|
||||
const fresh = prefixContactsWithLocalAccount(partial.items, localAccountId);
|
||||
return withFailedAccountsKept(new PartialAccountsError(fresh, partial.failedAccountIds, partial.primaryAccountId, error), held);
|
||||
}
|
||||
}),
|
||||
);
|
||||
@@ -694,7 +711,11 @@ export const useContactStore = create<ContactStore>()(
|
||||
return prefixAddressBooksWithLocalAccount(list, localAccountId);
|
||||
} catch (error) {
|
||||
debug.error(`Failed to fetch address books for account ${localAccountId}:`, error);
|
||||
return [] as AddressBook[];
|
||||
const held = get().addressBooks.filter((book) => book.localAccountId === localAccountId);
|
||||
const partial = partialAccounts<AddressBook>(error);
|
||||
if (!partial) return held;
|
||||
const fresh = prefixAddressBooksWithLocalAccount(partial.items, localAccountId);
|
||||
return withFailedAccountsKept(new PartialAccountsError(fresh, partial.failedAccountIds, partial.primaryAccountId, error), held);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -983,7 +983,9 @@ export const useFileStore = create<FileState>((set, get) => ({
|
||||
error: error instanceof Error ? error.message : 'Failed to list directory',
|
||||
hasListed: true,
|
||||
isLoading: false,
|
||||
resources: [],
|
||||
// A refresh that fails keeps what the folder showed. Another folder's
|
||||
// rows would be wrong here, so one being opened starts empty.
|
||||
resources: opensFolder ? [] : get().resources,
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user