feat: list files offline and keep pinned and recently opened files on the device

This commit is contained in:
Linus Rath
2026-11-03 05:47:39 +01:00
parent 7bb8447743
commit 698510fa08
10 changed files with 1504 additions and 1 deletions
+88
View File
@@ -0,0 +1,88 @@
// @vitest-environment node
/**
* The real JMAPClient listing files from the store, in each way a server says
* what a node is, plus what it does with a write.
*/
import { afterEach, describe, expect, it } from 'vitest';
import { JMAPClient } from '@/lib/jmap/client';
import { fileNodeKind } from '@/lib/jmap/filenode-kind';
import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks';
import { DEFAULT_SCOPE } from '../area';
import { AccountEngine } from '../engine/account-engine';
import { filesArea } from '../files';
import { FakeFiles, FILES, FILES_SESSION, SHARED_FILES_ACCOUNT, type FakeFilesOptions } from './helpers/fake-files';
import { openMemoryStore } from './helpers/node-adapter';
afterEach(() => setOfflineHooks(null));
async function offlineClient(server: FakeFiles): Promise<JMAPClient> {
const engine = new AccountEngine({
account: 'a@fake.test',
db: openMemoryStore(),
session: FILES_SESSION,
transport: server,
areas: [filesArea],
scope: DEFAULT_SCOPE,
sleep: async () => {},
});
await engine.requestSync();
engine.setOnline(false);
setOfflineHooks({
skipNetwork: () => true,
answer: (_client, calls) => engine.answer(calls),
record: () => {},
noteNetwork: () => {},
} as Partial<OfflineHooks> as OfflineHooks);
return JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', {
...FILES_SESSION,
capabilities: { ...FILES_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500 } },
});
}
describe('file listing offline', () => {
it.each<[string, FakeFilesOptions]>([
['nodeType (0.16.6 and later, 1.0 as tested)', { style: 'nodeType' }],
['blobId null only (0.16.0-5)', { style: 'blob' }],
['@type and mediaType (1.0.0)', { style: 'draft' }],
])('tells folders from files: %s', async (_label, options) => {
const server = new FakeFiles(options);
const folder = server.addFolder('Docs');
server.addFile('note.txt', folder.id);
const client = await offlineClient(server);
expect(client.supportsFiles()).toBe(true);
const nodes = await client.listAllFileNodes();
const byName = new Map(nodes.map((n) => [n.name, n]));
expect(fileNodeKind(byName.get('Docs') as never)).toBe('directory');
expect(fileNodeKind(byName.get('note.txt') as never)).toBe('file');
expect(byName.get('note.txt')).toMatchObject({ parentId: folder.id, type: 'text/plain', size: 10 });
expect((await client.listFileNodes(folder.id)).map((n) => n.name)).toEqual(['note.txt']);
expect((await client.listFileNodes(null)).map((n) => n.name)).toEqual(['Docs']);
});
it('lists an account larger than one /get, ending its paging cleanly', async () => {
const server = new FakeFiles({ getAll: 'refuse' });
const folder = server.addFolder('Docs');
for (let i = 0; i < 700; i += 1) server.addFile(`f${i}`, folder.id);
const client = await offlineClient(server);
expect(await client.listAllFileNodes()).toHaveLength(701);
});
it('lists shared accounts with namespaced ids', async () => {
const server = new FakeFiles();
server.addFolder('Mine');
const shared = server.addFolder('Theirs', null, undefined, SHARED_FILES_ACCOUNT);
const client = await offlineClient(server);
const nodes = await client.listAllFileNodesAcrossAccounts();
expect(nodes.find((n) => n.name === 'Theirs')).toMatchObject({ id: `${SHARED_FILES_ACCOUNT}:${shared.id}`, isShared: true });
expect(nodes.find((n) => n.name === 'Mine')).toMatchObject({ isShared: false });
});
it('fails a write with an error rather than losing it', async () => {
const server = new FakeFiles();
const client = await offlineClient(server);
await expect(client.createFileDirectory('New', null)).rejects.toThrow();
expect(FILES in FILES_SESSION.capabilities).toBe(true);
expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false);
});
});
+535
View File
@@ -0,0 +1,535 @@
// @vitest-environment node
import { describe, expect, it } from 'vitest';
import { DEFAULT_SCOPE, type StoreEvent, type StoreScope } from '../area';
import { AccountEngine } from '../engine/account-engine';
import { TransportError } from '../engine/transport';
import { fileBlob, FILE_CACHE_CAP_BYTES, filesArea } from '../files';
import type { StorageAdapter } from '../storage';
import { JMAP_ACCOUNT } from './helpers/fake-jmap';
import { FakeFiles, FILES_SESSION, SHARED_FILES_ACCOUNT } from './helpers/fake-files';
import { openMemoryStore } from './helpers/node-adapter';
const NOW = Date.UTC(2026, 10, 3);
/* eslint-disable @typescript-eslint/no-explicit-any -- answers are shaped like the server's */
interface Rig {
db: StorageAdapter;
engine: AccountEngine;
clock: { at: number };
events: StoreEvent[];
}
function rigFor(server: FakeFiles, scope: Partial<StoreScope> = {}): Rig {
const db = openMemoryStore();
const clock = { at: NOW };
const events: StoreEvent[] = [];
const engine = new AccountEngine({
account: 'a@fake.test',
db,
session: FILES_SESSION,
transport: server,
areas: [filesArea],
scope: { ...DEFAULT_SCOPE, ...scope },
emit: (event) => events.push(event),
now: () => clock.at,
sleep: async () => {},
});
return { db, engine, clock, events };
}
async function synced(server: FakeFiles, scope: Partial<StoreScope> = {}): Promise<Rig> {
const rig = rigFor(server, scope);
await rig.engine.requestSync();
return rig;
}
async function call(engine: AccountEngine, name: string, args: Record<string, unknown> = {}): Promise<any> {
const response = await engine.answer([[name, { accountId: JMAP_ACCOUNT, ...args }, '0']]);
if (!response) throw new Error('nothing answered');
const [answered, result] = response.methodResponses[0];
if (answered === 'error') throw new Error(result.type);
return result;
}
const ref = (id: string, jmap = JMAP_ACCOUNT) => ({ jmap, id });
function names(rows: Array<{ name?: unknown }>): unknown[] {
return rows.map((row) => row.name).sort();
}
describe('file metadata sync', () => {
it('stores small accounts from one FileNode/get', async () => {
const server = new FakeFiles();
const folder = server.addFolder('Docs');
server.addFile('a.txt', folder.id);
const { engine } = await synced(server);
expect(server.calls.filter(([n]) => n === 'FileNode/query')).toHaveLength(0);
expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['Docs', 'a.txt']);
});
it.each([
['0.16 cuts the list at 500', { getAll: 'cut' as const }],
['1.0 refuses a list over 500', { getAll: 'refuse' as const }],
])('pages past the /get limit: %s', async (_label, options) => {
const server = new FakeFiles(options);
const folder = server.addFolder('Docs');
for (let i = 0; i < 1234; i += 1) server.addFile(`f${i}.txt`, folder.id);
const { db } = await synced(server);
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(1235);
for (const [name, args] of server.calls) {
if (name === 'FileNode/get' && Array.isArray(args.ids)) expect(args.ids.length).toBeLessThanOrEqual(500);
}
});
it('finds folders the query leaves out on old servers, by the parents of what it did find', async () => {
const server = new FakeFiles({ queryOmitsFolders: true, style: 'blob' });
for (let i = 0; i < 600; i += 1) server.addFile(`early${i}.txt`);
const top = server.addFolder('Top');
const inner = server.addFolder('Inner', top.id);
server.addFile('deep.txt', inner.id);
const { db } = await synced(server);
const kinds = db.all<{ name: string; kind: string }>("SELECT name, kind FROM file_node WHERE name IN ('Top', 'Inner', 'deep.txt') ORDER BY name");
expect(kinds).toEqual([{ name: 'Inner', kind: 'directory' }, { name: 'Top', kind: 'directory' }, { name: 'deep.txt', kind: 'file' }]);
});
it('continues a stopped full sync from its cursor', async () => {
const server = new FakeFiles({ getAll: 'refuse' });
for (let i = 0; i < 6000; i += 1) server.addFile(`f${i}`);
const rig = rigFor(server);
let gets = 0;
server.beforeRequest = () => {
gets += 1;
// State capture, the refused list, a query page of 5000 and two requests of gets: then the network goes.
if (gets === 6) server.offline = true;
};
await rig.engine.requestSync();
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(5000);
server.offline = false;
server.beforeRequest = null;
server.calls.length = 0;
await rig.engine.requestSync();
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(6000);
expect(server.calls.find(([n]) => n === 'FileNode/query')?.[1].position).toBe(5000);
expect(server.calls.filter(([n, a]) => n === 'FileNode/get' && a.ids === null && a.accountId === JMAP_ACCOUNT)).toHaveLength(0);
});
it('follows changes through FileNode/changes', async () => {
const server = new FakeFiles();
const a = server.addFile('a.txt');
const b = server.addFile('b.txt');
const { engine } = await synced(server);
server.calls.length = 0;
server.renameNode(a.id, 'renamed.txt');
server.removeNode(b.id);
server.addFolder('New');
await engine.requestSync();
expect(server.calls.some(([n]) => n === 'FileNode/changes')).toBe(true);
expect(server.calls.some(([n, args]) => n === 'FileNode/get' && args.ids === null)).toBe(false);
expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['New', 'renamed.txt']);
});
it('starts over when the server can no longer give changes, and keeps a pin on a file still there', async () => {
const server = new FakeFiles();
const gone = server.addFile('gone.txt');
const kept = server.addFile('kept.txt');
const { db, engine } = await synced(server);
await engine.areaCall('files.pin', ref(kept.id));
await engine.requestSync();
server.removeNode(gone.id);
server.account().coll.truncatedBelow = server.account().coll.log.length;
await engine.requestSync();
expect(db.all('SELECT id FROM file_node').map((r) => r.id)).toEqual([kept.id]);
expect(db.get<{ pinned: number }>('SELECT pinned FROM file_node WHERE id = ?', [kept.id])?.pinned).toBe(1);
});
it('keeps accounts apart, even with the same ids', async () => {
const server = new FakeFiles();
server.addFile('own.txt', null, 5, { id: 'same' });
server.addFile('team.txt', null, 5, { id: 'same', blobId: 'team-blob' }, SHARED_FILES_ACCOUNT);
const { engine } = await synced(server);
expect(names((await call(engine, 'FileNode/get', { ids: ['same'] })).list)).toEqual(['own.txt']);
expect(names((await call(engine, 'FileNode/get', { ids: ['same'], accountId: SHARED_FILES_ACCOUNT })).list)).toEqual(['team.txt']);
});
});
describe('file answers', () => {
it('says so instead of an empty list when nothing was read yet', async () => {
const { engine } = rigFor(new FakeFiles());
expect(await engine.answer([['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '0']])).toBeNull();
});
it('gives the last server state and only asked-for properties', async () => {
const server = new FakeFiles();
const file = server.addFile('a.txt');
const { engine } = await synced(server);
const got = await call(engine, 'FileNode/get', { ids: [file.id, 'nope'], properties: ['name', 'parentId'] });
expect(got.state).toBe(server.account().coll.state());
expect(got.list).toEqual([{ id: file.id, name: 'a.txt', parentId: null }]);
expect(got.notFound).toEqual(['nope']);
});
it('answers a query by parent, also at the root, which the real server refuses to take', async () => {
const server = new FakeFiles();
const folder = server.addFolder('Docs');
const inside = server.addFile('in.txt', folder.id);
const root = server.addFile('root.txt');
const { engine } = await synced(server);
expect((await call(engine, 'FileNode/query', { filter: { parentId: folder.id } })).ids).toEqual([inside.id]);
expect((await call(engine, 'FileNode/query', { filter: { parentId: null } })).ids).toEqual([folder.id, root.id]);
});
it('pages and sorts a query', async () => {
const server = new FakeFiles();
server.addFile('b', null, 30);
server.addFile('a', null, 10);
server.addFile('c', null, 20);
const { engine } = await synced(server);
const page = await call(engine, 'FileNode/query', { filter: {}, position: 1, limit: 1, calculateTotal: true });
expect(page).toMatchObject({ position: 1, total: 3, ids: [expect.any(String)] });
const bySize = await call(engine, 'FileNode/query', { filter: {}, sort: [{ property: 'size', isAscending: false }] });
const list = (await call(engine, 'FileNode/get', { ids: bySize.ids })).list;
expect(list.map((n: { name: string }) => n.name)).toEqual(['b', 'c', 'a']);
});
it('refuses what it cannot read, for that call only', async () => {
const server = new FakeFiles();
server.addFile('a');
const { engine } = await synced(server);
const response = await engine.answer([
['FileNode/query', { accountId: JMAP_ACCOUNT, filter: { hasFoo: true } }, '0'],
['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'],
]);
expect(response?.methodResponses[0][1].type).toBe('unsupportedFilter');
expect(response?.methodResponses[1][0]).toBe('FileNode/get');
});
it('answers FileNode/set with a clear per-call error, not silence', async () => {
const server = new FakeFiles();
server.addFile('a');
const { engine } = await synced(server);
const response = await engine.answer([
['FileNode/set', { accountId: JMAP_ACCOUNT, create: { n0: { name: 'x' } } }, '0'],
['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'],
]);
expect(response?.methodResponses[0]).toEqual(['error', { type: 'serverUnavailable', description: 'FileNode/set needs the server' }, '0']);
expect(response?.methodResponses[1][0]).toBe('FileNode/get');
expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false);
});
});
describe('pinned files', () => {
it('fetches a pinned file and gives it back byte for byte, past the chunk size', async () => {
const server = new FakeFiles();
const content = new Uint8Array(3 * 1024 * 1024 + 17).map((_, i) => i % 251);
const file = server.addFile('big.bin', null, content);
const { db, engine } = await synced(server);
expect(server.downloads).toEqual([]);
expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: false });
await engine.requestSync();
expect(server.downloads).toEqual([{ jmap: JMAP_ACCOUNT, blobId: file.blobId }]);
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(4);
const read = await engine.areaCall('files.read', ref(file.id)) as any;
expect(read.name).toBe('big.bin');
expect(read.type).toBe('text/plain');
expect(Buffer.compare(read.data, content)).toBe(0);
expect(Buffer.compare(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)?.data as Uint8Array, content)).toBe(0);
expect(await engine.areaCall('files.blob', { jmap: JMAP_ACCOUNT, blobId: file.blobId })).toMatchObject({ size: content.length });
// Already kept: not fetched again.
await engine.requestSync();
expect(server.downloads).toHaveLength(1);
expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: true });
});
it('names a file with a %-encoded name from 0.16 WebDAV the way the client shows it', async () => {
const server = new FakeFiles();
const file = server.addFile('My%20Report.pdf');
const { engine } = await synced(server);
await engine.areaCall('files.pin', ref(file.id));
await engine.requestSync();
expect((await engine.areaCall('files.read', ref(file.id)) as any).name).toBe('My Report.pdf');
expect((await engine.areaCall('files.kept', undefined) as any).files[0].name).toBe('My Report.pdf');
// The stored node stays as the server sent it. The client decodes it once.
expect((await call(engine, 'FileNode/get', { ids: [file.id] })).list[0].name).toBe('My%20Report.pdf');
});
it('gives nothing for a file not kept', async () => {
const server = new FakeFiles();
const file = server.addFile('a.txt');
const { db, engine } = await synced(server);
expect(await engine.areaCall('files.read', ref(file.id))).toBeNull();
expect(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)).toBeNull();
});
it('refuses to pin a folder or an unknown id', async () => {
const server = new FakeFiles();
const folder = server.addFolder('Docs');
const { engine } = await synced(server);
await expect(engine.areaCall('files.pin', ref(folder.id))).rejects.toThrow('Only files');
await expect(engine.areaCall('files.pin', ref('nope'))).rejects.toThrow();
});
it('downloads one at a time', async () => {
const server = new FakeFiles();
const files = [server.addFile('a'), server.addFile('b'), server.addFile('c')];
const { engine } = await synced(server);
for (const file of files) await engine.areaCall('files.pin', ref(file.id));
await engine.requestSync();
expect(server.downloads).toHaveLength(3);
expect(server.mostAtOnce).toBe(1);
});
it('fetches a pinned file over the one-file limit, and skips an opened one', async () => {
const server = new FakeFiles();
const pinned = server.addFile('pinned.bin', null, 100);
const opened = server.addFile('opened.bin', null, 100);
const { engine } = await synced(server, { maxAttachmentBytes: 50 });
await engine.areaCall('files.pin', ref(pinned.id));
await engine.areaCall('files.noteOpened', ref(opened.id));
await engine.requestSync();
expect(server.downloads.map((d) => d.blobId)).toEqual([pinned.blobId]);
const status = await engine.areaCall('files.kept', undefined) as any;
const row = (name: string) => status.files.find((f: { name: string }) => f.name === name);
expect(row('pinned.bin')).toMatchObject({ pinned: true, kept: true, error: null });
expect(row('opened.bin')).toMatchObject({ pinned: false, kept: false, error: 'tooLarge' });
});
it('does not take pins past the total limit, and says so', async () => {
const server = new FakeFiles();
const a = server.addFile('a', null, 60);
const b = server.addFile('b', null, 60);
const { engine, clock } = await synced(server, { maxTotalBytes: 100 });
await engine.areaCall('files.pin', ref(a.id));
clock.at += 1;
await engine.areaCall('files.pin', ref(b.id));
await engine.requestSync();
expect(server.downloads).toHaveLength(1);
const status = await engine.areaCall('files.kept', undefined) as any;
expect(status.files.filter((f: { kept: boolean }) => f.kept)).toHaveLength(1);
expect(status.files.filter((f: { error: string | null }) => f.error === 'overLimit')).toHaveLength(1);
});
it('keeps nothing when the scope forbids it, and drops what was kept', async () => {
const server = new FakeFiles();
const file = server.addFile('a');
const rig = await synced(server);
await rig.engine.areaCall('files.pin', ref(file.id));
await rig.engine.requestSync();
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
rig.engine.setScope({ ...DEFAULT_SCOPE, attachments: false });
await rig.engine.requestSync();
expect(await rig.engine.areaCall('files.read', ref(file.id))).toBeNull();
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0);
expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ pinned: true, kept: false, error: 'notAllowed' });
rig.engine.setScope(DEFAULT_SCOPE);
await rig.engine.requestSync();
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
});
it('goes by the real size when the metadata has it wrong', async () => {
const server = new FakeFiles();
const opened = server.addFile('opened', null, new Uint8Array(100), { size: 1 });
const { engine } = await synced(server, { maxAttachmentBytes: 50 });
await engine.areaCall('files.noteOpened', ref(opened.id));
await engine.requestSync();
expect(server.downloads).toHaveLength(1);
expect(await engine.areaCall('files.read', ref(opened.id))).toBeNull();
expect((await engine.areaCall('files.kept', undefined) as any).files[0].error).toBe('tooLarge');
});
it('does not store a file unpinned while it was downloading', async () => {
const server = new FakeFiles();
const file = server.addFile('a');
const { engine } = await synced(server);
await engine.areaCall('files.pin', ref(file.id));
const original = server.download.bind(server);
server.download = async (jmap, blobId) => {
await engine.areaCall('files.unpin', ref(file.id));
return original(jmap, blobId);
};
await engine.requestSync();
expect(server.downloads).toHaveLength(1);
expect(await engine.areaCall('files.read', ref(file.id))).toBeNull();
});
it('lets a never-opened file go on unpin, and keeps an opened one as recent', async () => {
const server = new FakeFiles();
const never = server.addFile('never');
const opened = server.addFile('opened');
const { engine } = await synced(server);
await engine.areaCall('files.pin', ref(never.id));
await engine.areaCall('files.pin', ref(opened.id));
await engine.areaCall('files.noteOpened', ref(opened.id));
await engine.requestSync();
await engine.areaCall('files.unpin', ref(never.id));
await engine.areaCall('files.unpin', ref(opened.id));
expect(await engine.areaCall('files.read', ref(never.id))).toBeNull();
expect(await engine.areaCall('files.read', ref(opened.id))).not.toBeNull();
const status = await engine.areaCall('files.kept', undefined) as any;
expect(status.files.map((f: { name: string; pinned: boolean }) => [f.name, f.pinned])).toEqual([['opened', false]]);
});
it('drops the old content and fetches the new one when a pinned file changes', async () => {
const server = new FakeFiles();
const file = server.addFile('a.txt', null, new Uint8Array([1, 2, 3]));
const { db, engine } = await synced(server);
await engine.areaCall('files.pin', ref(file.id));
await engine.requestSync();
const old = file.blobId as string;
const fresh = server.replaceContent(file.id, new Uint8Array([9, 9]));
await engine.requestSync();
expect(fileBlob({ db }, JMAP_ACCOUNT, old)).toBeNull();
expect([...(fileBlob({ db }, JMAP_ACCOUNT, fresh)?.data ?? [])]).toEqual([9, 9]);
expect(Array.from((await engine.areaCall('files.read', ref(file.id)) as any).data)).toEqual([9, 9]);
});
it('drops the content when the file is removed on the server, unless another node shares it', async () => {
const server = new FakeFiles();
const a = server.addFile('a', null, 5, { blobId: 'shared-blob' });
const b = server.addFile('b', null, 5, { blobId: 'shared-blob' });
const { db, engine } = await synced(server);
await engine.areaCall('files.pin', ref(a.id));
await engine.areaCall('files.pin', ref(b.id));
await engine.requestSync();
expect(server.downloads).toHaveLength(1);
server.removeNode(a.id);
await engine.requestSync();
expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).not.toBeNull();
server.removeNode(b.id);
await engine.requestSync();
expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).toBeNull();
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0);
});
it('carries on past one bad file, and hands a network failure to the engine', async () => {
const server = new FakeFiles();
const bad = server.addFile('bad');
const good = server.addFile('good');
server.blobs.delete(bad.blobId as string);
const { engine, clock } = await synced(server);
await engine.areaCall('files.pin', ref(bad.id));
clock.at += 1;
await engine.areaCall('files.pin', ref(good.id));
await engine.requestSync();
expect(await engine.areaCall('files.read', ref(good.id))).not.toBeNull();
const status = await engine.areaCall('files.kept', undefined) as any;
expect(status.files.find((f: { name: string }) => f.name === 'bad').error).toBe('download');
});
it('hands a network failure to the engine, which ends the pass, and goes on next time', async () => {
const server = new FakeFiles();
const file = server.addFile('a');
const rig = await synced(server);
await rig.engine.areaCall('files.pin', ref(file.id));
const original = server.download.bind(server);
server.download = async () => {
throw new TransportError('network', 'Failed to fetch');
};
const syncedBefore = rig.events.filter((e) => e.type === 'synced').length;
await rig.engine.requestSync();
expect(rig.events.some((e) => e.type === 'error')).toBe(false);
expect(rig.events.filter((e) => e.type === 'synced').length).toBe(syncedBefore);
expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ kept: false, error: null });
server.download = original;
await rig.engine.requestSync();
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
});
});
describe('recently opened files', () => {
it('keeps an opened file without a pin', async () => {
const server = new FakeFiles();
const file = server.addFile('a');
const { engine } = await synced(server);
expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: false });
await engine.requestSync();
expect(await engine.areaCall('files.read', ref(file.id))).not.toBeNull();
expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: true });
});
it('pushes out the oldest unpinned first, and never a pinned one', async () => {
const server = new FakeFiles();
const files = Object.fromEntries(['old', 'pinnedOldest', 'mid', 'new', 'newest'].map((name) => [name, server.addFile(name, null, 10)]));
const rig = await synced(server, { maxTotalBytes: 30 });
const open = async (name: string) => {
rig.clock.at += 1000;
await rig.engine.areaCall('files.noteOpened', ref(files[name].id));
await rig.engine.requestSync();
};
await open('pinnedOldest');
await rig.engine.areaCall('files.pin', ref(files.pinnedOldest.id));
await open('old');
await open('mid');
const kept = async () => (await rig.engine.areaCall('files.kept', undefined) as any).files
.filter((f: { kept: boolean }) => f.kept).map((f: { name: string }) => f.name).sort();
expect(await kept()).toEqual(['mid', 'old', 'pinnedOldest']);
await open('new');
expect(await kept()).toEqual(['mid', 'new', 'pinnedOldest']);
await open('mid');
await open('newest');
expect(await kept()).toEqual(['mid', 'newest', 'pinnedOldest']);
expect((await rig.engine.areaCall('files.kept', undefined) as any).bytes).toBe(30);
// A pinned file was never pushed out and had to come back.
expect(server.downloads.filter((d) => d.blobId === files.pinnedOldest.blobId)).toHaveLength(1);
// Pushed out means forgotten: the next pass doesn't fetch it straight back.
expect(server.downloads.filter((d) => d.blobId === files.old.blobId)).toHaveLength(1);
const count = server.downloads.length;
await rig.engine.requestSync();
expect(server.downloads).toHaveLength(count);
expect((await rig.engine.areaCall('files.kept', undefined) as any).files.map((f: { name: string }) => f.name)).not.toContain('old');
});
it('pushes out recent files for a pin, and keeps a pin past the cap', async () => {
const server = new FakeFiles();
const recent = server.addFile('recent', null, 20);
const pin = server.addFile('pin', null, 20);
const rig = await synced(server, { maxTotalBytes: 25 });
await rig.engine.areaCall('files.noteOpened', ref(recent.id));
await rig.engine.requestSync();
rig.clock.at += 1000;
await rig.engine.areaCall('files.pin', ref(pin.id));
await rig.engine.requestSync();
expect(await rig.engine.areaCall('files.read', ref(recent.id))).toBeNull();
expect(await rig.engine.areaCall('files.read', ref(pin.id))).not.toBeNull();
});
it('keeps recently opened files under 500 MB whatever the total limit is', async () => {
const server = new FakeFiles();
const file = server.addFile('new', null, 10);
const rig = await synced(server, { maxTotalBytes: 2 * FILE_CACHE_CAP_BYTES });
// As if an older file already took all of it: only the row counts here.
rig.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [JMAP_ACCOUNT, 'huge', 'x', FILE_CACHE_CAP_BYTES, NOW - 1000]);
rig.db.run("INSERT INTO file_node (jmap, id, name, kind, blob_id, size, json, opened_at) VALUES (?, 'hugenode', 'huge', 'file', 'huge', ?, '{}', ?)", [JMAP_ACCOUNT, FILE_CACHE_CAP_BYTES, NOW - 1000]);
await rig.engine.areaCall('files.noteOpened', ref(file.id));
await rig.engine.requestSync();
expect(fileBlob({ db: rig.db }, JMAP_ACCOUNT, 'huge')).toBeNull();
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
expect((await rig.engine.areaCall('files.kept', undefined) as any).capBytes).toBe(FILE_CACHE_CAP_BYTES);
});
it('drops an old opened file that does not fit rather than a newer one', async () => {
const server = new FakeFiles();
const a = server.addFile('a', null, 20);
const b = server.addFile('b', null, 20);
const rig = await synced(server, { maxTotalBytes: 25 });
await rig.engine.areaCall('files.noteOpened', ref(a.id));
rig.clock.at += 1000;
await rig.engine.areaCall('files.noteOpened', ref(b.id));
await rig.engine.requestSync();
expect(await rig.engine.areaCall('files.read', ref(b.id))).not.toBeNull();
expect(await rig.engine.areaCall('files.read', ref(a.id))).toBeNull();
expect(server.downloads.map((d) => d.blobId)).toEqual([b.blobId, a.blobId]);
const count = server.downloads.length;
await rig.engine.requestSync();
expect(server.downloads).toHaveLength(count);
});
});
/* eslint-enable @typescript-eslint/no-explicit-any */
+192
View File
@@ -0,0 +1,192 @@
/**
* FakeJmap with files, as measured on both Stalwarts (2026-11-03) and as the
* client's comments record: `filter: {parentId: null}` is an HTTP 400 that
* fails the whole request, a folder has null blobId, type and size. What differs
* between versions is set per server:
* - `style` is how a node says what it is: `nodeType` (0.16.6 and later, 1.0 as
* tested), `blob` (0.16.0-5: nothing but blobId null), `draft` (1.0.0: `@type`
* and `mediaType`).
* - `getAll`: ids null past 500 is silently cut on 0.16 and refused on 1.0.
* - `queryOmitsFolders`: FileNode/query before 0.16.6.
*/
import { TransportError, type StoreSession } from '../../engine/transport';
import { changesFor } from './fake-contacts';
import { Collection, FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './fake-jmap';
export const FILES = 'urn:ietf:params:jmap:filenode';
export const SHARED_FILES_ACCOUNT = 'shared1';
export const FILES_SESSION: StoreSession = {
...FAKE_SESSION,
primaryAccounts: { ...FAKE_SESSION.primaryAccounts, [FILES]: JMAP_ACCOUNT },
accounts: {
[JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [FILES]: {} } },
[SHARED_FILES_ACCOUNT]: { name: 'team@example.org', isPersonal: false, accountCapabilities: { [FILES]: {} } },
},
capabilities: { [FILES]: {} },
};
export interface FakeNode {
id: string;
parentId: string | null;
name: string;
blobId: string | null;
type: string | null;
size: number | null;
modified?: string;
}
export type FolderStyle = 'nodeType' | 'blob' | 'draft';
export interface FakeFilesOptions {
style?: FolderStyle;
getAll?: 'cut' | 'refuse';
queryOmitsFolders?: boolean;
}
class FilesAccount {
readonly nodes = new Map<string, FakeNode>();
readonly coll = new Collection();
}
export class FakeFiles extends FakeJmap {
private readonly fileAccounts = new Map<string, FilesAccount>();
private readonly options: Required<FakeFilesOptions>;
private nextNode = 1;
readonly blobs = new Map<string, Uint8Array>();
/** In order, with the account. */
readonly downloads: Array<{ jmap: string; blobId: string }> = [];
downloading = 0;
mostAtOnce = 0;
constructor(options: FakeFilesOptions = {}) {
super();
this.options = { style: 'nodeType', getAll: 'cut', queryOmitsFolders: false, ...options };
}
account(id: string = JMAP_ACCOUNT): FilesAccount {
let found = this.fileAccounts.get(id);
if (!found) {
found = new FilesAccount();
this.fileAccounts.set(id, found);
}
return found;
}
addFolder(name: string, parentId: string | null = null, id?: string, accountId: string = JMAP_ACCOUNT): FakeNode {
const account = this.account(accountId);
const node: FakeNode = { id: id ?? `n${this.nextNode++}`, parentId, name, blobId: null, type: null, size: null };
account.nodes.set(node.id, node);
account.coll.push('created', node.id);
return node;
}
addFile(name: string, parentId: string | null = null, content: Uint8Array | number = 10, extra: Partial<FakeNode> = {}, accountId: string = JMAP_ACCOUNT): FakeNode {
const account = this.account(accountId);
const id = extra.id ?? `n${this.nextNode++}`;
const blobId = extra.blobId ?? `blob-${id}`;
const data = typeof content === 'number' ? new Uint8Array(content).fill(id.length) : content;
this.blobs.set(blobId, data);
const node: FakeNode = { id, parentId, name, blobId, type: 'text/plain', size: data.length, ...extra };
account.nodes.set(id, node);
account.coll.push('created', id);
return node;
}
/** A new version of the file: new blob, same node. */
replaceContent(id: string, content: Uint8Array, accountId: string = JMAP_ACCOUNT): string {
const account = this.account(accountId);
const node = account.nodes.get(id) as FakeNode;
const blobId = `${node.blobId as string}-v`;
this.blobs.set(blobId, content);
node.blobId = blobId;
node.size = content.length;
account.coll.push('updated', id);
return blobId;
}
renameNode(id: string, name: string, accountId: string = JMAP_ACCOUNT): void {
const account = this.account(accountId);
(account.nodes.get(id) as FakeNode).name = name;
account.coll.push('updated', id);
}
removeNode(id: string, accountId: string = JMAP_ACCOUNT): void {
const account = this.account(accountId);
account.nodes.delete(id);
account.coll.push('destroyed', id);
}
private wire(node: FakeNode, properties: string[] | undefined): Record<string, unknown> {
const isFolder = node.blobId === null;
const full: Record<string, unknown> = { id: node.id, parentId: node.parentId, name: node.name, blobId: node.blobId, size: node.size, created: '2026-10-01T00:00:00Z', modified: node.modified ?? '2026-10-01T00:00:00Z' };
if (this.options.style === 'draft') {
full['@type'] = isFolder ? 'Directory' : 'File';
full.mediaType = node.type;
} else {
full.type = node.type;
if (this.options.style === 'nodeType') full.nodeType = isFolder ? 'directory' : 'file';
}
if (!properties) return full;
const out: Record<string, unknown> = { id: node.id };
for (const p of properties) if (p in full) out[p] = full[p];
return out;
}
protected override handle(name: string, args: Record<string, unknown>): Record<string, unknown> {
if (!name.startsWith('FileNode/')) return super.handle(name, args);
const account = this.account(String(args.accountId));
switch (name) {
case 'FileNode/changes':
return changesFor(account.coll, args);
case 'FileNode/get': {
const ids = args.ids as string[] | null;
if (ids && ids.length > 500) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' });
if (!ids && account.nodes.size > 500 && this.options.getAll === 'refuse') {
throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' });
}
const chosen = ids ?? [...account.nodes.keys()].slice(0, 500);
const properties = args.properties as string[] | undefined;
return {
accountId: args.accountId,
state: account.coll.state(),
list: chosen.filter((id) => account.nodes.has(id)).map((id) => this.wire(account.nodes.get(id) as FakeNode, properties)),
notFound: chosen.filter((id) => !account.nodes.has(id)),
};
}
case 'FileNode/query': {
const filter = (args.filter ?? {}) as { parentId?: string | null };
// Measured on both servers: this fails the whole request.
if (filter.parentId === null) throw new TransportError('http', 'JMAP request failed: 400', 400, 'urn:ietf:params:jmap:error:notRequest');
let list = [...account.nodes.values()];
if (this.options.queryOmitsFolders) list = list.filter((n) => n.blobId !== null);
if (filter.parentId) list = list.filter((n) => n.parentId === filter.parentId);
const position = Number(args.position ?? 0);
const limit = Math.min(Number(args.limit ?? 5000) || 5000, 5000);
return {
accountId: args.accountId,
queryState: account.coll.state(),
canCalculateChanges: true,
position,
ids: list.slice(position, position + limit).map((n) => n.id),
...(args.calculateTotal ? { total: list.length } : {}),
};
}
default:
throw Object.assign(new Error(name), { jmapError: 'unknownMethod' });
}
}
override async download(jmap: string, blobId: string): Promise<Uint8Array> {
if (this.offline) throw new TransportError('network', 'Failed to fetch');
this.downloads.push({ jmap, blobId });
this.downloading += 1;
this.mostAtOnce = Math.max(this.mostAtOnce, this.downloading);
await Promise.resolve();
this.downloading -= 1;
const data = this.blobs.get(blobId);
if (!data) throw new TransportError('http', 'Download failed: 404', 404);
return data;
}
}
+2 -1
View File
@@ -2,6 +2,7 @@
import type { StoreArea } from './area';
import { contactsArea } from './contacts';
import { filesArea } from './files';
import { mailArea } from './mail';
export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea];
export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea, filesArea];
+27
View File
@@ -0,0 +1,27 @@
import type { StoreArea } from '../area';
import { fileBlob, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep';
import { FILE_METHODS } from './methods';
import { FILE_MIGRATIONS } from './schema';
import { filesSyncer } from './sync';
export { fileBlob, FILE_CACHE_CAP_BYTES } from './keep';
export const filesArea: StoreArea = {
name: 'files',
migrations: FILE_MIGRATIONS,
methods: FILE_METHODS,
syncers: [filesSyncer],
calls: {
/** Keep this file's content on the device until it's unpinned. */
'files.pin': (ctx, params: NodeRef) => pinFile(ctx, params),
'files.unpin': (ctx, params: NodeRef) => unpinFile(ctx, params),
/** The page opened this file: keep its content for a while. */
'files.noteOpened': (ctx, params: NodeRef) => noteOpened(ctx, params),
/** A kept file's content and name, or null. */
'files.read': (ctx, params: NodeRef) => readFile(ctx, params),
/** A kept content by blob id, or null. */
'files.blob': (ctx, params: { jmap: string; blobId: string }) => fileBlob(ctx, params.jmap, params.blobId),
/** Pinned and recently opened files, for a settings page. */
'files.kept': (ctx) => keptStatus(ctx),
},
};
+295
View File
@@ -0,0 +1,295 @@
/**
* File contents kept on the device: pinned files, and recently opened ones
* within a cap. Metadata for every file is in file_node regardless.
*
* What gets kept:
* - A pinned file stays until it's unpinned or removed on the server. The cap
* doesn't push it out and the size limit for one file doesn't apply. The
* account's total limit (scope.maxTotalBytes, which a policy can lower) does.
* - A recently opened file is kept if it's within scope.maxAttachmentBytes, and
* pushed out oldest first when the cap is reached. Its opened_at goes with
* it, so it isn't fetched straight back.
* - With scope.attachments off, nothing is kept.
*/
import { TransportError } from '../engine/transport';
import { MethodError, type StoreContext, type SyncContext } from '../area';
import type { StorageAdapter } from '../storage';
import { nodeMediaType, type WireNode } from './records';
export const FILE_CACHE_CAP_BYTES = 500 * 1024 * 1024;
const CHUNK_BYTES = 1024 * 1024;
export interface KeptFile { data: Uint8Array; type: string; size: number }
export interface NodeRef { jmap: string; id: string }
interface NodeRow {
jmap: string;
id: string;
name: string;
kind: string;
blob_id: string | null;
size: number | null;
json: string;
pinned: number;
opened_at: number | null;
keep_error: string | null;
}
interface BlobRow {
jmap: string;
blob_id: string;
size: number;
kept_at: number;
pinned: number;
opened_at: number | null;
}
function capBytes(ctx: Pick<StoreContext, 'scope'>): number {
return Math.min(FILE_CACHE_CAP_BYTES, ctx.scope.maxTotalBytes);
}
/** A kept file's content, for the client's blob lookup. */
export function fileBlob(ctx: Pick<StoreContext, 'db'>, jmap: string, blobId: string): KeptFile | null {
const row = ctx.db.get<{ type: string; size: number }>('SELECT type, size FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
if (!row) return null;
const data = new Uint8Array(Number(row.size));
let at = 0;
for (const chunk of ctx.db.all<{ data: Uint8Array }>('SELECT data FROM file_blob_chunk WHERE jmap = ? AND blob_id = ? ORDER BY seq', [jmap, blobId])) {
data.set(chunk.data, at);
at += chunk.data.length;
}
return { data, type: row.type, size: Number(row.size) };
}
function hasBlob(db: StorageAdapter, jmap: string, blobId: string): boolean {
return Boolean(db.get('SELECT 1 AS x FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]));
}
function dropBlob(db: StorageAdapter, jmap: string, blobId: string): void {
db.run('DELETE FROM file_blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
db.run('DELETE FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
}
function storeBlob(ctx: StoreContext, jmap: string, blobId: string, type: string, data: Uint8Array): void {
dropBlob(ctx.db, jmap, blobId);
ctx.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [jmap, blobId, type, data.length, ctx.now()]);
for (let seq = 0, at = 0; at < data.length; seq += 1, at += CHUNK_BYTES) {
ctx.db.run('INSERT INTO file_blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, data.subarray(at, at + CHUNK_BYTES)]);
}
}
/** All kept contents in the store, with what the nodes pointing at each say about it. */
function keptBlobs(db: StorageAdapter): BlobRow[] {
return db.all<BlobRow>(
`SELECT b.jmap, b.blob_id, b.size, b.kept_at, COALESCE(MAX(n.pinned), 0) AS pinned, MAX(n.opened_at) AS opened_at
FROM file_blob b LEFT JOIN file_node n ON n.jmap = b.jmap AND n.blob_id = b.blob_id
GROUP BY b.jmap, b.blob_id`,
);
}
function ageOf(blob: BlobRow): number {
return Number(blob.opened_at ?? blob.kept_at);
}
function pushOut(db: StorageAdapter, blob: BlobRow): void {
dropBlob(db, blob.jmap, blob.blob_id);
db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]);
}
/**
* Pushes out unpinned contents, oldest first, so `size` more fits under the
* cap. A recently opened file only pushes out older ones and says false when
* that isn't enough. A pinned one always gets in.
*/
function makeRoom(ctx: StoreContext, size: number, incoming: { pinned: boolean; openedAt: number }): boolean {
const cap = capBytes(ctx);
const blobs = keptBlobs(ctx.db);
let used = blobs.reduce((sum, blob) => sum + Number(blob.size), 0);
const candidates = blobs
.filter((blob) => !blob.pinned && (incoming.pinned || ageOf(blob) < incoming.openedAt))
.sort((a, b) => ageOf(a) - ageOf(b));
const chosen: BlobRow[] = [];
for (const blob of candidates) {
if (used + size <= cap) break;
used -= Number(blob.size);
chosen.push(blob);
}
if (used + size > cap && !incoming.pinned) return false;
for (const blob of chosen) pushOut(ctx.db, blob);
return true;
}
function pinnedBytes(db: StorageAdapter): number {
return keptBlobs(db).filter((blob) => blob.pinned).reduce((sum, blob) => sum + Number(blob.size), 0);
}
function setError(db: StorageAdapter, jmap: string, blobId: string, error: string | null): void {
db.run('UPDATE file_node SET keep_error = ? WHERE jmap = ? AND blob_id = ?', [error, jmap, blobId]);
}
/** Contents no node points at any more: the file was removed or has a new version. */
export function dropOrphanBlobs(db: StorageAdapter): void {
for (const blob of keptBlobs(db)) {
const used = db.get('SELECT 1 AS x FROM file_node WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]);
if (!used) dropBlob(db, blob.jmap, blob.blob_id);
}
}
interface Wanted { blob_id: string; id: string; name: string; size: number | null; json: string; pinned: number; opened_at: number | null }
function wantedRow(db: StorageAdapter, jmap: string, blobId: string): Wanted | undefined {
return db.get<Wanted>(
`SELECT blob_id, id, name, size, json, pinned, opened_at FROM file_node
WHERE jmap = ? AND blob_id = ? AND (pinned = 1 OR opened_at IS NOT NULL)
ORDER BY pinned DESC, opened_at DESC LIMIT 1`,
[jmap, blobId],
);
}
/** Fetches what should be kept and isn't yet, one file at a time, pinned files first. */
export async function fetchKeptFiles(ctx: SyncContext, jmap: string): Promise<void> {
const { db } = ctx;
if (!ctx.scope.attachments) {
db.transaction(() => {
for (const blob of keptBlobs(db)) dropBlob(db, blob.jmap, blob.blob_id);
db.run("UPDATE file_node SET keep_error = 'notAllowed' WHERE jmap = ? AND (pinned = 1 OR opened_at IS NOT NULL)", [jmap]);
});
return;
}
db.transaction(() => {
db.run("UPDATE file_node SET keep_error = NULL WHERE jmap = ? AND keep_error = 'notAllowed'", [jmap]);
// The limit may have gone down since last time.
makeRoom(ctx, 0, { pinned: true, openedAt: 0 });
});
const todo = db.all<{ blob_id: string }>(
`SELECT n.blob_id FROM file_node n
WHERE n.jmap = ? AND n.blob_id IS NOT NULL AND (n.pinned = 1 OR n.opened_at IS NOT NULL)
AND NOT EXISTS (SELECT 1 FROM file_blob b WHERE b.jmap = n.jmap AND b.blob_id = n.blob_id)
GROUP BY n.blob_id ORDER BY MAX(n.pinned) DESC, MAX(n.opened_at) DESC`,
[jmap],
).map((row) => String(row.blob_id));
let done = 0;
for (const blobId of todo) {
if (ctx.stopped()) return;
// The page may have changed its mind during the last download.
const want = wantedRow(db, jmap, blobId);
if (!want) continue;
const known = Number(want.size ?? 0);
if (!want.pinned && known > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) {
setError(db, jmap, blobId, 'tooLarge');
continue;
}
if (want.pinned && pinnedBytes(db) + known > ctx.scope.maxTotalBytes) {
setError(db, jmap, blobId, 'overLimit');
continue;
}
let data: Uint8Array;
const type = nodeMediaType(JSON.parse(want.json) as WireNode);
try {
data = await ctx.transport.download(jmap, blobId, type, want.name);
} catch (error) {
// No network or no credentials reach the engine. One bad file doesn't stop the others.
if (error instanceof TransportError && error.kind === 'http') {
setError(db, jmap, blobId, 'download');
continue;
}
throw error;
}
db.transaction(() => {
const now = wantedRow(db, jmap, blobId);
if (!now) return;
if (now.pinned ? pinnedBytes(db) + data.length > ctx.scope.maxTotalBytes : data.length > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) {
// The size in the metadata was wrong or missing. With the real one
// noted, the next pass sees this without downloading it again.
db.run('UPDATE file_node SET size = ? WHERE jmap = ? AND blob_id = ?', [data.length, jmap, blobId]);
setError(db, jmap, blobId, now.pinned ? 'overLimit' : 'tooLarge');
return;
}
if (!makeRoom(ctx, data.length, { pinned: Boolean(now.pinned), openedAt: Number(now.opened_at ?? 0) })) {
db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ? AND pinned = 0', [jmap, blobId]);
return;
}
storeBlob(ctx, jmap, blobId, type, data);
setError(db, jmap, blobId, null);
});
done += 1;
ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'contents', done, total: todo.length });
}
if (done > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] });
}
// ---- Calls from the page ----
function fileRow(db: StorageAdapter, { jmap, id }: NodeRef): NodeRow {
const row = db.get<NodeRow>('SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
if (!row) throw new MethodError('notFound', `No file ${id}`);
if (!row.blob_id || row.kind !== 'file') throw new MethodError('invalidArguments', 'Only files can be kept');
return row;
}
export function pinFile(ctx: StoreContext, ref: NodeRef): { kept: boolean } {
const row = fileRow(ctx.db, ref);
ctx.db.run('UPDATE file_node SET pinned = 1, keep_error = NULL WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]);
const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string);
if (!kept) void ctx.requestSync();
return { kept };
}
/** Back to being a recently opened file, or let go of it if it never was one. */
export function unpinFile(ctx: StoreContext, ref: NodeRef): void {
const row = fileRow(ctx.db, ref);
ctx.db.transaction(() => {
ctx.db.run('UPDATE file_node SET pinned = 0 WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]);
if (!wantedRow(ctx.db, ref.jmap, row.blob_id as string)) dropBlob(ctx.db, ref.jmap, row.blob_id as string);
});
}
export function noteOpened(ctx: StoreContext, ref: NodeRef): { kept: boolean } {
const row = fileRow(ctx.db, ref);
ctx.db.run('UPDATE file_node SET opened_at = ?, keep_error = NULL WHERE jmap = ? AND id = ?', [ctx.now(), ref.jmap, ref.id]);
const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string);
if (!kept) void ctx.requestSync();
return { kept };
}
export function readFile(ctx: StoreContext, ref: NodeRef): (KeptFile & { name: string }) | null {
const row = fileRow(ctx.db, ref);
const kept = fileBlob(ctx, ref.jmap, row.blob_id as string);
return kept ? { ...kept, name: row.name } : null;
}
export interface KeptFileStatus {
jmap: string;
id: string;
name: string;
size: number | null;
pinned: boolean;
openedAt: number | null;
kept: boolean;
/** Why it isn't kept: tooLarge, overLimit, notAllowed or download. */
error: string | null;
}
/** Pinned and recently opened files, and how much room their contents take. */
export function keptStatus(ctx: StoreContext): { files: KeptFileStatus[]; bytes: number; capBytes: number } {
const rows = ctx.db.all<NodeRow>(
`SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node
WHERE pinned = 1 OR opened_at IS NOT NULL ORDER BY pinned DESC, opened_at DESC, name`,
);
return {
files: rows.map((row) => ({
jmap: row.jmap,
id: row.id,
name: row.name,
size: row.size === null ? null : Number(row.size),
pinned: Boolean(row.pinned),
openedAt: row.opened_at === null ? null : Number(row.opened_at),
kept: row.blob_id !== null && hasBlob(ctx.db, row.jmap, row.blob_id),
error: row.keep_error,
})),
bytes: keptBlobs(ctx.db).reduce((sum, blob) => sum + Number(blob.size), 0),
capBytes: capBytes(ctx),
};
}
+108
View File
@@ -0,0 +1,108 @@
/**
* FileNode reads answered from the store, so listings work offline. Nothing
* here answers FileNode/set: the responder gives it a per-call
* serverUnavailable, since the design has no offline file writes.
*/
import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area';
import type { SqlValue } from '../storage';
import { FILES_CAPABILITY } from './records';
function jmapOf(ctx: MethodContext, args: Record<string, unknown>): string {
return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[FILES_CAPABILITY] ?? ctx.defaultJmap;
}
/** The last state from the server. Throws when the list was never read, so an empty answer always means empty. */
function syncedState(ctx: MethodContext, jmap: string): string | null {
const row = ctx.db.get<{ state: string | null; phase: string | null }>("SELECT state, phase FROM sync_state WHERE jmap = ? AND type = 'FileNode'", [jmap]);
if (!row || (!row.state && !row.phase)) throw new MethodError('serverUnavailable', 'Files not kept yet');
return row.state;
}
function pick(node: Record<string, unknown>, properties: unknown): Record<string, unknown> {
if (!Array.isArray(properties)) return node;
const out: Record<string, unknown> = { id: node.id };
for (const property of properties as string[]) if (property in node) out[property] = node[property];
return out;
}
const SORTS: Record<string, string> = {
name: 'name COLLATE NOCASE',
size: 'size',
created: "json_extract(json, '$.created')",
modified: "json_extract(json, '$.modified')",
};
function orderSql(sort: unknown): string {
const parts: string[] = [];
for (const level of (Array.isArray(sort) ? sort : []) as Array<{ property?: string; isAscending?: boolean }>) {
const column = SORTS[String(level.property)];
if (!column) throw new MethodError('unsupportedSort', `${String(level.property)} isn't available offline`);
parts.push(`${column} ${level.isAscending === false ? 'DESC' : 'ASC'}`);
}
parts.push('name COLLATE NOCASE ASC', 'id ASC');
return parts.join(', ');
}
const fileNodeGet: MethodHandler = (ctx, args) => {
const jmap = jmapOf(ctx, args);
const state = syncedState(ctx, jmap);
const list: Array<Record<string, unknown>> = [];
const notFound: string[] = [];
if (Array.isArray(args.ids)) {
for (const id of args.ids as string[]) {
const row = ctx.db.get<{ json: string }>('SELECT json FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
if (row) list.push(pick(JSON.parse(row.json) as Record<string, unknown>, args.properties));
else notFound.push(id);
}
} else {
// The whole tree at once. The client reads a long list as possibly cut,
// pages FileNode/query to check, and finds nothing missing.
for (const row of ctx.db.all<{ json: string }>('SELECT json FROM file_node WHERE jmap = ?', [jmap])) {
list.push(pick(JSON.parse(row.json) as Record<string, unknown>, args.properties));
}
}
return { accountId: jmap, ...(state ? { state } : {}), list, notFound };
};
const fileNodeQuery: MethodHandler = (ctx, args) => {
const jmap = jmapOf(ctx, args);
syncedState(ctx, jmap);
if (args.anchor !== undefined) throw new MethodError('unsupportedFilter', 'anchor needs the server');
const filter = (args.filter ?? {}) as Record<string, unknown>;
const conditions = ['jmap = ?'];
const params: SqlValue[] = [jmap];
for (const [key, value] of Object.entries(filter)) {
if (key !== 'parentId') throw new MethodError('unsupportedFilter', `${key} isn't available offline`);
if (value === null) {
conditions.push('parent_id IS NULL');
} else {
conditions.push('parent_id = ?');
params.push(String(value));
}
}
const where = conditions.join(' AND ');
const position = Math.max(0, Number(args.position ?? 0));
const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1;
const ids = ctx.db.all<{ id: string }>(
`SELECT id FROM file_node WHERE ${where} ORDER BY ${orderSql(args.sort)} LIMIT ? OFFSET ?`,
[...params, limit, position],
).map((row) => String(row.id));
const result: MethodResult = {
accountId: jmap,
// Local answers aren't a position in the server's log, so this is a token only.
queryState: 'local',
canCalculateChanges: false,
position,
ids,
};
if (args.calculateTotal) {
result.total = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM file_node WHERE ${where}`, params)?.n ?? 0);
}
return result;
};
export const FILE_METHODS: Readonly<Record<string, MethodHandler>> = {
'FileNode/get': fileNodeGet,
'FileNode/query': fileNodeQuery,
};
+50
View File
@@ -0,0 +1,50 @@
/** How a FileNode becomes a row. The server's own copy stays in `json`, in whichever draft it speaks. */
import { fileNodeKind, fromDraftFileNode } from '@/lib/jmap/filenode-kind';
import { decodeFileNodeName } from '@/lib/jmap/filenode-name';
import type { FileNode } from '@/lib/jmap/types';
import type { StorageAdapter } from '../storage';
export const FILES_CAPABILITY = 'urn:ietf:params:jmap:filenode';
export const FILES_USING = ['urn:ietf:params:jmap:core', FILES_CAPABILITY];
/** The client's list (lib/jmap/client.ts). Stalwart leaves out shareWith and myRights unless asked. */
export const FILE_NODE_PROPERTIES = [
'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified',
'nodeType', '@type', 'mediaType', 'shareWith', 'myRights',
];
export type WireNode = Record<string, unknown> & { id: string };
/** The node's media type, whichever draft named it. */
export function nodeMediaType(node: WireNode): string {
const { type } = fromDraftFileNode(node as unknown as Partial<FileNode> & { '@type'?: unknown; mediaType?: unknown });
return typeof type === 'string' && type ? type : 'application/octet-stream';
}
export function writeNode(db: StorageAdapter, jmap: string, node: WireNode, gen: number): void {
const normal = fromDraftFileNode(node as unknown as FileNode & { '@type'?: unknown; mediaType?: unknown });
db.run(
`INSERT INTO file_node (jmap, id, parent_id, name, kind, blob_id, size, json, gen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (jmap, id) DO UPDATE SET parent_id = excluded.parent_id, name = excluded.name, kind = excluded.kind,
size = excluded.size, json = excluded.json, gen = excluded.gen,
keep_error = CASE WHEN blob_id IS excluded.blob_id THEN keep_error END,
blob_id = excluded.blob_id`,
[
jmap,
node.id,
typeof node.parentId === 'string' ? node.parentId : null,
// The client decodes names itself from `json`. This column is for the store's own lists.
typeof node.name === 'string' ? decodeFileNodeName(node.name) : '',
fileNodeKind(normal),
typeof node.blobId === 'string' ? node.blobId : null,
typeof node.size === 'number' ? node.size : null,
JSON.stringify(node),
gen,
],
);
}
export function removeNode(db: StorageAdapter, jmap: string, id: string): void {
db.run('DELETE FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
}
+47
View File
@@ -0,0 +1,47 @@
import type { Migration } from '../storage';
export const FILE_MIGRATIONS: readonly Migration[] = [
{
id: 700,
name: 'files',
sql: `
CREATE TABLE file_node (
jmap TEXT NOT NULL,
id TEXT NOT NULL,
parent_id TEXT,
name TEXT NOT NULL DEFAULT '',
-- file, directory or symlink, whichever way the server says so.
kind TEXT NOT NULL,
blob_id TEXT,
size INTEGER,
json TEXT NOT NULL,
gen INTEGER NOT NULL DEFAULT 0,
pinned INTEGER NOT NULL DEFAULT 0,
-- When the page last opened it. Cleared when its content is pushed out.
opened_at INTEGER,
-- Why the content isn't kept: tooLarge, overLimit, notAllowed or download.
keep_error TEXT,
PRIMARY KEY (jmap, id)
) WITHOUT ROWID;
CREATE INDEX file_node_parent ON file_node (jmap, parent_id);
CREATE INDEX file_node_blob ON file_node (jmap, blob_id);
CREATE TABLE file_blob (
jmap TEXT NOT NULL,
blob_id TEXT NOT NULL,
type TEXT NOT NULL,
size INTEGER NOT NULL,
kept_at INTEGER NOT NULL,
PRIMARY KEY (jmap, blob_id)
) WITHOUT ROWID;
-- In pieces: a whole file bound at once would grow the WASM heap by its
-- size, and that memory never comes back.
CREATE TABLE file_blob_chunk (
jmap TEXT NOT NULL,
blob_id TEXT NOT NULL,
seq INTEGER NOT NULL,
data BLOB NOT NULL,
PRIMARY KEY (jmap, blob_id, seq)
);
`,
},
];
+160
View File
@@ -0,0 +1,160 @@
/**
* FileNode metadata for one JMAP account, then the contents worth keeping.
*
* A full pass reads the tree the way the client does (fetchAllFileNodes,
* lib/jmap/client.ts). `FileNode/get` with ids null returns everything, but
* 0.16 silently stops at maxObjectsInGet and 1.0 refuses a larger account with
* requestTooLarge. Past that, ids come from paging FileNode/query. Before
* 0.16.6 that query leaves out folders, so parents still unknown afterwards are
* read by id. The cursor is in sync_state, so a reload continues.
*/
import type { SyncContext, Syncer } from '../area';
import { runChangesLoop } from '../engine/changes';
import { JmapMethodError, responseFor, type MethodCall } from '../engine/transport';
import { dropOrphanBlobs, fetchKeptFiles } from './keep';
import { FILE_NODE_PROPERTIES, FILES_CAPABILITY, FILES_USING, removeNode, writeNode, type WireNode } from './records';
const MAX_GET = 500;
const GETS_PER_REQUEST = 8;
const QUERY_PAGE = 5000;
function getBatch(ctx: SyncContext): number {
const core = ctx.session.capabilities['urn:ietf:params:jmap:core'] as { maxObjectsInGet?: number } | undefined;
return Math.min(MAX_GET, core?.maxObjectsInGet || MAX_GET);
}
async function fetchNodes(ctx: SyncContext, jmap: string, ids: string[]): Promise<WireNode[]> {
const size = getBatch(ctx);
const nodes: WireNode[] = [];
for (let at = 0; at < ids.length;) {
const calls: MethodCall[] = [];
for (let i = 0; i < GETS_PER_REQUEST && at < ids.length; i += 1, at += size) {
calls.push(['FileNode/get', { accountId: jmap, ids: ids.slice(at, at + size), properties: FILE_NODE_PROPERTIES }, `g${i}`]);
}
const response = await ctx.transport.request(calls, FILES_USING);
calls.forEach(([, , callId]) => nodes.push(...responseFor(response, callId)[1].list as WireNode[]));
}
return nodes;
}
interface ListCursor { step: 'all' | 'query' | 'parents'; position: number; gen: number; captured: string }
async function startFullList(ctx: SyncContext, jmap: string): Promise<ListCursor> {
const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: [], properties: ['id'] }, 's']], FILES_USING);
const captured = String(responseFor(response, 's')[1].state);
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0) + 1;
const cursor: ListCursor = { step: 'all', position: 0, gen, captured };
ctx.db.transaction(() => ctx.setSyncState(jmap, 'FileNode', { state: null, phase: 'list', cursor: { ...cursor } }));
return cursor;
}
function saveStep(ctx: SyncContext, jmap: string, nodes: WireNode[], cursor: ListCursor): void {
ctx.db.transaction(() => {
for (const node of nodes) writeNode(ctx.db, jmap, node, cursor.gen);
ctx.setSyncState(jmap, 'FileNode', { cursor: { ...cursor } });
});
}
async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise<boolean> {
const cursor = { ...start };
if (cursor.step === 'all') {
let nodes: WireNode[] = [];
let complete = false;
try {
const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: null, properties: FILE_NODE_PROPERTIES }, 'a']], FILES_USING);
nodes = responseFor(response, 'a')[1].list as WireNode[];
complete = nodes.length < getBatch(ctx);
} catch (error) {
if (!(error instanceof JmapMethodError && error.type === 'requestTooLarge')) throw error;
}
if (!complete) cursor.step = 'query';
else cursor.step = 'parents';
saveStep(ctx, jmap, nodes, cursor);
}
while (cursor.step === 'query') {
if (ctx.stopped()) return false;
const response = await ctx.transport.request([['FileNode/query', { accountId: jmap, filter: {}, position: cursor.position, limit: QUERY_PAGE, calculateTotal: true }, 'q']], FILES_USING);
const [, query] = responseFor(response, 'q');
const ids = query.ids as string[];
const missing = ids.filter((id) => Number(ctx.db.get<{ gen: number }>('SELECT gen FROM file_node WHERE jmap = ? AND id = ?', [jmap, id])?.gen ?? -1) < cursor.gen);
const nodes = await fetchNodes(ctx, jmap, missing);
cursor.position += ids.length;
// The server may clamp the limit, so only an empty page or the total ends this.
if (ids.length === 0 || (typeof query.total === 'number' && cursor.position >= query.total)) cursor.step = 'parents';
saveStep(ctx, jmap, nodes, cursor);
ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'list', done: cursor.position, total: typeof query.total === 'number' ? query.total : null });
}
const asked = new Set<string>();
for (;;) {
if (ctx.stopped()) return false;
const parents = ctx.db.all<{ id: string }>(
`SELECT DISTINCT n.parent_id AS id FROM file_node n
WHERE n.jmap = ? AND n.gen = ? AND n.parent_id IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM file_node p WHERE p.jmap = n.jmap AND p.id = n.parent_id AND p.gen = ?)`,
[jmap, cursor.gen, cursor.gen],
).map((row) => String(row.id)).filter((id) => !asked.has(id));
if (parents.length === 0) break;
for (const id of parents) asked.add(id);
saveStep(ctx, jmap, await fetchNodes(ctx, jmap, parents), cursor);
}
ctx.db.transaction(() => {
// Whatever this pass didn't see is gone from the server.
ctx.db.run('DELETE FROM file_node WHERE jmap = ? AND gen < ?', [jmap, cursor.gen]);
ctx.setSyncState(jmap, 'FileNode', { state: cursor.captured, phase: null, cursor: null });
});
ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] });
return true;
}
async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: string | null): Promise<'done' | 'resync' | 'stopped'> {
const outcome = await runChangesLoop({
transport: ctx.transport,
type: 'FileNode',
jmap,
using: FILES_USING,
since,
target,
stopped: ctx.stopped,
apply: async (page) => {
const nodes = await fetchNodes(ctx, jmap, [...page.created, ...page.updated]);
ctx.db.transaction(() => {
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0);
for (const node of nodes) writeNode(ctx.db, jmap, node, gen);
for (const id of page.destroyed) removeNode(ctx.db, jmap, id);
ctx.setSyncState(jmap, 'FileNode', { state: page.newState });
});
if (nodes.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] });
},
});
if (outcome.outcome === 'resync') return 'resync';
return outcome.outcome === 'stopped' ? 'stopped' : 'done';
}
export const filesSyncer: Syncer = {
name: 'files',
capability: FILES_CAPABILITY,
async run(ctx, jmap) {
const row = ctx.syncState(jmap, 'FileNode');
let cursor = row.phase === 'list' ? row.cursor as unknown as ListCursor : null;
if (!cursor && !row.state) cursor = await startFullList(ctx, jmap);
if (!cursor && row.state) {
const outcome = await syncDelta(ctx, jmap, row.state, row.target);
if (outcome === 'stopped') return;
if (outcome === 'resync') cursor = await startFullList(ctx, jmap);
}
if (cursor) {
if (!(await runFullList(ctx, jmap, cursor))) return;
// Catch what changed while paging.
const after = ctx.syncState(jmap, 'FileNode');
if (after.state && (await syncDelta(ctx, jmap, after.state, after.target)) === 'stopped') return;
}
if (ctx.stopped()) return;
ctx.db.transaction(() => dropOrphanBlobs(ctx.db));
await fetchKeptFiles(ctx, jmap);
},
};