mirror of
https://github.com/bulwarkmail/webmail.git
synced 2026-11-04 08:15:41 +00:00
feat: list files offline and keep pinned and recently opened files on the device
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
// @vitest-environment node
|
||||
/**
|
||||
* The real JMAPClient listing files from the store, in each way a server says
|
||||
* what a node is, plus what it does with a write.
|
||||
*/
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { JMAPClient } from '@/lib/jmap/client';
|
||||
import { fileNodeKind } from '@/lib/jmap/filenode-kind';
|
||||
import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks';
|
||||
import { DEFAULT_SCOPE } from '../area';
|
||||
import { AccountEngine } from '../engine/account-engine';
|
||||
import { filesArea } from '../files';
|
||||
import { FakeFiles, FILES, FILES_SESSION, SHARED_FILES_ACCOUNT, type FakeFilesOptions } from './helpers/fake-files';
|
||||
import { openMemoryStore } from './helpers/node-adapter';
|
||||
|
||||
afterEach(() => setOfflineHooks(null));
|
||||
|
||||
async function offlineClient(server: FakeFiles): Promise<JMAPClient> {
|
||||
const engine = new AccountEngine({
|
||||
account: 'a@fake.test',
|
||||
db: openMemoryStore(),
|
||||
session: FILES_SESSION,
|
||||
transport: server,
|
||||
areas: [filesArea],
|
||||
scope: DEFAULT_SCOPE,
|
||||
sleep: async () => {},
|
||||
});
|
||||
await engine.requestSync();
|
||||
engine.setOnline(false);
|
||||
setOfflineHooks({
|
||||
skipNetwork: () => true,
|
||||
answer: (_client, calls) => engine.answer(calls),
|
||||
record: () => {},
|
||||
noteNetwork: () => {},
|
||||
} as Partial<OfflineHooks> as OfflineHooks);
|
||||
return JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', {
|
||||
...FILES_SESSION,
|
||||
capabilities: { ...FILES_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500 } },
|
||||
});
|
||||
}
|
||||
|
||||
describe('file listing offline', () => {
|
||||
it.each<[string, FakeFilesOptions]>([
|
||||
['nodeType (0.16.6 and later, 1.0 as tested)', { style: 'nodeType' }],
|
||||
['blobId null only (0.16.0-5)', { style: 'blob' }],
|
||||
['@type and mediaType (1.0.0)', { style: 'draft' }],
|
||||
])('tells folders from files: %s', async (_label, options) => {
|
||||
const server = new FakeFiles(options);
|
||||
const folder = server.addFolder('Docs');
|
||||
server.addFile('note.txt', folder.id);
|
||||
const client = await offlineClient(server);
|
||||
expect(client.supportsFiles()).toBe(true);
|
||||
|
||||
const nodes = await client.listAllFileNodes();
|
||||
const byName = new Map(nodes.map((n) => [n.name, n]));
|
||||
expect(fileNodeKind(byName.get('Docs') as never)).toBe('directory');
|
||||
expect(fileNodeKind(byName.get('note.txt') as never)).toBe('file');
|
||||
expect(byName.get('note.txt')).toMatchObject({ parentId: folder.id, type: 'text/plain', size: 10 });
|
||||
expect((await client.listFileNodes(folder.id)).map((n) => n.name)).toEqual(['note.txt']);
|
||||
expect((await client.listFileNodes(null)).map((n) => n.name)).toEqual(['Docs']);
|
||||
});
|
||||
|
||||
it('lists an account larger than one /get, ending its paging cleanly', async () => {
|
||||
const server = new FakeFiles({ getAll: 'refuse' });
|
||||
const folder = server.addFolder('Docs');
|
||||
for (let i = 0; i < 700; i += 1) server.addFile(`f${i}`, folder.id);
|
||||
const client = await offlineClient(server);
|
||||
expect(await client.listAllFileNodes()).toHaveLength(701);
|
||||
});
|
||||
|
||||
it('lists shared accounts with namespaced ids', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFolder('Mine');
|
||||
const shared = server.addFolder('Theirs', null, undefined, SHARED_FILES_ACCOUNT);
|
||||
const client = await offlineClient(server);
|
||||
const nodes = await client.listAllFileNodesAcrossAccounts();
|
||||
expect(nodes.find((n) => n.name === 'Theirs')).toMatchObject({ id: `${SHARED_FILES_ACCOUNT}:${shared.id}`, isShared: true });
|
||||
expect(nodes.find((n) => n.name === 'Mine')).toMatchObject({ isShared: false });
|
||||
});
|
||||
|
||||
it('fails a write with an error rather than losing it', async () => {
|
||||
const server = new FakeFiles();
|
||||
const client = await offlineClient(server);
|
||||
await expect(client.createFileDirectory('New', null)).rejects.toThrow();
|
||||
expect(FILES in FILES_SESSION.capabilities).toBe(true);
|
||||
expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,535 @@
|
||||
// @vitest-environment node
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { DEFAULT_SCOPE, type StoreEvent, type StoreScope } from '../area';
|
||||
import { AccountEngine } from '../engine/account-engine';
|
||||
import { TransportError } from '../engine/transport';
|
||||
import { fileBlob, FILE_CACHE_CAP_BYTES, filesArea } from '../files';
|
||||
import type { StorageAdapter } from '../storage';
|
||||
import { JMAP_ACCOUNT } from './helpers/fake-jmap';
|
||||
import { FakeFiles, FILES_SESSION, SHARED_FILES_ACCOUNT } from './helpers/fake-files';
|
||||
import { openMemoryStore } from './helpers/node-adapter';
|
||||
|
||||
const NOW = Date.UTC(2026, 10, 3);
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any -- answers are shaped like the server's */
|
||||
|
||||
interface Rig {
|
||||
db: StorageAdapter;
|
||||
engine: AccountEngine;
|
||||
clock: { at: number };
|
||||
events: StoreEvent[];
|
||||
}
|
||||
|
||||
function rigFor(server: FakeFiles, scope: Partial<StoreScope> = {}): Rig {
|
||||
const db = openMemoryStore();
|
||||
const clock = { at: NOW };
|
||||
const events: StoreEvent[] = [];
|
||||
const engine = new AccountEngine({
|
||||
account: 'a@fake.test',
|
||||
db,
|
||||
session: FILES_SESSION,
|
||||
transport: server,
|
||||
areas: [filesArea],
|
||||
scope: { ...DEFAULT_SCOPE, ...scope },
|
||||
emit: (event) => events.push(event),
|
||||
now: () => clock.at,
|
||||
sleep: async () => {},
|
||||
});
|
||||
return { db, engine, clock, events };
|
||||
}
|
||||
|
||||
async function synced(server: FakeFiles, scope: Partial<StoreScope> = {}): Promise<Rig> {
|
||||
const rig = rigFor(server, scope);
|
||||
await rig.engine.requestSync();
|
||||
return rig;
|
||||
}
|
||||
|
||||
async function call(engine: AccountEngine, name: string, args: Record<string, unknown> = {}): Promise<any> {
|
||||
const response = await engine.answer([[name, { accountId: JMAP_ACCOUNT, ...args }, '0']]);
|
||||
if (!response) throw new Error('nothing answered');
|
||||
const [answered, result] = response.methodResponses[0];
|
||||
if (answered === 'error') throw new Error(result.type);
|
||||
return result;
|
||||
}
|
||||
|
||||
const ref = (id: string, jmap = JMAP_ACCOUNT) => ({ jmap, id });
|
||||
|
||||
function names(rows: Array<{ name?: unknown }>): unknown[] {
|
||||
return rows.map((row) => row.name).sort();
|
||||
}
|
||||
|
||||
describe('file metadata sync', () => {
|
||||
it('stores small accounts from one FileNode/get', async () => {
|
||||
const server = new FakeFiles();
|
||||
const folder = server.addFolder('Docs');
|
||||
server.addFile('a.txt', folder.id);
|
||||
const { engine } = await synced(server);
|
||||
expect(server.calls.filter(([n]) => n === 'FileNode/query')).toHaveLength(0);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['Docs', 'a.txt']);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['0.16 cuts the list at 500', { getAll: 'cut' as const }],
|
||||
['1.0 refuses a list over 500', { getAll: 'refuse' as const }],
|
||||
])('pages past the /get limit: %s', async (_label, options) => {
|
||||
const server = new FakeFiles(options);
|
||||
const folder = server.addFolder('Docs');
|
||||
for (let i = 0; i < 1234; i += 1) server.addFile(`f${i}.txt`, folder.id);
|
||||
const { db } = await synced(server);
|
||||
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(1235);
|
||||
for (const [name, args] of server.calls) {
|
||||
if (name === 'FileNode/get' && Array.isArray(args.ids)) expect(args.ids.length).toBeLessThanOrEqual(500);
|
||||
}
|
||||
});
|
||||
|
||||
it('finds folders the query leaves out on old servers, by the parents of what it did find', async () => {
|
||||
const server = new FakeFiles({ queryOmitsFolders: true, style: 'blob' });
|
||||
for (let i = 0; i < 600; i += 1) server.addFile(`early${i}.txt`);
|
||||
const top = server.addFolder('Top');
|
||||
const inner = server.addFolder('Inner', top.id);
|
||||
server.addFile('deep.txt', inner.id);
|
||||
const { db } = await synced(server);
|
||||
const kinds = db.all<{ name: string; kind: string }>("SELECT name, kind FROM file_node WHERE name IN ('Top', 'Inner', 'deep.txt') ORDER BY name");
|
||||
expect(kinds).toEqual([{ name: 'Inner', kind: 'directory' }, { name: 'Top', kind: 'directory' }, { name: 'deep.txt', kind: 'file' }]);
|
||||
});
|
||||
|
||||
it('continues a stopped full sync from its cursor', async () => {
|
||||
const server = new FakeFiles({ getAll: 'refuse' });
|
||||
for (let i = 0; i < 6000; i += 1) server.addFile(`f${i}`);
|
||||
const rig = rigFor(server);
|
||||
let gets = 0;
|
||||
server.beforeRequest = () => {
|
||||
gets += 1;
|
||||
// State capture, the refused list, a query page of 5000 and two requests of gets: then the network goes.
|
||||
if (gets === 6) server.offline = true;
|
||||
};
|
||||
await rig.engine.requestSync();
|
||||
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(5000);
|
||||
server.offline = false;
|
||||
server.beforeRequest = null;
|
||||
server.calls.length = 0;
|
||||
await rig.engine.requestSync();
|
||||
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(6000);
|
||||
expect(server.calls.find(([n]) => n === 'FileNode/query')?.[1].position).toBe(5000);
|
||||
expect(server.calls.filter(([n, a]) => n === 'FileNode/get' && a.ids === null && a.accountId === JMAP_ACCOUNT)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('follows changes through FileNode/changes', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a.txt');
|
||||
const b = server.addFile('b.txt');
|
||||
const { engine } = await synced(server);
|
||||
server.calls.length = 0;
|
||||
server.renameNode(a.id, 'renamed.txt');
|
||||
server.removeNode(b.id);
|
||||
server.addFolder('New');
|
||||
await engine.requestSync();
|
||||
expect(server.calls.some(([n]) => n === 'FileNode/changes')).toBe(true);
|
||||
expect(server.calls.some(([n, args]) => n === 'FileNode/get' && args.ids === null)).toBe(false);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['New', 'renamed.txt']);
|
||||
});
|
||||
|
||||
it('starts over when the server can no longer give changes, and keeps a pin on a file still there', async () => {
|
||||
const server = new FakeFiles();
|
||||
const gone = server.addFile('gone.txt');
|
||||
const kept = server.addFile('kept.txt');
|
||||
const { db, engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(kept.id));
|
||||
await engine.requestSync();
|
||||
server.removeNode(gone.id);
|
||||
server.account().coll.truncatedBelow = server.account().coll.log.length;
|
||||
await engine.requestSync();
|
||||
expect(db.all('SELECT id FROM file_node').map((r) => r.id)).toEqual([kept.id]);
|
||||
expect(db.get<{ pinned: number }>('SELECT pinned FROM file_node WHERE id = ?', [kept.id])?.pinned).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps accounts apart, even with the same ids', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('own.txt', null, 5, { id: 'same' });
|
||||
server.addFile('team.txt', null, 5, { id: 'same', blobId: 'team-blob' }, SHARED_FILES_ACCOUNT);
|
||||
const { engine } = await synced(server);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: ['same'] })).list)).toEqual(['own.txt']);
|
||||
expect(names((await call(engine, 'FileNode/get', { ids: ['same'], accountId: SHARED_FILES_ACCOUNT })).list)).toEqual(['team.txt']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('file answers', () => {
|
||||
it('says so instead of an empty list when nothing was read yet', async () => {
|
||||
const { engine } = rigFor(new FakeFiles());
|
||||
expect(await engine.answer([['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '0']])).toBeNull();
|
||||
});
|
||||
|
||||
it('gives the last server state and only asked-for properties', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a.txt');
|
||||
const { engine } = await synced(server);
|
||||
const got = await call(engine, 'FileNode/get', { ids: [file.id, 'nope'], properties: ['name', 'parentId'] });
|
||||
expect(got.state).toBe(server.account().coll.state());
|
||||
expect(got.list).toEqual([{ id: file.id, name: 'a.txt', parentId: null }]);
|
||||
expect(got.notFound).toEqual(['nope']);
|
||||
});
|
||||
|
||||
it('answers a query by parent, also at the root, which the real server refuses to take', async () => {
|
||||
const server = new FakeFiles();
|
||||
const folder = server.addFolder('Docs');
|
||||
const inside = server.addFile('in.txt', folder.id);
|
||||
const root = server.addFile('root.txt');
|
||||
const { engine } = await synced(server);
|
||||
expect((await call(engine, 'FileNode/query', { filter: { parentId: folder.id } })).ids).toEqual([inside.id]);
|
||||
expect((await call(engine, 'FileNode/query', { filter: { parentId: null } })).ids).toEqual([folder.id, root.id]);
|
||||
});
|
||||
|
||||
it('pages and sorts a query', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('b', null, 30);
|
||||
server.addFile('a', null, 10);
|
||||
server.addFile('c', null, 20);
|
||||
const { engine } = await synced(server);
|
||||
const page = await call(engine, 'FileNode/query', { filter: {}, position: 1, limit: 1, calculateTotal: true });
|
||||
expect(page).toMatchObject({ position: 1, total: 3, ids: [expect.any(String)] });
|
||||
const bySize = await call(engine, 'FileNode/query', { filter: {}, sort: [{ property: 'size', isAscending: false }] });
|
||||
const list = (await call(engine, 'FileNode/get', { ids: bySize.ids })).list;
|
||||
expect(list.map((n: { name: string }) => n.name)).toEqual(['b', 'c', 'a']);
|
||||
});
|
||||
|
||||
it('refuses what it cannot read, for that call only', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
const response = await engine.answer([
|
||||
['FileNode/query', { accountId: JMAP_ACCOUNT, filter: { hasFoo: true } }, '0'],
|
||||
['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'],
|
||||
]);
|
||||
expect(response?.methodResponses[0][1].type).toBe('unsupportedFilter');
|
||||
expect(response?.methodResponses[1][0]).toBe('FileNode/get');
|
||||
});
|
||||
|
||||
it('answers FileNode/set with a clear per-call error, not silence', async () => {
|
||||
const server = new FakeFiles();
|
||||
server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
const response = await engine.answer([
|
||||
['FileNode/set', { accountId: JMAP_ACCOUNT, create: { n0: { name: 'x' } } }, '0'],
|
||||
['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'],
|
||||
]);
|
||||
expect(response?.methodResponses[0]).toEqual(['error', { type: 'serverUnavailable', description: 'FileNode/set needs the server' }, '0']);
|
||||
expect(response?.methodResponses[1][0]).toBe('FileNode/get');
|
||||
expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pinned files', () => {
|
||||
it('fetches a pinned file and gives it back byte for byte, past the chunk size', async () => {
|
||||
const server = new FakeFiles();
|
||||
const content = new Uint8Array(3 * 1024 * 1024 + 17).map((_, i) => i % 251);
|
||||
const file = server.addFile('big.bin', null, content);
|
||||
const { db, engine } = await synced(server);
|
||||
expect(server.downloads).toEqual([]);
|
||||
|
||||
expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: false });
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toEqual([{ jmap: JMAP_ACCOUNT, blobId: file.blobId }]);
|
||||
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(4);
|
||||
|
||||
const read = await engine.areaCall('files.read', ref(file.id)) as any;
|
||||
expect(read.name).toBe('big.bin');
|
||||
expect(read.type).toBe('text/plain');
|
||||
expect(Buffer.compare(read.data, content)).toBe(0);
|
||||
expect(Buffer.compare(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)?.data as Uint8Array, content)).toBe(0);
|
||||
expect(await engine.areaCall('files.blob', { jmap: JMAP_ACCOUNT, blobId: file.blobId })).toMatchObject({ size: content.length });
|
||||
// Already kept: not fetched again.
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: true });
|
||||
});
|
||||
|
||||
it('names a file with a %-encoded name from 0.16 WebDAV the way the client shows it', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('My%20Report.pdf');
|
||||
const { engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(file.id));
|
||||
await engine.requestSync();
|
||||
expect((await engine.areaCall('files.read', ref(file.id)) as any).name).toBe('My Report.pdf');
|
||||
expect((await engine.areaCall('files.kept', undefined) as any).files[0].name).toBe('My Report.pdf');
|
||||
// The stored node stays as the server sent it. The client decodes it once.
|
||||
expect((await call(engine, 'FileNode/get', { ids: [file.id] })).list[0].name).toBe('My%20Report.pdf');
|
||||
});
|
||||
|
||||
it('gives nothing for a file not kept', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a.txt');
|
||||
const { db, engine } = await synced(server);
|
||||
expect(await engine.areaCall('files.read', ref(file.id))).toBeNull();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)).toBeNull();
|
||||
});
|
||||
|
||||
it('refuses to pin a folder or an unknown id', async () => {
|
||||
const server = new FakeFiles();
|
||||
const folder = server.addFolder('Docs');
|
||||
const { engine } = await synced(server);
|
||||
await expect(engine.areaCall('files.pin', ref(folder.id))).rejects.toThrow('Only files');
|
||||
await expect(engine.areaCall('files.pin', ref('nope'))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('downloads one at a time', async () => {
|
||||
const server = new FakeFiles();
|
||||
const files = [server.addFile('a'), server.addFile('b'), server.addFile('c')];
|
||||
const { engine } = await synced(server);
|
||||
for (const file of files) await engine.areaCall('files.pin', ref(file.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(3);
|
||||
expect(server.mostAtOnce).toBe(1);
|
||||
});
|
||||
|
||||
it('fetches a pinned file over the one-file limit, and skips an opened one', async () => {
|
||||
const server = new FakeFiles();
|
||||
const pinned = server.addFile('pinned.bin', null, 100);
|
||||
const opened = server.addFile('opened.bin', null, 100);
|
||||
const { engine } = await synced(server, { maxAttachmentBytes: 50 });
|
||||
await engine.areaCall('files.pin', ref(pinned.id));
|
||||
await engine.areaCall('files.noteOpened', ref(opened.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads.map((d) => d.blobId)).toEqual([pinned.blobId]);
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
const row = (name: string) => status.files.find((f: { name: string }) => f.name === name);
|
||||
expect(row('pinned.bin')).toMatchObject({ pinned: true, kept: true, error: null });
|
||||
expect(row('opened.bin')).toMatchObject({ pinned: false, kept: false, error: 'tooLarge' });
|
||||
});
|
||||
|
||||
it('does not take pins past the total limit, and says so', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a', null, 60);
|
||||
const b = server.addFile('b', null, 60);
|
||||
const { engine, clock } = await synced(server, { maxTotalBytes: 100 });
|
||||
await engine.areaCall('files.pin', ref(a.id));
|
||||
clock.at += 1;
|
||||
await engine.areaCall('files.pin', ref(b.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
expect(status.files.filter((f: { kept: boolean }) => f.kept)).toHaveLength(1);
|
||||
expect(status.files.filter((f: { error: string | null }) => f.error === 'overLimit')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('keeps nothing when the scope forbids it, and drops what was kept', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const rig = await synced(server);
|
||||
await rig.engine.areaCall('files.pin', ref(file.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
|
||||
rig.engine.setScope({ ...DEFAULT_SCOPE, attachments: false });
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).toBeNull();
|
||||
expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ pinned: true, kept: false, error: 'notAllowed' });
|
||||
|
||||
rig.engine.setScope(DEFAULT_SCOPE);
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
});
|
||||
|
||||
it('goes by the real size when the metadata has it wrong', async () => {
|
||||
const server = new FakeFiles();
|
||||
const opened = server.addFile('opened', null, new Uint8Array(100), { size: 1 });
|
||||
const { engine } = await synced(server, { maxAttachmentBytes: 50 });
|
||||
await engine.areaCall('files.noteOpened', ref(opened.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
expect(await engine.areaCall('files.read', ref(opened.id))).toBeNull();
|
||||
expect((await engine.areaCall('files.kept', undefined) as any).files[0].error).toBe('tooLarge');
|
||||
});
|
||||
|
||||
it('does not store a file unpinned while it was downloading', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(file.id));
|
||||
const original = server.download.bind(server);
|
||||
server.download = async (jmap, blobId) => {
|
||||
await engine.areaCall('files.unpin', ref(file.id));
|
||||
return original(jmap, blobId);
|
||||
};
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
expect(await engine.areaCall('files.read', ref(file.id))).toBeNull();
|
||||
});
|
||||
|
||||
it('lets a never-opened file go on unpin, and keeps an opened one as recent', async () => {
|
||||
const server = new FakeFiles();
|
||||
const never = server.addFile('never');
|
||||
const opened = server.addFile('opened');
|
||||
const { engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(never.id));
|
||||
await engine.areaCall('files.pin', ref(opened.id));
|
||||
await engine.areaCall('files.noteOpened', ref(opened.id));
|
||||
await engine.requestSync();
|
||||
await engine.areaCall('files.unpin', ref(never.id));
|
||||
await engine.areaCall('files.unpin', ref(opened.id));
|
||||
expect(await engine.areaCall('files.read', ref(never.id))).toBeNull();
|
||||
expect(await engine.areaCall('files.read', ref(opened.id))).not.toBeNull();
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
expect(status.files.map((f: { name: string; pinned: boolean }) => [f.name, f.pinned])).toEqual([['opened', false]]);
|
||||
});
|
||||
|
||||
it('drops the old content and fetches the new one when a pinned file changes', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a.txt', null, new Uint8Array([1, 2, 3]));
|
||||
const { db, engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(file.id));
|
||||
await engine.requestSync();
|
||||
const old = file.blobId as string;
|
||||
const fresh = server.replaceContent(file.id, new Uint8Array([9, 9]));
|
||||
await engine.requestSync();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, old)).toBeNull();
|
||||
expect([...(fileBlob({ db }, JMAP_ACCOUNT, fresh)?.data ?? [])]).toEqual([9, 9]);
|
||||
expect(Array.from((await engine.areaCall('files.read', ref(file.id)) as any).data)).toEqual([9, 9]);
|
||||
});
|
||||
|
||||
it('drops the content when the file is removed on the server, unless another node shares it', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a', null, 5, { blobId: 'shared-blob' });
|
||||
const b = server.addFile('b', null, 5, { blobId: 'shared-blob' });
|
||||
const { db, engine } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(a.id));
|
||||
await engine.areaCall('files.pin', ref(b.id));
|
||||
await engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(1);
|
||||
server.removeNode(a.id);
|
||||
await engine.requestSync();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).not.toBeNull();
|
||||
server.removeNode(b.id);
|
||||
await engine.requestSync();
|
||||
expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).toBeNull();
|
||||
expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0);
|
||||
});
|
||||
|
||||
it('carries on past one bad file, and hands a network failure to the engine', async () => {
|
||||
const server = new FakeFiles();
|
||||
const bad = server.addFile('bad');
|
||||
const good = server.addFile('good');
|
||||
server.blobs.delete(bad.blobId as string);
|
||||
const { engine, clock } = await synced(server);
|
||||
await engine.areaCall('files.pin', ref(bad.id));
|
||||
clock.at += 1;
|
||||
await engine.areaCall('files.pin', ref(good.id));
|
||||
await engine.requestSync();
|
||||
expect(await engine.areaCall('files.read', ref(good.id))).not.toBeNull();
|
||||
const status = await engine.areaCall('files.kept', undefined) as any;
|
||||
expect(status.files.find((f: { name: string }) => f.name === 'bad').error).toBe('download');
|
||||
|
||||
});
|
||||
|
||||
it('hands a network failure to the engine, which ends the pass, and goes on next time', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const rig = await synced(server);
|
||||
await rig.engine.areaCall('files.pin', ref(file.id));
|
||||
const original = server.download.bind(server);
|
||||
server.download = async () => {
|
||||
throw new TransportError('network', 'Failed to fetch');
|
||||
};
|
||||
const syncedBefore = rig.events.filter((e) => e.type === 'synced').length;
|
||||
await rig.engine.requestSync();
|
||||
expect(rig.events.some((e) => e.type === 'error')).toBe(false);
|
||||
expect(rig.events.filter((e) => e.type === 'synced').length).toBe(syncedBefore);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ kept: false, error: null });
|
||||
|
||||
server.download = original;
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('recently opened files', () => {
|
||||
it('keeps an opened file without a pin', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('a');
|
||||
const { engine } = await synced(server);
|
||||
expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: false });
|
||||
await engine.requestSync();
|
||||
expect(await engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: true });
|
||||
});
|
||||
|
||||
it('pushes out the oldest unpinned first, and never a pinned one', async () => {
|
||||
const server = new FakeFiles();
|
||||
const files = Object.fromEntries(['old', 'pinnedOldest', 'mid', 'new', 'newest'].map((name) => [name, server.addFile(name, null, 10)]));
|
||||
const rig = await synced(server, { maxTotalBytes: 30 });
|
||||
const open = async (name: string) => {
|
||||
rig.clock.at += 1000;
|
||||
await rig.engine.areaCall('files.noteOpened', ref(files[name].id));
|
||||
await rig.engine.requestSync();
|
||||
};
|
||||
await open('pinnedOldest');
|
||||
await rig.engine.areaCall('files.pin', ref(files.pinnedOldest.id));
|
||||
await open('old');
|
||||
await open('mid');
|
||||
const kept = async () => (await rig.engine.areaCall('files.kept', undefined) as any).files
|
||||
.filter((f: { kept: boolean }) => f.kept).map((f: { name: string }) => f.name).sort();
|
||||
expect(await kept()).toEqual(['mid', 'old', 'pinnedOldest']);
|
||||
|
||||
await open('new');
|
||||
expect(await kept()).toEqual(['mid', 'new', 'pinnedOldest']);
|
||||
await open('mid');
|
||||
await open('newest');
|
||||
expect(await kept()).toEqual(['mid', 'newest', 'pinnedOldest']);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).bytes).toBe(30);
|
||||
|
||||
// A pinned file was never pushed out and had to come back.
|
||||
expect(server.downloads.filter((d) => d.blobId === files.pinnedOldest.blobId)).toHaveLength(1);
|
||||
// Pushed out means forgotten: the next pass doesn't fetch it straight back.
|
||||
expect(server.downloads.filter((d) => d.blobId === files.old.blobId)).toHaveLength(1);
|
||||
const count = server.downloads.length;
|
||||
await rig.engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(count);
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).files.map((f: { name: string }) => f.name)).not.toContain('old');
|
||||
});
|
||||
|
||||
it('pushes out recent files for a pin, and keeps a pin past the cap', async () => {
|
||||
const server = new FakeFiles();
|
||||
const recent = server.addFile('recent', null, 20);
|
||||
const pin = server.addFile('pin', null, 20);
|
||||
const rig = await synced(server, { maxTotalBytes: 25 });
|
||||
await rig.engine.areaCall('files.noteOpened', ref(recent.id));
|
||||
await rig.engine.requestSync();
|
||||
rig.clock.at += 1000;
|
||||
await rig.engine.areaCall('files.pin', ref(pin.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(recent.id))).toBeNull();
|
||||
expect(await rig.engine.areaCall('files.read', ref(pin.id))).not.toBeNull();
|
||||
});
|
||||
|
||||
it('keeps recently opened files under 500 MB whatever the total limit is', async () => {
|
||||
const server = new FakeFiles();
|
||||
const file = server.addFile('new', null, 10);
|
||||
const rig = await synced(server, { maxTotalBytes: 2 * FILE_CACHE_CAP_BYTES });
|
||||
// As if an older file already took all of it: only the row counts here.
|
||||
rig.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [JMAP_ACCOUNT, 'huge', 'x', FILE_CACHE_CAP_BYTES, NOW - 1000]);
|
||||
rig.db.run("INSERT INTO file_node (jmap, id, name, kind, blob_id, size, json, opened_at) VALUES (?, 'hugenode', 'huge', 'file', 'huge', ?, '{}', ?)", [JMAP_ACCOUNT, FILE_CACHE_CAP_BYTES, NOW - 1000]);
|
||||
await rig.engine.areaCall('files.noteOpened', ref(file.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(fileBlob({ db: rig.db }, JMAP_ACCOUNT, 'huge')).toBeNull();
|
||||
expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull();
|
||||
expect((await rig.engine.areaCall('files.kept', undefined) as any).capBytes).toBe(FILE_CACHE_CAP_BYTES);
|
||||
});
|
||||
|
||||
it('drops an old opened file that does not fit rather than a newer one', async () => {
|
||||
const server = new FakeFiles();
|
||||
const a = server.addFile('a', null, 20);
|
||||
const b = server.addFile('b', null, 20);
|
||||
const rig = await synced(server, { maxTotalBytes: 25 });
|
||||
await rig.engine.areaCall('files.noteOpened', ref(a.id));
|
||||
rig.clock.at += 1000;
|
||||
await rig.engine.areaCall('files.noteOpened', ref(b.id));
|
||||
await rig.engine.requestSync();
|
||||
expect(await rig.engine.areaCall('files.read', ref(b.id))).not.toBeNull();
|
||||
expect(await rig.engine.areaCall('files.read', ref(a.id))).toBeNull();
|
||||
expect(server.downloads.map((d) => d.blobId)).toEqual([b.blobId, a.blobId]);
|
||||
const count = server.downloads.length;
|
||||
await rig.engine.requestSync();
|
||||
expect(server.downloads).toHaveLength(count);
|
||||
});
|
||||
});
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* FakeJmap with files, as measured on both Stalwarts (2026-11-03) and as the
|
||||
* client's comments record: `filter: {parentId: null}` is an HTTP 400 that
|
||||
* fails the whole request, a folder has null blobId, type and size. What differs
|
||||
* between versions is set per server:
|
||||
* - `style` is how a node says what it is: `nodeType` (0.16.6 and later, 1.0 as
|
||||
* tested), `blob` (0.16.0-5: nothing but blobId null), `draft` (1.0.0: `@type`
|
||||
* and `mediaType`).
|
||||
* - `getAll`: ids null past 500 is silently cut on 0.16 and refused on 1.0.
|
||||
* - `queryOmitsFolders`: FileNode/query before 0.16.6.
|
||||
*/
|
||||
|
||||
import { TransportError, type StoreSession } from '../../engine/transport';
|
||||
import { changesFor } from './fake-contacts';
|
||||
import { Collection, FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './fake-jmap';
|
||||
|
||||
export const FILES = 'urn:ietf:params:jmap:filenode';
|
||||
export const SHARED_FILES_ACCOUNT = 'shared1';
|
||||
|
||||
export const FILES_SESSION: StoreSession = {
|
||||
...FAKE_SESSION,
|
||||
primaryAccounts: { ...FAKE_SESSION.primaryAccounts, [FILES]: JMAP_ACCOUNT },
|
||||
accounts: {
|
||||
[JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [FILES]: {} } },
|
||||
[SHARED_FILES_ACCOUNT]: { name: 'team@example.org', isPersonal: false, accountCapabilities: { [FILES]: {} } },
|
||||
},
|
||||
capabilities: { [FILES]: {} },
|
||||
};
|
||||
|
||||
export interface FakeNode {
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
name: string;
|
||||
blobId: string | null;
|
||||
type: string | null;
|
||||
size: number | null;
|
||||
modified?: string;
|
||||
}
|
||||
|
||||
export type FolderStyle = 'nodeType' | 'blob' | 'draft';
|
||||
|
||||
export interface FakeFilesOptions {
|
||||
style?: FolderStyle;
|
||||
getAll?: 'cut' | 'refuse';
|
||||
queryOmitsFolders?: boolean;
|
||||
}
|
||||
|
||||
class FilesAccount {
|
||||
readonly nodes = new Map<string, FakeNode>();
|
||||
readonly coll = new Collection();
|
||||
}
|
||||
|
||||
export class FakeFiles extends FakeJmap {
|
||||
private readonly fileAccounts = new Map<string, FilesAccount>();
|
||||
private readonly options: Required<FakeFilesOptions>;
|
||||
private nextNode = 1;
|
||||
readonly blobs = new Map<string, Uint8Array>();
|
||||
/** In order, with the account. */
|
||||
readonly downloads: Array<{ jmap: string; blobId: string }> = [];
|
||||
downloading = 0;
|
||||
mostAtOnce = 0;
|
||||
|
||||
constructor(options: FakeFilesOptions = {}) {
|
||||
super();
|
||||
this.options = { style: 'nodeType', getAll: 'cut', queryOmitsFolders: false, ...options };
|
||||
}
|
||||
|
||||
account(id: string = JMAP_ACCOUNT): FilesAccount {
|
||||
let found = this.fileAccounts.get(id);
|
||||
if (!found) {
|
||||
found = new FilesAccount();
|
||||
this.fileAccounts.set(id, found);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
addFolder(name: string, parentId: string | null = null, id?: string, accountId: string = JMAP_ACCOUNT): FakeNode {
|
||||
const account = this.account(accountId);
|
||||
const node: FakeNode = { id: id ?? `n${this.nextNode++}`, parentId, name, blobId: null, type: null, size: null };
|
||||
account.nodes.set(node.id, node);
|
||||
account.coll.push('created', node.id);
|
||||
return node;
|
||||
}
|
||||
|
||||
addFile(name: string, parentId: string | null = null, content: Uint8Array | number = 10, extra: Partial<FakeNode> = {}, accountId: string = JMAP_ACCOUNT): FakeNode {
|
||||
const account = this.account(accountId);
|
||||
const id = extra.id ?? `n${this.nextNode++}`;
|
||||
const blobId = extra.blobId ?? `blob-${id}`;
|
||||
const data = typeof content === 'number' ? new Uint8Array(content).fill(id.length) : content;
|
||||
this.blobs.set(blobId, data);
|
||||
const node: FakeNode = { id, parentId, name, blobId, type: 'text/plain', size: data.length, ...extra };
|
||||
account.nodes.set(id, node);
|
||||
account.coll.push('created', id);
|
||||
return node;
|
||||
}
|
||||
|
||||
/** A new version of the file: new blob, same node. */
|
||||
replaceContent(id: string, content: Uint8Array, accountId: string = JMAP_ACCOUNT): string {
|
||||
const account = this.account(accountId);
|
||||
const node = account.nodes.get(id) as FakeNode;
|
||||
const blobId = `${node.blobId as string}-v`;
|
||||
this.blobs.set(blobId, content);
|
||||
node.blobId = blobId;
|
||||
node.size = content.length;
|
||||
account.coll.push('updated', id);
|
||||
return blobId;
|
||||
}
|
||||
|
||||
renameNode(id: string, name: string, accountId: string = JMAP_ACCOUNT): void {
|
||||
const account = this.account(accountId);
|
||||
(account.nodes.get(id) as FakeNode).name = name;
|
||||
account.coll.push('updated', id);
|
||||
}
|
||||
|
||||
removeNode(id: string, accountId: string = JMAP_ACCOUNT): void {
|
||||
const account = this.account(accountId);
|
||||
account.nodes.delete(id);
|
||||
account.coll.push('destroyed', id);
|
||||
}
|
||||
|
||||
private wire(node: FakeNode, properties: string[] | undefined): Record<string, unknown> {
|
||||
const isFolder = node.blobId === null;
|
||||
const full: Record<string, unknown> = { id: node.id, parentId: node.parentId, name: node.name, blobId: node.blobId, size: node.size, created: '2026-10-01T00:00:00Z', modified: node.modified ?? '2026-10-01T00:00:00Z' };
|
||||
if (this.options.style === 'draft') {
|
||||
full['@type'] = isFolder ? 'Directory' : 'File';
|
||||
full.mediaType = node.type;
|
||||
} else {
|
||||
full.type = node.type;
|
||||
if (this.options.style === 'nodeType') full.nodeType = isFolder ? 'directory' : 'file';
|
||||
}
|
||||
if (!properties) return full;
|
||||
const out: Record<string, unknown> = { id: node.id };
|
||||
for (const p of properties) if (p in full) out[p] = full[p];
|
||||
return out;
|
||||
}
|
||||
|
||||
protected override handle(name: string, args: Record<string, unknown>): Record<string, unknown> {
|
||||
if (!name.startsWith('FileNode/')) return super.handle(name, args);
|
||||
const account = this.account(String(args.accountId));
|
||||
switch (name) {
|
||||
case 'FileNode/changes':
|
||||
return changesFor(account.coll, args);
|
||||
case 'FileNode/get': {
|
||||
const ids = args.ids as string[] | null;
|
||||
if (ids && ids.length > 500) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' });
|
||||
if (!ids && account.nodes.size > 500 && this.options.getAll === 'refuse') {
|
||||
throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' });
|
||||
}
|
||||
const chosen = ids ?? [...account.nodes.keys()].slice(0, 500);
|
||||
const properties = args.properties as string[] | undefined;
|
||||
return {
|
||||
accountId: args.accountId,
|
||||
state: account.coll.state(),
|
||||
list: chosen.filter((id) => account.nodes.has(id)).map((id) => this.wire(account.nodes.get(id) as FakeNode, properties)),
|
||||
notFound: chosen.filter((id) => !account.nodes.has(id)),
|
||||
};
|
||||
}
|
||||
case 'FileNode/query': {
|
||||
const filter = (args.filter ?? {}) as { parentId?: string | null };
|
||||
// Measured on both servers: this fails the whole request.
|
||||
if (filter.parentId === null) throw new TransportError('http', 'JMAP request failed: 400', 400, 'urn:ietf:params:jmap:error:notRequest');
|
||||
let list = [...account.nodes.values()];
|
||||
if (this.options.queryOmitsFolders) list = list.filter((n) => n.blobId !== null);
|
||||
if (filter.parentId) list = list.filter((n) => n.parentId === filter.parentId);
|
||||
const position = Number(args.position ?? 0);
|
||||
const limit = Math.min(Number(args.limit ?? 5000) || 5000, 5000);
|
||||
return {
|
||||
accountId: args.accountId,
|
||||
queryState: account.coll.state(),
|
||||
canCalculateChanges: true,
|
||||
position,
|
||||
ids: list.slice(position, position + limit).map((n) => n.id),
|
||||
...(args.calculateTotal ? { total: list.length } : {}),
|
||||
};
|
||||
}
|
||||
default:
|
||||
throw Object.assign(new Error(name), { jmapError: 'unknownMethod' });
|
||||
}
|
||||
}
|
||||
|
||||
override async download(jmap: string, blobId: string): Promise<Uint8Array> {
|
||||
if (this.offline) throw new TransportError('network', 'Failed to fetch');
|
||||
this.downloads.push({ jmap, blobId });
|
||||
this.downloading += 1;
|
||||
this.mostAtOnce = Math.max(this.mostAtOnce, this.downloading);
|
||||
await Promise.resolve();
|
||||
this.downloading -= 1;
|
||||
const data = this.blobs.get(blobId);
|
||||
if (!data) throw new TransportError('http', 'Download failed: 404', 404);
|
||||
return data;
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
|
||||
import type { StoreArea } from './area';
|
||||
import { contactsArea } from './contacts';
|
||||
import { filesArea } from './files';
|
||||
import { mailArea } from './mail';
|
||||
|
||||
export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea];
|
||||
export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea, filesArea];
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { StoreArea } from '../area';
|
||||
import { fileBlob, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep';
|
||||
import { FILE_METHODS } from './methods';
|
||||
import { FILE_MIGRATIONS } from './schema';
|
||||
import { filesSyncer } from './sync';
|
||||
|
||||
export { fileBlob, FILE_CACHE_CAP_BYTES } from './keep';
|
||||
|
||||
export const filesArea: StoreArea = {
|
||||
name: 'files',
|
||||
migrations: FILE_MIGRATIONS,
|
||||
methods: FILE_METHODS,
|
||||
syncers: [filesSyncer],
|
||||
calls: {
|
||||
/** Keep this file's content on the device until it's unpinned. */
|
||||
'files.pin': (ctx, params: NodeRef) => pinFile(ctx, params),
|
||||
'files.unpin': (ctx, params: NodeRef) => unpinFile(ctx, params),
|
||||
/** The page opened this file: keep its content for a while. */
|
||||
'files.noteOpened': (ctx, params: NodeRef) => noteOpened(ctx, params),
|
||||
/** A kept file's content and name, or null. */
|
||||
'files.read': (ctx, params: NodeRef) => readFile(ctx, params),
|
||||
/** A kept content by blob id, or null. */
|
||||
'files.blob': (ctx, params: { jmap: string; blobId: string }) => fileBlob(ctx, params.jmap, params.blobId),
|
||||
/** Pinned and recently opened files, for a settings page. */
|
||||
'files.kept': (ctx) => keptStatus(ctx),
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* File contents kept on the device: pinned files, and recently opened ones
|
||||
* within a cap. Metadata for every file is in file_node regardless.
|
||||
*
|
||||
* What gets kept:
|
||||
* - A pinned file stays until it's unpinned or removed on the server. The cap
|
||||
* doesn't push it out and the size limit for one file doesn't apply. The
|
||||
* account's total limit (scope.maxTotalBytes, which a policy can lower) does.
|
||||
* - A recently opened file is kept if it's within scope.maxAttachmentBytes, and
|
||||
* pushed out oldest first when the cap is reached. Its opened_at goes with
|
||||
* it, so it isn't fetched straight back.
|
||||
* - With scope.attachments off, nothing is kept.
|
||||
*/
|
||||
|
||||
import { TransportError } from '../engine/transport';
|
||||
import { MethodError, type StoreContext, type SyncContext } from '../area';
|
||||
import type { StorageAdapter } from '../storage';
|
||||
import { nodeMediaType, type WireNode } from './records';
|
||||
|
||||
export const FILE_CACHE_CAP_BYTES = 500 * 1024 * 1024;
|
||||
const CHUNK_BYTES = 1024 * 1024;
|
||||
|
||||
export interface KeptFile { data: Uint8Array; type: string; size: number }
|
||||
|
||||
export interface NodeRef { jmap: string; id: string }
|
||||
|
||||
interface NodeRow {
|
||||
jmap: string;
|
||||
id: string;
|
||||
name: string;
|
||||
kind: string;
|
||||
blob_id: string | null;
|
||||
size: number | null;
|
||||
json: string;
|
||||
pinned: number;
|
||||
opened_at: number | null;
|
||||
keep_error: string | null;
|
||||
}
|
||||
|
||||
interface BlobRow {
|
||||
jmap: string;
|
||||
blob_id: string;
|
||||
size: number;
|
||||
kept_at: number;
|
||||
pinned: number;
|
||||
opened_at: number | null;
|
||||
}
|
||||
|
||||
function capBytes(ctx: Pick<StoreContext, 'scope'>): number {
|
||||
return Math.min(FILE_CACHE_CAP_BYTES, ctx.scope.maxTotalBytes);
|
||||
}
|
||||
|
||||
/** A kept file's content, for the client's blob lookup. */
|
||||
export function fileBlob(ctx: Pick<StoreContext, 'db'>, jmap: string, blobId: string): KeptFile | null {
|
||||
const row = ctx.db.get<{ type: string; size: number }>('SELECT type, size FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
|
||||
if (!row) return null;
|
||||
const data = new Uint8Array(Number(row.size));
|
||||
let at = 0;
|
||||
for (const chunk of ctx.db.all<{ data: Uint8Array }>('SELECT data FROM file_blob_chunk WHERE jmap = ? AND blob_id = ? ORDER BY seq', [jmap, blobId])) {
|
||||
data.set(chunk.data, at);
|
||||
at += chunk.data.length;
|
||||
}
|
||||
return { data, type: row.type, size: Number(row.size) };
|
||||
}
|
||||
|
||||
function hasBlob(db: StorageAdapter, jmap: string, blobId: string): boolean {
|
||||
return Boolean(db.get('SELECT 1 AS x FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]));
|
||||
}
|
||||
|
||||
function dropBlob(db: StorageAdapter, jmap: string, blobId: string): void {
|
||||
db.run('DELETE FROM file_blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
|
||||
db.run('DELETE FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
|
||||
}
|
||||
|
||||
function storeBlob(ctx: StoreContext, jmap: string, blobId: string, type: string, data: Uint8Array): void {
|
||||
dropBlob(ctx.db, jmap, blobId);
|
||||
ctx.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [jmap, blobId, type, data.length, ctx.now()]);
|
||||
for (let seq = 0, at = 0; at < data.length; seq += 1, at += CHUNK_BYTES) {
|
||||
ctx.db.run('INSERT INTO file_blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, data.subarray(at, at + CHUNK_BYTES)]);
|
||||
}
|
||||
}
|
||||
|
||||
/** All kept contents in the store, with what the nodes pointing at each say about it. */
|
||||
function keptBlobs(db: StorageAdapter): BlobRow[] {
|
||||
return db.all<BlobRow>(
|
||||
`SELECT b.jmap, b.blob_id, b.size, b.kept_at, COALESCE(MAX(n.pinned), 0) AS pinned, MAX(n.opened_at) AS opened_at
|
||||
FROM file_blob b LEFT JOIN file_node n ON n.jmap = b.jmap AND n.blob_id = b.blob_id
|
||||
GROUP BY b.jmap, b.blob_id`,
|
||||
);
|
||||
}
|
||||
|
||||
function ageOf(blob: BlobRow): number {
|
||||
return Number(blob.opened_at ?? blob.kept_at);
|
||||
}
|
||||
|
||||
function pushOut(db: StorageAdapter, blob: BlobRow): void {
|
||||
dropBlob(db, blob.jmap, blob.blob_id);
|
||||
db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pushes out unpinned contents, oldest first, so `size` more fits under the
|
||||
* cap. A recently opened file only pushes out older ones and says false when
|
||||
* that isn't enough. A pinned one always gets in.
|
||||
*/
|
||||
function makeRoom(ctx: StoreContext, size: number, incoming: { pinned: boolean; openedAt: number }): boolean {
|
||||
const cap = capBytes(ctx);
|
||||
const blobs = keptBlobs(ctx.db);
|
||||
let used = blobs.reduce((sum, blob) => sum + Number(blob.size), 0);
|
||||
const candidates = blobs
|
||||
.filter((blob) => !blob.pinned && (incoming.pinned || ageOf(blob) < incoming.openedAt))
|
||||
.sort((a, b) => ageOf(a) - ageOf(b));
|
||||
const chosen: BlobRow[] = [];
|
||||
for (const blob of candidates) {
|
||||
if (used + size <= cap) break;
|
||||
used -= Number(blob.size);
|
||||
chosen.push(blob);
|
||||
}
|
||||
if (used + size > cap && !incoming.pinned) return false;
|
||||
for (const blob of chosen) pushOut(ctx.db, blob);
|
||||
return true;
|
||||
}
|
||||
|
||||
function pinnedBytes(db: StorageAdapter): number {
|
||||
return keptBlobs(db).filter((blob) => blob.pinned).reduce((sum, blob) => sum + Number(blob.size), 0);
|
||||
}
|
||||
|
||||
function setError(db: StorageAdapter, jmap: string, blobId: string, error: string | null): void {
|
||||
db.run('UPDATE file_node SET keep_error = ? WHERE jmap = ? AND blob_id = ?', [error, jmap, blobId]);
|
||||
}
|
||||
|
||||
/** Contents no node points at any more: the file was removed or has a new version. */
|
||||
export function dropOrphanBlobs(db: StorageAdapter): void {
|
||||
for (const blob of keptBlobs(db)) {
|
||||
const used = db.get('SELECT 1 AS x FROM file_node WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]);
|
||||
if (!used) dropBlob(db, blob.jmap, blob.blob_id);
|
||||
}
|
||||
}
|
||||
|
||||
interface Wanted { blob_id: string; id: string; name: string; size: number | null; json: string; pinned: number; opened_at: number | null }
|
||||
|
||||
function wantedRow(db: StorageAdapter, jmap: string, blobId: string): Wanted | undefined {
|
||||
return db.get<Wanted>(
|
||||
`SELECT blob_id, id, name, size, json, pinned, opened_at FROM file_node
|
||||
WHERE jmap = ? AND blob_id = ? AND (pinned = 1 OR opened_at IS NOT NULL)
|
||||
ORDER BY pinned DESC, opened_at DESC LIMIT 1`,
|
||||
[jmap, blobId],
|
||||
);
|
||||
}
|
||||
|
||||
/** Fetches what should be kept and isn't yet, one file at a time, pinned files first. */
|
||||
export async function fetchKeptFiles(ctx: SyncContext, jmap: string): Promise<void> {
|
||||
const { db } = ctx;
|
||||
if (!ctx.scope.attachments) {
|
||||
db.transaction(() => {
|
||||
for (const blob of keptBlobs(db)) dropBlob(db, blob.jmap, blob.blob_id);
|
||||
db.run("UPDATE file_node SET keep_error = 'notAllowed' WHERE jmap = ? AND (pinned = 1 OR opened_at IS NOT NULL)", [jmap]);
|
||||
});
|
||||
return;
|
||||
}
|
||||
db.transaction(() => {
|
||||
db.run("UPDATE file_node SET keep_error = NULL WHERE jmap = ? AND keep_error = 'notAllowed'", [jmap]);
|
||||
// The limit may have gone down since last time.
|
||||
makeRoom(ctx, 0, { pinned: true, openedAt: 0 });
|
||||
});
|
||||
const todo = db.all<{ blob_id: string }>(
|
||||
`SELECT n.blob_id FROM file_node n
|
||||
WHERE n.jmap = ? AND n.blob_id IS NOT NULL AND (n.pinned = 1 OR n.opened_at IS NOT NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM file_blob b WHERE b.jmap = n.jmap AND b.blob_id = n.blob_id)
|
||||
GROUP BY n.blob_id ORDER BY MAX(n.pinned) DESC, MAX(n.opened_at) DESC`,
|
||||
[jmap],
|
||||
).map((row) => String(row.blob_id));
|
||||
let done = 0;
|
||||
for (const blobId of todo) {
|
||||
if (ctx.stopped()) return;
|
||||
// The page may have changed its mind during the last download.
|
||||
const want = wantedRow(db, jmap, blobId);
|
||||
if (!want) continue;
|
||||
const known = Number(want.size ?? 0);
|
||||
if (!want.pinned && known > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) {
|
||||
setError(db, jmap, blobId, 'tooLarge');
|
||||
continue;
|
||||
}
|
||||
if (want.pinned && pinnedBytes(db) + known > ctx.scope.maxTotalBytes) {
|
||||
setError(db, jmap, blobId, 'overLimit');
|
||||
continue;
|
||||
}
|
||||
let data: Uint8Array;
|
||||
const type = nodeMediaType(JSON.parse(want.json) as WireNode);
|
||||
try {
|
||||
data = await ctx.transport.download(jmap, blobId, type, want.name);
|
||||
} catch (error) {
|
||||
// No network or no credentials reach the engine. One bad file doesn't stop the others.
|
||||
if (error instanceof TransportError && error.kind === 'http') {
|
||||
setError(db, jmap, blobId, 'download');
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
db.transaction(() => {
|
||||
const now = wantedRow(db, jmap, blobId);
|
||||
if (!now) return;
|
||||
if (now.pinned ? pinnedBytes(db) + data.length > ctx.scope.maxTotalBytes : data.length > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) {
|
||||
// The size in the metadata was wrong or missing. With the real one
|
||||
// noted, the next pass sees this without downloading it again.
|
||||
db.run('UPDATE file_node SET size = ? WHERE jmap = ? AND blob_id = ?', [data.length, jmap, blobId]);
|
||||
setError(db, jmap, blobId, now.pinned ? 'overLimit' : 'tooLarge');
|
||||
return;
|
||||
}
|
||||
if (!makeRoom(ctx, data.length, { pinned: Boolean(now.pinned), openedAt: Number(now.opened_at ?? 0) })) {
|
||||
db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ? AND pinned = 0', [jmap, blobId]);
|
||||
return;
|
||||
}
|
||||
storeBlob(ctx, jmap, blobId, type, data);
|
||||
setError(db, jmap, blobId, null);
|
||||
});
|
||||
done += 1;
|
||||
ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'contents', done, total: todo.length });
|
||||
}
|
||||
if (done > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] });
|
||||
}
|
||||
|
||||
// ---- Calls from the page ----
|
||||
|
||||
function fileRow(db: StorageAdapter, { jmap, id }: NodeRef): NodeRow {
|
||||
const row = db.get<NodeRow>('SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
if (!row) throw new MethodError('notFound', `No file ${id}`);
|
||||
if (!row.blob_id || row.kind !== 'file') throw new MethodError('invalidArguments', 'Only files can be kept');
|
||||
return row;
|
||||
}
|
||||
|
||||
export function pinFile(ctx: StoreContext, ref: NodeRef): { kept: boolean } {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
ctx.db.run('UPDATE file_node SET pinned = 1, keep_error = NULL WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]);
|
||||
const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string);
|
||||
if (!kept) void ctx.requestSync();
|
||||
return { kept };
|
||||
}
|
||||
|
||||
/** Back to being a recently opened file, or let go of it if it never was one. */
|
||||
export function unpinFile(ctx: StoreContext, ref: NodeRef): void {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
ctx.db.transaction(() => {
|
||||
ctx.db.run('UPDATE file_node SET pinned = 0 WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]);
|
||||
if (!wantedRow(ctx.db, ref.jmap, row.blob_id as string)) dropBlob(ctx.db, ref.jmap, row.blob_id as string);
|
||||
});
|
||||
}
|
||||
|
||||
export function noteOpened(ctx: StoreContext, ref: NodeRef): { kept: boolean } {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
ctx.db.run('UPDATE file_node SET opened_at = ?, keep_error = NULL WHERE jmap = ? AND id = ?', [ctx.now(), ref.jmap, ref.id]);
|
||||
const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string);
|
||||
if (!kept) void ctx.requestSync();
|
||||
return { kept };
|
||||
}
|
||||
|
||||
export function readFile(ctx: StoreContext, ref: NodeRef): (KeptFile & { name: string }) | null {
|
||||
const row = fileRow(ctx.db, ref);
|
||||
const kept = fileBlob(ctx, ref.jmap, row.blob_id as string);
|
||||
return kept ? { ...kept, name: row.name } : null;
|
||||
}
|
||||
|
||||
export interface KeptFileStatus {
|
||||
jmap: string;
|
||||
id: string;
|
||||
name: string;
|
||||
size: number | null;
|
||||
pinned: boolean;
|
||||
openedAt: number | null;
|
||||
kept: boolean;
|
||||
/** Why it isn't kept: tooLarge, overLimit, notAllowed or download. */
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** Pinned and recently opened files, and how much room their contents take. */
|
||||
export function keptStatus(ctx: StoreContext): { files: KeptFileStatus[]; bytes: number; capBytes: number } {
|
||||
const rows = ctx.db.all<NodeRow>(
|
||||
`SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node
|
||||
WHERE pinned = 1 OR opened_at IS NOT NULL ORDER BY pinned DESC, opened_at DESC, name`,
|
||||
);
|
||||
return {
|
||||
files: rows.map((row) => ({
|
||||
jmap: row.jmap,
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
size: row.size === null ? null : Number(row.size),
|
||||
pinned: Boolean(row.pinned),
|
||||
openedAt: row.opened_at === null ? null : Number(row.opened_at),
|
||||
kept: row.blob_id !== null && hasBlob(ctx.db, row.jmap, row.blob_id),
|
||||
error: row.keep_error,
|
||||
})),
|
||||
bytes: keptBlobs(ctx.db).reduce((sum, blob) => sum + Number(blob.size), 0),
|
||||
capBytes: capBytes(ctx),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* FileNode reads answered from the store, so listings work offline. Nothing
|
||||
* here answers FileNode/set: the responder gives it a per-call
|
||||
* serverUnavailable, since the design has no offline file writes.
|
||||
*/
|
||||
|
||||
import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area';
|
||||
import type { SqlValue } from '../storage';
|
||||
import { FILES_CAPABILITY } from './records';
|
||||
|
||||
function jmapOf(ctx: MethodContext, args: Record<string, unknown>): string {
|
||||
return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[FILES_CAPABILITY] ?? ctx.defaultJmap;
|
||||
}
|
||||
|
||||
/** The last state from the server. Throws when the list was never read, so an empty answer always means empty. */
|
||||
function syncedState(ctx: MethodContext, jmap: string): string | null {
|
||||
const row = ctx.db.get<{ state: string | null; phase: string | null }>("SELECT state, phase FROM sync_state WHERE jmap = ? AND type = 'FileNode'", [jmap]);
|
||||
if (!row || (!row.state && !row.phase)) throw new MethodError('serverUnavailable', 'Files not kept yet');
|
||||
return row.state;
|
||||
}
|
||||
|
||||
function pick(node: Record<string, unknown>, properties: unknown): Record<string, unknown> {
|
||||
if (!Array.isArray(properties)) return node;
|
||||
const out: Record<string, unknown> = { id: node.id };
|
||||
for (const property of properties as string[]) if (property in node) out[property] = node[property];
|
||||
return out;
|
||||
}
|
||||
|
||||
const SORTS: Record<string, string> = {
|
||||
name: 'name COLLATE NOCASE',
|
||||
size: 'size',
|
||||
created: "json_extract(json, '$.created')",
|
||||
modified: "json_extract(json, '$.modified')",
|
||||
};
|
||||
|
||||
function orderSql(sort: unknown): string {
|
||||
const parts: string[] = [];
|
||||
for (const level of (Array.isArray(sort) ? sort : []) as Array<{ property?: string; isAscending?: boolean }>) {
|
||||
const column = SORTS[String(level.property)];
|
||||
if (!column) throw new MethodError('unsupportedSort', `${String(level.property)} isn't available offline`);
|
||||
parts.push(`${column} ${level.isAscending === false ? 'DESC' : 'ASC'}`);
|
||||
}
|
||||
parts.push('name COLLATE NOCASE ASC', 'id ASC');
|
||||
return parts.join(', ');
|
||||
}
|
||||
|
||||
const fileNodeGet: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
const state = syncedState(ctx, jmap);
|
||||
const list: Array<Record<string, unknown>> = [];
|
||||
const notFound: string[] = [];
|
||||
if (Array.isArray(args.ids)) {
|
||||
for (const id of args.ids as string[]) {
|
||||
const row = ctx.db.get<{ json: string }>('SELECT json FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
if (row) list.push(pick(JSON.parse(row.json) as Record<string, unknown>, args.properties));
|
||||
else notFound.push(id);
|
||||
}
|
||||
} else {
|
||||
// The whole tree at once. The client reads a long list as possibly cut,
|
||||
// pages FileNode/query to check, and finds nothing missing.
|
||||
for (const row of ctx.db.all<{ json: string }>('SELECT json FROM file_node WHERE jmap = ?', [jmap])) {
|
||||
list.push(pick(JSON.parse(row.json) as Record<string, unknown>, args.properties));
|
||||
}
|
||||
}
|
||||
return { accountId: jmap, ...(state ? { state } : {}), list, notFound };
|
||||
};
|
||||
|
||||
const fileNodeQuery: MethodHandler = (ctx, args) => {
|
||||
const jmap = jmapOf(ctx, args);
|
||||
syncedState(ctx, jmap);
|
||||
if (args.anchor !== undefined) throw new MethodError('unsupportedFilter', 'anchor needs the server');
|
||||
const filter = (args.filter ?? {}) as Record<string, unknown>;
|
||||
const conditions = ['jmap = ?'];
|
||||
const params: SqlValue[] = [jmap];
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
if (key !== 'parentId') throw new MethodError('unsupportedFilter', `${key} isn't available offline`);
|
||||
if (value === null) {
|
||||
conditions.push('parent_id IS NULL');
|
||||
} else {
|
||||
conditions.push('parent_id = ?');
|
||||
params.push(String(value));
|
||||
}
|
||||
}
|
||||
const where = conditions.join(' AND ');
|
||||
const position = Math.max(0, Number(args.position ?? 0));
|
||||
const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1;
|
||||
const ids = ctx.db.all<{ id: string }>(
|
||||
`SELECT id FROM file_node WHERE ${where} ORDER BY ${orderSql(args.sort)} LIMIT ? OFFSET ?`,
|
||||
[...params, limit, position],
|
||||
).map((row) => String(row.id));
|
||||
const result: MethodResult = {
|
||||
accountId: jmap,
|
||||
// Local answers aren't a position in the server's log, so this is a token only.
|
||||
queryState: 'local',
|
||||
canCalculateChanges: false,
|
||||
position,
|
||||
ids,
|
||||
};
|
||||
if (args.calculateTotal) {
|
||||
result.total = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM file_node WHERE ${where}`, params)?.n ?? 0);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
export const FILE_METHODS: Readonly<Record<string, MethodHandler>> = {
|
||||
'FileNode/get': fileNodeGet,
|
||||
'FileNode/query': fileNodeQuery,
|
||||
};
|
||||
@@ -0,0 +1,50 @@
|
||||
/** How a FileNode becomes a row. The server's own copy stays in `json`, in whichever draft it speaks. */
|
||||
|
||||
import { fileNodeKind, fromDraftFileNode } from '@/lib/jmap/filenode-kind';
|
||||
import { decodeFileNodeName } from '@/lib/jmap/filenode-name';
|
||||
import type { FileNode } from '@/lib/jmap/types';
|
||||
import type { StorageAdapter } from '../storage';
|
||||
|
||||
export const FILES_CAPABILITY = 'urn:ietf:params:jmap:filenode';
|
||||
export const FILES_USING = ['urn:ietf:params:jmap:core', FILES_CAPABILITY];
|
||||
|
||||
/** The client's list (lib/jmap/client.ts). Stalwart leaves out shareWith and myRights unless asked. */
|
||||
export const FILE_NODE_PROPERTIES = [
|
||||
'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified',
|
||||
'nodeType', '@type', 'mediaType', 'shareWith', 'myRights',
|
||||
];
|
||||
|
||||
export type WireNode = Record<string, unknown> & { id: string };
|
||||
|
||||
/** The node's media type, whichever draft named it. */
|
||||
export function nodeMediaType(node: WireNode): string {
|
||||
const { type } = fromDraftFileNode(node as unknown as Partial<FileNode> & { '@type'?: unknown; mediaType?: unknown });
|
||||
return typeof type === 'string' && type ? type : 'application/octet-stream';
|
||||
}
|
||||
|
||||
export function writeNode(db: StorageAdapter, jmap: string, node: WireNode, gen: number): void {
|
||||
const normal = fromDraftFileNode(node as unknown as FileNode & { '@type'?: unknown; mediaType?: unknown });
|
||||
db.run(
|
||||
`INSERT INTO file_node (jmap, id, parent_id, name, kind, blob_id, size, json, gen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (jmap, id) DO UPDATE SET parent_id = excluded.parent_id, name = excluded.name, kind = excluded.kind,
|
||||
size = excluded.size, json = excluded.json, gen = excluded.gen,
|
||||
keep_error = CASE WHEN blob_id IS excluded.blob_id THEN keep_error END,
|
||||
blob_id = excluded.blob_id`,
|
||||
[
|
||||
jmap,
|
||||
node.id,
|
||||
typeof node.parentId === 'string' ? node.parentId : null,
|
||||
// The client decodes names itself from `json`. This column is for the store's own lists.
|
||||
typeof node.name === 'string' ? decodeFileNodeName(node.name) : '',
|
||||
fileNodeKind(normal),
|
||||
typeof node.blobId === 'string' ? node.blobId : null,
|
||||
typeof node.size === 'number' ? node.size : null,
|
||||
JSON.stringify(node),
|
||||
gen,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
export function removeNode(db: StorageAdapter, jmap: string, id: string): void {
|
||||
db.run('DELETE FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { Migration } from '../storage';
|
||||
|
||||
export const FILE_MIGRATIONS: readonly Migration[] = [
|
||||
{
|
||||
id: 700,
|
||||
name: 'files',
|
||||
sql: `
|
||||
CREATE TABLE file_node (
|
||||
jmap TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
parent_id TEXT,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
-- file, directory or symlink, whichever way the server says so.
|
||||
kind TEXT NOT NULL,
|
||||
blob_id TEXT,
|
||||
size INTEGER,
|
||||
json TEXT NOT NULL,
|
||||
gen INTEGER NOT NULL DEFAULT 0,
|
||||
pinned INTEGER NOT NULL DEFAULT 0,
|
||||
-- When the page last opened it. Cleared when its content is pushed out.
|
||||
opened_at INTEGER,
|
||||
-- Why the content isn't kept: tooLarge, overLimit, notAllowed or download.
|
||||
keep_error TEXT,
|
||||
PRIMARY KEY (jmap, id)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX file_node_parent ON file_node (jmap, parent_id);
|
||||
CREATE INDEX file_node_blob ON file_node (jmap, blob_id);
|
||||
CREATE TABLE file_blob (
|
||||
jmap TEXT NOT NULL,
|
||||
blob_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
kept_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (jmap, blob_id)
|
||||
) WITHOUT ROWID;
|
||||
-- In pieces: a whole file bound at once would grow the WASM heap by its
|
||||
-- size, and that memory never comes back.
|
||||
CREATE TABLE file_blob_chunk (
|
||||
jmap TEXT NOT NULL,
|
||||
blob_id TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
PRIMARY KEY (jmap, blob_id, seq)
|
||||
);
|
||||
`,
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* FileNode metadata for one JMAP account, then the contents worth keeping.
|
||||
*
|
||||
* A full pass reads the tree the way the client does (fetchAllFileNodes,
|
||||
* lib/jmap/client.ts). `FileNode/get` with ids null returns everything, but
|
||||
* 0.16 silently stops at maxObjectsInGet and 1.0 refuses a larger account with
|
||||
* requestTooLarge. Past that, ids come from paging FileNode/query. Before
|
||||
* 0.16.6 that query leaves out folders, so parents still unknown afterwards are
|
||||
* read by id. The cursor is in sync_state, so a reload continues.
|
||||
*/
|
||||
|
||||
import type { SyncContext, Syncer } from '../area';
|
||||
import { runChangesLoop } from '../engine/changes';
|
||||
import { JmapMethodError, responseFor, type MethodCall } from '../engine/transport';
|
||||
import { dropOrphanBlobs, fetchKeptFiles } from './keep';
|
||||
import { FILE_NODE_PROPERTIES, FILES_CAPABILITY, FILES_USING, removeNode, writeNode, type WireNode } from './records';
|
||||
|
||||
const MAX_GET = 500;
|
||||
const GETS_PER_REQUEST = 8;
|
||||
const QUERY_PAGE = 5000;
|
||||
|
||||
function getBatch(ctx: SyncContext): number {
|
||||
const core = ctx.session.capabilities['urn:ietf:params:jmap:core'] as { maxObjectsInGet?: number } | undefined;
|
||||
return Math.min(MAX_GET, core?.maxObjectsInGet || MAX_GET);
|
||||
}
|
||||
|
||||
async function fetchNodes(ctx: SyncContext, jmap: string, ids: string[]): Promise<WireNode[]> {
|
||||
const size = getBatch(ctx);
|
||||
const nodes: WireNode[] = [];
|
||||
for (let at = 0; at < ids.length;) {
|
||||
const calls: MethodCall[] = [];
|
||||
for (let i = 0; i < GETS_PER_REQUEST && at < ids.length; i += 1, at += size) {
|
||||
calls.push(['FileNode/get', { accountId: jmap, ids: ids.slice(at, at + size), properties: FILE_NODE_PROPERTIES }, `g${i}`]);
|
||||
}
|
||||
const response = await ctx.transport.request(calls, FILES_USING);
|
||||
calls.forEach(([, , callId]) => nodes.push(...responseFor(response, callId)[1].list as WireNode[]));
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
interface ListCursor { step: 'all' | 'query' | 'parents'; position: number; gen: number; captured: string }
|
||||
|
||||
async function startFullList(ctx: SyncContext, jmap: string): Promise<ListCursor> {
|
||||
const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: [], properties: ['id'] }, 's']], FILES_USING);
|
||||
const captured = String(responseFor(response, 's')[1].state);
|
||||
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0) + 1;
|
||||
const cursor: ListCursor = { step: 'all', position: 0, gen, captured };
|
||||
ctx.db.transaction(() => ctx.setSyncState(jmap, 'FileNode', { state: null, phase: 'list', cursor: { ...cursor } }));
|
||||
return cursor;
|
||||
}
|
||||
|
||||
function saveStep(ctx: SyncContext, jmap: string, nodes: WireNode[], cursor: ListCursor): void {
|
||||
ctx.db.transaction(() => {
|
||||
for (const node of nodes) writeNode(ctx.db, jmap, node, cursor.gen);
|
||||
ctx.setSyncState(jmap, 'FileNode', { cursor: { ...cursor } });
|
||||
});
|
||||
}
|
||||
|
||||
async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise<boolean> {
|
||||
const cursor = { ...start };
|
||||
|
||||
if (cursor.step === 'all') {
|
||||
let nodes: WireNode[] = [];
|
||||
let complete = false;
|
||||
try {
|
||||
const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: null, properties: FILE_NODE_PROPERTIES }, 'a']], FILES_USING);
|
||||
nodes = responseFor(response, 'a')[1].list as WireNode[];
|
||||
complete = nodes.length < getBatch(ctx);
|
||||
} catch (error) {
|
||||
if (!(error instanceof JmapMethodError && error.type === 'requestTooLarge')) throw error;
|
||||
}
|
||||
if (!complete) cursor.step = 'query';
|
||||
else cursor.step = 'parents';
|
||||
saveStep(ctx, jmap, nodes, cursor);
|
||||
}
|
||||
|
||||
while (cursor.step === 'query') {
|
||||
if (ctx.stopped()) return false;
|
||||
const response = await ctx.transport.request([['FileNode/query', { accountId: jmap, filter: {}, position: cursor.position, limit: QUERY_PAGE, calculateTotal: true }, 'q']], FILES_USING);
|
||||
const [, query] = responseFor(response, 'q');
|
||||
const ids = query.ids as string[];
|
||||
const missing = ids.filter((id) => Number(ctx.db.get<{ gen: number }>('SELECT gen FROM file_node WHERE jmap = ? AND id = ?', [jmap, id])?.gen ?? -1) < cursor.gen);
|
||||
const nodes = await fetchNodes(ctx, jmap, missing);
|
||||
cursor.position += ids.length;
|
||||
// The server may clamp the limit, so only an empty page or the total ends this.
|
||||
if (ids.length === 0 || (typeof query.total === 'number' && cursor.position >= query.total)) cursor.step = 'parents';
|
||||
saveStep(ctx, jmap, nodes, cursor);
|
||||
ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'list', done: cursor.position, total: typeof query.total === 'number' ? query.total : null });
|
||||
}
|
||||
|
||||
const asked = new Set<string>();
|
||||
for (;;) {
|
||||
if (ctx.stopped()) return false;
|
||||
const parents = ctx.db.all<{ id: string }>(
|
||||
`SELECT DISTINCT n.parent_id AS id FROM file_node n
|
||||
WHERE n.jmap = ? AND n.gen = ? AND n.parent_id IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM file_node p WHERE p.jmap = n.jmap AND p.id = n.parent_id AND p.gen = ?)`,
|
||||
[jmap, cursor.gen, cursor.gen],
|
||||
).map((row) => String(row.id)).filter((id) => !asked.has(id));
|
||||
if (parents.length === 0) break;
|
||||
for (const id of parents) asked.add(id);
|
||||
saveStep(ctx, jmap, await fetchNodes(ctx, jmap, parents), cursor);
|
||||
}
|
||||
|
||||
ctx.db.transaction(() => {
|
||||
// Whatever this pass didn't see is gone from the server.
|
||||
ctx.db.run('DELETE FROM file_node WHERE jmap = ? AND gen < ?', [jmap, cursor.gen]);
|
||||
ctx.setSyncState(jmap, 'FileNode', { state: cursor.captured, phase: null, cursor: null });
|
||||
});
|
||||
ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: string | null): Promise<'done' | 'resync' | 'stopped'> {
|
||||
const outcome = await runChangesLoop({
|
||||
transport: ctx.transport,
|
||||
type: 'FileNode',
|
||||
jmap,
|
||||
using: FILES_USING,
|
||||
since,
|
||||
target,
|
||||
stopped: ctx.stopped,
|
||||
apply: async (page) => {
|
||||
const nodes = await fetchNodes(ctx, jmap, [...page.created, ...page.updated]);
|
||||
ctx.db.transaction(() => {
|
||||
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0);
|
||||
for (const node of nodes) writeNode(ctx.db, jmap, node, gen);
|
||||
for (const id of page.destroyed) removeNode(ctx.db, jmap, id);
|
||||
ctx.setSyncState(jmap, 'FileNode', { state: page.newState });
|
||||
});
|
||||
if (nodes.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] });
|
||||
},
|
||||
});
|
||||
if (outcome.outcome === 'resync') return 'resync';
|
||||
return outcome.outcome === 'stopped' ? 'stopped' : 'done';
|
||||
}
|
||||
|
||||
export const filesSyncer: Syncer = {
|
||||
name: 'files',
|
||||
capability: FILES_CAPABILITY,
|
||||
async run(ctx, jmap) {
|
||||
const row = ctx.syncState(jmap, 'FileNode');
|
||||
let cursor = row.phase === 'list' ? row.cursor as unknown as ListCursor : null;
|
||||
if (!cursor && !row.state) cursor = await startFullList(ctx, jmap);
|
||||
if (!cursor && row.state) {
|
||||
const outcome = await syncDelta(ctx, jmap, row.state, row.target);
|
||||
if (outcome === 'stopped') return;
|
||||
if (outcome === 'resync') cursor = await startFullList(ctx, jmap);
|
||||
}
|
||||
if (cursor) {
|
||||
if (!(await runFullList(ctx, jmap, cursor))) return;
|
||||
// Catch what changed while paging.
|
||||
const after = ctx.syncState(jmap, 'FileNode');
|
||||
if (after.state && (await syncDelta(ctx, jmap, after.state, after.target)) === 'stopped') return;
|
||||
}
|
||||
if (ctx.stopped()) return;
|
||||
ctx.db.transaction(() => dropOrphanBlobs(ctx.db));
|
||||
await fetchKeptFiles(ctx, jmap);
|
||||
},
|
||||
};
|
||||
Reference in New Issue
Block a user