diff --git a/lib/store/__tests__/client-files.test.ts b/lib/store/__tests__/client-files.test.ts new file mode 100644 index 000000000..08efd48d4 --- /dev/null +++ b/lib/store/__tests__/client-files.test.ts @@ -0,0 +1,88 @@ +// @vitest-environment node +/** + * The real JMAPClient listing files from the store, in each way a server says + * what a node is, plus what it does with a write. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { JMAPClient } from '@/lib/jmap/client'; +import { fileNodeKind } from '@/lib/jmap/filenode-kind'; +import { setOfflineHooks, type OfflineHooks } from '@/lib/jmap/offline-hooks'; +import { DEFAULT_SCOPE } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { filesArea } from '../files'; +import { FakeFiles, FILES, FILES_SESSION, SHARED_FILES_ACCOUNT, type FakeFilesOptions } from './helpers/fake-files'; +import { openMemoryStore } from './helpers/node-adapter'; + +afterEach(() => setOfflineHooks(null)); + +async function offlineClient(server: FakeFiles): Promise { + const engine = new AccountEngine({ + account: 'a@fake.test', + db: openMemoryStore(), + session: FILES_SESSION, + transport: server, + areas: [filesArea], + scope: DEFAULT_SCOPE, + sleep: async () => {}, + }); + await engine.requestSync(); + engine.setOnline(false); + setOfflineHooks({ + skipNetwork: () => true, + answer: (_client, calls) => engine.answer(calls), + record: () => {}, + noteNetwork: () => {}, + } as Partial as OfflineHooks); + return JMAPClient.fromOfflineSession('https://fake.test', 'a@example.org', { + ...FILES_SESSION, + capabilities: { ...FILES_SESSION.capabilities, 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500 } }, + }); +} + +describe('file listing offline', () => { + it.each<[string, FakeFilesOptions]>([ + ['nodeType (0.16.6 and later, 1.0 as tested)', { style: 'nodeType' }], + ['blobId null only (0.16.0-5)', { style: 'blob' }], + ['@type and mediaType (1.0.0)', { style: 'draft' }], + ])('tells folders from files: %s', async (_label, options) => { + const server = new FakeFiles(options); + const folder = server.addFolder('Docs'); + server.addFile('note.txt', folder.id); + const client = await offlineClient(server); + expect(client.supportsFiles()).toBe(true); + + const nodes = await client.listAllFileNodes(); + const byName = new Map(nodes.map((n) => [n.name, n])); + expect(fileNodeKind(byName.get('Docs') as never)).toBe('directory'); + expect(fileNodeKind(byName.get('note.txt') as never)).toBe('file'); + expect(byName.get('note.txt')).toMatchObject({ parentId: folder.id, type: 'text/plain', size: 10 }); + expect((await client.listFileNodes(folder.id)).map((n) => n.name)).toEqual(['note.txt']); + expect((await client.listFileNodes(null)).map((n) => n.name)).toEqual(['Docs']); + }); + + it('lists an account larger than one /get, ending its paging cleanly', async () => { + const server = new FakeFiles({ getAll: 'refuse' }); + const folder = server.addFolder('Docs'); + for (let i = 0; i < 700; i += 1) server.addFile(`f${i}`, folder.id); + const client = await offlineClient(server); + expect(await client.listAllFileNodes()).toHaveLength(701); + }); + + it('lists shared accounts with namespaced ids', async () => { + const server = new FakeFiles(); + server.addFolder('Mine'); + const shared = server.addFolder('Theirs', null, undefined, SHARED_FILES_ACCOUNT); + const client = await offlineClient(server); + const nodes = await client.listAllFileNodesAcrossAccounts(); + expect(nodes.find((n) => n.name === 'Theirs')).toMatchObject({ id: `${SHARED_FILES_ACCOUNT}:${shared.id}`, isShared: true }); + expect(nodes.find((n) => n.name === 'Mine')).toMatchObject({ isShared: false }); + }); + + it('fails a write with an error rather than losing it', async () => { + const server = new FakeFiles(); + const client = await offlineClient(server); + await expect(client.createFileDirectory('New', null)).rejects.toThrow(); + expect(FILES in FILES_SESSION.capabilities).toBe(true); + expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false); + }); +}); diff --git a/lib/store/__tests__/files.test.ts b/lib/store/__tests__/files.test.ts new file mode 100644 index 000000000..acf81b483 --- /dev/null +++ b/lib/store/__tests__/files.test.ts @@ -0,0 +1,535 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { DEFAULT_SCOPE, type StoreEvent, type StoreScope } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError } from '../engine/transport'; +import { fileBlob, FILE_CACHE_CAP_BYTES, filesArea } from '../files'; +import type { StorageAdapter } from '../storage'; +import { JMAP_ACCOUNT } from './helpers/fake-jmap'; +import { FakeFiles, FILES_SESSION, SHARED_FILES_ACCOUNT } from './helpers/fake-files'; +import { openMemoryStore } from './helpers/node-adapter'; + +const NOW = Date.UTC(2026, 10, 3); + +/* eslint-disable @typescript-eslint/no-explicit-any -- answers are shaped like the server's */ + +interface Rig { + db: StorageAdapter; + engine: AccountEngine; + clock: { at: number }; + events: StoreEvent[]; +} + +function rigFor(server: FakeFiles, scope: Partial = {}): Rig { + const db = openMemoryStore(); + const clock = { at: NOW }; + const events: StoreEvent[] = []; + const engine = new AccountEngine({ + account: 'a@fake.test', + db, + session: FILES_SESSION, + transport: server, + areas: [filesArea], + scope: { ...DEFAULT_SCOPE, ...scope }, + emit: (event) => events.push(event), + now: () => clock.at, + sleep: async () => {}, + }); + return { db, engine, clock, events }; +} + +async function synced(server: FakeFiles, scope: Partial = {}): Promise { + const rig = rigFor(server, scope); + await rig.engine.requestSync(); + return rig; +} + +async function call(engine: AccountEngine, name: string, args: Record = {}): Promise { + const response = await engine.answer([[name, { accountId: JMAP_ACCOUNT, ...args }, '0']]); + if (!response) throw new Error('nothing answered'); + const [answered, result] = response.methodResponses[0]; + if (answered === 'error') throw new Error(result.type); + return result; +} + +const ref = (id: string, jmap = JMAP_ACCOUNT) => ({ jmap, id }); + +function names(rows: Array<{ name?: unknown }>): unknown[] { + return rows.map((row) => row.name).sort(); +} + +describe('file metadata sync', () => { + it('stores small accounts from one FileNode/get', async () => { + const server = new FakeFiles(); + const folder = server.addFolder('Docs'); + server.addFile('a.txt', folder.id); + const { engine } = await synced(server); + expect(server.calls.filter(([n]) => n === 'FileNode/query')).toHaveLength(0); + expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['Docs', 'a.txt']); + }); + + it.each([ + ['0.16 cuts the list at 500', { getAll: 'cut' as const }], + ['1.0 refuses a list over 500', { getAll: 'refuse' as const }], + ])('pages past the /get limit: %s', async (_label, options) => { + const server = new FakeFiles(options); + const folder = server.addFolder('Docs'); + for (let i = 0; i < 1234; i += 1) server.addFile(`f${i}.txt`, folder.id); + const { db } = await synced(server); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(1235); + for (const [name, args] of server.calls) { + if (name === 'FileNode/get' && Array.isArray(args.ids)) expect(args.ids.length).toBeLessThanOrEqual(500); + } + }); + + it('finds folders the query leaves out on old servers, by the parents of what it did find', async () => { + const server = new FakeFiles({ queryOmitsFolders: true, style: 'blob' }); + for (let i = 0; i < 600; i += 1) server.addFile(`early${i}.txt`); + const top = server.addFolder('Top'); + const inner = server.addFolder('Inner', top.id); + server.addFile('deep.txt', inner.id); + const { db } = await synced(server); + const kinds = db.all<{ name: string; kind: string }>("SELECT name, kind FROM file_node WHERE name IN ('Top', 'Inner', 'deep.txt') ORDER BY name"); + expect(kinds).toEqual([{ name: 'Inner', kind: 'directory' }, { name: 'Top', kind: 'directory' }, { name: 'deep.txt', kind: 'file' }]); + }); + + it('continues a stopped full sync from its cursor', async () => { + const server = new FakeFiles({ getAll: 'refuse' }); + for (let i = 0; i < 6000; i += 1) server.addFile(`f${i}`); + const rig = rigFor(server); + let gets = 0; + server.beforeRequest = () => { + gets += 1; + // State capture, the refused list, a query page of 5000 and two requests of gets: then the network goes. + if (gets === 6) server.offline = true; + }; + await rig.engine.requestSync(); + expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(5000); + server.offline = false; + server.beforeRequest = null; + server.calls.length = 0; + await rig.engine.requestSync(); + expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_node')?.n)).toBe(6000); + expect(server.calls.find(([n]) => n === 'FileNode/query')?.[1].position).toBe(5000); + expect(server.calls.filter(([n, a]) => n === 'FileNode/get' && a.ids === null && a.accountId === JMAP_ACCOUNT)).toHaveLength(0); + }); + + it('follows changes through FileNode/changes', async () => { + const server = new FakeFiles(); + const a = server.addFile('a.txt'); + const b = server.addFile('b.txt'); + const { engine } = await synced(server); + server.calls.length = 0; + server.renameNode(a.id, 'renamed.txt'); + server.removeNode(b.id); + server.addFolder('New'); + await engine.requestSync(); + expect(server.calls.some(([n]) => n === 'FileNode/changes')).toBe(true); + expect(server.calls.some(([n, args]) => n === 'FileNode/get' && args.ids === null)).toBe(false); + expect(names((await call(engine, 'FileNode/get', { ids: null })).list)).toEqual(['New', 'renamed.txt']); + }); + + it('starts over when the server can no longer give changes, and keeps a pin on a file still there', async () => { + const server = new FakeFiles(); + const gone = server.addFile('gone.txt'); + const kept = server.addFile('kept.txt'); + const { db, engine } = await synced(server); + await engine.areaCall('files.pin', ref(kept.id)); + await engine.requestSync(); + server.removeNode(gone.id); + server.account().coll.truncatedBelow = server.account().coll.log.length; + await engine.requestSync(); + expect(db.all('SELECT id FROM file_node').map((r) => r.id)).toEqual([kept.id]); + expect(db.get<{ pinned: number }>('SELECT pinned FROM file_node WHERE id = ?', [kept.id])?.pinned).toBe(1); + }); + + it('keeps accounts apart, even with the same ids', async () => { + const server = new FakeFiles(); + server.addFile('own.txt', null, 5, { id: 'same' }); + server.addFile('team.txt', null, 5, { id: 'same', blobId: 'team-blob' }, SHARED_FILES_ACCOUNT); + const { engine } = await synced(server); + expect(names((await call(engine, 'FileNode/get', { ids: ['same'] })).list)).toEqual(['own.txt']); + expect(names((await call(engine, 'FileNode/get', { ids: ['same'], accountId: SHARED_FILES_ACCOUNT })).list)).toEqual(['team.txt']); + }); +}); + +describe('file answers', () => { + it('says so instead of an empty list when nothing was read yet', async () => { + const { engine } = rigFor(new FakeFiles()); + expect(await engine.answer([['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '0']])).toBeNull(); + }); + + it('gives the last server state and only asked-for properties', async () => { + const server = new FakeFiles(); + const file = server.addFile('a.txt'); + const { engine } = await synced(server); + const got = await call(engine, 'FileNode/get', { ids: [file.id, 'nope'], properties: ['name', 'parentId'] }); + expect(got.state).toBe(server.account().coll.state()); + expect(got.list).toEqual([{ id: file.id, name: 'a.txt', parentId: null }]); + expect(got.notFound).toEqual(['nope']); + }); + + it('answers a query by parent, also at the root, which the real server refuses to take', async () => { + const server = new FakeFiles(); + const folder = server.addFolder('Docs'); + const inside = server.addFile('in.txt', folder.id); + const root = server.addFile('root.txt'); + const { engine } = await synced(server); + expect((await call(engine, 'FileNode/query', { filter: { parentId: folder.id } })).ids).toEqual([inside.id]); + expect((await call(engine, 'FileNode/query', { filter: { parentId: null } })).ids).toEqual([folder.id, root.id]); + }); + + it('pages and sorts a query', async () => { + const server = new FakeFiles(); + server.addFile('b', null, 30); + server.addFile('a', null, 10); + server.addFile('c', null, 20); + const { engine } = await synced(server); + const page = await call(engine, 'FileNode/query', { filter: {}, position: 1, limit: 1, calculateTotal: true }); + expect(page).toMatchObject({ position: 1, total: 3, ids: [expect.any(String)] }); + const bySize = await call(engine, 'FileNode/query', { filter: {}, sort: [{ property: 'size', isAscending: false }] }); + const list = (await call(engine, 'FileNode/get', { ids: bySize.ids })).list; + expect(list.map((n: { name: string }) => n.name)).toEqual(['b', 'c', 'a']); + }); + + it('refuses what it cannot read, for that call only', async () => { + const server = new FakeFiles(); + server.addFile('a'); + const { engine } = await synced(server); + const response = await engine.answer([ + ['FileNode/query', { accountId: JMAP_ACCOUNT, filter: { hasFoo: true } }, '0'], + ['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'], + ]); + expect(response?.methodResponses[0][1].type).toBe('unsupportedFilter'); + expect(response?.methodResponses[1][0]).toBe('FileNode/get'); + }); + + it('answers FileNode/set with a clear per-call error, not silence', async () => { + const server = new FakeFiles(); + server.addFile('a'); + const { engine } = await synced(server); + const response = await engine.answer([ + ['FileNode/set', { accountId: JMAP_ACCOUNT, create: { n0: { name: 'x' } } }, '0'], + ['FileNode/get', { accountId: JMAP_ACCOUNT, ids: null }, '1'], + ]); + expect(response?.methodResponses[0]).toEqual(['error', { type: 'serverUnavailable', description: 'FileNode/set needs the server' }, '0']); + expect(response?.methodResponses[1][0]).toBe('FileNode/get'); + expect(server.calls.some(([n]) => n === 'FileNode/set')).toBe(false); + }); +}); + +describe('pinned files', () => { + it('fetches a pinned file and gives it back byte for byte, past the chunk size', async () => { + const server = new FakeFiles(); + const content = new Uint8Array(3 * 1024 * 1024 + 17).map((_, i) => i % 251); + const file = server.addFile('big.bin', null, content); + const { db, engine } = await synced(server); + expect(server.downloads).toEqual([]); + + expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: false }); + await engine.requestSync(); + expect(server.downloads).toEqual([{ jmap: JMAP_ACCOUNT, blobId: file.blobId }]); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(4); + + const read = await engine.areaCall('files.read', ref(file.id)) as any; + expect(read.name).toBe('big.bin'); + expect(read.type).toBe('text/plain'); + expect(Buffer.compare(read.data, content)).toBe(0); + expect(Buffer.compare(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)?.data as Uint8Array, content)).toBe(0); + expect(await engine.areaCall('files.blob', { jmap: JMAP_ACCOUNT, blobId: file.blobId })).toMatchObject({ size: content.length }); + // Already kept: not fetched again. + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + expect(await engine.areaCall('files.pin', ref(file.id))).toEqual({ kept: true }); + }); + + it('names a file with a %-encoded name from 0.16 WebDAV the way the client shows it', async () => { + const server = new FakeFiles(); + const file = server.addFile('My%20Report.pdf'); + const { engine } = await synced(server); + await engine.areaCall('files.pin', ref(file.id)); + await engine.requestSync(); + expect((await engine.areaCall('files.read', ref(file.id)) as any).name).toBe('My Report.pdf'); + expect((await engine.areaCall('files.kept', undefined) as any).files[0].name).toBe('My Report.pdf'); + // The stored node stays as the server sent it. The client decodes it once. + expect((await call(engine, 'FileNode/get', { ids: [file.id] })).list[0].name).toBe('My%20Report.pdf'); + }); + + it('gives nothing for a file not kept', async () => { + const server = new FakeFiles(); + const file = server.addFile('a.txt'); + const { db, engine } = await synced(server); + expect(await engine.areaCall('files.read', ref(file.id))).toBeNull(); + expect(fileBlob({ db }, JMAP_ACCOUNT, file.blobId as string)).toBeNull(); + }); + + it('refuses to pin a folder or an unknown id', async () => { + const server = new FakeFiles(); + const folder = server.addFolder('Docs'); + const { engine } = await synced(server); + await expect(engine.areaCall('files.pin', ref(folder.id))).rejects.toThrow('Only files'); + await expect(engine.areaCall('files.pin', ref('nope'))).rejects.toThrow(); + }); + + it('downloads one at a time', async () => { + const server = new FakeFiles(); + const files = [server.addFile('a'), server.addFile('b'), server.addFile('c')]; + const { engine } = await synced(server); + for (const file of files) await engine.areaCall('files.pin', ref(file.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(3); + expect(server.mostAtOnce).toBe(1); + }); + + it('fetches a pinned file over the one-file limit, and skips an opened one', async () => { + const server = new FakeFiles(); + const pinned = server.addFile('pinned.bin', null, 100); + const opened = server.addFile('opened.bin', null, 100); + const { engine } = await synced(server, { maxAttachmentBytes: 50 }); + await engine.areaCall('files.pin', ref(pinned.id)); + await engine.areaCall('files.noteOpened', ref(opened.id)); + await engine.requestSync(); + expect(server.downloads.map((d) => d.blobId)).toEqual([pinned.blobId]); + const status = await engine.areaCall('files.kept', undefined) as any; + const row = (name: string) => status.files.find((f: { name: string }) => f.name === name); + expect(row('pinned.bin')).toMatchObject({ pinned: true, kept: true, error: null }); + expect(row('opened.bin')).toMatchObject({ pinned: false, kept: false, error: 'tooLarge' }); + }); + + it('does not take pins past the total limit, and says so', async () => { + const server = new FakeFiles(); + const a = server.addFile('a', null, 60); + const b = server.addFile('b', null, 60); + const { engine, clock } = await synced(server, { maxTotalBytes: 100 }); + await engine.areaCall('files.pin', ref(a.id)); + clock.at += 1; + await engine.areaCall('files.pin', ref(b.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + const status = await engine.areaCall('files.kept', undefined) as any; + expect(status.files.filter((f: { kept: boolean }) => f.kept)).toHaveLength(1); + expect(status.files.filter((f: { error: string | null }) => f.error === 'overLimit')).toHaveLength(1); + }); + + it('keeps nothing when the scope forbids it, and drops what was kept', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(file.id)); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + + rig.engine.setScope({ ...DEFAULT_SCOPE, attachments: false }); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).toBeNull(); + expect(Number(rig.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0); + expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ pinned: true, kept: false, error: 'notAllowed' }); + + rig.engine.setScope(DEFAULT_SCOPE); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + }); + + it('goes by the real size when the metadata has it wrong', async () => { + const server = new FakeFiles(); + const opened = server.addFile('opened', null, new Uint8Array(100), { size: 1 }); + const { engine } = await synced(server, { maxAttachmentBytes: 50 }); + await engine.areaCall('files.noteOpened', ref(opened.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + expect(await engine.areaCall('files.read', ref(opened.id))).toBeNull(); + expect((await engine.areaCall('files.kept', undefined) as any).files[0].error).toBe('tooLarge'); + }); + + it('does not store a file unpinned while it was downloading', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const { engine } = await synced(server); + await engine.areaCall('files.pin', ref(file.id)); + const original = server.download.bind(server); + server.download = async (jmap, blobId) => { + await engine.areaCall('files.unpin', ref(file.id)); + return original(jmap, blobId); + }; + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + expect(await engine.areaCall('files.read', ref(file.id))).toBeNull(); + }); + + it('lets a never-opened file go on unpin, and keeps an opened one as recent', async () => { + const server = new FakeFiles(); + const never = server.addFile('never'); + const opened = server.addFile('opened'); + const { engine } = await synced(server); + await engine.areaCall('files.pin', ref(never.id)); + await engine.areaCall('files.pin', ref(opened.id)); + await engine.areaCall('files.noteOpened', ref(opened.id)); + await engine.requestSync(); + await engine.areaCall('files.unpin', ref(never.id)); + await engine.areaCall('files.unpin', ref(opened.id)); + expect(await engine.areaCall('files.read', ref(never.id))).toBeNull(); + expect(await engine.areaCall('files.read', ref(opened.id))).not.toBeNull(); + const status = await engine.areaCall('files.kept', undefined) as any; + expect(status.files.map((f: { name: string; pinned: boolean }) => [f.name, f.pinned])).toEqual([['opened', false]]); + }); + + it('drops the old content and fetches the new one when a pinned file changes', async () => { + const server = new FakeFiles(); + const file = server.addFile('a.txt', null, new Uint8Array([1, 2, 3])); + const { db, engine } = await synced(server); + await engine.areaCall('files.pin', ref(file.id)); + await engine.requestSync(); + const old = file.blobId as string; + const fresh = server.replaceContent(file.id, new Uint8Array([9, 9])); + await engine.requestSync(); + expect(fileBlob({ db }, JMAP_ACCOUNT, old)).toBeNull(); + expect([...(fileBlob({ db }, JMAP_ACCOUNT, fresh)?.data ?? [])]).toEqual([9, 9]); + expect(Array.from((await engine.areaCall('files.read', ref(file.id)) as any).data)).toEqual([9, 9]); + }); + + it('drops the content when the file is removed on the server, unless another node shares it', async () => { + const server = new FakeFiles(); + const a = server.addFile('a', null, 5, { blobId: 'shared-blob' }); + const b = server.addFile('b', null, 5, { blobId: 'shared-blob' }); + const { db, engine } = await synced(server); + await engine.areaCall('files.pin', ref(a.id)); + await engine.areaCall('files.pin', ref(b.id)); + await engine.requestSync(); + expect(server.downloads).toHaveLength(1); + server.removeNode(a.id); + await engine.requestSync(); + expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).not.toBeNull(); + server.removeNode(b.id); + await engine.requestSync(); + expect(fileBlob({ db }, JMAP_ACCOUNT, 'shared-blob')).toBeNull(); + expect(Number(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM file_blob_chunk')?.n)).toBe(0); + }); + + it('carries on past one bad file, and hands a network failure to the engine', async () => { + const server = new FakeFiles(); + const bad = server.addFile('bad'); + const good = server.addFile('good'); + server.blobs.delete(bad.blobId as string); + const { engine, clock } = await synced(server); + await engine.areaCall('files.pin', ref(bad.id)); + clock.at += 1; + await engine.areaCall('files.pin', ref(good.id)); + await engine.requestSync(); + expect(await engine.areaCall('files.read', ref(good.id))).not.toBeNull(); + const status = await engine.areaCall('files.kept', undefined) as any; + expect(status.files.find((f: { name: string }) => f.name === 'bad').error).toBe('download'); + + }); + + it('hands a network failure to the engine, which ends the pass, and goes on next time', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(file.id)); + const original = server.download.bind(server); + server.download = async () => { + throw new TransportError('network', 'Failed to fetch'); + }; + const syncedBefore = rig.events.filter((e) => e.type === 'synced').length; + await rig.engine.requestSync(); + expect(rig.events.some((e) => e.type === 'error')).toBe(false); + expect(rig.events.filter((e) => e.type === 'synced').length).toBe(syncedBefore); + expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ kept: false, error: null }); + + server.download = original; + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + }); +}); + +describe('recently opened files', () => { + it('keeps an opened file without a pin', async () => { + const server = new FakeFiles(); + const file = server.addFile('a'); + const { engine } = await synced(server); + expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: false }); + await engine.requestSync(); + expect(await engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + expect(await engine.areaCall('files.noteOpened', ref(file.id))).toEqual({ kept: true }); + }); + + it('pushes out the oldest unpinned first, and never a pinned one', async () => { + const server = new FakeFiles(); + const files = Object.fromEntries(['old', 'pinnedOldest', 'mid', 'new', 'newest'].map((name) => [name, server.addFile(name, null, 10)])); + const rig = await synced(server, { maxTotalBytes: 30 }); + const open = async (name: string) => { + rig.clock.at += 1000; + await rig.engine.areaCall('files.noteOpened', ref(files[name].id)); + await rig.engine.requestSync(); + }; + await open('pinnedOldest'); + await rig.engine.areaCall('files.pin', ref(files.pinnedOldest.id)); + await open('old'); + await open('mid'); + const kept = async () => (await rig.engine.areaCall('files.kept', undefined) as any).files + .filter((f: { kept: boolean }) => f.kept).map((f: { name: string }) => f.name).sort(); + expect(await kept()).toEqual(['mid', 'old', 'pinnedOldest']); + + await open('new'); + expect(await kept()).toEqual(['mid', 'new', 'pinnedOldest']); + await open('mid'); + await open('newest'); + expect(await kept()).toEqual(['mid', 'newest', 'pinnedOldest']); + expect((await rig.engine.areaCall('files.kept', undefined) as any).bytes).toBe(30); + + // A pinned file was never pushed out and had to come back. + expect(server.downloads.filter((d) => d.blobId === files.pinnedOldest.blobId)).toHaveLength(1); + // Pushed out means forgotten: the next pass doesn't fetch it straight back. + expect(server.downloads.filter((d) => d.blobId === files.old.blobId)).toHaveLength(1); + const count = server.downloads.length; + await rig.engine.requestSync(); + expect(server.downloads).toHaveLength(count); + expect((await rig.engine.areaCall('files.kept', undefined) as any).files.map((f: { name: string }) => f.name)).not.toContain('old'); + }); + + it('pushes out recent files for a pin, and keeps a pin past the cap', async () => { + const server = new FakeFiles(); + const recent = server.addFile('recent', null, 20); + const pin = server.addFile('pin', null, 20); + const rig = await synced(server, { maxTotalBytes: 25 }); + await rig.engine.areaCall('files.noteOpened', ref(recent.id)); + await rig.engine.requestSync(); + rig.clock.at += 1000; + await rig.engine.areaCall('files.pin', ref(pin.id)); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(recent.id))).toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(pin.id))).not.toBeNull(); + }); + + it('keeps recently opened files under 500 MB whatever the total limit is', async () => { + const server = new FakeFiles(); + const file = server.addFile('new', null, 10); + const rig = await synced(server, { maxTotalBytes: 2 * FILE_CACHE_CAP_BYTES }); + // As if an older file already took all of it: only the row counts here. + rig.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [JMAP_ACCOUNT, 'huge', 'x', FILE_CACHE_CAP_BYTES, NOW - 1000]); + rig.db.run("INSERT INTO file_node (jmap, id, name, kind, blob_id, size, json, opened_at) VALUES (?, 'hugenode', 'huge', 'file', 'huge', ?, '{}', ?)", [JMAP_ACCOUNT, FILE_CACHE_CAP_BYTES, NOW - 1000]); + await rig.engine.areaCall('files.noteOpened', ref(file.id)); + await rig.engine.requestSync(); + expect(fileBlob({ db: rig.db }, JMAP_ACCOUNT, 'huge')).toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).not.toBeNull(); + expect((await rig.engine.areaCall('files.kept', undefined) as any).capBytes).toBe(FILE_CACHE_CAP_BYTES); + }); + + it('drops an old opened file that does not fit rather than a newer one', async () => { + const server = new FakeFiles(); + const a = server.addFile('a', null, 20); + const b = server.addFile('b', null, 20); + const rig = await synced(server, { maxTotalBytes: 25 }); + await rig.engine.areaCall('files.noteOpened', ref(a.id)); + rig.clock.at += 1000; + await rig.engine.areaCall('files.noteOpened', ref(b.id)); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(b.id))).not.toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(a.id))).toBeNull(); + expect(server.downloads.map((d) => d.blobId)).toEqual([b.blobId, a.blobId]); + const count = server.downloads.length; + await rig.engine.requestSync(); + expect(server.downloads).toHaveLength(count); + }); +}); +/* eslint-enable @typescript-eslint/no-explicit-any */ diff --git a/lib/store/__tests__/helpers/fake-files.ts b/lib/store/__tests__/helpers/fake-files.ts new file mode 100644 index 000000000..0ff22f4ed --- /dev/null +++ b/lib/store/__tests__/helpers/fake-files.ts @@ -0,0 +1,192 @@ +/** + * FakeJmap with files, as measured on both Stalwarts (2026-11-03) and as the + * client's comments record: `filter: {parentId: null}` is an HTTP 400 that + * fails the whole request, a folder has null blobId, type and size. What differs + * between versions is set per server: + * - `style` is how a node says what it is: `nodeType` (0.16.6 and later, 1.0 as + * tested), `blob` (0.16.0-5: nothing but blobId null), `draft` (1.0.0: `@type` + * and `mediaType`). + * - `getAll`: ids null past 500 is silently cut on 0.16 and refused on 1.0. + * - `queryOmitsFolders`: FileNode/query before 0.16.6. + */ + +import { TransportError, type StoreSession } from '../../engine/transport'; +import { changesFor } from './fake-contacts'; +import { Collection, FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './fake-jmap'; + +export const FILES = 'urn:ietf:params:jmap:filenode'; +export const SHARED_FILES_ACCOUNT = 'shared1'; + +export const FILES_SESSION: StoreSession = { + ...FAKE_SESSION, + primaryAccounts: { ...FAKE_SESSION.primaryAccounts, [FILES]: JMAP_ACCOUNT }, + accounts: { + [JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [FILES]: {} } }, + [SHARED_FILES_ACCOUNT]: { name: 'team@example.org', isPersonal: false, accountCapabilities: { [FILES]: {} } }, + }, + capabilities: { [FILES]: {} }, +}; + +export interface FakeNode { + id: string; + parentId: string | null; + name: string; + blobId: string | null; + type: string | null; + size: number | null; + modified?: string; +} + +export type FolderStyle = 'nodeType' | 'blob' | 'draft'; + +export interface FakeFilesOptions { + style?: FolderStyle; + getAll?: 'cut' | 'refuse'; + queryOmitsFolders?: boolean; +} + +class FilesAccount { + readonly nodes = new Map(); + readonly coll = new Collection(); +} + +export class FakeFiles extends FakeJmap { + private readonly fileAccounts = new Map(); + private readonly options: Required; + private nextNode = 1; + readonly blobs = new Map(); + /** In order, with the account. */ + readonly downloads: Array<{ jmap: string; blobId: string }> = []; + downloading = 0; + mostAtOnce = 0; + + constructor(options: FakeFilesOptions = {}) { + super(); + this.options = { style: 'nodeType', getAll: 'cut', queryOmitsFolders: false, ...options }; + } + + account(id: string = JMAP_ACCOUNT): FilesAccount { + let found = this.fileAccounts.get(id); + if (!found) { + found = new FilesAccount(); + this.fileAccounts.set(id, found); + } + return found; + } + + addFolder(name: string, parentId: string | null = null, id?: string, accountId: string = JMAP_ACCOUNT): FakeNode { + const account = this.account(accountId); + const node: FakeNode = { id: id ?? `n${this.nextNode++}`, parentId, name, blobId: null, type: null, size: null }; + account.nodes.set(node.id, node); + account.coll.push('created', node.id); + return node; + } + + addFile(name: string, parentId: string | null = null, content: Uint8Array | number = 10, extra: Partial = {}, accountId: string = JMAP_ACCOUNT): FakeNode { + const account = this.account(accountId); + const id = extra.id ?? `n${this.nextNode++}`; + const blobId = extra.blobId ?? `blob-${id}`; + const data = typeof content === 'number' ? new Uint8Array(content).fill(id.length) : content; + this.blobs.set(blobId, data); + const node: FakeNode = { id, parentId, name, blobId, type: 'text/plain', size: data.length, ...extra }; + account.nodes.set(id, node); + account.coll.push('created', id); + return node; + } + + /** A new version of the file: new blob, same node. */ + replaceContent(id: string, content: Uint8Array, accountId: string = JMAP_ACCOUNT): string { + const account = this.account(accountId); + const node = account.nodes.get(id) as FakeNode; + const blobId = `${node.blobId as string}-v`; + this.blobs.set(blobId, content); + node.blobId = blobId; + node.size = content.length; + account.coll.push('updated', id); + return blobId; + } + + renameNode(id: string, name: string, accountId: string = JMAP_ACCOUNT): void { + const account = this.account(accountId); + (account.nodes.get(id) as FakeNode).name = name; + account.coll.push('updated', id); + } + + removeNode(id: string, accountId: string = JMAP_ACCOUNT): void { + const account = this.account(accountId); + account.nodes.delete(id); + account.coll.push('destroyed', id); + } + + private wire(node: FakeNode, properties: string[] | undefined): Record { + const isFolder = node.blobId === null; + const full: Record = { id: node.id, parentId: node.parentId, name: node.name, blobId: node.blobId, size: node.size, created: '2026-10-01T00:00:00Z', modified: node.modified ?? '2026-10-01T00:00:00Z' }; + if (this.options.style === 'draft') { + full['@type'] = isFolder ? 'Directory' : 'File'; + full.mediaType = node.type; + } else { + full.type = node.type; + if (this.options.style === 'nodeType') full.nodeType = isFolder ? 'directory' : 'file'; + } + if (!properties) return full; + const out: Record = { id: node.id }; + for (const p of properties) if (p in full) out[p] = full[p]; + return out; + } + + protected override handle(name: string, args: Record): Record { + if (!name.startsWith('FileNode/')) return super.handle(name, args); + const account = this.account(String(args.accountId)); + switch (name) { + case 'FileNode/changes': + return changesFor(account.coll, args); + case 'FileNode/get': { + const ids = args.ids as string[] | null; + if (ids && ids.length > 500) throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' }); + if (!ids && account.nodes.size > 500 && this.options.getAll === 'refuse') { + throw Object.assign(new Error('requestTooLarge'), { jmapError: 'requestTooLarge' }); + } + const chosen = ids ?? [...account.nodes.keys()].slice(0, 500); + const properties = args.properties as string[] | undefined; + return { + accountId: args.accountId, + state: account.coll.state(), + list: chosen.filter((id) => account.nodes.has(id)).map((id) => this.wire(account.nodes.get(id) as FakeNode, properties)), + notFound: chosen.filter((id) => !account.nodes.has(id)), + }; + } + case 'FileNode/query': { + const filter = (args.filter ?? {}) as { parentId?: string | null }; + // Measured on both servers: this fails the whole request. + if (filter.parentId === null) throw new TransportError('http', 'JMAP request failed: 400', 400, 'urn:ietf:params:jmap:error:notRequest'); + let list = [...account.nodes.values()]; + if (this.options.queryOmitsFolders) list = list.filter((n) => n.blobId !== null); + if (filter.parentId) list = list.filter((n) => n.parentId === filter.parentId); + const position = Number(args.position ?? 0); + const limit = Math.min(Number(args.limit ?? 5000) || 5000, 5000); + return { + accountId: args.accountId, + queryState: account.coll.state(), + canCalculateChanges: true, + position, + ids: list.slice(position, position + limit).map((n) => n.id), + ...(args.calculateTotal ? { total: list.length } : {}), + }; + } + default: + throw Object.assign(new Error(name), { jmapError: 'unknownMethod' }); + } + } + + override async download(jmap: string, blobId: string): Promise { + if (this.offline) throw new TransportError('network', 'Failed to fetch'); + this.downloads.push({ jmap, blobId }); + this.downloading += 1; + this.mostAtOnce = Math.max(this.mostAtOnce, this.downloading); + await Promise.resolve(); + this.downloading -= 1; + const data = this.blobs.get(blobId); + if (!data) throw new TransportError('http', 'Download failed: 404', 404); + return data; + } +} diff --git a/lib/store/areas.ts b/lib/store/areas.ts index c8882f3ee..7fb966931 100644 --- a/lib/store/areas.ts +++ b/lib/store/areas.ts @@ -2,6 +2,7 @@ import type { StoreArea } from './area'; import { contactsArea } from './contacts'; +import { filesArea } from './files'; import { mailArea } from './mail'; -export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea]; +export const STORE_AREAS: readonly StoreArea[] = [mailArea, contactsArea, filesArea]; diff --git a/lib/store/files/index.ts b/lib/store/files/index.ts new file mode 100644 index 000000000..0957ced89 --- /dev/null +++ b/lib/store/files/index.ts @@ -0,0 +1,27 @@ +import type { StoreArea } from '../area'; +import { fileBlob, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep'; +import { FILE_METHODS } from './methods'; +import { FILE_MIGRATIONS } from './schema'; +import { filesSyncer } from './sync'; + +export { fileBlob, FILE_CACHE_CAP_BYTES } from './keep'; + +export const filesArea: StoreArea = { + name: 'files', + migrations: FILE_MIGRATIONS, + methods: FILE_METHODS, + syncers: [filesSyncer], + calls: { + /** Keep this file's content on the device until it's unpinned. */ + 'files.pin': (ctx, params: NodeRef) => pinFile(ctx, params), + 'files.unpin': (ctx, params: NodeRef) => unpinFile(ctx, params), + /** The page opened this file: keep its content for a while. */ + 'files.noteOpened': (ctx, params: NodeRef) => noteOpened(ctx, params), + /** A kept file's content and name, or null. */ + 'files.read': (ctx, params: NodeRef) => readFile(ctx, params), + /** A kept content by blob id, or null. */ + 'files.blob': (ctx, params: { jmap: string; blobId: string }) => fileBlob(ctx, params.jmap, params.blobId), + /** Pinned and recently opened files, for a settings page. */ + 'files.kept': (ctx) => keptStatus(ctx), + }, +}; diff --git a/lib/store/files/keep.ts b/lib/store/files/keep.ts new file mode 100644 index 000000000..a11b93455 --- /dev/null +++ b/lib/store/files/keep.ts @@ -0,0 +1,295 @@ +/** + * File contents kept on the device: pinned files, and recently opened ones + * within a cap. Metadata for every file is in file_node regardless. + * + * What gets kept: + * - A pinned file stays until it's unpinned or removed on the server. The cap + * doesn't push it out and the size limit for one file doesn't apply. The + * account's total limit (scope.maxTotalBytes, which a policy can lower) does. + * - A recently opened file is kept if it's within scope.maxAttachmentBytes, and + * pushed out oldest first when the cap is reached. Its opened_at goes with + * it, so it isn't fetched straight back. + * - With scope.attachments off, nothing is kept. + */ + +import { TransportError } from '../engine/transport'; +import { MethodError, type StoreContext, type SyncContext } from '../area'; +import type { StorageAdapter } from '../storage'; +import { nodeMediaType, type WireNode } from './records'; + +export const FILE_CACHE_CAP_BYTES = 500 * 1024 * 1024; +const CHUNK_BYTES = 1024 * 1024; + +export interface KeptFile { data: Uint8Array; type: string; size: number } + +export interface NodeRef { jmap: string; id: string } + +interface NodeRow { + jmap: string; + id: string; + name: string; + kind: string; + blob_id: string | null; + size: number | null; + json: string; + pinned: number; + opened_at: number | null; + keep_error: string | null; +} + +interface BlobRow { + jmap: string; + blob_id: string; + size: number; + kept_at: number; + pinned: number; + opened_at: number | null; +} + +function capBytes(ctx: Pick): number { + return Math.min(FILE_CACHE_CAP_BYTES, ctx.scope.maxTotalBytes); +} + +/** A kept file's content, for the client's blob lookup. */ +export function fileBlob(ctx: Pick, jmap: string, blobId: string): KeptFile | null { + const row = ctx.db.get<{ type: string; size: number }>('SELECT type, size FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + if (!row) return null; + const data = new Uint8Array(Number(row.size)); + let at = 0; + for (const chunk of ctx.db.all<{ data: Uint8Array }>('SELECT data FROM file_blob_chunk WHERE jmap = ? AND blob_id = ? ORDER BY seq', [jmap, blobId])) { + data.set(chunk.data, at); + at += chunk.data.length; + } + return { data, type: row.type, size: Number(row.size) }; +} + +function hasBlob(db: StorageAdapter, jmap: string, blobId: string): boolean { + return Boolean(db.get('SELECT 1 AS x FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId])); +} + +function dropBlob(db: StorageAdapter, jmap: string, blobId: string): void { + db.run('DELETE FROM file_blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + db.run('DELETE FROM file_blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); +} + +function storeBlob(ctx: StoreContext, jmap: string, blobId: string, type: string, data: Uint8Array): void { + dropBlob(ctx.db, jmap, blobId); + ctx.db.run('INSERT INTO file_blob (jmap, blob_id, type, size, kept_at) VALUES (?, ?, ?, ?, ?)', [jmap, blobId, type, data.length, ctx.now()]); + for (let seq = 0, at = 0; at < data.length; seq += 1, at += CHUNK_BYTES) { + ctx.db.run('INSERT INTO file_blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, data.subarray(at, at + CHUNK_BYTES)]); + } +} + +/** All kept contents in the store, with what the nodes pointing at each say about it. */ +function keptBlobs(db: StorageAdapter): BlobRow[] { + return db.all( + `SELECT b.jmap, b.blob_id, b.size, b.kept_at, COALESCE(MAX(n.pinned), 0) AS pinned, MAX(n.opened_at) AS opened_at + FROM file_blob b LEFT JOIN file_node n ON n.jmap = b.jmap AND n.blob_id = b.blob_id + GROUP BY b.jmap, b.blob_id`, + ); +} + +function ageOf(blob: BlobRow): number { + return Number(blob.opened_at ?? blob.kept_at); +} + +function pushOut(db: StorageAdapter, blob: BlobRow): void { + dropBlob(db, blob.jmap, blob.blob_id); + db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]); +} + +/** + * Pushes out unpinned contents, oldest first, so `size` more fits under the + * cap. A recently opened file only pushes out older ones and says false when + * that isn't enough. A pinned one always gets in. + */ +function makeRoom(ctx: StoreContext, size: number, incoming: { pinned: boolean; openedAt: number }): boolean { + const cap = capBytes(ctx); + const blobs = keptBlobs(ctx.db); + let used = blobs.reduce((sum, blob) => sum + Number(blob.size), 0); + const candidates = blobs + .filter((blob) => !blob.pinned && (incoming.pinned || ageOf(blob) < incoming.openedAt)) + .sort((a, b) => ageOf(a) - ageOf(b)); + const chosen: BlobRow[] = []; + for (const blob of candidates) { + if (used + size <= cap) break; + used -= Number(blob.size); + chosen.push(blob); + } + if (used + size > cap && !incoming.pinned) return false; + for (const blob of chosen) pushOut(ctx.db, blob); + return true; +} + +function pinnedBytes(db: StorageAdapter): number { + return keptBlobs(db).filter((blob) => blob.pinned).reduce((sum, blob) => sum + Number(blob.size), 0); +} + +function setError(db: StorageAdapter, jmap: string, blobId: string, error: string | null): void { + db.run('UPDATE file_node SET keep_error = ? WHERE jmap = ? AND blob_id = ?', [error, jmap, blobId]); +} + +/** Contents no node points at any more: the file was removed or has a new version. */ +export function dropOrphanBlobs(db: StorageAdapter): void { + for (const blob of keptBlobs(db)) { + const used = db.get('SELECT 1 AS x FROM file_node WHERE jmap = ? AND blob_id = ?', [blob.jmap, blob.blob_id]); + if (!used) dropBlob(db, blob.jmap, blob.blob_id); + } +} + +interface Wanted { blob_id: string; id: string; name: string; size: number | null; json: string; pinned: number; opened_at: number | null } + +function wantedRow(db: StorageAdapter, jmap: string, blobId: string): Wanted | undefined { + return db.get( + `SELECT blob_id, id, name, size, json, pinned, opened_at FROM file_node + WHERE jmap = ? AND blob_id = ? AND (pinned = 1 OR opened_at IS NOT NULL) + ORDER BY pinned DESC, opened_at DESC LIMIT 1`, + [jmap, blobId], + ); +} + +/** Fetches what should be kept and isn't yet, one file at a time, pinned files first. */ +export async function fetchKeptFiles(ctx: SyncContext, jmap: string): Promise { + const { db } = ctx; + if (!ctx.scope.attachments) { + db.transaction(() => { + for (const blob of keptBlobs(db)) dropBlob(db, blob.jmap, blob.blob_id); + db.run("UPDATE file_node SET keep_error = 'notAllowed' WHERE jmap = ? AND (pinned = 1 OR opened_at IS NOT NULL)", [jmap]); + }); + return; + } + db.transaction(() => { + db.run("UPDATE file_node SET keep_error = NULL WHERE jmap = ? AND keep_error = 'notAllowed'", [jmap]); + // The limit may have gone down since last time. + makeRoom(ctx, 0, { pinned: true, openedAt: 0 }); + }); + const todo = db.all<{ blob_id: string }>( + `SELECT n.blob_id FROM file_node n + WHERE n.jmap = ? AND n.blob_id IS NOT NULL AND (n.pinned = 1 OR n.opened_at IS NOT NULL) + AND NOT EXISTS (SELECT 1 FROM file_blob b WHERE b.jmap = n.jmap AND b.blob_id = n.blob_id) + GROUP BY n.blob_id ORDER BY MAX(n.pinned) DESC, MAX(n.opened_at) DESC`, + [jmap], + ).map((row) => String(row.blob_id)); + let done = 0; + for (const blobId of todo) { + if (ctx.stopped()) return; + // The page may have changed its mind during the last download. + const want = wantedRow(db, jmap, blobId); + if (!want) continue; + const known = Number(want.size ?? 0); + if (!want.pinned && known > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) { + setError(db, jmap, blobId, 'tooLarge'); + continue; + } + if (want.pinned && pinnedBytes(db) + known > ctx.scope.maxTotalBytes) { + setError(db, jmap, blobId, 'overLimit'); + continue; + } + let data: Uint8Array; + const type = nodeMediaType(JSON.parse(want.json) as WireNode); + try { + data = await ctx.transport.download(jmap, blobId, type, want.name); + } catch (error) { + // No network or no credentials reach the engine. One bad file doesn't stop the others. + if (error instanceof TransportError && error.kind === 'http') { + setError(db, jmap, blobId, 'download'); + continue; + } + throw error; + } + db.transaction(() => { + const now = wantedRow(db, jmap, blobId); + if (!now) return; + if (now.pinned ? pinnedBytes(db) + data.length > ctx.scope.maxTotalBytes : data.length > Math.min(ctx.scope.maxAttachmentBytes, capBytes(ctx))) { + // The size in the metadata was wrong or missing. With the real one + // noted, the next pass sees this without downloading it again. + db.run('UPDATE file_node SET size = ? WHERE jmap = ? AND blob_id = ?', [data.length, jmap, blobId]); + setError(db, jmap, blobId, now.pinned ? 'overLimit' : 'tooLarge'); + return; + } + if (!makeRoom(ctx, data.length, { pinned: Boolean(now.pinned), openedAt: Number(now.opened_at ?? 0) })) { + db.run('UPDATE file_node SET opened_at = NULL WHERE jmap = ? AND blob_id = ? AND pinned = 0', [jmap, blobId]); + return; + } + storeBlob(ctx, jmap, blobId, type, data); + setError(db, jmap, blobId, null); + }); + done += 1; + ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'contents', done, total: todo.length }); + } + if (done > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] }); +} + +// ---- Calls from the page ---- + +function fileRow(db: StorageAdapter, { jmap, id }: NodeRef): NodeRow { + const row = db.get('SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]); + if (!row) throw new MethodError('notFound', `No file ${id}`); + if (!row.blob_id || row.kind !== 'file') throw new MethodError('invalidArguments', 'Only files can be kept'); + return row; +} + +export function pinFile(ctx: StoreContext, ref: NodeRef): { kept: boolean } { + const row = fileRow(ctx.db, ref); + ctx.db.run('UPDATE file_node SET pinned = 1, keep_error = NULL WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]); + const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string); + if (!kept) void ctx.requestSync(); + return { kept }; +} + +/** Back to being a recently opened file, or let go of it if it never was one. */ +export function unpinFile(ctx: StoreContext, ref: NodeRef): void { + const row = fileRow(ctx.db, ref); + ctx.db.transaction(() => { + ctx.db.run('UPDATE file_node SET pinned = 0 WHERE jmap = ? AND id = ?', [ref.jmap, ref.id]); + if (!wantedRow(ctx.db, ref.jmap, row.blob_id as string)) dropBlob(ctx.db, ref.jmap, row.blob_id as string); + }); +} + +export function noteOpened(ctx: StoreContext, ref: NodeRef): { kept: boolean } { + const row = fileRow(ctx.db, ref); + ctx.db.run('UPDATE file_node SET opened_at = ?, keep_error = NULL WHERE jmap = ? AND id = ?', [ctx.now(), ref.jmap, ref.id]); + const kept = hasBlob(ctx.db, ref.jmap, row.blob_id as string); + if (!kept) void ctx.requestSync(); + return { kept }; +} + +export function readFile(ctx: StoreContext, ref: NodeRef): (KeptFile & { name: string }) | null { + const row = fileRow(ctx.db, ref); + const kept = fileBlob(ctx, ref.jmap, row.blob_id as string); + return kept ? { ...kept, name: row.name } : null; +} + +export interface KeptFileStatus { + jmap: string; + id: string; + name: string; + size: number | null; + pinned: boolean; + openedAt: number | null; + kept: boolean; + /** Why it isn't kept: tooLarge, overLimit, notAllowed or download. */ + error: string | null; +} + +/** Pinned and recently opened files, and how much room their contents take. */ +export function keptStatus(ctx: StoreContext): { files: KeptFileStatus[]; bytes: number; capBytes: number } { + const rows = ctx.db.all( + `SELECT jmap, id, name, kind, blob_id, size, json, pinned, opened_at, keep_error FROM file_node + WHERE pinned = 1 OR opened_at IS NOT NULL ORDER BY pinned DESC, opened_at DESC, name`, + ); + return { + files: rows.map((row) => ({ + jmap: row.jmap, + id: row.id, + name: row.name, + size: row.size === null ? null : Number(row.size), + pinned: Boolean(row.pinned), + openedAt: row.opened_at === null ? null : Number(row.opened_at), + kept: row.blob_id !== null && hasBlob(ctx.db, row.jmap, row.blob_id), + error: row.keep_error, + })), + bytes: keptBlobs(ctx.db).reduce((sum, blob) => sum + Number(blob.size), 0), + capBytes: capBytes(ctx), + }; +} diff --git a/lib/store/files/methods.ts b/lib/store/files/methods.ts new file mode 100644 index 000000000..0945a33eb --- /dev/null +++ b/lib/store/files/methods.ts @@ -0,0 +1,108 @@ +/** + * FileNode reads answered from the store, so listings work offline. Nothing + * here answers FileNode/set: the responder gives it a per-call + * serverUnavailable, since the design has no offline file writes. + */ + +import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area'; +import type { SqlValue } from '../storage'; +import { FILES_CAPABILITY } from './records'; + +function jmapOf(ctx: MethodContext, args: Record): string { + return (args.accountId as string | undefined) ?? ctx.session.primaryAccounts[FILES_CAPABILITY] ?? ctx.defaultJmap; +} + +/** The last state from the server. Throws when the list was never read, so an empty answer always means empty. */ +function syncedState(ctx: MethodContext, jmap: string): string | null { + const row = ctx.db.get<{ state: string | null; phase: string | null }>("SELECT state, phase FROM sync_state WHERE jmap = ? AND type = 'FileNode'", [jmap]); + if (!row || (!row.state && !row.phase)) throw new MethodError('serverUnavailable', 'Files not kept yet'); + return row.state; +} + +function pick(node: Record, properties: unknown): Record { + if (!Array.isArray(properties)) return node; + const out: Record = { id: node.id }; + for (const property of properties as string[]) if (property in node) out[property] = node[property]; + return out; +} + +const SORTS: Record = { + name: 'name COLLATE NOCASE', + size: 'size', + created: "json_extract(json, '$.created')", + modified: "json_extract(json, '$.modified')", +}; + +function orderSql(sort: unknown): string { + const parts: string[] = []; + for (const level of (Array.isArray(sort) ? sort : []) as Array<{ property?: string; isAscending?: boolean }>) { + const column = SORTS[String(level.property)]; + if (!column) throw new MethodError('unsupportedSort', `${String(level.property)} isn't available offline`); + parts.push(`${column} ${level.isAscending === false ? 'DESC' : 'ASC'}`); + } + parts.push('name COLLATE NOCASE ASC', 'id ASC'); + return parts.join(', '); +} + +const fileNodeGet: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + const state = syncedState(ctx, jmap); + const list: Array> = []; + const notFound: string[] = []; + if (Array.isArray(args.ids)) { + for (const id of args.ids as string[]) { + const row = ctx.db.get<{ json: string }>('SELECT json FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]); + if (row) list.push(pick(JSON.parse(row.json) as Record, args.properties)); + else notFound.push(id); + } + } else { + // The whole tree at once. The client reads a long list as possibly cut, + // pages FileNode/query to check, and finds nothing missing. + for (const row of ctx.db.all<{ json: string }>('SELECT json FROM file_node WHERE jmap = ?', [jmap])) { + list.push(pick(JSON.parse(row.json) as Record, args.properties)); + } + } + return { accountId: jmap, ...(state ? { state } : {}), list, notFound }; +}; + +const fileNodeQuery: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + syncedState(ctx, jmap); + if (args.anchor !== undefined) throw new MethodError('unsupportedFilter', 'anchor needs the server'); + const filter = (args.filter ?? {}) as Record; + const conditions = ['jmap = ?']; + const params: SqlValue[] = [jmap]; + for (const [key, value] of Object.entries(filter)) { + if (key !== 'parentId') throw new MethodError('unsupportedFilter', `${key} isn't available offline`); + if (value === null) { + conditions.push('parent_id IS NULL'); + } else { + conditions.push('parent_id = ?'); + params.push(String(value)); + } + } + const where = conditions.join(' AND '); + const position = Math.max(0, Number(args.position ?? 0)); + const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1; + const ids = ctx.db.all<{ id: string }>( + `SELECT id FROM file_node WHERE ${where} ORDER BY ${orderSql(args.sort)} LIMIT ? OFFSET ?`, + [...params, limit, position], + ).map((row) => String(row.id)); + const result: MethodResult = { + accountId: jmap, + // Local answers aren't a position in the server's log, so this is a token only. + queryState: 'local', + canCalculateChanges: false, + position, + ids, + }; + if (args.calculateTotal) { + result.total = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM file_node WHERE ${where}`, params)?.n ?? 0); + } + return result; +}; + +export const FILE_METHODS: Readonly> = { + 'FileNode/get': fileNodeGet, + 'FileNode/query': fileNodeQuery, +}; diff --git a/lib/store/files/records.ts b/lib/store/files/records.ts new file mode 100644 index 000000000..fd5cbcb51 --- /dev/null +++ b/lib/store/files/records.ts @@ -0,0 +1,50 @@ +/** How a FileNode becomes a row. The server's own copy stays in `json`, in whichever draft it speaks. */ + +import { fileNodeKind, fromDraftFileNode } from '@/lib/jmap/filenode-kind'; +import { decodeFileNodeName } from '@/lib/jmap/filenode-name'; +import type { FileNode } from '@/lib/jmap/types'; +import type { StorageAdapter } from '../storage'; + +export const FILES_CAPABILITY = 'urn:ietf:params:jmap:filenode'; +export const FILES_USING = ['urn:ietf:params:jmap:core', FILES_CAPABILITY]; + +/** The client's list (lib/jmap/client.ts). Stalwart leaves out shareWith and myRights unless asked. */ +export const FILE_NODE_PROPERTIES = [ + 'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified', + 'nodeType', '@type', 'mediaType', 'shareWith', 'myRights', +]; + +export type WireNode = Record & { id: string }; + +/** The node's media type, whichever draft named it. */ +export function nodeMediaType(node: WireNode): string { + const { type } = fromDraftFileNode(node as unknown as Partial & { '@type'?: unknown; mediaType?: unknown }); + return typeof type === 'string' && type ? type : 'application/octet-stream'; +} + +export function writeNode(db: StorageAdapter, jmap: string, node: WireNode, gen: number): void { + const normal = fromDraftFileNode(node as unknown as FileNode & { '@type'?: unknown; mediaType?: unknown }); + db.run( + `INSERT INTO file_node (jmap, id, parent_id, name, kind, blob_id, size, json, gen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (jmap, id) DO UPDATE SET parent_id = excluded.parent_id, name = excluded.name, kind = excluded.kind, + size = excluded.size, json = excluded.json, gen = excluded.gen, + keep_error = CASE WHEN blob_id IS excluded.blob_id THEN keep_error END, + blob_id = excluded.blob_id`, + [ + jmap, + node.id, + typeof node.parentId === 'string' ? node.parentId : null, + // The client decodes names itself from `json`. This column is for the store's own lists. + typeof node.name === 'string' ? decodeFileNodeName(node.name) : '', + fileNodeKind(normal), + typeof node.blobId === 'string' ? node.blobId : null, + typeof node.size === 'number' ? node.size : null, + JSON.stringify(node), + gen, + ], + ); +} + +export function removeNode(db: StorageAdapter, jmap: string, id: string): void { + db.run('DELETE FROM file_node WHERE jmap = ? AND id = ?', [jmap, id]); +} diff --git a/lib/store/files/schema.ts b/lib/store/files/schema.ts new file mode 100644 index 000000000..9a3f20e35 --- /dev/null +++ b/lib/store/files/schema.ts @@ -0,0 +1,47 @@ +import type { Migration } from '../storage'; + +export const FILE_MIGRATIONS: readonly Migration[] = [ + { + id: 700, + name: 'files', + sql: ` + CREATE TABLE file_node ( + jmap TEXT NOT NULL, + id TEXT NOT NULL, + parent_id TEXT, + name TEXT NOT NULL DEFAULT '', + -- file, directory or symlink, whichever way the server says so. + kind TEXT NOT NULL, + blob_id TEXT, + size INTEGER, + json TEXT NOT NULL, + gen INTEGER NOT NULL DEFAULT 0, + pinned INTEGER NOT NULL DEFAULT 0, + -- When the page last opened it. Cleared when its content is pushed out. + opened_at INTEGER, + -- Why the content isn't kept: tooLarge, overLimit, notAllowed or download. + keep_error TEXT, + PRIMARY KEY (jmap, id) + ) WITHOUT ROWID; + CREATE INDEX file_node_parent ON file_node (jmap, parent_id); + CREATE INDEX file_node_blob ON file_node (jmap, blob_id); + CREATE TABLE file_blob ( + jmap TEXT NOT NULL, + blob_id TEXT NOT NULL, + type TEXT NOT NULL, + size INTEGER NOT NULL, + kept_at INTEGER NOT NULL, + PRIMARY KEY (jmap, blob_id) + ) WITHOUT ROWID; + -- In pieces: a whole file bound at once would grow the WASM heap by its + -- size, and that memory never comes back. + CREATE TABLE file_blob_chunk ( + jmap TEXT NOT NULL, + blob_id TEXT NOT NULL, + seq INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (jmap, blob_id, seq) + ); + `, + }, +]; diff --git a/lib/store/files/sync.ts b/lib/store/files/sync.ts new file mode 100644 index 000000000..cc9d99918 --- /dev/null +++ b/lib/store/files/sync.ts @@ -0,0 +1,160 @@ +/** + * FileNode metadata for one JMAP account, then the contents worth keeping. + * + * A full pass reads the tree the way the client does (fetchAllFileNodes, + * lib/jmap/client.ts). `FileNode/get` with ids null returns everything, but + * 0.16 silently stops at maxObjectsInGet and 1.0 refuses a larger account with + * requestTooLarge. Past that, ids come from paging FileNode/query. Before + * 0.16.6 that query leaves out folders, so parents still unknown afterwards are + * read by id. The cursor is in sync_state, so a reload continues. + */ + +import type { SyncContext, Syncer } from '../area'; +import { runChangesLoop } from '../engine/changes'; +import { JmapMethodError, responseFor, type MethodCall } from '../engine/transport'; +import { dropOrphanBlobs, fetchKeptFiles } from './keep'; +import { FILE_NODE_PROPERTIES, FILES_CAPABILITY, FILES_USING, removeNode, writeNode, type WireNode } from './records'; + +const MAX_GET = 500; +const GETS_PER_REQUEST = 8; +const QUERY_PAGE = 5000; + +function getBatch(ctx: SyncContext): number { + const core = ctx.session.capabilities['urn:ietf:params:jmap:core'] as { maxObjectsInGet?: number } | undefined; + return Math.min(MAX_GET, core?.maxObjectsInGet || MAX_GET); +} + +async function fetchNodes(ctx: SyncContext, jmap: string, ids: string[]): Promise { + const size = getBatch(ctx); + const nodes: WireNode[] = []; + for (let at = 0; at < ids.length;) { + const calls: MethodCall[] = []; + for (let i = 0; i < GETS_PER_REQUEST && at < ids.length; i += 1, at += size) { + calls.push(['FileNode/get', { accountId: jmap, ids: ids.slice(at, at + size), properties: FILE_NODE_PROPERTIES }, `g${i}`]); + } + const response = await ctx.transport.request(calls, FILES_USING); + calls.forEach(([, , callId]) => nodes.push(...responseFor(response, callId)[1].list as WireNode[])); + } + return nodes; +} + +interface ListCursor { step: 'all' | 'query' | 'parents'; position: number; gen: number; captured: string } + +async function startFullList(ctx: SyncContext, jmap: string): Promise { + const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: [], properties: ['id'] }, 's']], FILES_USING); + const captured = String(responseFor(response, 's')[1].state); + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0) + 1; + const cursor: ListCursor = { step: 'all', position: 0, gen, captured }; + ctx.db.transaction(() => ctx.setSyncState(jmap, 'FileNode', { state: null, phase: 'list', cursor: { ...cursor } })); + return cursor; +} + +function saveStep(ctx: SyncContext, jmap: string, nodes: WireNode[], cursor: ListCursor): void { + ctx.db.transaction(() => { + for (const node of nodes) writeNode(ctx.db, jmap, node, cursor.gen); + ctx.setSyncState(jmap, 'FileNode', { cursor: { ...cursor } }); + }); +} + +async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise { + const cursor = { ...start }; + + if (cursor.step === 'all') { + let nodes: WireNode[] = []; + let complete = false; + try { + const response = await ctx.transport.request([['FileNode/get', { accountId: jmap, ids: null, properties: FILE_NODE_PROPERTIES }, 'a']], FILES_USING); + nodes = responseFor(response, 'a')[1].list as WireNode[]; + complete = nodes.length < getBatch(ctx); + } catch (error) { + if (!(error instanceof JmapMethodError && error.type === 'requestTooLarge')) throw error; + } + if (!complete) cursor.step = 'query'; + else cursor.step = 'parents'; + saveStep(ctx, jmap, nodes, cursor); + } + + while (cursor.step === 'query') { + if (ctx.stopped()) return false; + const response = await ctx.transport.request([['FileNode/query', { accountId: jmap, filter: {}, position: cursor.position, limit: QUERY_PAGE, calculateTotal: true }, 'q']], FILES_USING); + const [, query] = responseFor(response, 'q'); + const ids = query.ids as string[]; + const missing = ids.filter((id) => Number(ctx.db.get<{ gen: number }>('SELECT gen FROM file_node WHERE jmap = ? AND id = ?', [jmap, id])?.gen ?? -1) < cursor.gen); + const nodes = await fetchNodes(ctx, jmap, missing); + cursor.position += ids.length; + // The server may clamp the limit, so only an empty page or the total ends this. + if (ids.length === 0 || (typeof query.total === 'number' && cursor.position >= query.total)) cursor.step = 'parents'; + saveStep(ctx, jmap, nodes, cursor); + ctx.emit({ type: 'progress', account: ctx.account, area: 'files', phase: 'list', done: cursor.position, total: typeof query.total === 'number' ? query.total : null }); + } + + const asked = new Set(); + for (;;) { + if (ctx.stopped()) return false; + const parents = ctx.db.all<{ id: string }>( + `SELECT DISTINCT n.parent_id AS id FROM file_node n + WHERE n.jmap = ? AND n.gen = ? AND n.parent_id IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM file_node p WHERE p.jmap = n.jmap AND p.id = n.parent_id AND p.gen = ?)`, + [jmap, cursor.gen, cursor.gen], + ).map((row) => String(row.id)).filter((id) => !asked.has(id)); + if (parents.length === 0) break; + for (const id of parents) asked.add(id); + saveStep(ctx, jmap, await fetchNodes(ctx, jmap, parents), cursor); + } + + ctx.db.transaction(() => { + // Whatever this pass didn't see is gone from the server. + ctx.db.run('DELETE FROM file_node WHERE jmap = ? AND gen < ?', [jmap, cursor.gen]); + ctx.setSyncState(jmap, 'FileNode', { state: cursor.captured, phase: null, cursor: null }); + }); + ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] }); + return true; +} + +async function syncDelta(ctx: SyncContext, jmap: string, since: string, target: string | null): Promise<'done' | 'resync' | 'stopped'> { + const outcome = await runChangesLoop({ + transport: ctx.transport, + type: 'FileNode', + jmap, + using: FILES_USING, + since, + target, + stopped: ctx.stopped, + apply: async (page) => { + const nodes = await fetchNodes(ctx, jmap, [...page.created, ...page.updated]); + ctx.db.transaction(() => { + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM file_node WHERE jmap = ?', [jmap])?.gen ?? 0); + for (const node of nodes) writeNode(ctx.db, jmap, node, gen); + for (const id of page.destroyed) removeNode(ctx.db, jmap, id); + ctx.setSyncState(jmap, 'FileNode', { state: page.newState }); + }); + if (nodes.length + page.destroyed.length > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileNode'] }); + }, + }); + if (outcome.outcome === 'resync') return 'resync'; + return outcome.outcome === 'stopped' ? 'stopped' : 'done'; +} + +export const filesSyncer: Syncer = { + name: 'files', + capability: FILES_CAPABILITY, + async run(ctx, jmap) { + const row = ctx.syncState(jmap, 'FileNode'); + let cursor = row.phase === 'list' ? row.cursor as unknown as ListCursor : null; + if (!cursor && !row.state) cursor = await startFullList(ctx, jmap); + if (!cursor && row.state) { + const outcome = await syncDelta(ctx, jmap, row.state, row.target); + if (outcome === 'stopped') return; + if (outcome === 'resync') cursor = await startFullList(ctx, jmap); + } + if (cursor) { + if (!(await runFullList(ctx, jmap, cursor))) return; + // Catch what changed while paging. + const after = ctx.syncState(jmap, 'FileNode'); + if (after.state && (await syncDelta(ctx, jmap, after.state, after.target)) === 'stopped') return; + } + if (ctx.stopped()) return; + ctx.db.transaction(() => dropOrphanBlobs(ctx.db)); + await fetchKeptFiles(ctx, jmap); + }, +};