From fb40944c2d3a06bf15259843e955deec1a59ed08 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:28:31 +0100 Subject: [PATCH] feat: hold kept files to the scope at once, and count them towards what each account takes --- lib/store/__tests__/files.test.ts | 69 +++++++++++++++++++++++++++ lib/store/__tests__/mail-sync.test.ts | 2 +- lib/store/contacts/index.ts | 4 ++ lib/store/files/index.ts | 5 +- lib/store/files/keep.ts | 46 +++++++++++++++++- 5 files changed, 123 insertions(+), 3 deletions(-) diff --git a/lib/store/__tests__/files.test.ts b/lib/store/__tests__/files.test.ts index acf81b483..41d158e12 100644 --- a/lib/store/__tests__/files.test.ts +++ b/lib/store/__tests__/files.test.ts @@ -311,6 +311,75 @@ describe('pinned files', () => { expect(status.files.filter((f: { error: string | null }) => f.error === 'overLimit')).toHaveLength(1); }); + describe('a tightened scope, with no network', () => { + async function keptOffline(rig: Rig) { + rig.engine.setOnline(false); + } + + it('drops everything kept when files are forbidden, and the pins stay', async () => { + const server = new FakeFiles(); + const file = server.addFile('a', null, 60); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(file.id)); + await rig.engine.requestSync(); + await keptOffline(rig); + rig.engine.setScope({ ...DEFAULT_SCOPE, attachments: false }); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(file.id))).toBeNull(); + expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ pinned: true, kept: false, error: 'notAllowed' }); + }); + + it('drops an opened file over the new size, not a pinned one', async () => { + const server = new FakeFiles(); + const pinned = server.addFile('pinned', null, 60); + const opened = server.addFile('opened', null, 60); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(pinned.id)); + await rig.engine.areaCall('files.noteOpened', ref(opened.id)); + await rig.engine.requestSync(); + await keptOffline(rig); + rig.engine.setScope({ ...DEFAULT_SCOPE, maxAttachmentBytes: 10 }); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(opened.id))).toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(pinned.id))).not.toBeNull(); + }); + + it('holds even pins to a lower total, newest out first, and doesn\'t fetch them back', async () => { + const server = new FakeFiles(); + const older = server.addFile('older', null, 60); + const newer = server.addFile('newer', null, 60); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(older.id)); + await rig.engine.requestSync(); + rig.clock.at += 1000; + await rig.engine.areaCall('files.pin', ref(newer.id)); + await rig.engine.requestSync(); + await keptOffline(rig); + rig.engine.setScope({ ...DEFAULT_SCOPE, maxTotalBytes: 100 }); + await rig.engine.requestSync(); + expect(await rig.engine.areaCall('files.read', ref(older.id))).not.toBeNull(); + expect(await rig.engine.areaCall('files.read', ref(newer.id))).toBeNull(); + const status = (await rig.engine.areaCall('files.kept', undefined) as any).files; + expect(status.find((f: { id: string }) => f.id === newer.id).error).toBe('overLimit'); + + server.downloads.length = 0; + rig.engine.setOnline(true); + await rig.engine.requestSync(); + expect(server.downloads).toEqual([]); + }); + }); + + it('counts towards the storage page, and takes its part of the total from mail', async () => { + const server = new FakeFiles(); + const file = server.addFile('a', null, 60); + const rig = await synced(server); + await rig.engine.areaCall('files.pin', ref(file.id)); + await rig.engine.requestSync(); + const usage = await rig.engine.areaCall('store.usage', undefined) as { parts: Array<{ part: string; bytes: number; items: number }> }; + expect(usage.parts).toContainEqual({ part: 'files', bytes: 60, items: 1 }); + expect(filesArea.keptBytes?.({ db: rig.db })).toBe(60); + }); + it('keeps nothing when the scope forbids it, and drops what was kept', async () => { const server = new FakeFiles(); const file = server.addFile('a'); diff --git a/lib/store/__tests__/mail-sync.test.ts b/lib/store/__tests__/mail-sync.test.ts index e4424a4a3..4d7b61185 100644 --- a/lib/store/__tests__/mail-sync.test.ts +++ b/lib/store/__tests__/mail-sync.test.ts @@ -443,7 +443,7 @@ describe('query plans', () => { }, }; const answer = await answerRequest( - { account: 'a', db: spied as typeof db, session: FAKE_SESSION, scope: EVERYTHING, now: Date.now, emit: () => {}, overlay: (_jmap, _type, records) => records, keptByOthers: () => 0, defaultJmap: JMAP_ACCOUNT, invertedKeywordSort: false }, + { account: 'a', db: spied as typeof db, session: FAKE_SESSION, scope: EVERYTHING, now: Date.now, emit: () => {}, overlay: (_jmap, _type, records) => records, keptByOthers: () => 0, requestSync: async () => {}, defaultJmap: JMAP_ACCOUNT, invertedKeywordSort: false }, buildMethodTable([mailArea]), [['Email/query', { filter: INBOX, sort: [{ property: 'receivedAt', isAscending: false }], limit: 5 }, 'q']], ); diff --git a/lib/store/contacts/index.ts b/lib/store/contacts/index.ts index 01b129a8a..8c964775f 100644 --- a/lib/store/contacts/index.ts +++ b/lib/store/contacts/index.ts @@ -9,6 +9,10 @@ export const contactsArea: StoreArea = { migrations: CONTACT_MIGRATIONS, methods: CONTACT_METHODS, syncers: [contactsSyncer], + usage: (ctx) => { + const row = ctx.db.get<{ bytes: number | null; items: number }>('SELECT SUM(LENGTH(json)) AS bytes, COUNT(*) AS items FROM contact_card'); + return [{ part: 'contacts', bytes: Number(row?.bytes ?? 0), items: Number(row?.items ?? 0) }]; + }, calls: { /** Sends what's queued, in the next sync pass, and says what's left. */ 'contacts.flush': async (ctx) => { diff --git a/lib/store/files/index.ts b/lib/store/files/index.ts index 0957ced89..016d4b4f6 100644 --- a/lib/store/files/index.ts +++ b/lib/store/files/index.ts @@ -1,5 +1,5 @@ import type { StoreArea } from '../area'; -import { fileBlob, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep'; +import { applyFilesScope, fileBlob, filesUsage, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep'; import { FILE_METHODS } from './methods'; import { FILE_MIGRATIONS } from './schema'; import { filesSyncer } from './sync'; @@ -11,6 +11,9 @@ export const filesArea: StoreArea = { migrations: FILE_MIGRATIONS, methods: FILE_METHODS, syncers: [filesSyncer], + applyScope: applyFilesScope, + usage: filesUsage, + keptBytes: (ctx) => filesUsage(ctx)[0].bytes, calls: { /** Keep this file's content on the device until it's unpinned. */ 'files.pin': (ctx, params: NodeRef) => pinFile(ctx, params), diff --git a/lib/store/files/keep.ts b/lib/store/files/keep.ts index a11b93455..919d7cb97 100644 --- a/lib/store/files/keep.ts +++ b/lib/store/files/keep.ts @@ -13,7 +13,7 @@ */ import { TransportError } from '../engine/transport'; -import { MethodError, type StoreContext, type SyncContext } from '../area'; +import { MethodError, type ScopeChange, type StoreContext, type StoreUsagePart, type SyncContext } from '../area'; import type { StorageAdapter } from '../storage'; import { nodeMediaType, type WireNode } from './records'; @@ -220,6 +220,50 @@ export async function fetchKeptFiles(ctx: SyncContext, jmap: string): Promise 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] }); } +/** + * What the scope no longer allows leaves now, without the network. The pins + * stay, so a loosened scope fetches them back at the next pass. A pin ignores + * the size limit but not the total: the organisation's cap wins over it. + */ +export function applyFilesScope(ctx: SyncContext, _change: ScopeChange): void { + const { db, scope } = ctx; + let changed = false; + db.transaction(() => { + if (!scope.attachments) { + for (const blob of keptBlobs(db)) { + dropBlob(db, blob.jmap, blob.blob_id); + changed = true; + } + db.run("UPDATE file_node SET keep_error = 'notAllowed' WHERE pinned = 1 OR opened_at IS NOT NULL"); + return; + } + const largest = Math.min(scope.maxAttachmentBytes, capBytes(ctx)); + for (const blob of keptBlobs(db)) { + if (blob.pinned || Number(blob.size) <= largest) continue; + pushOut(db, blob); + changed = true; + } + const before = keptBlobs(db).length; + makeRoom(ctx, 0, { pinned: true, openedAt: 0 }); + if (keptBlobs(db).length !== before) changed = true; + // Past the total with pins alone: the newest kept go first, as they would have been refused. + let pinned = pinnedBytes(db); + for (const blob of keptBlobs(db).filter((row) => row.pinned).sort((a, b) => Number(b.kept_at) - Number(a.kept_at))) { + if (pinned <= scope.maxTotalBytes) break; + pinned -= Number(blob.size); + dropBlob(db, blob.jmap, blob.blob_id); + setError(db, blob.jmap, blob.blob_id, 'overLimit'); + changed = true; + } + }); + if (changed) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] }); +} + +export function filesUsage(ctx: Pick): StoreUsagePart[] { + const row = ctx.db.get<{ bytes: number | null; items: number }>('SELECT SUM(size) AS bytes, COUNT(*) AS items FROM file_blob'); + return [{ part: 'files', bytes: Number(row?.bytes ?? 0), items: Number(row?.items ?? 0) }]; +} + // ---- Calls from the page ---- function fileRow(db: StorageAdapter, { jmap, id }: NodeRef): NodeRow {