feat: hold kept files to the scope at once, and count them towards what each account takes

lib/store/__tests__/files.test.ts

@@ -311,6 +311,75 @@ describe('pinned files', () => {
     expect(status.files.filter((f: { error: string | null }) => f.error === 'overLimit')).toHaveLength(1);
   });
 
+  describe('a tightened scope, with no network', () => {
+    async function keptOffline(rig: Rig) {
+      rig.engine.setOnline(false);
+    }
+
+    it('drops everything kept when files are forbidden, and the pins stay', async () => {
+      const server = new FakeFiles();
+      const file = server.addFile('a', null, 60);
+      const rig = await synced(server);
+      await rig.engine.areaCall('files.pin', ref(file.id));
+      await rig.engine.requestSync();
+      await keptOffline(rig);
+      rig.engine.setScope({ ...DEFAULT_SCOPE, attachments: false });
+      await rig.engine.requestSync();
+      expect(await rig.engine.areaCall('files.read', ref(file.id))).toBeNull();
+      expect((await rig.engine.areaCall('files.kept', undefined) as any).files[0]).toMatchObject({ pinned: true, kept: false, error: 'notAllowed' });
+    });
+
+    it('drops an opened file over the new size, not a pinned one', async () => {
+      const server = new FakeFiles();
+      const pinned = server.addFile('pinned', null, 60);
+      const opened = server.addFile('opened', null, 60);
+      const rig = await synced(server);
+      await rig.engine.areaCall('files.pin', ref(pinned.id));
+      await rig.engine.areaCall('files.noteOpened', ref(opened.id));
+      await rig.engine.requestSync();
+      await keptOffline(rig);
+      rig.engine.setScope({ ...DEFAULT_SCOPE, maxAttachmentBytes: 10 });
+      await rig.engine.requestSync();
+      expect(await rig.engine.areaCall('files.read', ref(opened.id))).toBeNull();
+      expect(await rig.engine.areaCall('files.read', ref(pinned.id))).not.toBeNull();
+    });
+
+    it('holds even pins to a lower total, newest out first, and doesn\'t fetch them back', async () => {
+      const server = new FakeFiles();
+      const older = server.addFile('older', null, 60);
+      const newer = server.addFile('newer', null, 60);
+      const rig = await synced(server);
+      await rig.engine.areaCall('files.pin', ref(older.id));
+      await rig.engine.requestSync();
+      rig.clock.at += 1000;
+      await rig.engine.areaCall('files.pin', ref(newer.id));
+      await rig.engine.requestSync();
+      await keptOffline(rig);
+      rig.engine.setScope({ ...DEFAULT_SCOPE, maxTotalBytes: 100 });
+      await rig.engine.requestSync();
+      expect(await rig.engine.areaCall('files.read', ref(older.id))).not.toBeNull();
+      expect(await rig.engine.areaCall('files.read', ref(newer.id))).toBeNull();
+      const status = (await rig.engine.areaCall('files.kept', undefined) as any).files;
+      expect(status.find((f: { id: string }) => f.id === newer.id).error).toBe('overLimit');
+
+      server.downloads.length = 0;
+      rig.engine.setOnline(true);
+      await rig.engine.requestSync();
+      expect(server.downloads).toEqual([]);
+    });
+  });
+
+  it('counts towards the storage page, and takes its part of the total from mail', async () => {
+    const server = new FakeFiles();
+    const file = server.addFile('a', null, 60);
+    const rig = await synced(server);
+    await rig.engine.areaCall('files.pin', ref(file.id));
+    await rig.engine.requestSync();
+    const usage = await rig.engine.areaCall('store.usage', undefined) as { parts: Array<{ part: string; bytes: number; items: number }> };
+    expect(usage.parts).toContainEqual({ part: 'files', bytes: 60, items: 1 });
+    expect(filesArea.keptBytes?.({ db: rig.db })).toBe(60);
+  });
+
   it('keeps nothing when the scope forbids it, and drops what was kept', async () => {
     const server = new FakeFiles();
     const file = server.addFile('a');

lib/store/__tests__/mail-sync.test.ts

@@ -443,7 +443,7 @@ describe('query plans', () => {
       },
     };
     const answer = await answerRequest(
-      { account: 'a', db: spied as typeof db, session: FAKE_SESSION, scope: EVERYTHING, now: Date.now, emit: () => {}, overlay: (_jmap, _type, records) => records, keptByOthers: () => 0, defaultJmap: JMAP_ACCOUNT, invertedKeywordSort: false },
+      { account: 'a', db: spied as typeof db, session: FAKE_SESSION, scope: EVERYTHING, now: Date.now, emit: () => {}, overlay: (_jmap, _type, records) => records, keptByOthers: () => 0, requestSync: async () => {}, defaultJmap: JMAP_ACCOUNT, invertedKeywordSort: false },
       buildMethodTable([mailArea]),
       [['Email/query', { filter: INBOX, sort: [{ property: 'receivedAt', isAscending: false }], limit: 5 }, 'q']],
     );

lib/store/contacts/index.ts

@@ -9,6 +9,10 @@ export const contactsArea: StoreArea = {
   migrations: CONTACT_MIGRATIONS,
   methods: CONTACT_METHODS,
   syncers: [contactsSyncer],
+  usage: (ctx) => {
+    const row = ctx.db.get<{ bytes: number | null; items: number }>('SELECT SUM(LENGTH(json)) AS bytes, COUNT(*) AS items FROM contact_card');
+    return [{ part: 'contacts', bytes: Number(row?.bytes ?? 0), items: Number(row?.items ?? 0) }];
+  },
   calls: {
     /** Sends what's queued, in the next sync pass, and says what's left. */
     'contacts.flush': async (ctx) => {

lib/store/files/index.ts

@@ -1,5 +1,5 @@
 import type { StoreArea } from '../area';
-import { fileBlob, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep';
+import { applyFilesScope, fileBlob, filesUsage, keptStatus, noteOpened, pinFile, readFile, unpinFile, type NodeRef } from './keep';
 import { FILE_METHODS } from './methods';
 import { FILE_MIGRATIONS } from './schema';
 import { filesSyncer } from './sync';
@@ -11,6 +11,9 @@ export const filesArea: StoreArea = {
   migrations: FILE_MIGRATIONS,
   methods: FILE_METHODS,
   syncers: [filesSyncer],
+  applyScope: applyFilesScope,
+  usage: filesUsage,
+  keptBytes: (ctx) => filesUsage(ctx)[0].bytes,
   calls: {
     /** Keep this file's content on the device until it's unpinned. */
     'files.pin': (ctx, params: NodeRef) => pinFile(ctx, params),

lib/store/files/keep.ts

@@ -13,7 +13,7 @@
  */
 
 import { TransportError } from '../engine/transport';
-import { MethodError, type StoreContext, type SyncContext } from '../area';
+import { MethodError, type ScopeChange, type StoreContext, type StoreUsagePart, type SyncContext } from '../area';
 import type { StorageAdapter } from '../storage';
 import { nodeMediaType, type WireNode } from './records';
 
@@ -220,6 +220,50 @@ export async function fetchKeptFiles(ctx: SyncContext, jmap: string): Promise<vo
   if (done > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] });
 }
 
+/**
+ * What the scope no longer allows leaves now, without the network. The pins
+ * stay, so a loosened scope fetches them back at the next pass. A pin ignores
+ * the size limit but not the total: the organisation's cap wins over it.
+ */
+export function applyFilesScope(ctx: SyncContext, _change: ScopeChange): void {
+  const { db, scope } = ctx;
+  let changed = false;
+  db.transaction(() => {
+    if (!scope.attachments) {
+      for (const blob of keptBlobs(db)) {
+        dropBlob(db, blob.jmap, blob.blob_id);
+        changed = true;
+      }
+      db.run("UPDATE file_node SET keep_error = 'notAllowed' WHERE pinned = 1 OR opened_at IS NOT NULL");
+      return;
+    }
+    const largest = Math.min(scope.maxAttachmentBytes, capBytes(ctx));
+    for (const blob of keptBlobs(db)) {
+      if (blob.pinned || Number(blob.size) <= largest) continue;
+      pushOut(db, blob);
+      changed = true;
+    }
+    const before = keptBlobs(db).length;
+    makeRoom(ctx, 0, { pinned: true, openedAt: 0 });
+    if (keptBlobs(db).length !== before) changed = true;
+    // Past the total with pins alone: the newest kept go first, as they would have been refused.
+    let pinned = pinnedBytes(db);
+    for (const blob of keptBlobs(db).filter((row) => row.pinned).sort((a, b) => Number(b.kept_at) - Number(a.kept_at))) {
+      if (pinned <= scope.maxTotalBytes) break;
+      pinned -= Number(blob.size);
+      dropBlob(db, blob.jmap, blob.blob_id);
+      setError(db, blob.jmap, blob.blob_id, 'overLimit');
+      changed = true;
+    }
+  });
+  if (changed) ctx.emit({ type: 'changed', account: ctx.account, collections: ['FileContent'] });
+}
+
+export function filesUsage(ctx: Pick<StoreContext, 'db'>): StoreUsagePart[] {
+  const row = ctx.db.get<{ bytes: number | null; items: number }>('SELECT SUM(size) AS bytes, COUNT(*) AS items FROM file_blob');
+  return [{ part: 'files', bytes: Number(row?.bytes ?? 0), items: Number(row?.items ?? 0) }];
+}
+
 // ---- Calls from the page ----
 
 function fileRow(db: StorageAdapter, { jmap, id }: NodeRef): NodeRow {