fix: list files on Stalwart 1.0 when the account holds more nodes than one get allows #1385

src/api/__tests__/files.test.ts

@@ -304,6 +304,8 @@ function fakeFilesServer(nodes: FakeNode[], opts: {
   // Stalwart before 0.16.6 returns leaf files only.
   queryOmitsFolders?: boolean;
   queryFails?: boolean;
+  // Stalwart 1.0 refuses a return-all /get past the limit (#1385).
+  refusesReturnAll?: boolean;
 }): MethodCall[][] {
   mockMaxObjectsInGet.mockReturnValue(opts.maxObjectsInGet);
   if (opts.maxCallsInRequest) mockMaxCallsInRequest.mockReturnValue(opts.maxCallsInRequest);
@@ -313,6 +315,9 @@ function fakeFilesServer(nodes: FakeNode[], opts: {
     const methodResponses = methodCalls.map(([method, args, callId]) => {
       if (method === 'FileNode/get') {
         const ids = args.ids as string[] | null;
+        if (ids === null && opts.refusesReturnAll && nodes.length > opts.maxObjectsInGet) {
+          return ['error', { type: 'requestTooLarge' }, callId];
+        }
         if (ids === null) return [method, { list: nodes.slice(0, opts.maxObjectsInGet) }, callId];
         if (ids.length > opts.maxObjectsInGet) return ['error', { type: 'requestTooLarge' }, callId];
         return [method, {
@@ -420,6 +425,27 @@ describe('listing past maxObjectsInGet (#1069)', () => {
     expect(askedForHidden).toHaveLength(1);
   });
 
+  // #1385: Stalwart 1.0 answers the return-all /get of a larger account with
+  // requestTooLarge instead of a first page.
+  it('lists every node when Stalwart 1.0 refuses the return-all /get', async () => {
+    const all = [
+      fakeFolder('d1'),
+      ...Array.from({ length: 6 }, (_, i) => fakeFile(`f${i}`, 'd1')),
+    ];
+    fakeFilesServer(all, { maxObjectsInGet: 3, refusesReturnAll: true });
+
+    const nodes = await getAllFileNodes();
+    expect(sortedIds(nodes)).toEqual(sortedIds(all));
+  });
+
+  it('reports a failed listing of the own drive instead of an empty one', async () => {
+    fakeFilesServer([fakeFile('f0'), fakeFile('f1'), fakeFile('f2')], {
+      maxObjectsInGet: 2, refusesReturnAll: true, queryFails: true,
+    });
+
+    await expect(getAllFileNodesAcrossAccounts()).rejects.toThrow();
+  });
+
   it('keeps the first page when the query is refused', async () => {
     fakeFilesServer([fakeFile('f0'), fakeFile('f1'), fakeFile('f2')], { maxObjectsInGet: 2, queryFails: true });
 

src/api/__tests__/live-stalwart-files.test.ts

@@ -179,4 +179,32 @@ live('live Stalwart files', () => {
     expect(result.created?.l).toBeUndefined();
     expect(result.notCreated?.l?.properties).toEqual(['target']);
   }, 30_000);
+
+  // #1385: past maxObjectsInGet, 0.16 cuts the return-all /get short and 1.0
+  // refuses it with requestTooLarge.
+  it('lists an account holding more nodes than one FileNode/get returns', async () => {
+    const max = client.getMaxObjectsInGet();
+    const perSet = client.getMaxObjectsInSet();
+    const bulk = await createFolder('bulk', scratch);
+    const count = max + 5;
+    const made: string[] = [];
+    for (let start = 0; start < count; start += perSet) {
+      const create: Record<string, Record<string, unknown>> = {};
+      for (let i = start; i < Math.min(count, start + perSet); i++) {
+        create[`n${i}`] = { name: `n${i}`, parentId: bulk.id, ...(draft18() ? { '@type': 'Directory' } : {}) };
+      }
+      const res = await client.request([['FileNode/set', { accountId: filesAccountId(), create }, '0']],
+        [CAPABILITIES.CORE, CAPABILITIES.FILES]);
+      const [, result] = res.methodResponses[0] as unknown as [string, { created?: Record<string, { id: string }> }];
+      made.push(...Object.values(result.created ?? {}).map((c) => c.id));
+    }
+    expect(made).toHaveLength(count);
+
+    const all = await nodes();
+    const ids = new Set(all.map((n) => n.id));
+    expect(made.filter((id) => !ids.has(id))).toEqual([]);
+    expect(ids.has(scratch)).toBe(true);
+
+    await deleteFileNodes([bulk.id]);
+  }, 180_000);
 });

src/api/files.ts

@@ -192,12 +192,15 @@ async function fetchAllFileNodes(accountId: string): Promise<FileNode[]> {
     using,
   );
   const result = res.methodResponses[0];
-  if (!result || result[0] === 'error') {
+  // Stalwart 1.0 refuses the return-all /get instead of cutting it short
+  // (#1385); then every node comes from the query below.
+  const tooMany = result?.[0] === 'error' && result[1]?.type === 'requestTooLarge';
+  if (!result || (result[0] === 'error' && !tooMany)) {
     throw new Error(result?.[1]?.description || 'FileNode list failed');
   }
-  const firstPage = ((result[1].list ?? []) as FileNode[]).map(fromDraftFileNode);
+  const firstPage = tooMany ? [] : ((result[1].list ?? []) as FileNode[]).map(fromDraftFileNode);
   const maxObjects = jmapClient.getMaxObjectsInGet();
-  if (firstPage.length < maxObjects) return firstPage;
+  if (!tooMany && firstPage.length < maxObjects) return firstPage;
 
   const known = new Map(firstPage.map((node) => [node.id, node]));
   // Several /get calls share one request, so a large account costs a few
@@ -256,6 +259,8 @@ async function fetchAllFileNodes(accountId: string): Promise<FileNode[]> {
     }
   } catch (err) {
     // A partial tree beats none: keep what was read, as before #1069.
+    // With nothing read there is no tree to show, so say why.
+    if (known.size === 0) throw err;
     console.warn(`[files] listing for account ${accountId} is incomplete past ${maxObjects} nodes`, err);
   }
 
@@ -311,7 +316,10 @@ export async function getAllFileNodesAcrossAccounts(): Promise<FileNode[]> {
           ...(!isPrimary && isDiscoverableOnly(node) ? { discoverableOnly: true } : {}),
         });
       }
-    } catch {
+    } catch (err) {
+      // A failed listing of the user's own drive is an error to show, not
+      // an empty drive (#1385 looked like "No files yet").
+      if (isPrimary) throw err;
       // A single unreachable shared account shouldn't hide the user's own
       // files; skip it and keep aggregating.
     }