From 6dc9d5a584b3338f410be5ab7705a2f6cb7bb9cf Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:04:26 +0200 Subject: [PATCH] fix: refetch mail when the server's change state goes backwards Stalwart's change ids only grow. An upgrade that keeps the old change log but restarts the counters lower (1.0 counts per collection where 0.16 counted per account) makes Email/changes and Mailbox/changes accept an old state and answer with a lower `newState` and no changes, so the delta sync would silently skip everything that changed since. Read Stalwart `s` states (LEB128 in Stalwart's base32) and treat a `newState` below `sinceState` like `cannotCalculateChanges`: fetch in full. The check runs only on accounts that list `urn:stalwart:jmap`, only for states in the exact form Stalwart writes, and never trips while states go up. --- lib/__tests__/helpers/stalwart-state.ts | 84 +++++++++++++++++++ lib/__tests__/jmap-client-extensions.test.ts | 74 +++++++++++++++++ lib/__tests__/stalwart-state.test.ts | 87 ++++++++++++++++++++ lib/jmap/client.ts | 21 ++++- lib/jmap/stalwart-state.ts | 87 ++++++++++++++++++++ 5 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 lib/__tests__/helpers/stalwart-state.ts create mode 100644 lib/__tests__/stalwart-state.test.ts create mode 100644 lib/jmap/stalwart-state.ts diff --git a/lib/__tests__/helpers/stalwart-state.ts b/lib/__tests__/helpers/stalwart-state.ts new file mode 100644 index 000000000..95a6b03f6 --- /dev/null +++ b/lib/__tests__/helpers/stalwart-state.ts @@ -0,0 +1,84 @@ +// Stalwart's JMAP state writer, ported line by line for tests: Base32Writer +// (utils/src/codec/base32_custom.rs) fed by write_leb128 (utils/src/codec/ +// leb128.rs), as `impl Display for State` (jmap-proto/src/types/state.rs) +// uses them. Kept apart from lib/jmap/stalwart-state.ts on purpose, so tests +// check the decoder against an encoder written the way the server's is. + +const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz792013'; + +class Base32Writer { + private lastByte = 0; + private pos = 0; + private result = ''; + + pushChar(ch: string): void { + this.result += ch; + } + + private pushByte(byte: number, isRemainder: boolean): void { + let ch1: number; + let ch2: number | null; + switch (this.pos % 5) { + case 0: + ch1 = (byte & 0xf8) >> 3; + ch2 = null; + break; + case 1: + ch1 = ((this.lastByte & 0x07) << 2) | ((byte & 0xc0) >> 6); + ch2 = (byte & 0x3e) >> 1; + break; + case 2: + ch1 = ((this.lastByte & 0x01) << 4) | ((byte & 0xf0) >> 4); + ch2 = null; + break; + case 3: + ch1 = ((this.lastByte & 0x0f) << 1) | (byte >> 7); + ch2 = (byte & 0x7c) >> 2; + break; + default: + ch1 = ((this.lastByte & 0x03) << 3) | ((byte & 0xe0) >> 5); + ch2 = byte & 0x1f; + } + this.result += BASE32_ALPHABET[ch1]; + if (!isRemainder) { + if (ch2 !== null) this.result += BASE32_ALPHABET[ch2]; + this.lastByte = byte; + this.pos += 1; + } + } + + writeLeb128(value: bigint): void { + let rest = value; + for (;;) { + if (rest < BigInt(0x80)) { + this.pushByte(Number(rest), false); + return; + } + this.pushByte(Number((rest & BigInt(0x7f)) | BigInt(0x80)), false); + rest >>= BigInt(7); + } + } + + finalize(): string { + if (this.pos % 5 !== 0) this.pushByte(0, true); + return this.result; + } +} + +/** `State::Exact(id).to_string()` */ +export function stalwartExactState(id: bigint | number): string { + const writer = new Base32Writer(); + writer.pushChar('s'); + writer.writeLeb128(BigInt(id)); + return writer.finalize(); +} + +/** `State::Intermediate { from_id, to_id, items_sent }.to_string()` */ +export function stalwartIntermediateState(fromId: number, toId: number, itemsSent: number): string { + const writer = new Base32Writer(); + writer.pushChar('r'); + writer.writeLeb128(BigInt(fromId)); + writer.writeLeb128(BigInt(toId - fromId)); + writer.writeLeb128(BigInt(itemsSent)); + return writer.finalize(); +} diff --git a/lib/__tests__/jmap-client-extensions.test.ts b/lib/__tests__/jmap-client-extensions.test.ts index 85f774fac..e1abe33bf 100644 --- a/lib/__tests__/jmap-client-extensions.test.ts +++ b/lib/__tests__/jmap-client-extensions.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { JMAPClient, submissionEnvelopeParameters } from '../jmap/client'; import { isJmapSetError, jmapErrorType } from '../jmap/jmap-error'; +import { stalwartExactState, stalwartIntermediateState } from './helpers/stalwart-state'; describe('submissionEnvelopeParameters', () => { it('is empty without options', () => { @@ -142,6 +143,79 @@ describe('JMAP client extensions', () => { expect(page.state).toBe('e9'); expect(page.emails).toHaveLength(1); }); + + // Stalwart lists urn:stalwart:jmap on every account (0.16.6 through 1.0). + const STALWART = { 'urn:stalwart:jmap': {} }; + const changes = (type: string, oldState: string, newState: string) => ({ + methodResponses: [[ + `${type}/changes`, + { accountId: 'acct-1', oldState, newState, hasMoreChanges: false, created: [], updated: [], destroyed: [] }, + '0', + ]], + }); + + it.each(['Email', 'Mailbox'])( + 'falls back to a full fetch when Stalwart answers %s/changes with a lower state (upgrade with restarted counters)', + async (type) => { + await setup(STALWART); + // The old change log survived with its ids, the counter restarted + // lower: Stalwart accepts the old state and reports nothing since. + const since = stalwartExactState(12_345_678); + answer(() => changes(type, since, stalwartExactState(16_384))); + + const delta = type === 'Email' ? await client.getEmailChanges(since) : await client.getMailboxChanges(since); + + expect(delta).toBeNull(); + }, + ); + + it('trips on the states Stalwart 0.16.23 and 1.0 actually return', async () => { + await setup(STALWART); + // `sba` (8) from 0.16.23 answered with `sau` (5) from 1.0. + answer(() => changes('Email', 'sba', 'sau')); + expect(await client.getEmailChanges('sba')).toBeNull(); + + answer(() => changes('Email', 'sau', 'sba')); + expect(await client.getEmailChanges('sau')).toMatchObject({ oldState: 'sau', newState: 'sba' }); + }); + + it('never trips while Stalwart states go up or stay put', async () => { + await setup(STALWART); + const big = (n: number | string) => BigInt(n); + // Every LEB128 byte boundary, and ids past 2^53. + const ids = [ + big(0), big(1), big(126), big(127), big(128), big(129), big(16_383), big(16_384), + big(2_097_151), big(2_097_152), big(2) ** big(32) - big(1), big(2) ** big(32), + big(2) ** big(53), big(2) ** big(53) + big(1), big(2) ** big(64) - big(2), big(2) ** big(64) - big(1), + ]; + for (let i = 0; i < ids.length; i++) { + for (const next of [ids[i], ids[i + 1]]) { + if (next === undefined) continue; + const since = stalwartExactState(ids[i]); + const newState = stalwartExactState(next); + answer(() => changes('Email', since, newState)); + expect(await client.getEmailChanges(since)).toMatchObject({ oldState: since, newState }); + } + } + }); + + it('leaves states in any other format alone', async () => { + await setup(STALWART); + // An intermediate state (a delta cut short by maxChanges) on either side. + const intermediate = stalwartIntermediateState(1024, 2048, 100); + answer(() => changes('Email', stalwartExactState(5000), intermediate)); + expect(await client.getEmailChanges(stalwartExactState(5000))).toMatchObject({ newState: intermediate }); + + answer(() => changes('Email', intermediate, stalwartExactState(1))); + expect(await client.getEmailChanges(intermediate)).toMatchObject({ newState: stalwartExactState(1) }); + }); + + it('does not read states of servers other than Stalwart', async () => { + await setup(); + // Same strings, but the session doesn't list urn:stalwart:jmap. + answer(() => changes('Email', 'sba', 'sau')); + expect(await client.getEmailChanges('sba')).toMatchObject({ newState: 'sau' }); + }); }); describe('search snippets', () => { diff --git a/lib/__tests__/stalwart-state.test.ts b/lib/__tests__/stalwart-state.test.ts new file mode 100644 index 000000000..a3bc0b8c4 --- /dev/null +++ b/lib/__tests__/stalwart-state.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest'; +import { stalwartChangeId } from '@/lib/jmap/stalwart-state'; +import { stalwartExactState, stalwartIntermediateState } from './helpers/stalwart-state'; + +const big = (n: number | string) => BigInt(n); +const U64_MAX = big(2) ** big(64) - big(1); + +// What the stw10 test servers answered to Mailbox/get, Email/get and +// Thread/get with `ids: []` for usera and the team group (2026-09-25). +const CAPTURED_1_0: Array<[string, number]> = [['sae', 1], ['sai', 2], ['saq', 4], ['sau', 5]]; +const CAPTURED_0_16: Array<[string, number]> = [['sae', 1], ['sai', 2], ['say', 6], ['sba', 8]]; + +// Written by a separate port of Stalwart's writer, one per LEB128 length and +// at each byte boundary. +const WRITTEN: Array<[string, bigint]> = [ + ['saa', big(0)], + ['sd2', big(31)], + ['sp2', big(127)], + ['sqaaq', big(128)], + ['sqeaq', big(129)], + ['s32aq', big(255)], + ['svqba', big(300)], + ['s303q', big(16383)], + ['sqcaac', big(16384)], + ['syccd7', big(1000000)], + ['sz9bpcbi', big(12345678)], + ['s333333yp', big(2) ** big(32) - big(1)], + ['sqgaibaeaqcaba', big(2) ** big(53) + big(1)], + ['s33333333333333yb', U64_MAX], +]; + +// A small deterministic generator for ids across the whole u64 range. +function* seededIds(count: number): Generator { + let seed = big(0x2545f491); + for (let i = 0; i < count; i++) { + seed = (seed * big('6364136223846793005') + big('1442695040888963407')) & U64_MAX; + yield seed >> big(i % 64); + } +} + +describe('stalwartChangeId', () => { + it('reads the change ids of states captured from Stalwart 1.0 and 0.16.23', () => { + for (const [state, id] of [...CAPTURED_1_0, ...CAPTURED_0_16]) { + expect(stalwartChangeId(state)).toBe(big(id)); + } + }); + + it('reads ids of every LEB128 length, up to u64::MAX', () => { + for (const [state, id] of WRITTEN) { + expect(stalwartExactState(id)).toBe(state); + expect(stalwartChangeId(state)).toBe(id); + } + }); + + it('reads back every id written the way Stalwart writes it', () => { + for (let id = 0; id < 20_000; id++) { + expect(stalwartChangeId(stalwartExactState(id))).toBe(big(id)); + } + for (const id of seededIds(5_000)) { + expect(stalwartChangeId(stalwartExactState(id))).toBe(id); + } + }); + + it('reads nothing from other state formats', () => { + for (const state of [ + 'n', // Initial + stalwartIntermediateState(1024, 2048, 100), // an intermediate `r` state + '', 's', + 's1', // '1' is 30: a LEB128 byte that never ends + 'sZZ', 's!', 'sa e', // not the alphabet + '12345', 'J1234', '2d7a4b56-6c1e-4f2b-9d57-6a3c9e1f0b42', // other servers + ]) { + expect(stalwartChangeId(state)).toBeNull(); + } + }); + + it('reads nothing Stalwart would not have written for that id', () => { + // Stalwart's reader takes these too, but its writer never produces them. + expect(stalwartChangeId('sa')).toBeNull(); // 0 is `saa` + expect(stalwartChangeId('sb')).toBeNull(); // 8 is `sba` + expect(stalwartChangeId('saaaaaaaa')).toBeNull(); + expect(stalwartChangeId('sbb')).toBeNull(); // nonzero padding bits + // Past a u64: ten bytes that overflow it, and eleven. + expect(stalwartChangeId(stalwartExactState(big(2) ** big(64)))).toBeNull(); + expect(stalwartChangeId(stalwartExactState(big(2) ** big(70)))).toBeNull(); + }); +}); diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index 790a2d78d..9c787279e 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -22,6 +22,7 @@ import { acceptedFileName, fileNameRulesFrom, type FileNameRules } from "@/lib/f import { getEffectiveTimeZone, toLocalDateTime } from "@/lib/timezone"; import { buildEmailSort, compareEmails, hasKeywordLevels, type KeywordSortPolarity, type SortLevel } from "@/lib/message-list-order"; import type { AccountInfo } from "./types"; +import { stalwartChangeId } from "./stalwart-state"; // Cap for the follow-up Email/get issued when a displayed body part comes // back truncated at the normal 256000-byte limit (see refetchTruncatedBodyValues @@ -1877,9 +1878,10 @@ export class JMAPClient implements IJMAPClient { /** * Generic `Foo/changes` (RFC 8620 ยง5.2). Returns null when the server can * no longer compute a delta from `sinceState` (`cannotCalculateChanges`, - * a too-old state, or any other error) - the caller then falls back to a - * full re-fetch. `hasMoreChanges` is surfaced rather than paged through: - * a delta that large is cheaper to resolve with a fresh query. + * a too-old state, a state that went backwards, or any other error) - the + * caller then falls back to a full re-fetch. `hasMoreChanges` is surfaced + * rather than paged through: a delta that large is cheaper to resolve with + * a fresh query. */ private async getCollectionChanges(type: 'Email' | 'Mailbox', sinceState: string, accountId?: string, maxChanges?: number): Promise { const targetAccountId = accountId || this.accountId; @@ -1905,6 +1907,19 @@ export class JMAPClient implements IJMAPClient { updatedProperties?: string[] | null; }; if (typeof r.newState !== 'string') return null; + // Stalwart's change ids only grow. A newState below sinceState means its + // counters went back - a restore, or an upgrade that kept the old change + // log but restarted the counters lower (1.0 counts per collection where + // 0.16 counted per account) - and it then answers an old state with an + // empty delta, skipping every change since. Take the full fetch instead. + if (this.hasAccountCapability('urn:stalwart:jmap', targetAccountId)) { + const since = stalwartChangeId(sinceState); + const next = stalwartChangeId(r.newState); + if (since !== null && next !== null && next < since) { + debug.warn('jmap', `[JMAP] ${type}/changes went back from ${sinceState} to ${r.newState}, falling back to a full fetch`); + return null; + } + } return { oldState: r.oldState ?? sinceState, newState: r.newState, diff --git a/lib/jmap/stalwart-state.ts b/lib/jmap/stalwart-state.ts new file mode 100644 index 000000000..d4a1f3e01 --- /dev/null +++ b/lib/jmap/stalwart-state.ts @@ -0,0 +1,87 @@ +/** + * Stalwart's JMAP state strings, read back into the change id they carry. + * + * Stalwart writes a state as `n` (initial), `s` + the change id, or `r` + an + * intermediate position (a /changes call cut short by maxChanges). The change + * id is LEB128-encoded, then written in base32 with Stalwart's own alphabet + * (jmap-proto types/state.rs, utils codec/base32_custom.rs; the same from + * 0.16.0 through 1.0). Change ids only grow on a healthy server, which is what + * lets a client notice a state that went backwards. + */ + +const STALWART_BASE32 = 'abcdefghijklmnopqrstuvwxyz792013'; + +// A u64 needs at most ten LEB128 bytes. +const MAX_LEB128_BYTES = 10; + +function base32Encode(bytes: number[]): string { + let out = ''; + let buffer = 0; + let bits = 0; + for (const byte of bytes) { + buffer = (buffer << 8) | byte; + bits += 8; + while (bits >= 5) { + bits -= 5; + out += STALWART_BASE32[(buffer >> bits) & 31]; + } + buffer &= (1 << bits) - 1; + } + if (bits > 0) out += STALWART_BASE32[(buffer << (5 - bits)) & 31]; + return out; +} + +function base32Decode(text: string): number[] | null { + const bytes: number[] = []; + let buffer = 0; + let bits = 0; + for (const ch of text) { + const value = STALWART_BASE32.indexOf(ch); + if (value < 0) return null; + buffer = (buffer << 5) | value; + bits += 5; + if (bits >= 8) { + bits -= 8; + bytes.push((buffer >> bits) & 0xff); + buffer &= (1 << bits) - 1; + } + } + return bytes; +} + +function leb128Encode(value: bigint): number[] { + const out: number[] = []; + let rest = value; + const low7 = BigInt(0x7f); + const shift = BigInt(7); + while (rest > low7) { + out.push(Number(rest & low7) | 0x80); + rest >>= shift; + } + out.push(Number(rest)); + return out; +} + +function leb128Decode(bytes: number[]): bigint | null { + let value = BigInt(0); + for (let i = 0; i < bytes.length && i < MAX_LEB128_BYTES; i++) { + value |= BigInt(bytes[i] & 0x7f) << BigInt(7 * i); + // Stalwart reads into a u64 and drops what doesn't fit. + if ((bytes[i] & 0x80) === 0) return BigInt.asUintN(64, value); + } + return null; +} + +/** + * The change id of a Stalwart `s` state, or null for any other string: an + * initial or intermediate state, another server's format, or anything + * Stalwart would not have written for that id (a non-canonical encoding). + * A BigInt, because ids are u64. + */ +export function stalwartChangeId(state: string): bigint | null { + if (state.length < 2 || state[0] !== 's') return null; + const bytes = base32Decode(state.slice(1)); + const id = bytes ? leb128Decode(bytes) : null; + if (id === null) return null; + return `s${base32Encode(leb128Encode(id))}` === state ? id : null; +}