diff --git a/lib/__tests__/helpers/stalwart-state.ts b/lib/__tests__/helpers/stalwart-state.ts new file mode 100644 index 000000000..95a6b03f6 --- /dev/null +++ b/lib/__tests__/helpers/stalwart-state.ts @@ -0,0 +1,84 @@ +// Stalwart's JMAP state writer, ported line by line for tests: Base32Writer +// (utils/src/codec/base32_custom.rs) fed by write_leb128 (utils/src/codec/ +// leb128.rs), as `impl Display for State` (jmap-proto/src/types/state.rs) +// uses them. Kept apart from lib/jmap/stalwart-state.ts on purpose, so tests +// check the decoder against an encoder written the way the server's is. + +const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz792013'; + +class Base32Writer { + private lastByte = 0; + private pos = 0; + private result = ''; + + pushChar(ch: string): void { + this.result += ch; + } + + private pushByte(byte: number, isRemainder: boolean): void { + let ch1: number; + let ch2: number | null; + switch (this.pos % 5) { + case 0: + ch1 = (byte & 0xf8) >> 3; + ch2 = null; + break; + case 1: + ch1 = ((this.lastByte & 0x07) << 2) | ((byte & 0xc0) >> 6); + ch2 = (byte & 0x3e) >> 1; + break; + case 2: + ch1 = ((this.lastByte & 0x01) << 4) | ((byte & 0xf0) >> 4); + ch2 = null; + break; + case 3: + ch1 = ((this.lastByte & 0x0f) << 1) | (byte >> 7); + ch2 = (byte & 0x7c) >> 2; + break; + default: + ch1 = ((this.lastByte & 0x03) << 3) | ((byte & 0xe0) >> 5); + ch2 = byte & 0x1f; + } + this.result += BASE32_ALPHABET[ch1]; + if (!isRemainder) { + if (ch2 !== null) this.result += BASE32_ALPHABET[ch2]; + this.lastByte = byte; + this.pos += 1; + } + } + + writeLeb128(value: bigint): void { + let rest = value; + for (;;) { + if (rest < BigInt(0x80)) { + this.pushByte(Number(rest), false); + return; + } + this.pushByte(Number((rest & BigInt(0x7f)) | BigInt(0x80)), false); + rest >>= BigInt(7); + } + } + + finalize(): string { + if (this.pos % 5 !== 0) this.pushByte(0, true); + return this.result; + } +} + +/** `State::Exact(id).to_string()` */ +export function stalwartExactState(id: bigint | number): string { + const writer = new Base32Writer(); + writer.pushChar('s'); + writer.writeLeb128(BigInt(id)); + return writer.finalize(); +} + +/** `State::Intermediate { from_id, to_id, items_sent }.to_string()` */ +export function stalwartIntermediateState(fromId: number, toId: number, itemsSent: number): string { + const writer = new Base32Writer(); + writer.pushChar('r'); + writer.writeLeb128(BigInt(fromId)); + writer.writeLeb128(BigInt(toId - fromId)); + writer.writeLeb128(BigInt(itemsSent)); + return writer.finalize(); +} diff --git a/lib/__tests__/jmap-client-extensions.test.ts b/lib/__tests__/jmap-client-extensions.test.ts index 85f774fac..e1abe33bf 100644 --- a/lib/__tests__/jmap-client-extensions.test.ts +++ b/lib/__tests__/jmap-client-extensions.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { JMAPClient, submissionEnvelopeParameters } from '../jmap/client'; import { isJmapSetError, jmapErrorType } from '../jmap/jmap-error'; +import { stalwartExactState, stalwartIntermediateState } from './helpers/stalwart-state'; describe('submissionEnvelopeParameters', () => { it('is empty without options', () => { @@ -142,6 +143,79 @@ describe('JMAP client extensions', () => { expect(page.state).toBe('e9'); expect(page.emails).toHaveLength(1); }); + + // Stalwart lists urn:stalwart:jmap on every account (0.16.6 through 1.0). + const STALWART = { 'urn:stalwart:jmap': {} }; + const changes = (type: string, oldState: string, newState: string) => ({ + methodResponses: [[ + `${type}/changes`, + { accountId: 'acct-1', oldState, newState, hasMoreChanges: false, created: [], updated: [], destroyed: [] }, + '0', + ]], + }); + + it.each(['Email', 'Mailbox'])( + 'falls back to a full fetch when Stalwart answers %s/changes with a lower state (upgrade with restarted counters)', + async (type) => { + await setup(STALWART); + // The old change log survived with its ids, the counter restarted + // lower: Stalwart accepts the old state and reports nothing since. + const since = stalwartExactState(12_345_678); + answer(() => changes(type, since, stalwartExactState(16_384))); + + const delta = type === 'Email' ? await client.getEmailChanges(since) : await client.getMailboxChanges(since); + + expect(delta).toBeNull(); + }, + ); + + it('trips on the states Stalwart 0.16.23 and 1.0 actually return', async () => { + await setup(STALWART); + // `sba` (8) from 0.16.23 answered with `sau` (5) from 1.0. + answer(() => changes('Email', 'sba', 'sau')); + expect(await client.getEmailChanges('sba')).toBeNull(); + + answer(() => changes('Email', 'sau', 'sba')); + expect(await client.getEmailChanges('sau')).toMatchObject({ oldState: 'sau', newState: 'sba' }); + }); + + it('never trips while Stalwart states go up or stay put', async () => { + await setup(STALWART); + const big = (n: number | string) => BigInt(n); + // Every LEB128 byte boundary, and ids past 2^53. + const ids = [ + big(0), big(1), big(126), big(127), big(128), big(129), big(16_383), big(16_384), + big(2_097_151), big(2_097_152), big(2) ** big(32) - big(1), big(2) ** big(32), + big(2) ** big(53), big(2) ** big(53) + big(1), big(2) ** big(64) - big(2), big(2) ** big(64) - big(1), + ]; + for (let i = 0; i < ids.length; i++) { + for (const next of [ids[i], ids[i + 1]]) { + if (next === undefined) continue; + const since = stalwartExactState(ids[i]); + const newState = stalwartExactState(next); + answer(() => changes('Email', since, newState)); + expect(await client.getEmailChanges(since)).toMatchObject({ oldState: since, newState }); + } + } + }); + + it('leaves states in any other format alone', async () => { + await setup(STALWART); + // An intermediate state (a delta cut short by maxChanges) on either side. + const intermediate = stalwartIntermediateState(1024, 2048, 100); + answer(() => changes('Email', stalwartExactState(5000), intermediate)); + expect(await client.getEmailChanges(stalwartExactState(5000))).toMatchObject({ newState: intermediate }); + + answer(() => changes('Email', intermediate, stalwartExactState(1))); + expect(await client.getEmailChanges(intermediate)).toMatchObject({ newState: stalwartExactState(1) }); + }); + + it('does not read states of servers other than Stalwart', async () => { + await setup(); + // Same strings, but the session doesn't list urn:stalwart:jmap. + answer(() => changes('Email', 'sba', 'sau')); + expect(await client.getEmailChanges('sba')).toMatchObject({ newState: 'sau' }); + }); }); describe('search snippets', () => { diff --git a/lib/__tests__/stalwart-state.test.ts b/lib/__tests__/stalwart-state.test.ts new file mode 100644 index 000000000..a3bc0b8c4 --- /dev/null +++ b/lib/__tests__/stalwart-state.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest'; +import { stalwartChangeId } from '@/lib/jmap/stalwart-state'; +import { stalwartExactState, stalwartIntermediateState } from './helpers/stalwart-state'; + +const big = (n: number | string) => BigInt(n); +const U64_MAX = big(2) ** big(64) - big(1); + +// What the stw10 test servers answered to Mailbox/get, Email/get and +// Thread/get with `ids: []` for usera and the team group (2026-09-25). +const CAPTURED_1_0: Array<[string, number]> = [['sae', 1], ['sai', 2], ['saq', 4], ['sau', 5]]; +const CAPTURED_0_16: Array<[string, number]> = [['sae', 1], ['sai', 2], ['say', 6], ['sba', 8]]; + +// Written by a separate port of Stalwart's writer, one per LEB128 length and +// at each byte boundary. +const WRITTEN: Array<[string, bigint]> = [ + ['saa', big(0)], + ['sd2', big(31)], + ['sp2', big(127)], + ['sqaaq', big(128)], + ['sqeaq', big(129)], + ['s32aq', big(255)], + ['svqba', big(300)], + ['s303q', big(16383)], + ['sqcaac', big(16384)], + ['syccd7', big(1000000)], + ['sz9bpcbi', big(12345678)], + ['s333333yp', big(2) ** big(32) - big(1)], + ['sqgaibaeaqcaba', big(2) ** big(53) + big(1)], + ['s33333333333333yb', U64_MAX], +]; + +// A small deterministic generator for ids across the whole u64 range. +function* seededIds(count: number): Generator { + let seed = big(0x2545f491); + for (let i = 0; i < count; i++) { + seed = (seed * big('6364136223846793005') + big('1442695040888963407')) & U64_MAX; + yield seed >> big(i % 64); + } +} + +describe('stalwartChangeId', () => { + it('reads the change ids of states captured from Stalwart 1.0 and 0.16.23', () => { + for (const [state, id] of [...CAPTURED_1_0, ...CAPTURED_0_16]) { + expect(stalwartChangeId(state)).toBe(big(id)); + } + }); + + it('reads ids of every LEB128 length, up to u64::MAX', () => { + for (const [state, id] of WRITTEN) { + expect(stalwartExactState(id)).toBe(state); + expect(stalwartChangeId(state)).toBe(id); + } + }); + + it('reads back every id written the way Stalwart writes it', () => { + for (let id = 0; id < 20_000; id++) { + expect(stalwartChangeId(stalwartExactState(id))).toBe(big(id)); + } + for (const id of seededIds(5_000)) { + expect(stalwartChangeId(stalwartExactState(id))).toBe(id); + } + }); + + it('reads nothing from other state formats', () => { + for (const state of [ + 'n', // Initial + stalwartIntermediateState(1024, 2048, 100), // an intermediate `r` state + '', 's', + 's1', // '1' is 30: a LEB128 byte that never ends + 'sZZ', 's!', 'sa e', // not the alphabet + '12345', 'J1234', '2d7a4b56-6c1e-4f2b-9d57-6a3c9e1f0b42', // other servers + ]) { + expect(stalwartChangeId(state)).toBeNull(); + } + }); + + it('reads nothing Stalwart would not have written for that id', () => { + // Stalwart's reader takes these too, but its writer never produces them. + expect(stalwartChangeId('sa')).toBeNull(); // 0 is `saa` + expect(stalwartChangeId('sb')).toBeNull(); // 8 is `sba` + expect(stalwartChangeId('saaaaaaaa')).toBeNull(); + expect(stalwartChangeId('sbb')).toBeNull(); // nonzero padding bits + // Past a u64: ten bytes that overflow it, and eleven. + expect(stalwartChangeId(stalwartExactState(big(2) ** big(64)))).toBeNull(); + expect(stalwartChangeId(stalwartExactState(big(2) ** big(70)))).toBeNull(); + }); +}); diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index 790a2d78d..9c787279e 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -22,6 +22,7 @@ import { acceptedFileName, fileNameRulesFrom, type FileNameRules } from "@/lib/f import { getEffectiveTimeZone, toLocalDateTime } from "@/lib/timezone"; import { buildEmailSort, compareEmails, hasKeywordLevels, type KeywordSortPolarity, type SortLevel } from "@/lib/message-list-order"; import type { AccountInfo } from "./types"; +import { stalwartChangeId } from "./stalwart-state"; // Cap for the follow-up Email/get issued when a displayed body part comes // back truncated at the normal 256000-byte limit (see refetchTruncatedBodyValues @@ -1877,9 +1878,10 @@ export class JMAPClient implements IJMAPClient { /** * Generic `Foo/changes` (RFC 8620 ยง5.2). Returns null when the server can * no longer compute a delta from `sinceState` (`cannotCalculateChanges`, - * a too-old state, or any other error) - the caller then falls back to a - * full re-fetch. `hasMoreChanges` is surfaced rather than paged through: - * a delta that large is cheaper to resolve with a fresh query. + * a too-old state, a state that went backwards, or any other error) - the + * caller then falls back to a full re-fetch. `hasMoreChanges` is surfaced + * rather than paged through: a delta that large is cheaper to resolve with + * a fresh query. */ private async getCollectionChanges(type: 'Email' | 'Mailbox', sinceState: string, accountId?: string, maxChanges?: number): Promise { const targetAccountId = accountId || this.accountId; @@ -1905,6 +1907,19 @@ export class JMAPClient implements IJMAPClient { updatedProperties?: string[] | null; }; if (typeof r.newState !== 'string') return null; + // Stalwart's change ids only grow. A newState below sinceState means its + // counters went back - a restore, or an upgrade that kept the old change + // log but restarted the counters lower (1.0 counts per collection where + // 0.16 counted per account) - and it then answers an old state with an + // empty delta, skipping every change since. Take the full fetch instead. + if (this.hasAccountCapability('urn:stalwart:jmap', targetAccountId)) { + const since = stalwartChangeId(sinceState); + const next = stalwartChangeId(r.newState); + if (since !== null && next !== null && next < since) { + debug.warn('jmap', `[JMAP] ${type}/changes went back from ${sinceState} to ${r.newState}, falling back to a full fetch`); + return null; + } + } return { oldState: r.oldState ?? sinceState, newState: r.newState, diff --git a/lib/jmap/stalwart-state.ts b/lib/jmap/stalwart-state.ts new file mode 100644 index 000000000..d4a1f3e01 --- /dev/null +++ b/lib/jmap/stalwart-state.ts @@ -0,0 +1,87 @@ +/** + * Stalwart's JMAP state strings, read back into the change id they carry. + * + * Stalwart writes a state as `n` (initial), `s` + the change id, or `r` + an + * intermediate position (a /changes call cut short by maxChanges). The change + * id is LEB128-encoded, then written in base32 with Stalwart's own alphabet + * (jmap-proto types/state.rs, utils codec/base32_custom.rs; the same from + * 0.16.0 through 1.0). Change ids only grow on a healthy server, which is what + * lets a client notice a state that went backwards. + */ + +const STALWART_BASE32 = 'abcdefghijklmnopqrstuvwxyz792013'; + +// A u64 needs at most ten LEB128 bytes. +const MAX_LEB128_BYTES = 10; + +function base32Encode(bytes: number[]): string { + let out = ''; + let buffer = 0; + let bits = 0; + for (const byte of bytes) { + buffer = (buffer << 8) | byte; + bits += 8; + while (bits >= 5) { + bits -= 5; + out += STALWART_BASE32[(buffer >> bits) & 31]; + } + buffer &= (1 << bits) - 1; + } + if (bits > 0) out += STALWART_BASE32[(buffer << (5 - bits)) & 31]; + return out; +} + +function base32Decode(text: string): number[] | null { + const bytes: number[] = []; + let buffer = 0; + let bits = 0; + for (const ch of text) { + const value = STALWART_BASE32.indexOf(ch); + if (value < 0) return null; + buffer = (buffer << 5) | value; + bits += 5; + if (bits >= 8) { + bits -= 8; + bytes.push((buffer >> bits) & 0xff); + buffer &= (1 << bits) - 1; + } + } + return bytes; +} + +function leb128Encode(value: bigint): number[] { + const out: number[] = []; + let rest = value; + const low7 = BigInt(0x7f); + const shift = BigInt(7); + while (rest > low7) { + out.push(Number(rest & low7) | 0x80); + rest >>= shift; + } + out.push(Number(rest)); + return out; +} + +function leb128Decode(bytes: number[]): bigint | null { + let value = BigInt(0); + for (let i = 0; i < bytes.length && i < MAX_LEB128_BYTES; i++) { + value |= BigInt(bytes[i] & 0x7f) << BigInt(7 * i); + // Stalwart reads into a u64 and drops what doesn't fit. + if ((bytes[i] & 0x80) === 0) return BigInt.asUintN(64, value); + } + return null; +} + +/** + * The change id of a Stalwart `s` state, or null for any other string: an + * initial or intermediate state, another server's format, or anything + * Stalwart would not have written for that id (a non-canonical encoding). + * A BigInt, because ids are u64. + */ +export function stalwartChangeId(state: string): bigint | null { + if (state.length < 2 || state[0] !== 's') return null; + const bytes = base32Decode(state.slice(1)); + const id = bytes ? leb128Decode(bytes) : null; + if (id === null) return null; + return `s${base32Encode(leb128Encode(id))}` === state ? id : null; +}