diff --git a/app/api/auth/token/route.ts b/app/api/auth/token/route.ts index 3e6a90667..ba2d3d9ee 100644 --- a/app/api/auth/token/route.ts +++ b/app/api/auth/token/route.ts @@ -273,21 +273,29 @@ export async function PUT(request: NextRequest) { // The refresh token may have been minted by the password+TOTP login route, // which works without a configured OAuth client by falling back to the // default client id - refreshing must fall back the same way (#873). + // Discovery or the token endpoint out of reach is an outage: the client + // waits it out. A 500 here would count toward signing the user out. + let tokenResponse: Response; const upstream = IS_DESKTOP_BUILD ? await readCustomUpstream(cookieStore, slot) : null; - const tokenEndpoint = await getTokenEndpoint(serverId, upstream ? { upstream } : { fallbackClientId: DEFAULT_CLIENT_ID }); + try { + const tokenEndpoint = await getTokenEndpoint(serverId, upstream ? { upstream } : { fallbackClientId: DEFAULT_CLIENT_ID }); - const params = buildOAuthParams({ - grant_type: 'refresh_token', - refresh_token: refreshToken, - }, serverId, await defaultClientOptions(serverId, upstream)); + const params = buildOAuthParams({ + grant_type: 'refresh_token', + refresh_token: refreshToken, + }, serverId, await defaultClientOptions(serverId, upstream)); - const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: params.toString(), - // See revokeRefreshToken: the body holds the refresh token. - redirect: 'error', - })); + tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: params.toString(), + // See revokeRefreshToken: the body holds the refresh token. + redirect: 'error', + })); + } catch (error) { + logger.warn('Token refresh: provider unreachable', { error: error instanceof Error ? error.message : 'Unknown error' }); + return NextResponse.json({ error: 'Token endpoint unavailable' }, { status: 503 }); + } if (!tokenResponse.ok) { const errorText = await tokenResponse.text(); @@ -302,7 +310,10 @@ export async function PUT(request: NextRequest) { cookieStore.delete(accessTokenCookieName(slot)); clearCustomUpstream(cookieStore, slot); clearIdToken(cookieStore, slot, request.nextUrl.basePath); - return NextResponse.json({ error: 'Refresh failed' }, { status: 401 }); + // Only invalid_grant says the login itself is over. invalid_client, or + // a provider refusing an early refresh (#552), says nothing about it, + // and the client keeps the account's local data for those. + return NextResponse.json({ error: oauthErrorCode(errorText) === 'invalid_grant' ? 'invalid_grant' : 'Refresh failed' }, { status: 401 }); } return NextResponse.json({ error: 'Token endpoint unavailable' }, { status: 503 }); } @@ -350,6 +361,15 @@ export async function PUT(request: NextRequest) { } } +function oauthErrorCode(body: string): string | null { + try { + const code = (JSON.parse(body) as { error?: unknown }).error; + return typeof code === 'string' ? code : null; + } catch { + return null; + } +} + /** * Revoke a slot's refresh token and clear its token cookies. With * `endSession`, also returns the URL that ends the provider's session, or null diff --git a/app/api/offline/shell/route.ts b/app/api/offline/shell/route.ts index 43a88d3ec..ac0fc757a 100644 --- a/app/api/offline/shell/route.ts +++ b/app/api/offline/shell/route.ts @@ -7,8 +7,10 @@ export const dynamic = 'force-dynamic'; const BASE_PATH = (process.env.NEXT_PUBLIC_BASE_PATH ?? '').replace(/\/+$/, ''); /** Files a page can load: scripts, styles, fonts and the images they name. */ const SHELL_FILE = /\.(js|css|woff2?|svg|png|webp)$/; +/** The local store's engine in public/. */ +const ENGINE_FILE = /\.(mjs|wasm)$/; -async function listFiles(dir: string, prefix: string, out: string[]): Promise { +async function listFiles(dir: string, prefix: string, out: string[], wanted: RegExp = SHELL_FILE): Promise { let entries; try { entries = await readdir(dir, { withFileTypes: true }); @@ -16,8 +18,8 @@ async function listFiles(dir: string, prefix: string, out: string[]): Promise state.timeFormat); const plainTextModeDefault = useSettingsStore((state) => state.plainTextMode); @@ -4213,6 +4215,8 @@ export function EmailComposer({ } else if (err instanceof SubmissionQuotaError) { // At a full submission quota only an administrator can help. toast.error(sendFailureMessage(err, t)); + } else if (isServerOutage(err)) { + toast.error(t('send_failed'), { message: tOfflineNotes('server_not_answering') }); } else if (err instanceof Error && OFFLINE_SEND_ERRORS[err.name]) { // Offline mode (#523) refused to keep it: says why, in the user's language. toast.error(t('send_failed'), { message: tOffline(OFFLINE_SEND_ERRORS[err.name]) }); diff --git a/components/offline/__tests__/offline-host.test.tsx b/components/offline/__tests__/offline-host.test.tsx new file mode 100644 index 000000000..1cceb0a23 --- /dev/null +++ b/components/offline/__tests__/offline-host.test.tsx @@ -0,0 +1,55 @@ +import 'fake-indexeddb/auto'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { OfflineHost } from '../offline-host'; +import { queueSend } from '@/lib/offline/queue'; +import { useOfflineStore, writeEnabled } from '@/lib/offline/state'; +import { passSignOutGates } from '@/lib/sign-out-cleanup'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; +import { wipeEverything } from '@/lib/offline/offline-mode'; +import { useAccountStore } from '@/stores/account-store'; + +let store: InProcessStore; + +beforeEach(() => { + store = new InProcessStore(); + setStoreBackend(store); + localStorage.clear(); + useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [] }); + useAccountStore.setState({ accounts: [{ id: 'acc' }] as never }); + writeEnabled(['acc']); +}); + +afterEach(async () => { + cleanup(); + await wipeEverything(); + setStoreBackend(null); +}); + +describe('the question before a sign-out', () => { + it('counts what the queue holds, though the screen has seen none of it', async () => { + await queueSend('acc', { subject: 'Hello', to: [] }, { kind: 'email', args: [] }); + useOfflineStore.setState({ outbox: [] }); + render(); + const answer = passSignOutGates('acc'); + fireEvent.click(await screen.findByRole('button', { name: /signout\.confirm/ })); + expect(await answer).toBe(true); + }); + + it('asks about a change that hasn\'t reached the server, on its own', async () => { + await store.ensureOpen('acc'); + store.dbs.get('acc')!.run("INSERT INTO calendar_pending (jmap, record_id, kind, patch, queued_at) VALUES ('j1', 'ev', 'update', '{}', 1)"); + render(); + const answer = passSignOutGates('acc'); + expect(await screen.findByText(/signout_changes_title/)).toBeTruthy(); + fireEvent.click(await screen.findByRole('button', { name: /signout\.confirm/ })); + expect(await answer).toBe(true); + }); + + it('asks nothing with nothing waiting', async () => { + render(); + expect(await passSignOutGates('acc')).toBe(true); + expect(screen.queryByRole('button', { name: /signout\.confirm/ })).toBeNull(); + }); +}); diff --git a/components/offline/__tests__/offline-settings.test.tsx b/components/offline/__tests__/offline-settings.test.tsx index 7cace4d96..5a9e35962 100644 --- a/components/offline/__tests__/offline-settings.test.tsx +++ b/components/offline/__tests__/offline-settings.test.tsx @@ -14,6 +14,7 @@ import { useOfflineStore } from '@/lib/offline/state'; import { useAuthStore } from '@/stores/auth-store'; import { usePolicyStore } from '@/stores/policy-store'; import { DEFAULT_ORGANISATION_POLICY } from '@/lib/admin/organisation-policy'; +import { setStoreBackend, type StoreBackend } from '@/lib/store/client/api'; function setNoOffline(noOffline: boolean) { usePolicyStore.setState({ @@ -24,6 +25,8 @@ function setNoOffline(noOffline: boolean) { } beforeEach(() => { + // jsdom has no OPFS: the store's page side says it's there, as a browser would. + setStoreBackend({ supported: () => true } as unknown as StoreBackend); offline.enableOffline.mockClear(); offline.disableOffline.mockClear(); useAuthStore.setState({ activeAccountId: 'acc' }); @@ -32,6 +35,17 @@ beforeEach(() => { }); describe('Settings > Offline', () => { + it('says so in a private window, with no switch to turn on', () => { + useOfflineStore.setState({ storageBlocked: true }); + try { + render(); + expect(screen.getByTestId('offline-private')).toBeTruthy(); + expect(screen.queryByTestId('offline-switch')).toBeNull(); + } finally { + useOfflineStore.setState({ storageBlocked: false }); + } + }); + it('is off until the user turns it on in this browser', () => { render(); const toggle = screen.getByTestId('offline-switch'); diff --git a/components/offline/__tests__/outbox-conflicts.test.tsx b/components/offline/__tests__/outbox-conflicts.test.tsx new file mode 100644 index 000000000..871b77cbf --- /dev/null +++ b/components/offline/__tests__/outbox-conflicts.test.tsx @@ -0,0 +1,68 @@ +import 'fake-indexeddb/auto'; +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setStoreBackend, type StoreBackend } from '@/lib/store/client/api'; +import { useOfflineStore } from '@/lib/offline/state'; +import { useAuthStore } from '@/stores/auth-store'; +import { OutboxList } from '../outbox-list'; +import { OfflineBanner } from '../offline-banner'; + +const conflict = (id: number, reason: 'changed' | 'gone' | 'refused', op: 'update' | 'destroy' = 'update', detail?: string) => ({ + id, + account: 'acc', + recordId: `m${id}`, + createdAt: Date.UTC(2026, 10, 3), + summary: { subject: `Subject ${id}`, op, reason, ...(detail ? { detail } : {}) }, +}); + +let area: ReturnType; + +beforeEach(() => { + area = vi.fn(async (_account: string, name: string) => (name === 'outbox.read' ? { patches: 0, entries: [], conflicts: [] } : undefined)); + setStoreBackend({ supported: () => true, area } as unknown as StoreBackend); + useAuthStore.setState({ activeAccountId: 'acc' }); + useOfflineStore.setState({ enabled: ['acc'], outbox: [], pendingChanges: 0, conflicts: [], browserOffline: false, downServers: [], startedOffline: false }); +}); + +afterEach(() => { + setStoreBackend(null); + useOfflineStore.setState({ enabled: [], conflicts: [] }); +}); + +describe('changes that did not go through', () => { + it('lists each with what happened to it', () => { + useOfflineStore.setState({ conflicts: [conflict(1, 'changed'), conflict(2, 'changed', 'destroy'), conflict(3, 'gone'), conflict(4, 'refused', 'update', 'Read only')] }); + render(); + const rows = screen.getAllByTestId('offline-conflict-row'); + expect(rows.map((row) => row.getAttribute('data-reason'))).toEqual(['changed', 'changed', 'gone', 'refused']); + expect(rows[0].textContent).toContain('Subject 1'); + expect(rows[0].textContent).toContain('changed'); + expect(rows[1].textContent).toContain('changed_delete'); + expect(rows[2].textContent).toContain('gone'); + expect(rows[3].textContent).toContain('refused'); + }); + + it('shows nothing when there are none', () => { + render(); + expect(screen.queryByTestId('offline-conflicts')).toBeNull(); + }); + + it('dismisses one in the store and reads the queue again', async () => { + useOfflineStore.setState({ conflicts: [conflict(7, 'changed')] }); + render(); + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'dismiss_label' })); + }); + expect(area).toHaveBeenCalledWith('acc', 'outbox.dismissConflict', { id: 7 }); + expect(area).toHaveBeenCalledWith('acc', 'outbox.read'); + expect(useOfflineStore.getState().conflicts).toEqual([]); + }); + + it('the banner points to them while online, with nothing else to say', () => { + useOfflineStore.setState({ conflicts: [conflict(1, 'changed')] }); + render(); + expect(screen.getByTestId('offline-banner-conflicts')).toBeTruthy(); + fireEvent.click(screen.getByTestId('offline-banner-conflicts')); + expect(screen.getByTestId('offline-conflict-row')).toBeTruthy(); + }); +}); diff --git a/components/offline/offline-banner.tsx b/components/offline/offline-banner.tsx index c04def228..dc179e362 100644 --- a/components/offline/offline-banner.tsx +++ b/components/offline/offline-banner.tsx @@ -24,9 +24,11 @@ export function OfflineBanner() { const browserOffline = useOfflineStore((s) => s.browserOffline || s.startedOffline); const serverDown = useOfflineStore((s) => !!server && s.downServers.includes(serverKey(server))); const waiting = useOfflineStore((s) => s.outbox.length); + const conflicts = useOfflineStore((s) => s.conflicts.length); + const tConflicts = useTranslations('offlineConflicts'); const [open, setOpen] = useState(false); const offline = browserOffline || serverDown; - if (IS_LITE || !active || (!offline && waiting === 0 && !open)) return null; + if (IS_LITE || !active || (!offline && waiting === 0 && conflicts === 0 && !open)) return null; return ( <>
)} + {conflicts > 0 && ( + + )}
setOpen(false)} title={t('outbox.title')} testId="offline-outbox-dialog"> diff --git a/components/offline/offline-host.tsx b/components/offline/offline-host.tsx index 63e797918..0ddb51046 100644 --- a/components/offline/offline-host.tsx +++ b/components/offline/offline-host.tsx @@ -6,8 +6,7 @@ import { ConfirmDialog } from '@/components/ui/confirm-dialog'; import { useConfirmDialog } from '@/hooks/use-confirm-dialog'; import { addSignOutGate } from '@/lib/sign-out-cleanup'; import { reloadToNewBuild } from '@/lib/app-update'; -import { confirmLoss, setOfflineNotifier, startOfflineMode, type OfflineNotice } from '@/lib/offline/offline-mode'; -import { useOfflineStore } from '@/lib/offline/state'; +import { confirmLoss, setOfflineNotifier, startOfflineMode, unsentCount, type OfflineNotice } from '@/lib/offline/offline-mode'; import { toast } from '@/stores/toast-store'; /** @@ -20,6 +19,9 @@ export function OfflineHost() { const t = useTranslations('offline'); const tRef = useRef(t); tRef.current = t; + const tConflicts = useTranslations('offlineConflicts'); + const tConflictsRef = useRef(tConflicts); + tConflictsRef.current = tConflicts; const { dialogProps, confirm } = useConfirmDialog(); const confirmRef = useRef(confirm); confirmRef.current = confirm; @@ -45,6 +47,21 @@ export function OfflineHost() { case 'lost': toast.warning(tr('toast.lost', { subjects: notice.subjects.map((s) => s || tr('outbox.no_subject')).join(', ') }), { duration: 0 }); break; + case 'conflict': + toast.warning(tConflictsRef.current('toast', { count: notice.count })); + break; + case 'storage-failed': + toast.warning(tConflictsRef.current('storage_failed'), { duration: 0 }); + break; + case 'legacy-stranded': + toast.warning(tConflictsRef.current('legacy_stranded'), { duration: 0 }); + break; + case 'store-reset': + toast.warning(tConflictsRef.current('store_reset'), { duration: 0 }); + break; + case 'store-unavailable': + toast.warning(tConflictsRef.current('store_unavailable'), { duration: 0 }); + break; case 'refused-delete': toast.warning(tr('toast.refused_delete')); break; @@ -66,12 +83,16 @@ export function OfflineHost() { const stop = startOfflineMode(); // Messages that would be lost: ask before a sign-out the user started. const removeGate = addSignOutGate(async (accountId) => { - const unsent = useOfflineStore.getState().outbox.filter((row) => accountId === 'all' || row.account === accountId).length; - if (unsent === 0) return true; + const { messages, changes } = await unsentCount(accountId); + if (messages === 0 && changes === 0) return true; const tr = tRef.current; + const trConflicts = tConflictsRef.current; const ok = await confirmRef.current({ - title: tr('signout.title'), - message: tr('signout.message', { count: unsent }), + title: messages > 0 ? tr('signout.title') : trConflicts('signout_changes_title'), + message: [ + ...(messages > 0 ? [tr('signout.message', { count: messages })] : []), + ...(changes > 0 ? [trConflicts('signout_changes', { count: changes })] : []), + ].join(' '), confirmText: tr('signout.confirm'), variant: 'destructive', }); diff --git a/components/offline/offline-settings.tsx b/components/offline/offline-settings.tsx index 013e37ecb..d8c1a0439 100644 --- a/components/offline/offline-settings.tsx +++ b/components/offline/offline-settings.tsx @@ -30,6 +30,8 @@ export function OfflineSettings() { const syncing = useOfflineStore((s) => !!account && s.syncing.includes(account)); const lastSynced = useOfflineStore((s) => (account ? s.lastSyncedAt[account] : undefined)); const unsent = useOfflineStore((s) => s.outbox.filter((row) => row.account === account).length); + const blocked = useOfflineStore((s) => s.storageBlocked); + const tNotes = useTranslations('offlineConflicts'); const [confirmOff, setConfirmOff] = useState(false); if (IS_LITE) return null; @@ -42,7 +44,9 @@ export function OfflineSettings() { return ( - {!offlineSupported() ? ( + {blocked ? ( +

{tNotes('store_private')}

+ ) : !offlineSupported() ? (

{t('unsupported')}

) : !allowed ? (
diff --git a/components/offline/outbox-list.tsx b/components/offline/outbox-list.tsx index 54745baf3..7e8c3a864 100644 --- a/components/offline/outbox-list.tsx +++ b/components/offline/outbox-list.tsx @@ -6,6 +6,7 @@ import { Button } from '@/components/ui/button'; import { ConfirmDialog } from '@/components/ui/confirm-dialog'; import { deleteOutboxEntry, retryOutboxEntry } from '@/lib/offline/flush'; import { refreshQueueState, scheduleFlushAll } from '@/lib/offline/offline-mode'; +import { dismissConflict, type ConflictEntry } from '@/lib/offline/queue'; import { isOfflineNow, useOfflineStore, type OutboxRow } from '@/lib/offline/state'; import { useAccountStore } from '@/stores/account-store'; @@ -15,13 +16,16 @@ import { useAccountStore } from '@/stores/account-store'; * be sent again or deleted; one whose connection broke on the way is never * sent again on its own (it may have gone out); one held because the * organisation's rules changed can only be deleted (and written again). + * Changes that didn't go through ("changed on another device") are listed too. */ export function OutboxList({ heading = true }: { heading?: boolean }) { const t = useTranslations('offline.outbox'); const tErrors = useTranslations('offline.errors'); + const tConflicts = useTranslations('offlineConflicts'); const format = useFormatter(); const rows = useOfflineStore((s) => s.outbox); const changes = useOfflineStore((s) => s.pendingChanges); + const conflicts = useOfflineStore((s) => s.conflicts); const accounts = useAccountStore((s) => s.accounts); const [deleting, setDeleting] = useState(null); const several = new Set(rows.map((r) => r.account)).size > 1 || accounts.length > 1; @@ -37,11 +41,22 @@ export function OutboxList({ heading = true }: { heading?: boolean }) { }; const retry = async (row: OutboxRow) => { - await retryOutboxEntry(row.id); + await retryOutboxEntry(row.account, row.id); await refreshQueueState(); if (!isOfflineNow()) scheduleFlushAll(0); }; + const conflictText = (conflict: ConflictEntry) => { + if (conflict.summary.reason === 'refused') return tConflicts('refused', { reason: conflict.summary.detail ?? '' }); + if (conflict.summary.reason === 'gone') return tConflicts('gone'); + return tConflicts(conflict.summary.op === 'destroy' ? 'changed_delete' : 'changed'); + }; + + const dismiss = async (conflict: ConflictEntry) => { + await dismissConflict(conflict.account, conflict.id); + await refreshQueueState(); + }; + return (
{heading &&

{t('title')}

} @@ -51,7 +66,7 @@ export function OutboxList({ heading = true }: { heading?: boolean }) { ) : (
    {rows.map((row) => ( -
  • +
  • {row.subject || t('no_subject')}

    @@ -77,13 +92,33 @@ export function OutboxList({ heading = true }: { heading?: boolean }) { {changes > 0 && (

    {t('changes', { count: changes })}

    )} + {conflicts.length > 0 && ( +
    +

    {tConflicts('description')}

    +
      + {conflicts.map((conflict) => ( +
    • +
      +

      {conflict.summary.subject || t('no_subject')}

      +

      + {conflictText(conflict)} · {format.dateTime(conflict.createdAt, { dateStyle: 'short', timeStyle: 'short' })} +

      +
      +
      + +
      +
    • + ))} +
    +
    + )} setDeleting(null)} onConfirm={() => { const row = deleting; setDeleting(null); - if (row) void deleteOutboxEntry(row.id).then(refreshQueueState); + if (row) void deleteOutboxEntry(row.account, row.id).then(refreshQueueState); }} title={t('delete_confirm_title')} message={t('delete_confirm_message')} diff --git a/integration/tests/22-ent-offline.spec.ts b/integration/tests/22-ent-offline.spec.ts index 0491db978..3a32f0af6 100644 --- a/integration/tests/22-ent-offline.spec.ts +++ b/integration/tests/22-ent-offline.spec.ts @@ -24,6 +24,9 @@ import { JMAP_URL } from './helpers/config'; * - a mail written offline waits in the outbox; the required classification * label still stops it; once online it goes out with SIO-Label; * - read, flag and delete to Trash made offline reach the server; + * - a change the server can no longer take shows as changed on another device; + * - the Trash emptied offline is emptied on the server after; + * - a send whose response got lost is looked up and goes out once; * - a sign-out the user starts asks first when unsent mail would be lost, * then wipes the device; the idle sign-out wipes it and the login page * names the lost message; @@ -148,24 +151,69 @@ async function send(page: Page): Promise { await page.keyboard.press('Control+Enter'); } -/** Whether the offline database exists, without opening it (opening would create it). */ +/** Whether anything of the local store is left, without opening it (opening would create it). */ async function offlineDatabaseExists(page: Page): Promise { - return page.evaluate(async () => (await indexedDB.databases()).some((d) => d.name === 'bulwark-offline')); + return page.evaluate(async () => { + if ((await indexedDB.databases()).some((d) => d.name === 'bulwark-store' || d.name === 'bulwark-offline')) return true; + const root = await navigator.storage.getDirectory(); + return root.getDirectoryHandle('.bulwark-store').then(() => true, () => false); + }); } -async function offlineDatabase(page: Page): Promise<{ exists: boolean; emails: number }> { +/** + * The store's files as bytes, with no worker holding them: which of `needles` + * occur in them. Goes to a same-origin file without scripts first, so that + * leaving the app ends the owner. + */ +async function rawStoreFiles(page: Page, needles: string[]): Promise<{ bytes: number; found: string[] }> { + await page.goto(new URL('/sw.js', page.url()).href); + return page.evaluate(async (wanted) => { + const read = async (dir: FileSystemDirectoryHandle): Promise => { + const parts: string[] = []; + for await (const handle of (dir as unknown as { values(): AsyncIterable }).values()) { + if (handle.kind === 'directory') parts.push(...await read(handle as FileSystemDirectoryHandle)); + else parts.push(new TextDecoder('latin1').decode(await (await (handle as FileSystemFileHandle).getFile()).arrayBuffer())); + } + return parts; + }; + const root = await navigator.storage.getDirectory(); + const files = await read(await root.getDirectoryHandle('.bulwark-store')); + return { + bytes: files.reduce((sum, file) => sum + file.length, 0), + found: wanted.filter((needle) => files.some((file) => file.includes(needle))), + }; + }, needles); +} + +/** The file of the account's store, from the key database. */ +async function storeFileName(page: Page): Promise { + return page.evaluate(() => new Promise((resolve, reject) => { + const open = indexedDB.open('bulwark-store'); + open.onerror = () => reject(open.error); + open.onsuccess = () => { + const all = open.result.transaction('accounts').objectStore('accounts').getAll(); + all.onsuccess = () => { + open.result.close(); + resolve((all.result as Array<{ file: string }>)[0].file); + }; + }; + })); +} + +/** The accounts the local store holds a key for. Their mail is in encrypted files nothing here can read. */ +async function offlineDatabase(page: Page): Promise<{ exists: boolean; accounts: number }> { return page.evaluate(async () => { const dbs = await indexedDB.databases(); - if (!dbs.some((d) => d.name === 'bulwark-offline')) return { exists: false, emails: 0 }; - return new Promise<{ exists: boolean; emails: number }>((resolve) => { - const req = indexedDB.open('bulwark-offline'); + if (!dbs.some((d) => d.name === 'bulwark-store')) return { exists: false, accounts: 0 }; + return new Promise<{ exists: boolean; accounts: number }>((resolve) => { + const req = indexedDB.open('bulwark-store'); req.onsuccess = () => { const db = req.result; - if (!db.objectStoreNames.contains('emails')) { db.close(); resolve({ exists: true, emails: 0 }); return; } - const count = db.transaction('emails').objectStore('emails').count(); - count.onsuccess = () => { db.close(); resolve({ exists: true, emails: count.result }); }; + if (!db.objectStoreNames.contains('accounts')) { db.close(); resolve({ exists: true, accounts: 0 }); return; } + const count = db.transaction('accounts').objectStore('accounts').count(); + count.onsuccess = () => { db.close(); resolve({ exists: true, accounts: count.result }); }; }; - req.onerror = () => resolve({ exists: false, emails: 0 }); + req.onerror = () => resolve({ exists: false, accounts: 0 }); }); }); } @@ -230,6 +278,45 @@ test('with the browser offline, kept mail opens, also after a reload without a n await page.context().close(); }); +test('the files on the device hold no readable mail, and a store whose key went starts over', async ({ browser }) => { + const subject = `Offline encrypted ${RUN}`; + const secret = `Secret body ${RUN}`; + await deliverAndWait(subject, secret); + const page = await userPage(browser, ENT.alice); + await enableOffline(page); + await toMail(page); + await expect.poll(async () => (await offlineDatabase(page)).accounts, { timeout: 30_000 }).toBeGreaterThan(0); + await openMessage(page, subject); + await expect(body(page)).toContainText(secret, { timeout: 20_000 }); + // Long enough for the store to have the message and its text on disk. + await page.waitForTimeout(5_000); + const file = await storeFileName(page); + + const raw = await rawStoreFiles(page, [subject, secret, 'SQLite format 3', file]); + expect(raw.bytes).toBeGreaterThan(100_000); + // The name of the file is the one thing here in the clear: the search works. + expect(raw.found).toEqual([file]); + + // The browser clears the key database and leaves the files. + await page.evaluate(() => new Promise((resolve) => { + const request = indexedDB.deleteDatabase('bulwark-store'); + request.onsuccess = () => resolve(); + request.onerror = () => resolve(); + request.onblocked = () => resolve(); + })); + await page.goto(new URL('/', page.url()).href); + // Once on screen and once for screen readers. + await expect(page.locator('p', { hasText: "couldn't be read" }).first()).toBeVisible({ timeout: 60_000 }); + await toMail(page); + await expect(page.locator('[data-testid="email-list-item"]', { hasText: subject }).first()).toBeVisible({ timeout: 60_000 }); + await expect.poll(async () => (await offlineDatabase(page)).accounts, { timeout: 30_000 }).toBeGreaterThan(0); + const newFile = await storeFileName(page); + expect(newFile).not.toBe(file); + const after = await rawStoreFiles(page, [file, newFile]); + expect(after.found).toEqual([newFile]); + await page.context().close(); +}); + test('a mail written offline waits in the outbox, the required label still stops it, and it goes out labelled when back online', async ({ browser }) => { const subject = `Outbox label ${RUN}`; await setPolicy(browser, { organisation: { labels: { list: LABELS, defaultId: '', required: true, extraHeader: '' } } }); @@ -332,6 +419,111 @@ test('read, flag and delete to Trash made offline reach the server once back onl await page.context().close(); }); +test('a change the server can no longer take is shown as changed on another device', async ({ browser }) => { + const subject = `Offline conflict ${RUN}`; + await deliverAndWait(subject, 'Moved twice'); + const page = await userPage(browser, ENT.alice); + await enableOffline(page); + await toMail(page); + await page.context().setOffline(true); + + await openMessage(page, subject); + await page.keyboard.press('Delete'); + await expect(page.locator('[data-testid="email-list-item"]', { hasText: subject })).toHaveCount(0, { timeout: 15_000 }); + + // Meanwhile another device files it elsewhere. + const other = await jmapAs(ENT.alice); + const junk = (await other.mailboxByRole('junk'))!.id; + const id = (await waitForMessage(ENT.alice, (m) => m.subject === subject)).id; + await other.request([['Email/set', { accountId: other.accountId, update: { [id]: { mailboxIds: { [junk]: true } } } }, '0']]); + + await page.context().setOffline(false); + await page.getByTestId('offline-banner-conflicts').click({ timeout: 90_000 }); + const row = page.getByTestId('offline-conflict-row').filter({ hasText: subject }); + await expect(row).toHaveAttribute('data-reason', 'changed'); + await expect(row).toContainText('Changed on another device'); + // The other device's move stands. + expect(Object.keys((await waitForMessage(ENT.alice, (m) => m.subject === subject)).mailboxIds)).toEqual([junk]); + await shot(page, 'f-offline-conflict'); + await row.getByRole('button', { name: /^Dismiss/ }).click(); + await expect(page.getByTestId('offline-conflict-row').filter({ hasText: subject })).toHaveCount(0); + await page.context().close(); +}); + +test('emptying the Trash offline shows at once and reaches the server once back online', async ({ browser }) => { + const subject = `Offline empty trash ${RUN}`; + await deliverAndWait(subject, 'Gone for good'); + const client = await jmapAs(ENT.alice); + const trash = (await client.mailboxByRole('trash'))!.id; + const id = (await waitForMessage(ENT.alice, (m) => m.subject === subject)).id; + await client.request([['Email/set', { accountId: client.accountId, update: { [id]: { mailboxIds: { [trash]: true } } } }, '0']]); + const page = await userPage(browser, ENT.alice); + await enableOffline(page); + await toMail(page); + await page.context().setOffline(true); + + await page.getByText('Trash', { exact: true }).first().click(); + await expect(page.locator('[data-testid="email-list-item"]', { hasText: subject }).first()).toBeVisible({ timeout: 30_000 }); + await page.getByRole('button', { name: 'Empty folder' }).first().click(); + await page.getByRole('alertdialog').getByRole('button', { name: 'Empty folder' }).click(); + await expect(page.locator('[data-testid="email-list-item"]', { hasText: subject })).toHaveCount(0, { timeout: 15_000 }); + // Nothing reached the server yet. + expect((await messages(ENT.alice, RUN)).some((m) => m.subject === subject)).toBe(true); + + await page.context().setOffline(false); + await expect.poll(async () => (await messages(ENT.alice, RUN)).some((m) => m.subject === subject), { timeout: 90_000 }).toBe(false); + await page.context().close(); +}); + +/** A send taken from the outbox gets no answer. `arrives` is what reached the server of it. */ +async function loseResponse(browser: Browser, subject: string, arrives: (body: { methodCalls: Array<[string, unknown, string]> }) => void) { + const page = await userPage(browser, ENT.alice); + await enableOffline(page); + await toMail(page); + const context = page.context(); + await blockMailServer(context); + await waitUntilOffline(page); + await compose(page, ENT.bob, subject, 'Only one copy'); + await send(page); + await expect(page.getByTestId('offline-banner-outbox')).toHaveText('1 message in the outbox', { timeout: 30_000 }); + + // The server comes back, and takes the send, but its answer never arrives. + const origin = `${new URL(JMAP_URL).origin}/**`; + let lost = 0; + await context.unroute(origin); + await context.route(origin, async (route) => { + const sent = route.request().postData() ?? ''; + if (lost === 0 && sent.includes('EmailSubmission/set')) { + lost += 1; + const body = JSON.parse(sent); + arrives(body); + await route.fetch({ postData: JSON.stringify(body) }); + await route.abort('connectionreset'); + return; + } + await route.continue(); + }); + + await expect.poll(() => lost, { timeout: 120_000 }).toBe(1); + // Looked up, found in Sent, done: the row goes without another submission. + await expect(page.getByTestId('offline-banner-outbox')).toHaveCount(0, { timeout: 120_000 }); + await waitForMessage(ENT.bob, (m) => m.subject === subject); + const copies = (await messages(ENT.alice, RUN)).filter((m) => m.subject === subject); + expect(copies).toHaveLength(1); + expect(copies[0].keywords.$draft).toBeFalsy(); + await context.close(); +} + +test('a send whose response got lost goes out once', async ({ browser }) => { + await loseResponse(browser, `Offline lost response ${RUN}`, () => {}); +}); + +test('a send of which only the draft arrived is submitted, and goes out once', async ({ browser }) => { + await loseResponse(browser, `Offline lost submission ${RUN}`, (body) => { + body.methodCalls = body.methodCalls.filter(([name]) => name === 'Email/set'); + }); +}); + test('signing out asks first when unsent mail would be lost, then wipes the device', async ({ browser }) => { const subject = `Unsent at sign-out ${RUN}`; const page = await userPage(browser, ENT.alice); @@ -342,7 +534,7 @@ test('signing out asks first when unsent mail would be lost, then wipes the devi await compose(page, ENT.bob, subject, 'Never sent'); await send(page); await expect(page.getByTestId('offline-banner-outbox')).toHaveText('1 message in the outbox', { timeout: 30_000 }); - expect((await offlineDatabase(page)).emails).toBeGreaterThan(0); + expect((await offlineDatabase(page)).accounts).toBeGreaterThan(0); // Sign out from the account menu: asked first, and Cancel keeps everything. await accountSwitcher(page).click(); @@ -417,17 +609,20 @@ test('forbidding offline mode wipes the device at the next policy fetch and hide await enableOffline(carol); await toMail(alice); const loadedAt = Date.now(); - expect((await offlineDatabase(alice)).emails).toBeGreaterThan(0); + expect((await offlineDatabase(alice)).accounts).toBeGreaterThan(0); + // Waiting starts now: the app may fetch the policy again at any time, not + // only on focus, and the notice doesn't stay. + const notice = alice.getByText('Your organisation turned off offline mode', { exact: false }).first().waitFor({ state: 'visible', timeout: 180_000 }); // ent-team (alice, not carol) may no longer keep mail on devices. await setPolicy(browser, { scopes: [{ id: 'no-offline', label: 'No offline', match: { groups: [ENT.team] }, policy: { organisation: { devices: { noOffline: true } } } }], }); // The open tab fetches the policy again when it gets focus (at most once a minute). await alice.waitForTimeout(Math.max(0, 62_000 - (Date.now() - loadedAt))); - await alice.evaluate(() => window.dispatchEvent(new Event('focus'))); - await expect(alice.getByText('Your organisation turned off offline mode', { exact: false }).first()).toBeVisible({ timeout: 30_000 }); - await expect.poll(async () => (await offlineDatabase(alice)).emails, { timeout: 20_000 }).toBe(0); + await alice.evaluate(() => window.dispatchEvent(new Event('focus'))).catch(() => {}); + await notice; + await expect.poll(async () => (await offlineDatabase(alice)).accounts, { timeout: 20_000 }).toBe(0); await alice.goto('/settings/backup/offline'); await expect(alice.getByTestId('offline-forbidden')).toBeVisible({ timeout: 30_000 }); await expect(alice.getByTestId('offline-switch')).toHaveCount(0); @@ -436,7 +631,7 @@ test('forbidding offline mode wipes the device at the next policy fetch and hide // carol is outside the scope: her switch and her mail stay. await carol.reload(); await expect(carol.getByTestId('offline-switch')).toHaveAttribute('aria-checked', 'true', { timeout: 30_000 }); - expect((await offlineDatabase(carol)).emails).toBeGreaterThan(0); + expect((await offlineDatabase(carol)).accounts).toBeGreaterThan(0); await alice.context().close(); await carol.context().close(); await setPolicy(browser, {}); diff --git a/lib/__tests__/lite-scripts.test.ts b/lib/__tests__/lite-scripts.test.ts index 5bf2edb5d..2b7e49d95 100644 --- a/lib/__tests__/lite-scripts.test.ts +++ b/lib/__tests__/lite-scripts.test.ts @@ -59,6 +59,8 @@ import { stalwartPrefixProblems, stalwartRewriteIndex, stalwartZipEntryProblems, + unexpectedEngineFiles, + LITE_REMOVED_PATHS, unexpectedApiStrings, withScriptHashCsp, } from '../../scripts/lite/lib.mjs'; @@ -343,6 +345,33 @@ describe('lite build helpers', () => { }); }); +describe('the local store in an export', () => { + it('finds every engine file, and nothing else', () => { + expect(unexpectedEngineFiles([ + 'store-worker.js', + 'sqlite3mc/2.6.1-3.54.0/sqlite3.mjs', + 'sqlite3mc\\2.6.1-3.54.0\\sqlite3.wasm', + '_next/static/media/sqlite3.wasm', + 'pst-worker.js', + 'en/mail/index.html', + '_next/static/chunks/app.js', + ])).toEqual([ + 'store-worker.js', + 'sqlite3mc/2.6.1-3.54.0/sqlite3.mjs', + 'sqlite3mc/2.6.1-3.54.0/sqlite3.wasm', + '_next/static/media/sqlite3.wasm', + ]); + }); + + it('removes what a dev checkout holds before a build, and offline mode never starts in Lite', () => { + expect(LITE_REMOVED_PATHS).toEqual(expect.arrayContaining(['public/store-worker.js', 'public/sqlite3mc'])); + const offlineMode = readFileSync(join(process.cwd(), 'lib/offline/offline-mode.ts'), 'utf8'); + expect(offlineMode).toMatch(/if \(IS_LITE \|\| !hasIndexedDb\(\)\) return false;/); + const build = readFileSync(join(process.cwd(), 'scripts/lite/build.mjs'), 'utf8'); + expect(build).not.toMatch(/build-store-worker/); + }); +}); + describe('prepare / postbuild / verify against a fake checkout', () => { it('plans the server-only trees and every test file, skipping node_modules, repos and scripts', () => { const root = makeRepo(); diff --git a/lib/__tests__/oauth-token-route.test.ts b/lib/__tests__/oauth-token-route.test.ts index c12d8185a..0170c197b 100644 --- a/lib/__tests__/oauth-token-route.test.ts +++ b/lib/__tests__/oauth-token-route.test.ts @@ -166,7 +166,39 @@ describe('oauth token route - access token cache (#552)', () => { const { status } = await callPut(); - expect(status).toBe(500); + expect(status).toBe(503); + expect(cookieStore.get('jmap_rt')).toBeTruthy(); + }); + + it('names a real invalid_grant, and only that, for the client to wipe on', async () => { + const refuse = (status: number, body: string) => fetchMock.mockResolvedValueOnce(new Response(body, { status })); + cookieStore.set('jmap_rt', 'refresh-token'); + refuse(400, JSON.stringify({ error: 'invalid_grant' })); + expect(await callPut({ force: 'true' })).toMatchObject({ status: 401, body: { error: 'invalid_grant' } }); + + for (const body of [JSON.stringify({ error: 'invalid_client' }), JSON.stringify({ error: 'invalid_request', error_description: 'Token is not valid yet' }), 'Forbidden']) { + cookieStore.set('jmap_rt', 'refresh-token'); + refuse(403, body); + const { status, body: answer } = await callPut({ force: 'true' }); + expect(status).toBe(401); + expect(answer.error).not.toBe('invalid_grant'); + } + }); + + it('answers 503 when the token endpoint is out of reach, which the client waits out', async () => { + cookieStore.set('jmap_rt', 'refresh-token'); + fetchMock.mockRejectedValue(new TypeError('fetch failed')); + const { status } = await callPut({ force: 'true' }); + expect(status).toBe(503); + expect(cookieStore.get('jmap_rt')).toBeTruthy(); + }); + + it('answers 503 when discovery fails, also for a server it has no config for', async () => { + cookieStore.set('jmap_rt', 'refresh-token'); + getTokenEndpoint.mockRejectedValue(new Error('OAuth misconfigured')); + const { status } = await callPut({ force: 'true' }); + expect(status).toBe(503); + expect(fetchMock).not.toHaveBeenCalled(); expect(cookieStore.get('jmap_rt')).toBeTruthy(); }); diff --git a/lib/__tests__/offline-shell-route.test.ts b/lib/__tests__/offline-shell-route.test.ts new file mode 100644 index 000000000..cacd327d7 --- /dev/null +++ b/lib/__tests__/offline-shell-route.test.ts @@ -0,0 +1,38 @@ +// @vitest-environment node +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +describe('the list of files a start without a network needs', () => { + let dir = ''; + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + it('names the store worker with its build id, and the database engine with its .mjs and .wasm', async () => { + dir = mkdtempSync(join(os.tmpdir(), 'shell-')); + mkdirSync(join(dir, '.next', 'static', 'chunks'), { recursive: true }); + writeFileSync(join(dir, '.next', 'static', 'chunks', 'app.js'), ''); + writeFileSync(join(dir, '.next', 'static', 'chunks', 'app.js.map'), ''); + mkdirSync(join(dir, 'public', 'sqlite3mc', '1.2'), { recursive: true }); + writeFileSync(join(dir, 'public', 'sqlite3mc', '1.2', 'sqlite3.mjs'), ''); + writeFileSync(join(dir, 'public', 'sqlite3mc', '1.2', 'sqlite3.wasm'), ''); + writeFileSync(join(dir, 'public', 'sqlite3mc', '1.2', 'LICENSE'), ''); + vi.spyOn(process, 'cwd').mockReturnValue(dir); + vi.stubEnv('NEXT_PUBLIC_BUILD_ID', 'build a'); + + const { GET } = await import('@/app/api/offline/shell/route'); + const { files, buildId } = await (await GET()).json() as { files: string[]; buildId: string }; + expect(buildId).toBe('build a'); + expect(files.sort()).toEqual([ + '/_next/static/chunks/app.js', + '/sqlite3mc/1.2/sqlite3.mjs', + '/sqlite3mc/1.2/sqlite3.wasm', + '/store-worker.js?v=build%20a', + ]); + }); +}); diff --git a/lib/__tests__/sw-offline-shell.test.ts b/lib/__tests__/sw-offline-shell.test.ts index a032751e4..7785e6c28 100644 --- a/lib/__tests__/sw-offline-shell.test.ts +++ b/lib/__tests__/sw-offline-shell.test.ts @@ -165,6 +165,31 @@ describe('service worker with offline mode on', () => { expect(await worker.request({ url: `${ORIGIN}/mail`, mode: 'navigate' })).toBeNull(); }); + it('keeps the store\'s worker and database engine, so the store starts offline', async () => { + const caches = new FakeCaches(); + await shellOn(caches, true); + let online = true; + const fetchMock = vi.fn(async (url: unknown) => { + if (!online) throw new TypeError('Failed to fetch'); + return sameOrigin(`file ${String(url)}`, 'text/javascript'); + }); + const worker = loadWorker({ buildId: 'a', caches, clients: [], fetch: fetchMock }); + await settle(); + const urls = ['/store-worker.js?v=a', '/sqlite3mc/1.2/sqlite3.mjs', '/sqlite3mc/1.2/sqlite3.wasm']; + await worker.dispatch('message', { data: { type: 'offline-shell', on: true, urls, buildId: 'a' } }); + const shell = await caches.open('bulwark-shell-a'); + await vi.waitFor(async () => { + for (const url of urls) expect(await shell.match(`${ORIGIN}${url}`)).toBeDefined(); + }); + online = false; + for (const url of urls) { + const answer = await worker.request({ url: `${ORIGIN}${url}` }); + expect(await answer!.text()).toContain(url); + } + // Only those: another root file still goes to the network. + expect(await worker.request({ url: `${ORIGIN}/other.js` })).toBeNull(); + }); + it('keeps a newer tab’s shell in that tab’s build cache, not its own', async () => { const caches = new FakeCaches(); const fetchMock = vi.fn(async () => sameOrigin('x', 'text/javascript')); diff --git a/lib/jmap/client-interface.ts b/lib/jmap/client-interface.ts index 9b28a08eb..0c6e9c5fb 100644 --- a/lib/jmap/client-interface.ts +++ b/lib/jmap/client-interface.ts @@ -499,8 +499,8 @@ export interface IJMAPClient { humanText?: string; }): Promise; - sendRawEmail(blob: Blob, identityId: string, sentMailboxId: string, draftMailboxId?: string, delayedUntil?: string, envelopeRecipients?: string[]): Promise; - submitRawEmail(blob: Blob, identityId: string, delayedUntil?: string, envelopeRecipients?: string[]): Promise; + sendRawEmail(blob: Blob, identityId: string, sentMailboxId: string, draftMailboxId?: string, delayedUntil?: string, envelopeRecipients?: string[], existingCopyId?: string): Promise; + submitRawEmail(blob: Blob, identityId: string, delayedUntil?: string, envelopeRecipients?: string[], existingCopyId?: string): Promise; getScheduledEmails(limit?: number, position?: number): Promise<{ emails: ScheduledEmail[]; hasMore: boolean; total: number; totalByAccount?: Record; nextPosition: number }>; /** Cancels a pending submission, then destroys its record. */ cancelEmailSubmission(submissionId: string, accountId?: string): Promise; diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index dc9c63c39..914772ce6 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -1407,12 +1407,14 @@ function stripMessageIdBrackets(id: string): string { return id.trim().replace(/^<+/, '').replace(/>+$/, '').trim(); } +const GATEWAY_STATUSES = new Set([502, 503, 504]); + // Generate a Message-ID for outgoing mail (bare msg-id, no angle brackets, per // RFC 8621 §4.1.2.3). Without one the server synthesizes it from its OS // hostname, which leaks internal names (e.g. @ip-10-0-12-97.ec2.internal) into // headers — an anti-spam signal and an information disclosure. Use the sender's // domain instead, matching what receivers expect a Message-ID to look like. -function generateMessageId(fromEmail: string): string { +export function generateMessageId(fromEmail: string): string { const at = fromEmail.lastIndexOf('@'); const domain = at > 0 ? fromEmail.slice(at + 1) : 'localhost'; // FTM fix (2026-08-22): crypto.randomUUID() only exists in SECURE contexts (https/localhost). @@ -2219,20 +2221,22 @@ export class JMAPClient implements IJMAPClient { * `transfer`: a mail import or export's own request. A 429 throws * RateLimitError for the transfer to wait on, without pausing the rest of * the client, and a pause the client is in doesn't hold the transfer up. + * `networkOnly`: only the server's own word counts. A send or a change the + * outbox hands over must never be answered by the device: the device would + * queue it again, or answer for the server. */ - private async request(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean; skipOfflineRecord?: boolean }): Promise { + private async request(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean; skipOfflineRecord?: boolean; networkOnly?: boolean }): Promise { if (!this.apiUrl) { throw new Error('Not connected. Call connect() first.'); } // Error reports keep the method names of the last calls, nothing else. recordJmapCall(methodCalls.map(([name]) => name)); // Offline mode (#523, lib/offline/): answered from this device when the - // browser is offline or the network fails, and every answer from the - // server goes to the device's cache. Without offline mode no hooks are - // set and this is the plain network request. + // browser is offline or the network fails. Without offline mode no hooks + // are set and this is the plain network request. const offline = getOfflineHooks(); if (this.offlineOnly || offline?.skipNetwork(this)) { - const local = offline ? await offline.answer(this, methodCalls) : null; + const local = offline && !opts?.networkOnly ? await offline.answer(this, methodCalls) : null; if (local) { this.lastAnswerFromDevice = true; return local as JMAPResponse; @@ -2246,7 +2250,7 @@ export class JMAPClient implements IJMAPClient { if (standIns.length > 0) { const resolved = await offline!.resolveStandIns!(this, standIns); if (standIns.some((id) => !resolved[id])) { - const local = await offline!.answer(this, methodCalls); + const local = opts?.networkOnly ? null : await offline!.answer(this, methodCalls); if (local) { this.lastAnswerFromDevice = true; return local as JMAPResponse; @@ -2263,8 +2267,8 @@ export class JMAPClient implements IJMAPClient { } catch (error) { // Only when the server is really gone: a refusal without CORS // headers looks the same to the browser and must stay an error. - if (offline && isNetworkFailure(error) && await offline.serverGone(this)) { - const local = await offline.answer(this, methodCalls); + if (offline && isNetworkFailure(error) && await offline.serverGone(this, error)) { + const local = opts?.networkOnly ? null : await offline.answer(this, methodCalls); if (local) { this.lastAnswerFromDevice = true; return asAsked(local as JMAPResponse); @@ -2286,6 +2290,22 @@ export class JMAPClient implements IJMAPClient { return this.lastAnswerFromDevice; } + /** A request only the server may answer: for the outbox, which must see outages as outages. */ + requestNetworkOnly(methodCalls: JMAPMethodCall[], using?: string[]): Promise { + return this.request(methodCalls, using, { networkOnly: true, skipOfflineRecord: true }); + } + + /** The account that holds the submissions of mail in `accountId`. */ + submissionAccountFor(accountId?: string): string { + return this.getSubmissionAccountId(accountId); + } + + /** Whether this server reads keyword sort the other way. Null until the client has probed it. */ + keywordSortInverted(accountId?: string): boolean | null { + const polarity = this.keywordSortPolarity.get(accountId ?? this.accountId)?.value; + return polarity ? polarity === 'inverted' : null; + } + private async networkRequest(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean }): Promise { const requestBody = { @@ -2323,7 +2343,10 @@ export class JMAPClient implements IJMAPClient { console.error('Request failed:', response.status, responseText); // The status only: the answer may quote what was sent. if (!UNREPORTED_STATUSES.has(response.status)) reportJmapFailure({ status: response.status }); - throw new Error(`Request failed: ${response.status} - ${responseText.substring(0, 200)}`); + const failure = new Error(`Request failed: ${response.status} - ${responseText.substring(0, 200)}`); + // A proxy in front of a server that's down: to the app, the same as no network. + if (GATEWAY_STATUSES.has(response.status)) failure.name = 'GatewayError'; + throw failure; } let parsed: JMAPResponse; @@ -3349,16 +3372,6 @@ export class JMAPClient implements IJMAPClient { return out; } - /** Changes made offline, sent as they were made: the ids the server refused. */ - async applyEmailPatches(accountId: string | undefined, update: Record>): Promise { - const response = await this.request([ - ["Email/set", { accountId: accountId || this.accountId, update }, "0"], - ]); - const [name, result] = response.methodResponses?.[0] ?? []; - if (name !== "Email/set") throw jmapError(methodErrorMessage(response, 'Failed to update emails'), result); - return Object.keys((result?.notUpdated ?? {}) as Record); - } - // The 256000-byte body cap keeps list/thread fetches fast for the common // case, but truncation lands mid-tag on large HTML bodies (e.g. a data:image // src cut in half), which drops the whole element and can render blank. Only @@ -5229,8 +5242,10 @@ export class JMAPClient implements IJMAPClient { delayedUntil?: string, envelopeMailFrom?: string, // `draftAccountId`: the account `draftId` lives in when it is not this - // one (see createDraft). - options?: SubmissionSendOptions & { draftAccountId?: string } + // one (see createDraft). `messageId` and `existingCopyId` come from the + // offline outbox: a Message-ID fixed when the message was queued, and the + // copy already on the server that only needs submitting (lib/store/outbox/dedupe.ts). + options?: SubmissionSendOptions & { draftAccountId?: string; messageId?: string; existingCopyId?: string } ): Promise { const holdForSeconds = delayedUntil ? this.validateDelayedUntil(delayedUntil) : undefined; const emailId = `send-${Date.now()}`; @@ -5338,7 +5353,7 @@ export class JMAPClient implements IJMAPClient { cc: cc?.length ? cc.map(parseRecipientString) : undefined, bcc: bcc?.length ? bcc.map(parseRecipientString) : undefined, subject, - messageId: [generateMessageId(senderEmail)], + messageId: [options?.messageId ?? generateMessageId(senderEmail)], inReplyTo: normalizedInReplyTo?.length ? normalizedInReplyTo : undefined, references: normalizedReferences?.length ? normalizedReferences : undefined, keywords: { "$seen": true, "$draft": true }, @@ -5433,13 +5448,16 @@ export class JMAPClient implements IJMAPClient { // same request meant a failed create/submission - e.g. the outgoing email // referencing part blobs of that very draft, which die with it - still // deleted the user's only copy of the message. - methodCalls.push(["Email/set", { - accountId: targetAccountId, - create: { [emailId]: emailCreate }, - }, "0"]); + const existingCopyId = options?.existingCopyId; + if (!existingCopyId) { + methodCalls.push(["Email/set", { + accountId: targetAccountId, + create: { [emailId]: emailCreate }, + }, "0"]); + } methodCalls.push(["EmailSubmission/set", { accountId: submissionAccountId, - create: buildSubmissionCreate(`#${emailId}`, firstMailFrom), + create: buildSubmissionCreate(existingCopyId ?? `#${emailId}`, firstMailFrom), ...onSuccess, }, "1"]); // Which recipients the server took. A refused one is lost without a @@ -5457,7 +5475,7 @@ export class JMAPClient implements IJMAPClient { // the identity's address - the composer warns about this before sending // (#1009). // Stalwart checks the quota first, so both can happen, one after the other. - const draftCopyId = response.methodResponses + const draftCopyId = existingCopyId ?? response.methodResponses ?.find(([name]) => name === 'Email/set')?.[1]?.created?.[emailId]?.id as string | undefined; let mailFrom = firstMailFrom; let madeRoom = false; @@ -5480,7 +5498,7 @@ export class JMAPClient implements IJMAPClient { accountId: submissionAccountId, create: buildSubmissionCreate(draftCopyId, mailFrom), ...onSuccess, - }, "1"], deliveryStatusCall(submissionAccountId, "1")]); + }, "1"], deliveryStatusCall(submissionAccountId, "1")], undefined, { networkOnly: true }); response = { ...retry, methodResponses: [ @@ -5495,7 +5513,7 @@ export class JMAPClient implements IJMAPClient { const statusReadBack = takeDeliveryStatus(response); response = statusReadBack.response; - let createdEmailId: string | undefined; + let createdEmailId: string | undefined = existingCopyId; let emailSubmissionId: string | undefined; let serverSendAt: string | undefined; let filingError: string | undefined; @@ -6828,17 +6846,18 @@ export class JMAPClient implements IJMAPClient { try { return await step(); } catch (error) { - if (isNetworkFailure(error) || (error instanceof Error && (error.name === 'RequestTimeoutError' || error.name === 'OfflineUnavailableError'))) { - throw new SendNotStartedError(error); - } + if (isNetworkFailure(error)) throw new SendNotStartedError(error); throw error; } } - /** The request that sends: only a refusal without touching the network means it didn't start. */ + /** + * The request that sends: only a refusal without touching the network means + * it didn't start. The device never answers it: the send may be on its way. + */ private async submissionRequest(methodCalls: JMAPMethodCall[]): Promise { try { - return await this.request(methodCalls); + return await this.request(methodCalls, undefined, { networkOnly: true }); } catch (error) { if (error instanceof Error && error.name === 'OfflineUnavailableError') throw new SendNotStartedError(error); throw error; @@ -11256,6 +11275,7 @@ export class JMAPClient implements IJMAPClient { if (eventType === 'state' && dataLines.length > 0) { try { const change = JSON.parse(dataLines.join('\n')) as StateChange; + getOfflineHooks()?.stateChange?.(this, change); this.stateChangeCallback?.(change); } catch { // Malformed SSE data - ignore @@ -11430,8 +11450,9 @@ export class JMAPClient implements IJMAPClient { this.pollingStates[key] = state; } - if (Object.keys(changedByAccount).length > 0 && this.stateChangeCallback) { - this.stateChangeCallback({ '@type': 'StateChange', changed: changedByAccount }); + if (Object.keys(changedByAccount).length > 0) { + getOfflineHooks()?.stateChange?.(this, { changed: changedByAccount }); + this.stateChangeCallback?.({ '@type': 'StateChange', changed: changedByAccount }); } } catch { // Silently fail - polling will retry @@ -11856,11 +11877,13 @@ export class JMAPClient implements IJMAPClient { draftMailboxId?: string, delayedUntil?: string, envelopeRecipients?: string[], + /** The offline outbox found this message's copy on the server: submit that one. */ + existingCopyId?: string, ): Promise { const holdForSeconds = delayedUntil ? this.validateDelayedUntil(delayedUntil) : undefined; // Upload the raw message const file = new File([blob], 'message.eml', { type: 'message/rfc822' }); - const { blobId } = await this.beforeSubmission(() => this.uploadBlob(file)); + const blobId = existingCopyId ? undefined : (await this.beforeSubmission(() => this.uploadBlob(file))).blobId; // Import into Drafts first, then move to Sent after submission succeeds. // This avoids encrypt-on-append affecting the SMTP send. See #188. @@ -11873,7 +11896,7 @@ export class JMAPClient implements IJMAPClient { const submissionAccountId = this.getSubmissionAccountId(); const { emailId, submissionId: emailSubmissionId, sendAt, rejected } = await this.importAndSubmitRaw( - ['Email/import', { + existingCopyId ? null : ['Email/import', { accountId: this.accountId, emails: { 'raw-import': { @@ -11883,7 +11906,7 @@ export class JMAPClient implements IJMAPClient { }, }, }, '0'], - 'raw-import', + existingCopyId ?? 'raw-import', submissionAccountId, (submitEmailId) => ({ accountId: submissionAccountId, @@ -11926,6 +11949,8 @@ export class JMAPClient implements IJMAPClient { identityId: string, delayedUntil?: string, envelopeRecipients?: string[], + /** The offline outbox found this message's copy on the server: submit that one. */ + existingCopyId?: string, ): Promise { const mailboxes = await this.beforeSubmission(() => this.getMailboxes()); const draftsMailbox = mailboxes.find(mb => mb.role === 'drafts'); @@ -11936,7 +11961,7 @@ export class JMAPClient implements IJMAPClient { const holdForSeconds = delayedUntil ? this.validateDelayedUntil(delayedUntil) : undefined; const file = new File([blob], 'message.eml', { type: 'message/rfc822' }); - const { blobId } = await this.beforeSubmission(() => this.uploadBlob(file)); + const blobId = existingCopyId ? undefined : (await this.beforeSubmission(() => this.uploadBlob(file))).blobId; const identities = await this.beforeSubmission(() => this.getIdentities()); const identity = identities.find(item => item.id === identityId); @@ -11946,7 +11971,7 @@ export class JMAPClient implements IJMAPClient { //Temporarily import the raw email into Drafts to satisfy JMAP's requirement that an EmailSubmission references an existing Email. // The Email will be destroyed after submission. const { submissionId: emailSubmissionId, sendAt, rejected } = await this.importAndSubmitRaw( - ['Email/import', { + existingCopyId ? null : ['Email/import', { accountId: this.accountId, emails: { 'temp-submit': { @@ -11956,7 +11981,7 @@ export class JMAPClient implements IJMAPClient { }, }, }, '0'], - 'temp-submit', + existingCopyId ?? 'temp-submit', submissionAccountId, (submitEmailId) => ({ accountId: submissionAccountId, @@ -11993,29 +12018,31 @@ export class JMAPClient implements IJMAPClient { * (or in Sent, where it passes for a sent message). So does a submission * whose recipients were all refused (#1123): it throws * RecipientsRejectedError; the ones refused while others were taken come - * back as `rejected`. + * back as `rejected`. Without `importCall`, `importCreationId` is the id of a + * copy already on the server. */ private async importAndSubmitRaw( - importCall: JMAPMethodCall, + importCall: JMAPMethodCall | null, importCreationId: string, submissionAccountId: string, submissionSet: (emailId: string) => Record, failureMessage: string, ): Promise<{ emailId?: string; submissionId?: string; sendAt?: string; rejected: RejectedRecipient[] }> { - const firstSet = submissionSet(`#${importCreationId}`); + const firstSet = submissionSet(importCall ? `#${importCreationId}` : importCreationId); let response = await this.submissionRequest([ - importCall, + ...(importCall ? [importCall] : []), ['EmailSubmission/set', firstSet, '1'], deliveryStatusCall(submissionAccountId, 'raw-submit'), ]); - const importedId = response.methodResponses - ?.find(([name]) => name === 'Email/import')?.[1]?.created?.[importCreationId]?.id as string | undefined; + const importedId = importCall + ? response.methodResponses?.find(([name]) => name === 'Email/import')?.[1]?.created?.[importCreationId]?.id as string | undefined + : importCreationId; if (importedId && submissionCreateError(response, 'raw-submit')?.type === 'overQuota') { await this.makeRoomForSubmission(submissionAccountId); const retry = await this.request([ ['EmailSubmission/set', submissionSet(importedId), '1'], deliveryStatusCall(submissionAccountId, 'raw-submit'), - ]); + ], undefined, { networkOnly: true }); response = { ...retry, methodResponses: [ diff --git a/lib/jmap/offline-hooks.ts b/lib/jmap/offline-hooks.ts index ab746f3bc..fed004080 100644 --- a/lib/jmap/offline-hooks.ts +++ b/lib/jmap/offline-hooks.ts @@ -1,7 +1,7 @@ /** * Where the JMAP client hands over to offline mode (#523, lib/offline/). - * The client knows nothing about the cache: it asks these hooks whether to - * skip the network, gives them what the server answered, and asks them for + * The client knows nothing about the local store: it asks these hooks whether + * to skip the network, gives them what the server answered, and asks them for * an answer when the network is gone. Nothing is registered while offline * mode is off for every account, so the client then behaves as before. * @@ -22,6 +22,8 @@ export interface OfflineClientRef { getServerUrl(): string; getUsername(): string; getAccountId(): string; + /** Answers from this device only: a local copy on screen at a start. */ + isOfflineOnly?(): boolean; } export interface OfflineHooks { @@ -36,9 +38,11 @@ export interface OfflineHooks { /** * After a request failed on the network: whether the server can't be * reached at all (then the device answers), or it only refused that - * request (then it stays an error). + * request (then it stays an error). `error` is what the request failed with. */ - serverGone(client: OfflineClientRef): Promise; + serverGone(client: OfflineClientRef, error?: unknown): Promise; + /** The server pushed new states, for the local store to catch up. */ + stateChange?(client: OfflineClientRef, change: { changed?: Record> }): void; /** A blob kept on the device, or null. */ blob(client: OfflineClientRef, blobId: string, accountId: string): Promise; /** A blob the server sent, for the cache. */ @@ -57,6 +61,8 @@ export interface OfflineHooks { * ids as they are. */ resolveStandIns?(client: OfflineClientRef, ids: string[]): Promise>; + /** The user opened this file: the local store keeps recently opened files. */ + fileOpened?(client: OfflineClientRef, accountId: string, id: string): void; } const KEY = '__bulwarkOfflineHooks'; @@ -71,13 +77,24 @@ export function getOfflineHooks(): OfflineHooks | null { return (globalThis as HookGlobal)[KEY] ?? null; } -/** The network refused the request (not an HTTP error): offline, DNS, a dropped connection. */ +/** + * The server can't be talked to: offline, DNS, a dropped connection, a request + * that hung until its deadline, a proxy answering for a server that's down. + * Not an answer from the server itself. + */ export function isNetworkFailure(error: unknown): boolean { if (!(error instanceof Error)) return false; - if (error.name === 'OfflineUnavailableError') return true; + if (error.name === 'OfflineUnavailableError' || error.name === 'RequestTimeoutError' || error.name === 'GatewayError') return true; return error.name === 'TypeError' && /fetch|network|load failed|internet/i.test(error.message); } +/** The connection is fine and a proxy answered: the mail server behind it is what's gone. */ +export function isServerOutage(error: unknown): boolean { + if (!(error instanceof Error)) return false; + if (error.name === 'GatewayError') return true; + return error.name === 'SendNotStartedError' && (error as SendNotStartedError).cause instanceof Error && ((error as SendNotStartedError).cause as Error).name === 'GatewayError'; +} + /** Thrown for a request the device can't answer while the server can't be reached. */ export class OfflineUnavailableError extends Error { constructor(message = 'Not available offline') { diff --git a/lib/offline/__tests__/cache.test.ts b/lib/offline/__tests__/cache.test.ts deleted file mode 100644 index 53ffced76..000000000 --- a/lib/offline/__tests__/cache.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import 'fake-indexeddb/auto'; -import { beforeEach, describe, expect, it } from 'vitest'; -import { - cacheUsage, - emailIndex, - emailKey, - enforceCap, - forgetIndexes, - getBlob, - getBody, - getMisc, - patchEmails, - putBlob, - putEmails, - putMisc, - wipeAccountCache, -} from '../cache'; -import { forgetDeviceKey, openJson, sealJson } from '../crypto'; -import { allowOfflineDbCreation, deleteOfflineDb, idbGet, idbGetAll } from '../db'; - -const day = 24 * 60 * 60 * 1000; -const email = (id: string, ageDays: number, extra: Record = {}) => ({ - id, - threadId: `t-${id}`, - mailboxIds: { inbox: true }, - keywords: {}, - receivedAt: new Date(Date.now() - ageDays * day).toISOString(), - subject: `Subject ${id}`, - ...extra, -}); - -beforeEach(async () => { - forgetIndexes(); - forgetDeviceKey(); - await deleteOfflineDb(); - allowOfflineDbCreation(); -}); - -describe('offline encryption', () => { - it('keeps nothing readable and a key that cannot be exported', async () => { - const sealed = await sealJson({ subject: 'Quarterly numbers' }); - expect(new TextDecoder().decode(sealed.data)).not.toContain('Quarterly'); - expect(await openJson(sealed)).toEqual({ subject: 'Quarterly numbers' }); - const stored = await idbGet<{ key: CryptoKey }>('keys', 'aes'); - expect(stored?.key.extractable).toBe(false); - await expect(crypto.subtle.exportKey('raw', stored!.key)).rejects.toThrow(); - }); - - it('stores messages encrypted, list part and body apart', async () => { - await putEmails('acc', 'j1', [email('m1', 1, { bodyValues: { 1: { value: 'Secret text' } }, textBody: [{ partId: '1' }] })]); - const records = await idbGetAll<{ data: ArrayBuffer }>('emails'); - expect(records).toHaveLength(1); - expect(new TextDecoder().decode(records[0].data)).not.toContain('Subject m1'); - const bodies = await idbGetAll<{ data: ArrayBuffer }>('bodies'); - expect(new TextDecoder().decode(bodies[0].data)).not.toContain('Secret text'); - - forgetIndexes(); - const index = await emailIndex('acc'); - const meta = index.get(emailKey('acc', 'j1', 'm1'))!; - expect(meta.subject).toBe('Subject m1'); - expect(meta).not.toHaveProperty('bodyValues'); - expect((await getBody('acc', 'j1', 'm1'))?.bodyValues).toEqual({ 1: { value: 'Secret text' } }); - }); - - it('merges a later list fetch into the kept message', async () => { - await putEmails('acc', 'j1', [email('m1', 1, { bodyValues: { 1: { value: 'x' } } })]); - await putEmails('acc', 'j1', [{ id: 'm1', keywords: { $seen: true } }]); - const meta = (await emailIndex('acc')).get(emailKey('acc', 'j1', 'm1'))!; - expect(meta.keywords).toEqual({ $seen: true }); - expect(meta.subject).toBe('Subject m1'); - expect(await getBody('acc', 'j1', 'm1')).not.toBeNull(); - }); - - it('applies keyword and folder patches', async () => { - await putEmails('acc', 'j1', [email('m1', 1)]); - await patchEmails('acc', 'j1', { m1: { 'keywords/$flagged': true, 'mailboxIds/inbox': null, 'mailboxIds/archive': true } }); - const meta = (await emailIndex('acc')).get(emailKey('acc', 'j1', 'm1'))!; - expect(meta.keywords).toEqual({ $flagged: true }); - expect(meta.mailboxIds).toEqual({ archive: true }); - }); -}); - -describe('the cache cap', () => { - it('drops the oldest messages first, with their bodies and attachments', async () => { - await putEmails('acc', 'j1', [ - email('old', 20, { bodyValues: { 1: { value: 'a'.repeat(4000) } } }), - email('mid', 10, { bodyValues: { 1: { value: 'b'.repeat(4000) } } }), - email('new', 1, { bodyValues: { 1: { value: 'c'.repeat(4000) } } }), - ]); - await putBlob('acc', 'j1', 'blob-old', new Blob(['x'.repeat(4000)]), Date.now() - 20 * day, emailKey('acc', 'j1', 'old')); - const { total } = await cacheUsage(); - // Room for about two messages. - const freed = await enforceCap(total - 6000); - expect(freed).toBeGreaterThan(0); - const index = await emailIndex('acc'); - expect(index.has(emailKey('acc', 'j1', 'old'))).toBe(false); - expect(index.has(emailKey('acc', 'j1', 'new'))).toBe(true); - expect(await getBody('acc', 'j1', 'old')).toBeNull(); - expect(await getBlob('acc', 'j1', 'blob-old')).toBeNull(); - expect(await getBody('acc', 'j1', 'new')).not.toBeNull(); - expect((await cacheUsage()).total).toBeLessThanOrEqual(total - 6000); - }); - - it('leaves the cache alone under the cap', async () => { - await putEmails('acc', 'j1', [email('m1', 1)]); - expect(await enforceCap()).toBe(0); - }); -}); - -describe('wiping one account', () => { - it('removes that account only', async () => { - await putEmails('a', 'j1', [email('m1', 1)]); - await putEmails('b', 'j1', [email('m2', 1)]); - await putMisc('a', 'session', { apiUrl: 'x' }); - await wipeAccountCache('a'); - expect((await emailIndex('a')).size).toBe(0); - expect(await getMisc('a', 'session')).toBeNull(); - forgetIndexes(); - expect((await emailIndex('b')).size).toBe(1); - }); -}); diff --git a/lib/offline/__tests__/client-fallback.test.ts b/lib/offline/__tests__/client-fallback.test.ts index f7d49e49b..a4aa5ffa5 100644 --- a/lib/offline/__tests__/client-fallback.test.ts +++ b/lib/offline/__tests__/client-fallback.test.ts @@ -5,36 +5,44 @@ import { useAuthStore } from '@/stores/auth-store'; import { useAccountStore } from '@/stores/account-store'; import { usePolicyStore } from '@/stores/policy-store'; import { DEFAULT_ORGANISATION_POLICY } from '@/lib/admin/organisation-policy'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { FakeJmapMail } from '@/lib/store/__tests__/helpers/fake-jmap-mail'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; import { startOfflineMode, wipeEverything } from '../offline-mode'; -import { forgetIndexes, putEmails, putMisc, getMisc, emailIndex } from '../cache'; -import { forgetDeviceKey } from '../crypto'; -import { allowOfflineDbCreation, deleteOfflineDb } from '../db'; +import { forgetLegacyMove } from '../migrate-legacy'; import { useOfflineStore, writeEnabled } from '../state'; -import { syncAccount } from '../sync'; const SERVER = 'https://mail.example.org'; const SESSION = { apiUrl: `${SERVER}/jmap/`, downloadUrl: `${SERVER}/jmap/download/{accountId}/{blobId}/{name}?accept={type}`, uploadUrl: `${SERVER}/jmap/upload/{accountId}/`, - accounts: { j1: { name: 'alice', isPersonal: true, accountCapabilities: {} } }, - primaryAccounts: { 'urn:ietf:params:jmap:mail': 'j1' }, - capabilities: { 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500 } }, + accounts: { acc1: { name: 'alice', isPersonal: true, accountCapabilities: { 'urn:ietf:params:jmap:mail': {}, 'urn:ietf:params:jmap:submission': {} } } }, + primaryAccounts: { 'urn:ietf:params:jmap:mail': 'acc1' }, + capabilities: { 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500, maxObjectsInSet: 500 }, 'urn:ietf:params:jmap:mail': {} }, username: 'alice@example.org', }; let probeOk = true; -let apiFails = false; +let apiFails: 'no' | 'network' | 'gateway' = 'no'; +let server: FakeJmapMail; +let store: InProcessStore; beforeEach(async () => { - forgetIndexes(); - forgetDeviceKey(); - await deleteOfflineDb(); - allowOfflineDbCreation(); - localStorage.clear(); probeOk = true; - apiFails = false; - useOfflineStore.setState({ downServers: [], browserOffline: false }); + apiFails = 'no'; + server = new FakeJmapMail({ mailboxes: [ + { id: 'inbox', name: 'Inbox', role: 'inbox', totalEmails: 2, unreadEmails: 2 }, + { id: 'trash', name: 'Trash', role: 'trash', totalEmails: 0, unreadEmails: 0 }, + ] }); + server.identities.set('i1', { id: 'i1', email: 'alice@example.org', name: 'Alice' }); + server.addEmail({ id: 'm1', subject: 'Kept one', threadId: 't1', keywords: {} }); + server.addEmail({ id: 'm2', subject: 'Kept two', threadId: 't2', keywords: {} }); + store = new InProcessStore(server, SESSION); + setStoreBackend(store); + forgetLegacyMove(); + localStorage.clear(); + useOfflineStore.setState({ enabled: [], downServers: [], browserOffline: false, outbox: [], pendingChanges: 0, conflicts: [] }); usePolicyStore.setState({ loaded: true, subject: null, policy: { ...usePolicyStore.getState().policy, organisation: DEFAULT_ORGANISATION_POLICY } }); useAccountStore.setState({ accounts: [{ id: 'acc', serverUrl: SERVER, username: 'alice@example.org' } as never] }); vi.stubGlobal('fetch', vi.fn(async (url: string, init?: RequestInit) => { @@ -43,175 +51,243 @@ beforeEach(async () => { return new Response(null, { status: 200 }); } if (String(url).endsWith('/.well-known/jmap')) return new Response(JSON.stringify(SESSION), { status: 200 }); - if (apiFails) throw new TypeError('Failed to fetch'); - return new Response(JSON.stringify({ methodResponses: [['Email/get', { accountId: 'j1', list: [], notFound: ['m1'] }, '0']] }), { status: 200 }); + if (apiFails === 'network') throw new TypeError('Failed to fetch'); + if (apiFails === 'gateway') return new Response('Bad Gateway', { status: 502 }); + const body = JSON.parse(String(init?.body)) as { methodCalls: never[] }; + return new Response(JSON.stringify(await server.request(body.methodCalls)), { status: 200 }); })); }); afterEach(async () => { vi.unstubAllGlobals(); await wipeEverything(); + setStoreBackend(null); }); -async function connectedClient(): Promise { +async function ready() { const client = new JMAPClient(SERVER, 'alice@example.org', 'pw'); await client.connect(); useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); - return client; + writeEnabled(['acc']); + const stop = startOfflineMode(); + await store.sync('acc'); + return { client, stop }; +} + +async function offline(run: () => Promise): Promise { + vi.stubGlobal('navigator', { onLine: false }); + try { + await run(); + } finally { + vi.stubGlobal('navigator', window.navigator); + } } describe('falling back to the device', () => { - it('answers from the device when the server is gone', async () => { - const client = await connectedClient(); - writeEnabled(['acc']); - const stop = startOfflineMode(); - await putEmails('acc', 'j1', [{ id: 'm1', mailboxIds: { inbox: true }, receivedAt: new Date().toISOString(), subject: 'Kept', bodyValues: { 1: { value: 'x' } } }]); - apiFails = true; + it('answers from the store when the server is gone', async () => { + const { client, stop } = await ready(); + apiFails = 'network'; probeOk = false; const email = await client.getEmail('m1'); - expect(email?.subject).toBe('Kept'); + expect(email?.subject).toBe('Kept one'); expect(useOfflineStore.getState().downServers).toEqual(['https://mail.example.org']); stop(); }); it('keeps a refused request an error while the server answers', async () => { - const client = await connectedClient(); - writeEnabled(['acc']); - const stop = startOfflineMode(); - await putEmails('acc', 'j1', [{ id: 'm1', mailboxIds: { inbox: true }, receivedAt: new Date().toISOString(), subject: 'Kept', bodyValues: { 1: { value: 'x' } } }]); - apiFails = true; + const { client, stop } = await ready(); + apiFails = 'network'; probeOk = true; // getEmail swallows errors into null: the device's copy isn't passed off as the server's answer. expect(await client.getEmail('m1')).toBeNull(); expect(useOfflineStore.getState().downServers).toEqual([]); stop(); }); + + it('counts a proxy\'s 5xx as the server being gone, though a probe would pass', async () => { + const { client, stop } = await ready(); + apiFails = 'gateway'; + probeOk = true; + const email = await client.getEmail('m1'); + expect(email?.subject).toBe('Kept one'); + expect(useOfflineStore.getState().downServers).toEqual(['https://mail.example.org']); + // A later good answer ends it. + apiFails = 'no'; + await client.getEmail('m1'); + expect(useOfflineStore.getState().downServers).toEqual([]); + stop(); + }); }); -describe('mailbox paging with the device cache', () => { - const mailboxes = Array.from({ length: 5 }, (_, i) => ({ id: `box-${i}`, name: `Folder ${i}`, parentId: null })); - - function mailboxServer(failLaterPage = false) { - const positions: number[] = []; - vi.stubGlobal('fetch', vi.fn(async (url: string, init?: RequestInit) => { - if (String(url).endsWith('/.well-known/jmap')) { - return new Response(JSON.stringify({ - ...SESSION, - capabilities: { 'urn:ietf:params:jmap:core': { maxObjectsInGet: 2 } }, - }), { status: 200 }); - } - const { methodCalls } = JSON.parse(String(init?.body)); - const query = methodCalls.find(([name]: [string]) => name === 'Mailbox/query'); - const position = query?.[1].position ?? 0; - if (query) positions.push(position); - const page = mailboxes.slice(position, position + 2); - const methodResponses = methodCalls.map(([name, , id]: [string, unknown, string]) => { - if (name === 'Identity/get') return [name, { accountId: 'j1', list: [{ id: 'recording-barrier' }] }, id]; - if (query && failLaterPage && position > 0) return ['error', { type: 'serverFail' }, id]; - if (name === 'Mailbox/query') return [name, { - accountId: 'j1', queryState: 'query-state', position, - ids: page.map((box) => box.id), total: mailboxes.length, - }, id]; - return [name, { accountId: 'j1', state: 'mailbox-state', list: page, notFound: [] }, id]; - }); - return new Response(JSON.stringify({ methodResponses }), { status: 200 }); - })); - return positions; - } - - // Identity recording shares the per-account queue with mailbox recording. - // Waiting for this marker also detects erroneous writes queued by failed pages. - async function recorded(client: JMAPClient) { - await client.getIdentities(); - await vi.waitFor(async () => { - expect(await getMisc('acc', 'identities')).toEqual({ j1: [{ id: 'recording-barrier' }] }); +describe('per-call errors, end to end through the client', () => { + it('keeps the results when one call in the request can\'t be answered', async () => { + const { client, stop } = await ready(); + await offline(async () => { + // A search with text: Email/query, Email/get, and SearchSnippet/get, which the store refuses. + const result = await client.searchEmails('kept'); + expect(result.emails.map((email) => email.id).sort()).toEqual(['m1', 'm2']); + expect(result.emails[0].subject).toBeTruthy(); }); - } + stop(); + }); - it('returns a cached list larger than maxObjectsInGet without inventing a collection state', async () => { - mailboxServer(); - const client = await connectedClient(); - writeEnabled(['acc']); - const stop = startOfflineMode(); - try { - await putMisc('acc', 'mailboxes', { j1: mailboxes }); - vi.stubGlobal('navigator', { onLine: false }); + it('does a full fetch when /changes is refused: the client reads null', async () => { + const { client, stop } = await ready(); + await offline(async () => { + expect(await client.getEmailChanges('s1')).toBeNull(); + expect(await client.getMailboxChanges('s1')).toBeNull(); + }); + stop(); + }); + + it('a draft can\'t be saved yet, and says so the way the composer expects', async () => { + const { client, stop } = await ready(); + await offline(async () => { + await expect(client.createDraft(['b@example.org'], 'S', 'x', undefined, undefined, undefined, 'alice@example.org')).rejects.toThrow(); + }); + stop(); + }); + + it('queues a delete for good and an emptied folder, and the client sees them done', async () => { + const { client, stop } = await ready(); + await offline(async () => { + await client.deleteEmail('m1'); + expect(await client.getEmail('m1')).toBeNull(); + await client.moveToTrash('m2', 'trash'); + expect(await client.emptyMailbox('trash')).toBe(1); + const boxes = await client.getMailboxes(); + expect(boxes.find((box) => box.id === 'inbox')).toMatchObject({ totalEmails: 0, unreadEmails: 0 }); + expect(boxes.find((box) => box.id === 'trash')).toMatchObject({ totalEmails: 0 }); + }); + stop(); + // Nothing reached the server yet. + expect(server.emails.size).toBe(2); + }); + + it('marks read offline and the change reaches the message and the counts', async () => { + const { client, stop } = await ready(); + await offline(async () => { + await client.markAsRead('m1', true); + expect((await client.getEmail('m1'))?.keywords).toMatchObject({ $seen: true }); + const boxes = await client.getMailboxes(); + expect(boxes.find((box) => box.id === 'inbox')?.unreadEmails).toBe(1); + }); + stop(); + }); +}); + +describe('what the store answers and is told', () => { + it('answers identities', async () => { + const { client, stop } = await ready(); + await offline(async () => { + expect((await client.getIdentities()).map((identity) => identity.id)).toEqual(['i1']); + }); + stop(); + }); + + it('answers the folder list with the server\'s real state', async () => { + const { client, stop } = await ready(); + await offline(async () => { const result = await client.getAllMailboxesWithState(); - expect(result.mailboxes.map((box) => box.id)).toEqual(mailboxes.map((box) => box.id)); - expect(result.states).toEqual({}); - // Only session discovery touched the network; both query/get pages are local. - expect(fetch).toHaveBeenCalledTimes(1); - } finally { - stop(); - } + expect(result.mailboxes.map((box) => box.id).sort()).toEqual(['inbox', 'trash']); + expect(Object.values(result.states)).toEqual([server.coll.Mailbox.state()]); + }); + stop(); }); - it('records the complete live multi-page list rather than its final page', async () => { - const positions = mailboxServer(); - const client = await connectedClient(); - writeEnabled(['acc']); - const stop = startOfflineMode(); - try { - const result = await client.getMailboxes(); - expect(result.map((box) => box.id)).toEqual(mailboxes.map((box) => box.id)); - expect(positions).toEqual([0, 2, 4]); - await recorded(client); - expect(await getMisc('acc', 'mailboxes')).toEqual({ j1: mailboxes }); - } finally { - stop(); - } + it('never takes a refusal from the device for the answer to a send', async () => { + const { client, stop } = await ready(); + await client.getMailboxes(); + await client.getIdentities(); + const fetchMock = fetch as unknown as ReturnType; + const passThrough = fetchMock.getMockImplementation() as (url: string, init?: RequestInit) => Promise; + fetchMock.mockImplementation(async (url: string, init?: RequestInit) => { + if (String(init?.body ?? '').includes('EmailSubmission/set')) { + apiFails = 'network'; + probeOk = false; + } + return passThrough(url, init); + }); + server.mailboxes.set('drafts', { id: 'drafts', role: 'drafts' }); + server.mailboxes.set('sent', { id: 'sent', role: 'sent' }); + const error = await client.sendEmail(['bob@example.org'], 'Hi', 'Body', undefined, undefined, 'i1', 'alice@example.org').catch((e: unknown) => e) as Error; + // The connection broke: "may have gone", not a refusal, and not "never started". + expect(error).toBeInstanceOf(TypeError); + expect(error.message).toBe('Failed to fetch'); + stop(); }); - it('retains the previous complete cache when a later live page fails', async () => { - const positions = mailboxServer(true); - const client = await connectedClient(); - writeEnabled(['acc']); - const stop = startOfflineMode(); - const previous = [{ id: 'old', name: 'Previously cached', parentId: null }]; - try { - await putMisc('acc', 'mailboxes', { j1: previous }); - await expect(client.getMailboxes()).rejects.toThrow('Failed to load the complete mailbox list'); - expect(positions).toEqual([0, 2]); - await recorded(client); - expect(await getMisc('acc', 'mailboxes')).toEqual({ j1: previous }); - } finally { - stop(); - } + it('never lets the device answer for the server when the outbox asks', async () => { + const { client, stop } = await ready(); + apiFails = 'network'; + probeOk = false; + await expect(client.requestNetworkOnly([['Email/set', { accountId: 'acc1', update: { m1: { 'keywords/$seen': true } } }, '0']])).rejects.toThrow('Failed to fetch'); + expect(await store.area('acc', 'outbox.read')).toMatchObject({ patches: 0 }); + stop(); }); -}); - -describe('the sync', () => { - function fakeClient(opts: { failQuery?: boolean }) { - return { - getSessionSnapshot: () => SESSION, - getAccountId: () => 'j1', - getAllMailboxes: vi.fn(async () => { await putMisc('acc', 'mailboxes', { j1: [{ id: 'inbox', role: 'inbox' }] }); return []; }), - getMailboxes: vi.fn(async () => []), - getIdentities: vi.fn(async () => []), - getRecentEmailIds: vi.fn(async (mailboxId: string) => { if (opts.failQuery && mailboxId === 'archive') throw new Error('refused'); return mailboxId === 'inbox' ? ['new'] : []; }), - getSomeEmails: vi.fn(async () => []), - getFullEmails: vi.fn(async (ids: string[]) => { - await putEmails('acc', 'j1', ids.map((id) => ({ id, mailboxIds: { inbox: true }, receivedAt: new Date().toISOString(), bodyValues: {} }))); - return []; - }), - fetchBlob: vi.fn(), - }; - } - const deps = { stillActive: () => true, recorded: async () => {}, scopeMailboxes: async () => [{ jmap: 'j1', mailboxId: 'inbox', isInbox: true }] }; - - it('fetches what is missing and drops what left the Inbox', async () => { - await putEmails('acc', 'j1', [{ id: 'gone', mailboxIds: { inbox: true }, receivedAt: new Date().toISOString() }]); - const client = fakeClient({}); - await syncAccount('acc', client as never, deps); - expect(client.getFullEmails).toHaveBeenCalledWith(['new'], 'j1'); - const index = await emailIndex('acc'); - expect([...index.values()].map((e) => e.id)).toEqual(['new']); + + it('keeps a message opened from outside the window', async () => { + server.addEmail({ id: 'old', subject: 'Old', receivedAt: '2019-01-01T00:00:00Z', bodyValues: { 1: { value: 'Old text' } } }); + const { client, stop } = await ready(); + store.dbs.get('acc')!.run("DELETE FROM email WHERE id = 'old'"); + await client.getEmail('old'); + await vi.waitFor(async () => { + const answer = await store.answer('acc', [['Email/get', { ids: ['old'], properties: ['subject', 'bodyValues'], fetchAllBodyValues: true }, '0']]); + expect(answer?.methodResponses[0][1].list[0]?.subject).toBe('Old'); + }); + stop(); + }); + + it('tells the store what the server pushed', async () => { + const { client, stop } = await ready(); + const note = vi.spyOn(store, 'noteStateChange'); + const { getOfflineHooks } = await import('@/lib/jmap/offline-hooks'); + getOfflineHooks()?.stateChange?.(client, { changed: { acc1: { Email: 's99' } } }); + expect(note).toHaveBeenCalledWith('acc', { acc1: { Email: 's99' } }); + stop(); + }); + + it('holds a file while offline and hands it back for the message', async () => { + const { client, stop } = await ready(); + await offline(async () => { + const held = await client.uploadBlob(new File(['hello'], 'note.txt', { type: 'text/plain' })); + expect(held.blobId).toMatch(/^bulwark-local-/); + const blob = await client.fetchBlob(held.blobId); + expect(await blob.text()).toBe('hello'); + }); + stop(); + }); + + it('hands the other areas\u2019 lookups to the store', async () => { + const { client, stop } = await ready(); + const { getOfflineHooks } = await import('@/lib/jmap/offline-hooks'); + const area = vi.spyOn(store, 'area'); + const hooks = getOfflineHooks(); + + area.mockResolvedValueOnce({ 'bulwark-new-1': 'real-1' }); + expect(await hooks?.resolveStandIns?.(client, ['bulwark-new-1'])).toEqual({ 'bulwark-new-1': 'real-1' }); + expect(area).toHaveBeenLastCalledWith('acc', 'calendar.resolveStandIns', { ids: ['bulwark-new-1'] }); + + hooks?.fileOpened?.(client, 'acc1', 'node-1'); + expect(area).toHaveBeenLastCalledWith('acc', 'files.noteOpened', { jmap: 'acc1', id: 'node-1' }); + + // Not an attachment of a kept message: maybe a kept file. + area.mockResolvedValueOnce(null).mockResolvedValueOnce({ type: 'text/plain', data: new TextEncoder().encode('kept') }); + const blob = await hooks?.blob(client, 'blob-1', 'acc1'); + expect(await blob?.text()).toBe('kept'); + expect(area).toHaveBeenLastCalledWith('acc', 'files.blob', { jmap: 'acc1', blobId: 'blob-1' }); + area.mockRestore(); + stop(); }); - it('keeps everything when a folder couldn\'t be asked', async () => { - await putEmails('acc', 'j1', [{ id: 'kept', mailboxIds: { archive: true }, receivedAt: new Date().toISOString() }]); - const both = { ...deps, scopeMailboxes: async () => [{ jmap: 'j1', mailboxId: 'inbox', isInbox: true }, { jmap: 'j1', mailboxId: 'archive', isInbox: false }] }; - await syncAccount('acc', fakeClient({ failQuery: true }) as never, both); - expect([...(await emailIndex('acc')).values()].map((e) => e.id).sort()).toEqual(['kept', 'new']); + it('answers a search for the merge with the server', async () => { + const { client, stop } = await ready(); + const { getOfflineHooks } = await import('@/lib/jmap/offline-hooks'); + const answer = await getOfflineHooks()?.searchLocal?.(client, [['Email/query', { filter: { text: 'kept' } }, '0']]); + expect(answer?.methodResponses[0][1].ids).toHaveLength(2); + useAccountStore.setState({ accounts: [] }); + useAuthStore.setState({ client: null, activeAccountId: null }); + expect(await getOfflineHooks()?.searchLocal?.(client, [['Email/query', {}, '0']])).toBeNull(); + stop(); }); }); diff --git a/lib/offline/__tests__/client-send.test.ts b/lib/offline/__tests__/client-send.test.ts index 05f6e003e..aaed29409 100644 --- a/lib/offline/__tests__/client-send.test.ts +++ b/lib/offline/__tests__/client-send.test.ts @@ -34,10 +34,10 @@ beforeEach(() => { if (names.includes('EmailSubmission/set')) { submissions.push(request); if (mode.submissionFails) throw new TypeError('Failed to fetch'); - const create = request.methodCalls.find(([name]) => name === 'Email/set')?.[1].create as Record; - const key = Object.keys(create)[0]; + const create = request.methodCalls.find(([name]) => name === 'Email/set')?.[1].create as Record | undefined; + const key = Object.keys(create ?? {})[0]; return answer({ methodResponses: [ - ['Email/set', { accountId: 'j1', created: { [key]: { id: 'e1' } } }, '0'], + ...(key ? [['Email/set', { accountId: 'j1', created: { [key]: { id: 'e1' } } }, '0']] : []), ['EmailSubmission/set', { accountId: 'j1', created: { 1: { id: 'sub1' } } }, '1'], ] }); } @@ -85,6 +85,27 @@ describe('telling a send that never started from one that may have gone', () => expect(submissions).toHaveLength(1); }); + it('sends with a Message-ID fixed by the outbox, and makes one up otherwise', async () => { + const c = await client(); + await c.sendEmail(['bob@example.org'], 'Hi', 'Body', undefined, undefined, 'i1', 'alice@example.org', undefined, 'Alice', undefined, undefined, undefined, undefined, undefined, undefined, { messageId: 'fixed@example.org' }); + await send(c); + const created = (submissions as Array<{ methodCalls: Array<[string, { create?: Record }, string]> }>) + .map((request) => Object.values(request.methodCalls.find(([name]) => name === 'Email/set')?.[1].create ?? {})[0]?.messageId?.[0]); + expect(created[0]).toBe('fixed@example.org'); + expect(created[1]).toMatch(/@example\.org$/); + expect(created[1]).not.toBe('fixed@example.org'); + }); + + it('submits a copy the lookup found, without creating another', async () => { + const c = await client(); + const result = await c.sendEmail(['bob@example.org'], 'Hi', 'Body', undefined, undefined, 'i1', 'alice@example.org', undefined, 'Alice', undefined, undefined, undefined, undefined, undefined, undefined, { existingCopyId: 'copy-1' }); + expect(result.emailSubmissionId).toBe('sub1'); + const [request] = submissions as Array<{ methodCalls: Array<[string, Record, string]> }>; + expect(request.methodCalls.map(([name]) => name)).not.toContain('Email/set'); + const set = request.methodCalls.find(([name]) => name === 'EmailSubmission/set')?.[1] as { create: Record }; + expect(set.create['1'].emailId).toBe('copy-1'); + }); + it('a held send whose hold can’t be read back afterwards still counts as sent', async () => { const c = await client(); mode.readBackFails = true; diff --git a/lib/offline/__tests__/handoff-wipes.test.ts b/lib/offline/__tests__/handoff-wipes.test.ts index 84c58ba08..534a060c8 100644 --- a/lib/offline/__tests__/handoff-wipes.test.ts +++ b/lib/offline/__tests__/handoff-wipes.test.ts @@ -5,9 +5,8 @@ import { usePolicyStore } from '@/stores/policy-store'; import { DEFAULT_ORGANISATION_POLICY } from '@/lib/admin/organisation-policy'; import { handOffFiles, pendingHandoffs } from '@/lib/launch-handoff'; import { disableOffline, enforcePolicy, wipeEverything } from '../offline-mode'; -import { forgetIndexes } from '../cache'; -import { forgetDeviceKey } from '../crypto'; -import { allowOfflineDbCreation, deleteOfflineDb } from '../db'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; import { useOfflineStore, writeEnabled } from '../state'; // Offline mode's wipes take what was on its way into an account along. @@ -18,10 +17,7 @@ function waiting() { } beforeEach(async () => { - forgetIndexes(); - forgetDeviceKey(); - await deleteOfflineDb(); - allowOfflineDbCreation(); + setStoreBackend(new InProcessStore()); localStorage.clear(); useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, browserOffline: false, downServers: [], startedOffline: false }); usePolicyStore.setState({ loaded: true, subject: null, subjectPending: false, policy: { ...usePolicyStore.getState().policy, organisation: DEFAULT_ORGANISATION_POLICY } } as never); @@ -30,6 +26,7 @@ beforeEach(async () => { afterEach(async () => { await wipeEverything(); + setStoreBackend(null); }); describe('offline wipes and handoffs', () => { diff --git a/lib/offline/__tests__/migrate-legacy.test.ts b/lib/offline/__tests__/migrate-legacy.test.ts new file mode 100644 index 000000000..0639de69d --- /dev/null +++ b/lib/offline/__tests__/migrate-legacy.test.ts @@ -0,0 +1,192 @@ +import 'fake-indexeddb/auto'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; +import { forgetLegacyMove, LEGACY_DB_NAME, moveLegacyOutbox } from '../migrate-legacy'; +import { readQueue } from '../queue'; +import { useOfflineStore } from '../state'; + +/** The old database, exactly as lib/offline/db.ts laid it out. */ +const STORES = ['keys', 'emails', 'bodies', 'blobs', 'misc', 'ledger', 'queue', 'uploads']; + +function openOld(): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(LEGACY_DB_NAME, 1); + request.onupgradeneeded = () => { + const db = request.result; + for (const name of STORES) { + if (name === 'queue') db.createObjectStore(name, { keyPath: 'id', autoIncrement: true }).createIndex('account', 'account'); + else if (name === 'keys') db.createObjectStore(name, { keyPath: 'id' }); + else db.createObjectStore(name, { keyPath: 'key' }).createIndex('account', 'account'); + } + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); +} + +function put(db: IDBDatabase, store: string, value: unknown): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readwrite'); + tx.objectStore(store).put(value); + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); +} + +let key: CryptoKey; + +async function seal(value: unknown): Promise<{ iv: Uint8Array; data: ArrayBuffer }> { + const iv = crypto.getRandomValues(new Uint8Array(12)); + const bytes = ArrayBuffer.isView(value) ? value as Uint8Array : new TextEncoder().encode(JSON.stringify(value)); + return { iv, data: await crypto.subtle.encrypt({ name: 'AES-GCM', iv: iv as BufferSource }, key, bytes as BufferSource) }; +} + +const RAW = 'Subject: Signed\r\nMessage-ID: \r\n\r\nbody'; + +async function seedOld(): Promise { + key = await crypto.subtle.generateKey({ name: 'AES-GCM', length: 256 }, false, ['encrypt', 'decrypt']); + const db = await openOld(); + await put(db, 'keys', { id: 'aes', key }); + await put(db, 'misc', { key: 'acc\u0001misc\u0001session', account: 'acc', ...(await seal({ apiUrl: 'https://old/jmap', accounts: {}, primaryAccounts: {}, capabilities: {}, downloadUrl: '', uploadUrl: '' })) }); + await put(db, 'queue', { id: 1, account: 'acc', kind: 'patch', createdAt: 1, ...(await seal({ kind: 'patch', jmap: 'acc1', update: { m1: { 'keywords/$seen': true } } })) }); + await put(db, 'queue', { id: 2, account: 'acc', kind: 'patch', createdAt: 2, ...(await seal({ kind: 'patch', jmap: 'acc1', update: { m1: { 'keywords/$flagged': true } } })) }); + await put(db, 'queue', { + id: 3, account: 'acc', kind: 'send', createdAt: 3, status: 'waiting', + ...(await seal({ summary: { subject: 'Waiting', to: ['b@example.org'] }, item: { kind: 'email', args: [['b@example.org'], 'Waiting', 'x', null, null, 'i', 'a@example.org'] }, fingerprint: { labelRequired: false, hasLabel: false, guards: [] } })), + }); + await put(db, 'queue', { id: 4, account: 'acc', kind: 'send', createdAt: 4, status: 'sending', sendingSince: 4, ...(await seal({ summary: { subject: 'On its way', to: [] }, item: { kind: 'email', args: [] } })) }); + await put(db, 'queue', { id: 5, account: 'acc', kind: 'send', createdAt: 5, status: 'failed', ...(await seal({ summary: { subject: 'Refused', to: [] }, item: { kind: 'email', args: [] }, error: { text: 'No' } })) }); + await put(db, 'queue', { id: 6, account: 'acc', kind: 'send', createdAt: 6, status: 'waiting', ...(await seal({ summary: { subject: 'Signed', to: [] }, item: { kind: 'raw', blobId: 'bulwark-local-raw', identityId: 'i' } })) }); + await put(db, 'uploads', { key: 'acc\u0001bulwark-local-raw', account: 'acc', createdAt: 6, meta: await seal({ name: 'message.eml', type: 'message/rfc822' }), ...(await seal(new TextEncoder().encode(RAW))) }); + // Someone else's, with offline mode off here. + await put(db, 'queue', { id: 7, account: 'stranger', kind: 'send', createdAt: 7, status: 'waiting', ...(await seal({ summary: { subject: 'Not moved', to: [] }, item: { kind: 'email', args: [] } })) }); + db.close(); +} + +const oldExists = async () => (await indexedDB.databases()).some((d) => d.name === LEGACY_DB_NAME); + +let store: InProcessStore; + +beforeEach(async () => { + store = new InProcessStore(); + setStoreBackend(store); + forgetLegacyMove(); + useOfflineStore.setState({ enabled: ['acc'] }); + await new Promise((resolve) => { + const request = indexedDB.deleteDatabase(LEGACY_DB_NAME); + request.onsuccess = () => resolve(); + }); +}); + +afterEach(() => { + setStoreBackend(null); +}); + +describe('moving the old outbox into the store', () => { + it('gives a sender written "Name
    " a Message-ID with a clean domain', async () => { + await seedOld(); + const db = await new Promise((resolve) => { + const request = indexedDB.open(LEGACY_DB_NAME); + request.onsuccess = () => resolve(request.result); + }); + await put(db, 'queue', { + id: 20, account: 'acc', kind: 'send', createdAt: 20, status: 'waiting', + ...(await seal({ summary: { subject: 'Named', to: [] }, item: { kind: 'email', args: [[], 'Named', 'x', null, null, 'i', 'Alice '] } })), + }); + db.close(); + await moveLegacyOutbox(); + const named = (await readQueue('acc')).outbox.find((entry) => entry.summary.subject === 'Named'); + expect(named?.messageId).toMatch(/@example\.org$/); + }); + + it('moves everything that can\'t be fetched again, then deletes the old database', async () => { + await seedOld(); + await moveLegacyOutbox(); + const { outbox, patches } = await readQueue('acc'); + expect(outbox.map((entry) => [entry.summary.subject, entry.status])).toEqual([ + ['Waiting', 'waiting'], + ['On its way', 'uncertain'], + ['Refused', 'failed'], + ['Signed', 'waiting'], + ]); + expect(outbox[0].createdAt).toBe(3); + expect(outbox[0].fingerprint).toEqual({ labelRequired: false, hasLabel: false, guards: [] }); + expect(outbox[2].error).toEqual({ text: 'No' }); + // The patches of one message merge. + expect(patches).toBe(1); + const rows = await store.area>('acc', 'outbox.pendingPatches', {}); + expect(rows[0].patch).toEqual({ 'keywords/$seen': true, 'keywords/$flagged': true }); + // A waiting message gets a fixed id; one that may have left under another id doesn't pretend. + expect(outbox[0].messageId).toMatch(/@example\.org$/); + expect(outbox[1].messageId).toBeUndefined(); + expect(outbox[3].messageId).toBe('raw-1@example.org'); + const file = await store.area<{ name: string; data: Uint8Array }>('acc', 'outbox.heldFile', { blobId: 'bulwark-local-raw' }); + expect(file.name).toBe('message.eml'); + expect(new TextDecoder().decode(file.data)).toBe(RAW); + expect(await oldExists()).toBe(false); + expect(store.has('stranger')).toBe(false); + }); + + it('keeps the old database when the store can\'t take it, and moves it next time without doubling', async () => { + await seedOld(); + store.failArea = (name) => name === 'outbox.importLegacy'; + await moveLegacyOutbox(); + expect(await oldExists()).toBe(true); + expect((await readQueue('acc')).outbox).toEqual([]); + + store.failArea = null; + forgetLegacyMove(); + await moveLegacyOutbox(); + expect((await readQueue('acc')).outbox).toHaveLength(4); + expect(await oldExists()).toBe(false); + }); + + it('adds nothing when a later account failed and the first runs again', async () => { + await seedOld(); + const db = await openOld(); + await put(db, 'queue', { id: 8, account: 'second', kind: 'send', createdAt: 8, status: 'waiting', ...(await seal({ summary: { subject: 'Second', to: [] }, item: { kind: 'email', args: [] } })) }); + db.close(); + useOfflineStore.setState({ enabled: ['acc', 'second'] }); + const realArea = store.area.bind(store); + let failSecond = true; + store.area = (async (account: string, name: string, params?: unknown) => { + if (failSecond && account === 'second' && name === 'outbox.importLegacy') throw new Error('full'); + return realArea(account, name, params); + }) as typeof store.area; + await moveLegacyOutbox(); + expect(await oldExists()).toBe(true); + expect((await readQueue('acc')).outbox).toHaveLength(4); + + failSecond = false; + forgetLegacyMove(); + await moveLegacyOutbox(); + expect((await readQueue('acc')).outbox).toHaveLength(4); + expect((await readQueue('second')).outbox).toHaveLength(1); + expect(await oldExists()).toBe(false); + }); + + it('hands the kept session to the store, for a first start without a network', async () => { + await seedOld(); + await moveLegacyOutbox(); + expect(await store.area('acc', 'mail.session')).toMatchObject({ apiUrl: 'https://old/jmap' }); + }); + + it('never creates the old database when it isn\'t there', async () => { + await moveLegacyOutbox(); + expect(await oldExists()).toBe(false); + expect(store.has('acc')).toBe(false); + }); + + it('skips records sealed with a key that\'s gone, and finishes', async () => { + await seedOld(); + const db = await openOld(); + const other = await crypto.subtle.generateKey({ name: 'AES-GCM', length: 256 }, false, ['encrypt', 'decrypt']); + const iv = crypto.getRandomValues(new Uint8Array(12)); + await put(db, 'queue', { id: 9, account: 'acc', kind: 'send', createdAt: 9, status: 'waiting', iv, data: await crypto.subtle.encrypt({ name: 'AES-GCM', iv: iv as BufferSource }, other, new Uint8Array([1])) }); + db.close(); + await moveLegacyOutbox(); + expect((await readQueue('acc')).outbox).toHaveLength(4); + expect(await oldExists()).toBe(false); + }); +}); diff --git a/lib/offline/__tests__/outbox.test.ts b/lib/offline/__tests__/outbox.test.ts index 141217fdb..59fe37929 100644 --- a/lib/offline/__tests__/outbox.test.ts +++ b/lib/offline/__tests__/outbox.test.ts @@ -6,25 +6,33 @@ import { useEmailStore } from '@/stores/email-store'; import { usePolicyStore } from '@/stores/policy-store'; import { DEFAULT_ORGANISATION_POLICY } from '@/lib/admin/organisation-policy'; import { passSignOutGates, addSignOutGate, purgeSignedOutData } from '@/lib/sign-out-cleanup'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { FakeJmapMail } from '@/lib/store/__tests__/helpers/fake-jmap-mail'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; import { enforcePolicy, startOfflineMode, wipeEverything } from '../offline-mode'; import { flushQueue } from '../flush'; -import { holdFile, queuePatch, readQueue } from '../queue'; -import { emailIndex, forgetIndexes, putEmails } from '../cache'; -import { forgetDeviceKey } from '../crypto'; -import { allowOfflineDbCreation, deleteOfflineDb } from '../db'; +import { holdFile, readQueue } from '../queue'; +import { forgetLegacyMove } from '../migrate-legacy'; import { useOfflineStore, writeEnabled, OFFLINE_ACCOUNTS_KEY } from '../state'; import type { JMAPClient } from '@/lib/jmap/client'; +let server: FakeJmapMail; +let store: InProcessStore; + function fakeClient(overrides: Record = {}) { return { getServerUrl: () => 'https://mail.example.org', getUsername: () => 'alice@example.org', - getAccountId: () => 'j1', + getAccountId: () => 'acc1', isOfflineOnly: () => false, - getSessionSnapshot: () => ({ apiUrl: 'https://mail.example.org/jmap' }), + getSessionSnapshot: () => null, + getAuthHeader: () => 'Basic x', + keywordSortInverted: () => null, sendEmail: vi.fn(async () => ({ scheduled: false, emailId: 'sent-1', emailSubmissionId: 'sub-1' })), uploadBlob: vi.fn(async (file: File) => ({ blobId: `server-${file.name}`, size: file.size, type: file.type })), - applyEmailPatches: vi.fn(async () => []), + requestNetworkOnly: vi.fn((calls: never) => server.request(calls)), + resolveSendAccountId: vi.fn(async () => 'acc1'), + submissionAccountFor: () => 'acc1', pruneEmailSubmissions: vi.fn(async () => {}), getMailboxes: vi.fn(async () => [{ id: 'sent', role: 'sent' }, { id: 'drafts', role: 'drafts' }]), sendRawEmail: vi.fn(async () => ({ scheduled: false })), @@ -41,20 +49,23 @@ function setOnline(online: boolean) { let stop: () => void = () => {}; beforeEach(async () => { - forgetIndexes(); - forgetDeviceKey(); - await deleteOfflineDb(); - allowOfflineDbCreation(); + server = new FakeJmapMail({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'drafts', role: 'drafts' }, { id: 'sent', role: 'sent' }] }); + server.addEmail({ id: 'm1', keywords: {}, subject: 'One' }); + store = new InProcessStore(server); + setStoreBackend(store); + forgetLegacyMove(); localStorage.clear(); - useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, browserOffline: false, downServers: [], startedOffline: false }); + useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, conflicts: [], browserOffline: false, downServers: [], startedOffline: false }); usePolicyStore.setState({ loaded: true, subject: null, policy: { ...usePolicyStore.getState().policy, organisation: DEFAULT_ORGANISATION_POLICY } }); useAccountStore.setState({ accounts: [{ id: 'acc', label: 'alice', serverUrl: 'https://mail.example.org', username: 'alice@example.org' } as never] }); }); afterEach(async () => { stop(); + stop = () => {}; setOnline(true); await wipeEverything(); + setStoreBackend(null); }); const sendArgs = (client: unknown, options: Record) => useEmailStore.getState().sendEmail( @@ -62,8 +73,10 @@ const sendArgs = (client: unknown, options: Record) => useEmail 'draft-1', 'Alice', '

    Hi Bob

    ', [{ blobId: 'blob-1', name: 'a.txt', type: 'text/plain', size: 3 }], undefined, undefined, undefined, undefined, options as never, ); +const patchOffline = (update: Record) => store.answer('acc', [['Email/set', { update }, '0']]); + describe('the outbox', () => { - it('keeps a composer send while offline, with its label, and sends nothing', async () => { + it('keeps a composer send while offline, with its label and a fixed Message-ID, and sends nothing', async () => { const client = fakeClient(); useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); writeEnabled(['acc']); @@ -77,11 +90,44 @@ describe('the outbox', () => { expect(outbox).toHaveLength(1); expect(outbox[0].summary).toEqual({ subject: 'Quarterly numbers', to: ['bob@example.org'] }); expect(outbox[0].item).toMatchObject({ kind: 'email' }); + expect(outbox[0].messageId).toMatch(/@example\.org$/); const options = (outbox[0].item as { args: unknown[] }).args[15] as Record; expect(options.labelHeaders).toEqual({ 'header:SIO-Label:asRaw': ' marking="CONFIDENTIAL"' }); expect(options).not.toHaveProperty('outbox'); }); + it('takes the domain of a Message-ID from a sender written with a name', async () => { + const client = fakeClient(); + useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); + writeEnabled(['acc']); + stop = startOfflineMode(); + setOnline(false); + await useEmailStore.getState().sendEmail(client as never, ['b@example.org'], 'S', 'x', undefined, undefined, 'i', 'Alice ', undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, { outbox: true } as never); + expect((await readQueue('acc')).outbox[0].messageId).toMatch(/@example\.net$/); + }); + + it('sends with the Message-ID fixed at queueing, and the same one after a retry', async () => { + const client = fakeClient({ + sendEmail: vi.fn() + .mockRejectedValueOnce(Object.assign(new Error('x'), { name: 'SendNotStartedError' })) + .mockResolvedValue({ scheduled: false }), + }); + useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); + writeEnabled(['acc']); + stop = startOfflineMode(); + setOnline(false); + await sendArgs(client, { outbox: true }); + const fixed = (await readQueue('acc')).outbox[0].messageId; + setOnline(true); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + const calls = (client.sendEmail as ReturnType).mock.calls as unknown[][]; + expect(calls).toHaveLength(2); + expect((calls[0][15] as { messageId: string }).messageId).toBe(fixed); + expect((calls[1][15] as { messageId: string }).messageId).toBe(fixed); + expect((await readQueue('acc')).outbox).toEqual([]); + }); + it('leaves sends from elsewhere (plugin forward, phishing report) to the network', async () => { const client = fakeClient({ sendEmail: vi.fn(async () => { throw new TypeError('Failed to fetch'); }) }); useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); @@ -105,14 +151,18 @@ describe('the outbox', () => { it('sends what waited in order once the server answers: changes first, then mail', async () => { const order: string[] = []; const client = fakeClient({ - applyEmailPatches: vi.fn(async () => { order.push('patch'); return []; }), + requestNetworkOnly: vi.fn(async (calls: Array<[string]>) => { + if (calls[0][0] === 'Email/set') order.push('patch'); + return server.request(calls as never); + }), sendEmail: vi.fn(async () => { order.push('send'); return { scheduled: false }; }), }); useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); writeEnabled(['acc']); stop = startOfflineMode(); + await store.sync('acc'); setOnline(false); - await queuePatch('acc', 'j1', { m1: { 'keywords/$seen': true } }); + await patchOffline({ m1: { 'keywords/$seen': true } }); const held = await holdFile('acc', new File(['hello'], 'note.txt', { type: 'text/plain' }), 'note.txt'); await useEmailStore.getState().sendEmail(client as never, ['bob@example.org'], 'With file', 'x', undefined, undefined, 'id-1', 'alice@example.org', undefined, 'Alice', undefined, [{ blobId: held.blobId, name: 'note.txt', type: 'text/plain', size: 5 }], undefined, undefined, new Date(Date.now() + 5000).toISOString(), undefined, { outbox: true } as never); @@ -121,7 +171,7 @@ describe('the outbox', () => { const notify = vi.fn(); await flushQueue('acc', client as unknown as JMAPClient, { notify, allowed: () => true }); expect(order).toEqual(['patch', 'send']); - expect(client.applyEmailPatches).toHaveBeenCalledWith('j1', { m1: { 'keywords/$seen': true } }); + expect(server.emails.get('m1')?.keywords).toEqual({ $seen: true }); const args = (client.sendEmail as ReturnType).mock.calls[0] as unknown[]; expect(args[1]).toBe('With file'); // The file attached offline was uploaded first; the passed undo hold was dropped. @@ -130,7 +180,28 @@ describe('the outbox', () => { expect(notify).toHaveBeenCalledWith({ kind: 'sent', subject: 'With file' }); const left = await readQueue('acc'); expect(left.outbox).toHaveLength(0); - expect(left.patches).toHaveLength(0); + expect(left.patches).toBe(0); + }); + + it('a change made after a send goes after it', async () => { + const order: string[] = []; + const client = fakeClient({ + requestNetworkOnly: vi.fn(async (calls: Array<[string]>) => { + if (calls[0][0] === 'Email/set') order.push('patch'); + return server.request(calls as never); + }), + sendEmail: vi.fn(async () => { order.push('send'); return { scheduled: false }; }), + }); + useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); + writeEnabled(['acc']); + stop = startOfflineMode(); + await store.sync('acc'); + setOnline(false); + await sendArgs(client, { outbox: true }); + await patchOffline({ m1: { 'keywords/$answered': true } }); + setOnline(true); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(order).toEqual(['send', 'patch']); }); it('keeps a send the server refused, marked failed, and one cut off on the way, marked uncertain', async () => { @@ -146,10 +217,13 @@ describe('the outbox', () => { expect(outbox[0].status).toBe('failed'); expect(outbox[0].error).toEqual({ text: 'Recipient address rejected' }); - const cut = fakeClient({ sendEmail: vi.fn(async () => { throw new TypeError('Failed to fetch'); }) }); - await wipeEverything(); - allowOfflineDbCreation(); + store.stop(); writeEnabled(['acc']); + // The lookup that follows can't reach the server either, so it stays uncertain. + const cut = fakeClient({ + sendEmail: vi.fn(async () => { throw new TypeError('Failed to fetch'); }), + requestNetworkOnly: vi.fn(async () => { throw new TypeError('Failed to fetch'); }), + }); useAuthStore.setState({ client: cut as never, activeAccountId: 'acc' }); setOnline(false); await sendArgs(cut, { outbox: true }); @@ -160,13 +234,187 @@ describe('the outbox', () => { // Not tried again on its own. await flushQueue('acc', cut as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); expect(cut.sendEmail).toHaveBeenCalledTimes(1); + + // With the server still there, nothing else would start the lookup. + const entry = (await readQueue('acc')).outbox[0]; + await store.area('acc', 'outbox.update', { id: entry.id, status: 'waiting' }); + const retryLater = vi.fn(); + await flushQueue('acc', cut as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true, retryLater }); + expect(retryLater).toHaveBeenCalledTimes(1); }); it('sends nothing once the organisation forbids offline mode', async () => { const client = fakeClient(); - await queuePatch('acc', 'j1', { m1: { 'keywords/$seen': true } }); + await store.sync('acc'); + await patchOffline({ m1: { 'keywords/$seen': true } }); await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => false }); - expect(client.applyEmailPatches).not.toHaveBeenCalled(); + expect(client.requestNetworkOnly).not.toHaveBeenCalled(); + expect(server.emails.get('m1')?.keywords).toEqual({}); + }); +}); + +describe('a send whose response got lost', () => { + async function uncertainSend(client: ReturnType) { + useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); + writeEnabled(['acc']); + stop = startOfflineMode(); + setOnline(false); + await sendArgs(client, { outbox: true }); + setOnline(true); + const entry = (await readQueue('acc')).outbox[0]; + await store.area('acc', 'outbox.claim', { id: entry.id }); + await store.area('acc', 'outbox.update', { id: entry.id, status: 'uncertain' }); + // The wait before a lookup has passed. + store.dbs.get('acc')!.run('UPDATE outbox SET last_attempt_at = ?', [Date.now() - 31_000]); + return entry; + } + + const flush = (client: ReturnType, notify = vi.fn()) => flushQueue('acc', client as unknown as JMAPClient, { notify, allowed: () => true }); + + const copy = (messageId: string | undefined, over: Record) => server.addEmail({ + id: 'copy', messageId: [messageId as string], receivedAt: new Date().toISOString(), ...over, + }); + + it('never arrived: sends it again, with the same Message-ID', async () => { + const client = fakeClient(); + const entry = await uncertainSend(client); + const notify = vi.fn(); + await flush(client, notify); + expect(client.sendEmail).toHaveBeenCalledTimes(1); + const args = (client.sendEmail as ReturnType).mock.calls[0] as unknown[]; + expect((args[15] as { messageId?: string; existingCopyId?: string })).toMatchObject({ messageId: entry.messageId }); + expect(args[15]).not.toHaveProperty('existingCopyId'); + expect(notify).toHaveBeenCalledWith({ kind: 'sent', subject: 'Quarterly numbers' }); + }); + + it('draft arrived, submission didn\'t: submits that copy and uploads nothing again', async () => { + const client = fakeClient(); + const entry = await uncertainSend(client); + copy(entry.messageId, { mailboxIds: { drafts: true }, keywords: { $draft: true } }); + await flush(client); + expect(client.sendEmail).toHaveBeenCalledTimes(1); + const args = (client.sendEmail as ReturnType).mock.calls[0] as unknown[]; + expect(args[15]).toMatchObject({ messageId: entry.messageId, existingCopyId: 'copy' }); + expect(client.uploadBlob).not.toHaveBeenCalled(); + }); + + it('both arrived: done, nothing sent, and the old draft goes', async () => { + const client = fakeClient(); + const entry = await uncertainSend(client); + copy(entry.messageId, { mailboxIds: { sent: true }, keywords: {} }); + server.addEmail({ id: 'draft-1', mailboxIds: { drafts: true }, keywords: { $draft: true } }); + const notify = vi.fn(); + await flush(client, notify); + expect(client.sendEmail).not.toHaveBeenCalled(); + expect((await readQueue('acc')).outbox).toEqual([]); + expect(server.emails.has('draft-1')).toBe(false); + expect(notify).toHaveBeenCalledWith({ kind: 'sent', subject: 'Quarterly numbers' }); + }); + + it('scheduled: done, the submission stays', async () => { + const client = fakeClient(); + const entry = await uncertainSend(client); + copy(entry.messageId, { mailboxIds: { drafts: true }, keywords: { $draft: true } }); + server.addSubmission('copy', 'pending'); + await flush(client); + expect(client.sendEmail).not.toHaveBeenCalled(); + expect((await readQueue('acc')).outbox).toEqual([]); + }); + + it('canceled: done without a "sent" notice, and it doesn\'t go', async () => { + const client = fakeClient(); + const entry = await uncertainSend(client); + copy(entry.messageId, { mailboxIds: { drafts: true }, keywords: { $draft: true } }); + server.addSubmission('copy', 'canceled'); + const notify = vi.fn(); + await flush(client, notify); + expect(client.sendEmail).not.toHaveBeenCalled(); + expect((await readQueue('acc')).outbox).toEqual([]); + expect(notify).not.toHaveBeenCalledWith(expect.objectContaining({ kind: 'sent' })); + }); + + it('a lookup that can\'t decide leaves it uncertain for the user, who can still send it', async () => { + const client = fakeClient(); + const entry = await uncertainSend(client); + server.failNext.set('Email/query', 'forbidden'); + await flush(client); + expect(client.sendEmail).not.toHaveBeenCalled(); + expect((await readQueue('acc')).outbox[0].status).toBe('uncertain'); + + const { retryOutboxEntry } = await import('../flush'); + await retryOutboxEntry('acc', entry.id); + expect((await readQueue('acc')).outbox[0].status).toBe('waiting'); + await flush(client); + expect(client.sendEmail).toHaveBeenCalledTimes(1); + }); + + it('a send that fails on the server clears the copy, so a retry starts clean', async () => { + const client = fakeClient({ sendEmail: vi.fn(async () => { throw new Error('Refused'); }) }); + const entry = await uncertainSend(client); + copy(entry.messageId, { mailboxIds: { drafts: true }, keywords: { $draft: true } }); + await flush(client); + const [after] = (await readQueue('acc')).outbox; + expect(after.status).toBe('failed'); + expect(after.copyId).toBeUndefined(); + }); +}); + +describe('what the other areas queued', () => { + const flushWith = async (tally: unknown) => { + const { flushAll } = await import('../offline-mode'); + const client = fakeClient(); + useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); + usePolicyStore.setState({ fetchPolicy: vi.fn(async () => {}), source: 'server' } as never); + const realArea = store.area.bind(store); + const calls: string[] = []; + store.area = (async (account: string, name: string, params?: unknown) => { + if (name.endsWith('.flush')) { + calls.push(name); + return tally; + } + return realArea(account, name, params); + }) as typeof store.area; + const { subscribePushStateChanges } = await import('@/lib/push-events'); + const pushes: unknown[] = []; + const unsubscribe = subscribePushStateChanges((change, _client, role) => pushes.push([change.changed, role])); + useAccountStore.setState({ accounts: [{ id: 'acc', serverUrl: 'https://mail.example.org', username: 'alice@example.org' } as never] }); + useAuthStore.setState({ getClientForAccount: () => client } as never); + writeEnabled(['acc']); + await flushAll(); + unsubscribe(); + return { calls, pushes }; + }; + + it('flushes them, also with nothing in the outbox, and has the page read again', async () => { + const { calls, pushes } = await flushWith({ sent: 1, conflicts: 0, pending: 0 }); + expect(calls).toEqual(['calendar.flush', 'contacts.flush']); + expect(pushes).toEqual([ + [{ acc1: { Calendar: 'flushed', CalendarEvent: 'flushed' } }, 'active'], + [{ acc1: { AddressBook: 'flushed', ContactCard: 'flushed' } }, 'active'], + ]); + }); + + it('doesn\u2019t have the page read again when nothing went out', async () => { + const { pushes } = await flushWith({ sent: 0, conflicts: 0, pending: 0 }); + expect(pushes).toEqual([]); + }); +}); + +describe('conflicts reach the page', () => { + it('lists a refused change and tells the user', async () => { + const client = fakeClient(); + useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); + writeEnabled(['acc']); + stop = startOfflineMode(); + await store.sync('acc'); + await patchOffline({ m1: { 'keywords/$seen': true } }); + server.refuse.set('m1', { type: 'forbidden' }); + const notify = vi.fn(); + await flushQueue('acc', client as unknown as JMAPClient, { notify, allowed: () => true }); + expect(notify).toHaveBeenCalledWith({ kind: 'conflict', count: 1 }); + const { conflicts, patches } = await readQueue('acc'); + expect(patches).toBe(0); + expect(conflicts).toEqual([expect.objectContaining({ account: 'acc', recordId: 'm1', summary: expect.objectContaining({ subject: 'One', reason: 'refused' }) })]); }); }); @@ -176,39 +424,66 @@ describe('wiping offline data', () => { useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); writeEnabled(['acc']); stop = startOfflineMode(); - await putEmails('acc', 'j1', [{ id: 'm1', mailboxIds: { inbox: true }, receivedAt: new Date().toISOString() }]); usePolicyStore.setState({ policy: { ...usePolicyStore.getState().policy, organisation: { ...DEFAULT_ORGANISATION_POLICY, devices: { ...DEFAULT_ORGANISATION_POLICY.devices, noOffline: true } } } }); enforcePolicy(); await vi.waitFor(async () => expect(useOfflineStore.getState().enabled).toEqual([])); - // Its only account: the whole database goes. - await vi.waitFor(async () => expect((await indexedDB.databases()).map((d) => d.name)).not.toContain('bulwark-offline')); + // Its only account: the whole store goes. + await vi.waitFor(() => expect(store.stopped).toBeGreaterThan(0)); expect(localStorage.getItem(OFFLINE_ACCOUNTS_KEY)).toBeNull(); }); - it('wipes an account that signs out', async () => { + it('wipes an account that signs out or is removed on purpose', async () => { + useAccountStore.setState({ accounts: [{ id: 'acc' } as never, { id: 'other' } as never] }); writeEnabled(['acc', 'other']); stop = startOfflineMode(); - await putEmails('acc', 'j1', [{ id: 'm1', mailboxIds: { inbox: true }, receivedAt: new Date().toISOString() }]); - await putEmails('other', 'j1', [{ id: 'm2', mailboxIds: { inbox: true }, receivedAt: new Date().toISOString() }]); - useAccountStore.setState({ accounts: [{ id: 'other' } as never] }); + await store.sync('acc'); + await store.sync('other'); + useAccountStore.getState().removeAccount('acc'); await vi.waitFor(() => expect(useOfflineStore.getState().enabled).toEqual(['other'])); - await vi.waitFor(async () => { - forgetIndexes(); - expect((await emailIndex('acc')).size).toBe(0); - }); - expect((await emailIndex('other')).size).toBe(1); + await vi.waitFor(() => expect(store.removed).toEqual(['acc'])); + expect(store.has('acc')).toBe(false); + expect(store.has('other')).toBe(true); }); - it('deletes the database and the switch at a full sign-out', async () => { + it('keeps the data of an account the app dropped by itself', async () => { + useAccountStore.setState({ accounts: [{ id: 'acc' } as never, { id: 'other' } as never] }); + writeEnabled(['acc', 'other']); + stop = startOfflineMode(); + await store.sync('acc'); + useAccountStore.getState().removeAccount('acc', 'evicted'); + await new Promise((r) => setTimeout(r, 20)); + expect(useAccountStore.getState().accounts.map((a) => a.id)).toEqual(['other']); + expect(useOfflineStore.getState().enabled).toEqual(['acc', 'other']); + expect(store.removed).toEqual([]); + }); + + it('deletes the old database, the key database and the switch at a full sign-out', async () => { writeEnabled(['acc']); - await putEmails('acc', 'j1', [{ id: 'm1', mailboxIds: { inbox: true }, receivedAt: new Date().toISOString() }]); - forgetIndexes(); - purgeSignedOutData(); - expect(localStorage.getItem(OFFLINE_ACCOUNTS_KEY)).toBeNull(); - await vi.waitFor(async () => { - const dbs = await indexedDB.databases(); - expect(dbs.map((d) => d.name)).not.toContain('bulwark-offline'); + await new Promise((resolve) => { + const request = indexedDB.open('bulwark-offline'); + request.onsuccess = () => { request.result.close(); resolve(); }; }); + await new Promise((resolve) => { + const request = indexedDB.open('bulwark-store'); + request.onsuccess = () => { request.result.close(); resolve(); }; + }); + const removeEntry = vi.fn(async () => {}); + const getDirectory = vi.fn(async () => ({ removeEntry })); + Object.defineProperty(window.navigator, 'storage', { configurable: true, value: { getDirectory } }); + try { + purgeSignedOutData(); + expect(localStorage.getItem(OFFLINE_ACCOUNTS_KEY)).toBeNull(); + await vi.waitFor(async () => { + const dbs = (await indexedDB.databases()).map((d) => d.name); + expect(dbs).not.toContain('bulwark-offline'); + expect(dbs).not.toContain('bulwark-store'); + }); + // With no worker running: straight from the page. + await vi.waitFor(() => expect(removeEntry).toHaveBeenCalledWith('.bulwark-store', { recursive: true })); + expect(store.stopped).toBeGreaterThan(0); + } finally { + delete (window.navigator as { storage?: unknown }).storage; + } }); }); @@ -222,13 +497,14 @@ describe('signing out with unsent mail', () => { }); describe('after a sign-out', () => { - it('starting offline mode on the sign-in page doesn\'t create the database again', async () => { - const stop = startOfflineMode(); + it('starting offline mode on the sign-in page doesn\'t create the store again', async () => { + const stopHere = startOfflineMode(); await new Promise((r) => setTimeout(r, 50)); const { refreshQueueState, refreshUsage } = await import('../offline-mode'); await refreshQueueState(); await refreshUsage(); - stop(); - expect((await indexedDB.databases()).map((d) => d.name)).not.toContain('bulwark-offline'); + stopHere(); + expect(store.engines.size).toBe(0); + expect((await indexedDB.databases()).map((d) => d.name)).not.toContain('bulwark-store'); }); }); diff --git a/lib/offline/__tests__/queue-sync.test.ts b/lib/offline/__tests__/queue-sync.test.ts index a31f5616e..da3099e06 100644 --- a/lib/offline/__tests__/queue-sync.test.ts +++ b/lib/offline/__tests__/queue-sync.test.ts @@ -1,35 +1,53 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { queuePatch, queueSend } from '../queue'; +import 'fake-indexeddb/auto'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; +import { getOfflineHooks } from '@/lib/jmap/offline-hooks'; +import { usePolicyStore } from '@/stores/policy-store'; +import { useAccountStore } from '@/stores/account-store'; +import { DEFAULT_ORGANISATION_POLICY } from '@/lib/admin/organisation-policy'; +import { queueSend, readQueue } from '../queue'; +import { registerOfflineHooks } from '../offline-mode'; +import { writeEnabled } from '../state'; -// Whatever goes into the queue asks for a Background Sync, once it's stored. +// Whatever goes into the outbox asks for a Background Sync, once it's stored. -const requestOutboxSync = vi.fn(); -const stored: unknown[] = []; -vi.mock('@/lib/pwa-notify/outbox-sync', () => ({ requestOutboxSync: () => requestOutboxSync(stored.length) })); -vi.mock('../db', () => ({ - idbAdd: vi.fn(async (_store: string, record: unknown) => { stored.push(record); return stored.length; }), - idbDeleteAccount: vi.fn(), idbGet: vi.fn(), idbGetAll: vi.fn(), idbGetAllByAccount: vi.fn(), idbUpdate: vi.fn(), idbWrite: vi.fn(), -})); -vi.mock('../crypto', () => ({ - sealJson: vi.fn(async () => ({ iv: 'i', data: 'd' })), - openJson: vi.fn(), openBytes: vi.fn(), sealBytes: vi.fn(), -})); +const seen = vi.hoisted(() => ({ calls: 0, onCall: (() => {}) as () => void })); +vi.mock('@/lib/pwa-notify/outbox-sync', () => ({ requestOutboxSync: () => { seen.calls += 1; seen.onCall(); } })); + +let store: InProcessStore; beforeEach(() => { - stored.length = 0; - requestOutboxSync.mockClear(); + store = new InProcessStore(); + setStoreBackend(store); + seen.calls = 0; + seen.onCall = () => {}; + usePolicyStore.setState({ loaded: true, subject: null, policy: { ...usePolicyStore.getState().policy, organisation: DEFAULT_ORGANISATION_POLICY } }); + useAccountStore.setState({ accounts: [{ id: 'acc', serverUrl: 'https://mail.example.org', username: 'alice@example.org' } as never] }); }); -describe('the offline queue', () => { - it('asks for a sync after storing a change', async () => { - await queuePatch('acc', 'jmap', {} as never); - expect(requestOutboxSync).toHaveBeenCalledTimes(1); - expect(requestOutboxSync).toHaveBeenCalledWith(1); - }); +afterEach(() => { + setStoreBackend(null); + writeEnabled([]); +}); +describe('the outbox', () => { it('asks for a sync after storing a message, and still returns its id', async () => { - const id = await queueSend('acc', {} as never, {} as never); + let storedWhenAsked = -1; + seen.onCall = () => { void readQueue('acc').then(({ outbox }) => { storedWhenAsked = outbox.length; }); }; + const id = await queueSend('acc', { subject: 'S', to: [] }, { kind: 'email', args: [] }); expect(id).toBe(1); - expect(requestOutboxSync).toHaveBeenCalledWith(1); + expect(seen.calls).toBe(1); + await vi.waitFor(() => expect(storedWhenAsked).toBe(1)); + }); + + it('asks for a sync after keeping a change the client made, not after a read', async () => { + writeEnabled(['acc']); + registerOfflineHooks(); + const client = { getServerUrl: () => 'https://mail.example.org', getUsername: () => 'alice@example.org', getAccountId: () => 'acc1' }; + await getOfflineHooks()?.answer(client, [['Email/set', { update: { m: { 'keywords/$seen': true } } }, '0']]); + expect(seen.calls).toBe(1); + await getOfflineHooks()?.answer(client, [['Email/get', { ids: [] }, '0']]); + expect(seen.calls).toBe(1); }); }); diff --git a/lib/offline/__tests__/responder.test.ts b/lib/offline/__tests__/responder.test.ts deleted file mode 100644 index b8617fdd1..000000000 --- a/lib/offline/__tests__/responder.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import 'fake-indexeddb/auto'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { answerOffline, type ResponderContext } from '../responder'; -import { emailIndex, emailKey, forgetIndexes, putEmails, putMisc } from '../cache'; -import { forgetDeviceKey } from '../crypto'; -import { allowOfflineDbCreation, deleteOfflineDb } from '../db'; - -const day = 24 * 60 * 60 * 1000; -const at = (ageDays: number) => new Date(Date.now() - ageDays * day).toISOString(); - -let queued: Array<{ jmap: string; update: Record> }>; -let refused: string[]; -const ctx: ResponderContext = { - account: 'acc', - defaultJmap: 'j1', - queuePatch: async (jmap, update) => { queued.push({ jmap, update }); }, - onRefused: (what) => { refused.push(what); }, -}; - -beforeEach(async () => { - forgetIndexes(); - forgetDeviceKey(); - await deleteOfflineDb(); - allowOfflineDbCreation(); - queued = []; - refused = []; - await putEmails('acc', 'j1', [ - { id: 'a', threadId: 't1', mailboxIds: { inbox: true }, keywords: {}, receivedAt: at(3), subject: 'Older', bodyValues: { 1: { value: 'Body A' } }, textBody: [{ partId: '1' }] }, - { id: 'b', threadId: 't1', mailboxIds: { inbox: true }, keywords: { $flagged: true }, receivedAt: at(1), subject: 'Newer' }, - { id: 'c', threadId: 't2', mailboxIds: { archive: true }, keywords: {}, receivedAt: at(2), subject: 'Archived' }, - ]); - await putMisc('acc', 'mailboxes', { j1: [ - { id: 'inbox', name: 'Inbox', role: 'inbox', totalEmails: 2, unreadEmails: 1 }, - { id: 'archive', name: 'Archive', role: 'archive', totalEmails: 1, unreadEmails: 1 }, - ] }); -}); - -describe('answering a folder list offline', () => { - it('answers the list request the client makes, back-references included', async () => { - const response = await answerOffline(ctx, [ - ['Email/query', { accountId: 'j1', filter: { inMailbox: 'inbox' }, sort: [{ property: 'receivedAt', isAscending: false }], limit: 50, position: 0, calculateTotal: true }, '0'], - ['Email/get', { accountId: 'j1', '#ids': { resultOf: '0', name: 'Email/query', path: '/ids' }, properties: ['id', 'threadId', 'subject', 'keywords'] }, '1'], - ['Thread/get', { accountId: 'j1', '#ids': { resultOf: '1', name: 'Email/get', path: '/list/*/threadId' } }, '2'], - ]); - expect(response).not.toBeNull(); - const [query, get, thread] = response!.methodResponses; - expect(query[1].ids).toEqual(['b', 'a']); - expect(query[1].total).toBe(2); - expect(get[1].list).toEqual([ - { id: 'b', threadId: 't1', subject: 'Newer', keywords: { $flagged: true } }, - { id: 'a', threadId: 't1', subject: 'Older', keywords: {} }, - ]); - expect(thread[1].list).toEqual([{ id: 't1', emailIds: ['a', 'b'] }]); - }); - - it('pins flagged mail first when the list sorts that way', async () => { - const response = await answerOffline(ctx, [ - ['Email/query', { filter: { inMailbox: 'inbox' }, sort: [{ property: 'receivedAt', isAscending: true }] }, '0'], - ]); - expect(response!.methodResponses[0][1].ids).toEqual(['a', 'b']); - const pinned = await answerOffline(ctx, [ - ['Email/query', { filter: { inMailbox: 'inbox' }, sort: [{ property: 'hasKeyword', keyword: '$flagged', isAscending: false }, { property: 'receivedAt', isAscending: true }] }, '0'], - ]); - expect(pinned!.methodResponses[0][1].ids).toEqual(['b', 'a']); - }); - - it('opens a kept message with its body and refuses one without', async () => { - const full = { properties: ['id', 'subject', 'bodyValues', 'textBody'], fetchTextBodyValues: true }; - const response = await answerOffline(ctx, [['Email/get', { ids: ['a', 'b'], ...full }, '0']]); - const result = response!.methodResponses[0][1]; - expect(result.list).toEqual([{ id: 'a', subject: 'Older', bodyValues: { 1: { value: 'Body A' } }, textBody: [{ partId: '1' }] }]); - expect(result.notFound).toEqual(['b']); - }); - - it('answers the folder list', async () => { - const response = await answerOffline(ctx, [['Mailbox/get', { accountId: 'j1' }, '0']]); - expect(response!.methodResponses[0][1].list.map((m: { id: string }) => m.id)).toEqual(['inbox', 'archive']); - }); - - it('leaves search to the server', async () => { - expect(await answerOffline(ctx, [['Email/query', { filter: { text: 'invoice' } }, '0']])).toBeNull(); - expect(await answerOffline(ctx, [['SearchSnippet/get', {}, '0']])).toBeNull(); - }); -}); - -describe('changes made offline', () => { - it('queues read, flag and move, and shows them at once', async () => { - const response = await answerOffline(ctx, [ - ['Email/set', { accountId: 'j1', update: { a: { 'keywords/$seen': true, 'keywords/$flagged': true }, c: { 'mailboxIds/archive': null, 'mailboxIds/inbox': true } } }, '0'], - ]); - expect(response!.methodResponses[0][1].updated).toEqual({ a: null, c: null }); - expect(queued).toEqual([{ jmap: 'j1', update: { a: { 'keywords/$seen': true, 'keywords/$flagged': true }, c: { 'mailboxIds/archive': null, 'mailboxIds/inbox': true } } }]); - const index = await emailIndex('acc'); - expect(index.get(emailKey('acc', 'j1', 'a'))?.keywords).toEqual({ $seen: true, $flagged: true }); - const inbox = await answerOffline(ctx, [['Email/query', { filter: { inMailbox: 'inbox' } }, '0']]); - expect(inbox!.methodResponses[0][1].ids).toEqual(['b', 'c', 'a']); - }); - - it('keeps the folder counts in step', async () => { - await answerOffline(ctx, [['Email/set', { update: { a: { 'keywords/$seen': true }, c: { 'mailboxIds/archive': null, 'mailboxIds/inbox': true } } }, '0']]); - const boxes = (await answerOffline(ctx, [['Mailbox/get', {}, '0']]))!.methodResponses[0][1].list; - expect(boxes.find((b: { id: string }) => b.id === 'inbox')).toMatchObject({ totalEmails: 3, unreadEmails: 1 }); - expect(boxes.find((b: { id: string }) => b.id === 'archive')).toMatchObject({ totalEmails: 0, unreadEmails: 0 }); - }); - - it('refuses deleting for good and emptying a folder', async () => { - expect(await answerOffline(ctx, [['Email/set', { destroy: ['a'] }, '0']])).toBeNull(); - expect(await answerOffline(ctx, [['Email/set', { update: { a: { mailboxIds: {} } } }, '0']])).toBeNull(); - expect(await answerOffline(ctx, [['Email/set', { update: { c: { 'mailboxIds/archive': null } } }, '0']])).toBeNull(); - expect(refused).toEqual(['destroy', 'destroy', 'destroy']); - expect(queued).toEqual([]); - expect((await emailIndex('acc')).size).toBe(3); - }); - - it('refuses other changes and new messages without queueing', async () => { - const onRefused = vi.fn(); - expect(await answerOffline({ ...ctx, onRefused }, [['Email/set', { update: { a: { subject: 'x' } } }, '0']])).toBeNull(); - expect(await answerOffline({ ...ctx, onRefused }, [['Email/set', { create: { d: { subject: 'draft' } } }, '0']])).toBeNull(); - expect(onRefused).toHaveBeenCalledTimes(1); - expect(queued).toEqual([]); - }); -}); diff --git a/lib/offline/__tests__/review-fixes.test.ts b/lib/offline/__tests__/review-fixes.test.ts index db4e4e7ab..a5bbc6a0d 100644 --- a/lib/offline/__tests__/review-fixes.test.ts +++ b/lib/offline/__tests__/review-fixes.test.ts @@ -9,31 +9,35 @@ import { runBeforeSignOut } from '@/lib/sign-out-cleanup'; import { setSendGuardSource } from '@/lib/plugin-send-guards'; import { clearAllHooks, emailHooks } from '@/lib/plugin-hooks'; import { beginComposeTakeover, completesComposeTakeover, endComposeTakeover, resetComposeTakeovers } from '@/lib/outgoing/compose-takeover'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; +import { FakeJmapMail } from '@/lib/store/__tests__/helpers/fake-jmap-mail'; +import { JMAP_ACCOUNT } from '@/lib/store/__tests__/helpers/fake-jmap'; import type { JMAPClient } from '@/lib/jmap/client'; import type { InstalledPlugin } from '@/lib/plugin-types'; import { confirmLoss, enforcePolicy, - inScope, offlineSessionFor, + offlineSupported, pluginsNotRunning, - recordAnswer, - scopeOf, + refreshQueueState, + setOfflineNotifier, startOfflineMode, stricterNow, + unsentCount, wipeEverything, } from '../offline-mode'; -import { flushQueue, settleStaleSends, SENDING_STALE_MS } from '../flush'; -import { claimForSending, holdFile, queueSend, readQueue } from '../queue'; -import { emailIndex, emailKey, forgetIndexes, getBlob, messageOfBlob, putBlob, putEmails, putMisc, removeEmails } from '../cache'; -import { forgetDeviceKey } from '../crypto'; -import { allowOfflineDbCreation, deleteOfflineDb, idbGetAll, openOfflineDb } from '../db'; +import { flushQueue, forgetOrphanCheck, LOOKUP_WAIT_MS, ORPHAN_FILE_AGE_MS, settleStaleSends, SENDING_STALE_MS } from '../flush'; +import { installMayPass } from '@/lib/store/adapters/wasm'; +import { isServerOutage, SendNotStartedError } from '@/lib/jmap/offline-hooks'; +import { claimForSending, holdFile, LOCAL_BLOB_PREFIX, queueSend, readQueue, updateOutboxEntry } from '../queue'; +import { forgetLegacyMove, strandedLegacyMessages } from '../migrate-legacy'; import { consumeLost } from '../lost'; +import { getOutboxHooks } from '../outbox-hooks'; import { shellFiles } from '../shell'; -import { isOfflineNow, OFFLINE_ACCOUNTS_KEY, setServerDown, useOfflineStore, writeEnabled } from '../state'; - -const day = 24 * 60 * 60 * 1000; -const recent = () => new Date(Date.now() - day).toISOString(); +import { DEFAULT_SCOPE } from '@/lib/store/area'; +import { isOfflineNow, OFFLINE_ACCOUNTS_KEY, OFFLINE_EVICTED_KEY, OFFLINE_SCOPE_KEY, offlineScopeFor, setServerDown, useOfflineStore, writeEnabled } from '../state'; function fakeClient(server = 'https://mail.example.org', overrides: Record = {}) { return { @@ -42,9 +46,10 @@ function fakeClient(server = 'https://mail.example.org', overrides: Record 'j1', isOfflineOnly: () => false, getSessionSnapshot: () => ({ apiUrl: `${server}/jmap` }), + getAuthHeader: () => 'Basic x', + keywordSortInverted: () => null, sendEmail: vi.fn(async () => ({ scheduled: false, emailId: 'sent-1', emailSubmissionId: 'sub-1' })), uploadBlob: vi.fn(async (file: File) => ({ blobId: `server-${file.name}`, size: file.size, type: file.type })), - applyEmailPatches: vi.fn(async () => []), pruneEmailSubmissions: vi.fn(async () => {}), ...overrides, }; @@ -59,16 +64,17 @@ function setOnline(online: boolean) { } let stop: () => void = () => {}; +const useRealClient = useAuthStore.getState().getClientForAccount; +let store: InProcessStore; beforeEach(async () => { - forgetIndexes(); - forgetDeviceKey(); - await deleteOfflineDb(); - allowOfflineDbCreation(); + store = new InProcessStore(); + setStoreBackend(store); + forgetLegacyMove(); localStorage.clear(); sessionStorage.clear(); setOnline(true); - useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, browserOffline: false, downServers: [], startedOffline: false }); + useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, conflicts: [], browserOffline: false, downServers: [], startedOffline: false }); setPolicyOrg({}); useAccountStore.setState({ accounts: [{ id: 'acc', label: 'alice', serverUrl: 'https://mail.example.org', username: 'alice@example.org' } as never] }); useAuthStore.setState({ isAuthenticated: true, activeAccountId: 'acc', client: null }); @@ -82,19 +88,20 @@ afterEach(async () => { setOnline(true); setSendGuardSource(null); await wipeEverything(); + setStoreBackend(null); }); describe('1: two tabs never send the same message', () => { it('only one claim wins', async () => { const id = await queueSend('acc', { subject: 'S', to: [] }, { kind: 'email', args: [] }); - const results = await Promise.all([claimForSending(id), claimForSending(id)]); + const results = await Promise.all([claimForSending('acc', id), claimForSending('acc', id)]); expect(results.filter(Boolean)).toHaveLength(1); expect((await readQueue('acc')).outbox[0].status).toBe('sending'); }); it('a message another tab took isn’t sent here', async () => { const id = await queueSend('acc', { subject: 'S', to: [] }, { kind: 'email', args: [['b@example.org'], 'S', 'x'] }); - await claimForSending(id); + await claimForSending('acc', id); const client = fakeClient(); await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); expect(client.sendEmail).not.toHaveBeenCalled(); @@ -112,7 +119,7 @@ describe('1: two tabs never send the same message', () => { }); }); -describe('2: a send that may have reached the server is never queued or sent again', () => { +describe('2: a send that may have reached the server is never queued or sent again blindly', () => { it('puts a send back to waiting only when it never started', async () => { const notStarted = Object.assign(new Error('x'), { name: 'SendNotStartedError' }); const client = fakeClient(undefined, { sendEmail: vi.fn(async () => { throw notStarted; }) }); @@ -158,7 +165,7 @@ describe('3: every sign-out the user didn’t agree to names the lost messages', setOfflineNotifier((n) => notices.push(n)); stop = startOfflineMode(); await queueSend('acc', { subject: 'Lost one', to: [] }, { kind: 'email', args: [] }); - useAccountStore.setState({ accounts: [{ id: 'other' } as never] }); + useAccountStore.getState().removeAccount('acc'); await vi.waitFor(() => expect(notices).toContainEqual({ kind: 'lost', subjects: ['Lost one'] })); setOfflineNotifier(() => {}); }); @@ -186,16 +193,17 @@ describe('4: no queueing while a plugin isn’t running', () => { }); describe('5: another tab switching it off sticks', () => { - it('follows the switch from another tab and doesn’t create the database again', async () => { + it('follows the switch from another tab and doesn’t open the store for it again', async () => { writeEnabled(['acc']); stop = startOfflineMode(); - await putEmails('acc', 'j1', [{ id: 'm1', mailboxIds: { inbox: true }, receivedAt: recent() }]); - // The other tab: switched off, database deleted. - await deleteOfflineDb(); + await vi.waitFor(() => expect(store.has('acc')).toBe(true)); + await new Promise((r) => setTimeout(r, 10)); + store.stop(); window.dispatchEvent(new StorageEvent('storage', { key: OFFLINE_ACCOUNTS_KEY, newValue: null })); expect(useOfflineStore.getState().enabled).toEqual([]); - await expect(openOfflineDb()).rejects.toMatchObject({ name: 'OfflineDbMissingError' }); - expect((await indexedDB.databases()).map((d) => d.name)).not.toContain('bulwark-offline'); + await new Promise((r) => setTimeout(r, 20)); + expect(store.has('acc')).toBe(false); + expect((await indexedDB.databases()).map((d) => d.name)).not.toContain('bulwark-store'); }); }); @@ -216,7 +224,7 @@ describe('6: queued mail and a stricter policy', () => { describe('7: a send of a closed tab isn’t stuck', () => { it('becomes uncertain after a minute', async () => { const id = await queueSend('acc', { subject: 'S', to: [] }, { kind: 'email', args: [] }); - await claimForSending(id); + await claimForSending('acc', id); await settleStaleSends('acc'); expect((await readQueue('acc')).outbox[0].status).toBe('sending'); await settleStaleSends('acc', Date.now() + SENDING_STALE_MS + 1000); @@ -225,21 +233,31 @@ describe('7: a send of a closed tab isn’t stuck', () => { }); describe('8: only parts of kept messages are kept, and go with them', () => { - it('knows which message a blob belongs to and deletes it with the message', async () => { - await putEmails('acc', 'j1', [{ id: 'm1', mailboxIds: { inbox: true }, receivedAt: recent(), bodyValues: {}, attachments: [{ blobId: 'b1', name: 'a.pdf' }] }]); - expect(await messageOfBlob('acc', 'j1', 'b1')).toMatchObject({ key: emailKey('acc', 'j1', 'm1') }); - expect(await messageOfBlob('acc', 'j1', 'export-of-a-whole-message')).toBeNull(); - await putBlob('acc', 'j1', 'b1', new Blob(['x']), Date.now(), emailKey('acc', 'j1', 'm1')); - await removeEmails('acc', [emailKey('acc', 'j1', 'm1')]); - expect(await getBlob('acc', 'j1', 'b1')).toBeNull(); + it('files a blob under its message and deletes it with the message', async () => { + const server = new FakeJmapMail(); + server.addEmail({ id: 'm1', attachments: [{ blobId: 'b1', name: 'a.pdf', size: 99_999_999 }] }); + store = new InProcessStore(server); + setStoreBackend(store); + await store.sync('acc'); + expect(await store.area('acc', 'mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'export-of-a-whole-message', type: null, data: new Uint8Array([1]) })).toBe(false); + expect(await store.area('acc', 'mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'b1', type: null, data: new Uint8Array([1]) })).toBe(true); + server.destroyEmail('m1'); + await store.sync('acc'); + expect(await store.area('acc', 'mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b1' })).toBeNull(); }); }); describe('9: folders that went away', () => { - it('drops opened folders the folder list no longer has', async () => { - await putMisc('acc', 'mailboxes', { j1: [{ id: 'inbox', role: 'inbox' }, { id: 'kept' }] }); - await putMisc('acc', 'opened', { j1: ['kept', 'deleted'] }); - expect((await scopeOf('acc')).opened).toEqual({ j1: ['kept'] }); + it('drops a folder the server no longer has', async () => { + const server = new FakeJmapMail({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'kept' }, { id: 'deleted' }] }); + store = new InProcessStore(server); + setStoreBackend(store); + await store.sync('acc'); + server.mailboxes.delete('deleted'); + server.coll.Mailbox.push('destroyed', 'deleted'); + await store.sync('acc'); + const answer = await store.answer('acc', [['Mailbox/get', {}, '0']]); + expect(answer?.methodResponses[0][1].list.map((box: { id: string }) => box.id).sort()).toEqual(['inbox', 'kept']); }); }); @@ -259,7 +277,6 @@ describe('10: no decision on an account whose groups aren’t known yet', () => describe('11: starting from the device', () => { it('uses the policy kept from the last fetch, never the defaults', async () => { writeEnabled(['acc']); - await putMisc('acc', 'session', { apiUrl: 'x' }); localStorage.setItem('bulwark-policy-last', JSON.stringify({ ...usePolicyStore.getState().policy, organisation: { ...DEFAULT_ORGANISATION_POLICY, devices: { ...DEFAULT_ORGANISATION_POLICY.devices, noOffline: true } } })); usePolicyStore.setState({ loaded: false, fetching: false, source: 'none' }); vi.stubGlobal('fetch', vi.fn(async () => { throw new TypeError('Failed to fetch'); })); @@ -270,6 +287,390 @@ describe('11: starting from the device', () => { vi.unstubAllGlobals(); } }); + + it('hands out the session kept with the account', async () => { + writeEnabled(['acc']); + expect(await offlineSessionFor('acc')).toMatchObject({ primaryAccounts: { 'urn:ietf:params:jmap:mail': JMAP_ACCOUNT } }); + }); + + it('gives up on a policy fetch that never answers', async () => { + writeEnabled(['acc']); + usePolicyStore.setState({ loaded: false, fetching: false, source: 'none' }); + vi.useFakeTimers(); + vi.stubGlobal('fetch', vi.fn(() => new Promise(() => {}))); + try { + const session = offlineSessionFor('acc'); + await vi.advanceTimersByTimeAsync(10_000); + expect(await session).toBeNull(); + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + + it('doesn\'t wait for the old database to finish moving', async () => { + writeEnabled(['acc']); + forgetLegacyMove(); + const legacy = indexedDB.open; + vi.spyOn(indexedDB, 'open').mockImplementation(((name: string, version?: number) => { + // A database that never opens. + if (name === 'bulwark-offline') return { result: null } as never; + return legacy.call(indexedDB, name, version); + }) as never); + vi.useFakeTimers(); + try { + const session = offlineSessionFor('acc'); + await vi.advanceTimersByTimeAsync(10_000); + expect(await session).toMatchObject({ primaryAccounts: { 'urn:ietf:params:jmap:mail': JMAP_ACCOUNT } }); + } finally { + vi.useRealTimers(); + vi.restoreAllMocks(); + } + }); + + it('doesn\'t offer a quick start before the store has its folders', async () => { + store = new InProcessStore(new FakeJmapMail()); + setStoreBackend(store); + writeEnabled(['acc']); + expect(await offlineSessionFor('acc', { withFolders: true })).toBeNull(); + await store.sync('acc'); + expect(await offlineSessionFor('acc', { withFolders: true })).toMatchObject({ primaryAccounts: { 'urn:ietf:params:jmap:mail': JMAP_ACCOUNT } }); + }); +}); + +describe('what a sign-out would delete', () => { + beforeEach(() => writeEnabled(['acc', 'other'])); + + const addCalendarEdit = async (account: string) => { + await store.ensureOpen(account); + store.dbs.get(account)!.run("INSERT INTO calendar_pending (jmap, record_id, kind, patch, queued_at) VALUES ('j1', 'ev', 'update', '{}', 1)"); + }; + + it('counts the queue itself, not what the screen last saw', async () => { + await queueSend('acc', { subject: 'A', to: [] }, { kind: 'email', args: [] }); + await queueSend('other', { subject: 'B', to: [] }, { kind: 'email', args: [] }); + expect(useOfflineStore.getState().outbox).toEqual([]); + expect(await unsentCount('acc')).toEqual({ messages: 1, changes: 0 }); + expect(await unsentCount('all')).toEqual({ messages: 2, changes: 0 }); + }); + + it('counts changes and calendar edits, for the account asked about', async () => { + await addCalendarEdit('acc'); + await store.area('acc', 'outbox.queueSend', { summary: { subject: 'A', to: [] }, item: { kind: 'email', args: [] } }); + expect(await unsentCount('other')).toEqual({ messages: 0, changes: 0 }); + expect(await unsentCount('acc')).toEqual({ messages: 1, changes: 1 }); + }); + + it('counts an area that isn\'t here as nothing', async () => { + store.failArea = (name) => name === 'contacts.pending'; + await addCalendarEdit('acc'); + expect((await unsentCount('acc')).changes).toBe(1); + }); + + it('answers with the last known count when the store hangs', async () => { + useOfflineStore.setState({ outbox: [{ id: 1, account: 'acc', createdAt: 1, subject: 'A', to: [], status: 'waiting' }] as never, pendingChanges: 2 }); + const hung: Array<(error: Error) => void> = []; + vi.spyOn(store, 'area').mockImplementation(() => new Promise((_resolve, reject) => { hung.push(reject); })); + expect(await unsentCount('acc', 10)).toEqual({ messages: 1, changes: 2 }); + for (const reject of hung) reject(new Error('Gone')); + }); + + it('doesn\'t count another signed-out account\'s waiting mail', async () => { + await queueSend('other', { subject: 'B', to: [] }, { kind: 'email', args: [] }); + localStorage.setItem(OFFLINE_EVICTED_KEY, JSON.stringify({ other: Date.now() })); + expect(await unsentCount('all')).toEqual({ messages: 0, changes: 0 }); + }); +}); + +describe('a store that won\'t open', () => { + beforeEach(() => writeEnabled(['acc', 'other'])); + + it('holds up no other account, and keeps what it last showed', async () => { + await queueSend('acc', { subject: 'A', to: [] }, { kind: 'email', args: [] }); + await queueSend('other', { subject: 'B', to: [] }, { kind: 'email', args: [] }); + await refreshQueueState(); + const area = store.area.bind(store); + vi.spyOn(store, 'area').mockImplementation(((account: string, name: string, params: unknown) => (account === 'other' ? Promise.reject(new Error('The local store could not start')) : area(account, name, params))) as never); + const notify = vi.fn(); + setOfflineNotifier(notify); + await refreshQueueState(); + expect(useOfflineStore.getState().outbox.map((row) => row.subject).sort()).toEqual(['A', 'B']); + expect(notify).toHaveBeenCalledWith({ kind: 'store-unavailable' }); + // Not again right away. + notify.mockClear(); + await refreshQueueState(); + expect(notify).not.toHaveBeenCalled(); + }); + + it('doesn\'t call an unreadable queue empty', async () => { + await queueSend('acc', { subject: 'A', to: [] }, { kind: 'email', args: [] }); + await refreshQueueState(); + vi.spyOn(store, 'area').mockRejectedValue(new Error('The local store could not start')); + await refreshQueueState(); + expect(useOfflineStore.getState().outbox.map((row) => row.subject)).toEqual(['A']); + await expect(readQueue()).rejects.toThrow('could not start'); + }); +}); + +describe('a send the server couldn\'t take right now', () => { + const statusAfter = async (error: Error) => { + writeEnabled(['acc']); + await queueSend('acc', { subject: 'Hello', to: [] }, { kind: 'email', args: [] }); + const notify = vi.fn(); + const retryLater = vi.fn(); + const sendEmail = vi.fn(async () => { throw error; }); + await flushQueue('acc', fakeClient('https://mail.example.org', { sendEmail }) as unknown as JMAPClient, { notify, allowed: () => true, retryLater }); + const row = (await readQueue('acc')).outbox[0]; + return { row, notify, retryLater }; + }; + + it.each([ + ['a rate limit', Object.assign(new Error('Rate limited by server'), { name: 'RateLimitError' })], + ['a login that needs renewing', new Error('Request failed: 401 - ')], + ['a store that went away', Object.assign(new Error('gone'), { name: 'StoreOwnerGoneError' })], + ['a full disk', Object.assign(new Error('full'), { name: 'QuotaExceededError' })], + ])('goes back to waiting, on a timer, after %s', async (_name, error) => { + const { row, notify, retryLater } = await statusAfter(error); + expect(row.status).toBe('waiting'); + expect(retryLater).toHaveBeenCalled(); + expect(notify).not.toHaveBeenCalled(); + }); + + it.each(['Request failed: 500 - oops', 'Request failed: 501 - nope'])('is looked up after %s, which may have got through', async (message) => { + const { row, notify, retryLater } = await statusAfter(new Error(message)); + expect(row.status).toBe('uncertain'); + expect(retryLater).toHaveBeenCalled(); + expect(notify).not.toHaveBeenCalled(); + }); + + it('fails, for the user to see, on a refusal', async () => { + const { row, notify } = await statusAfter(new Error('Request failed: 400 - {"type":"urn:ietf:params:jmap:error:notRequest"}')); + expect(row.status).toBe('failed'); + expect(notify).toHaveBeenCalledWith(expect.objectContaining({ kind: 'send-failed' })); + }); + + it('keeps the copy on the server through a wait', async () => { + writeEnabled(['acc']); + const id = await queueSend('acc', { subject: 'Hello', to: [] }, { kind: 'email', args: [] }); + const [entry] = (await readQueue('acc')).outbox; + await updateOutboxEntry(entry, { status: 'waiting', copyId: 'copy-1' }); + const sendEmail = vi.fn(async () => { throw new Error('Request failed: 429 - '); }); + await flushQueue('acc', fakeClient('https://mail.example.org', { sendEmail }) as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + const row = (await readQueue('acc')).outbox.find((candidate) => candidate.id === id); + expect(row).toMatchObject({ status: 'waiting', copyId: 'copy-1' }); + }); + + it('doesn\'t hand an online send a stand-in for a file the upload held again', async () => { + writeEnabled(['acc']); + stop = startOfflineMode(); + const { blobId } = await holdFile('acc', new File(['x'], 'x.txt', { type: 'text/plain' }), 'x.txt'); + const client = fakeClient('https://mail.example.org', { uploadBlob: vi.fn(async () => ({ blobId: `${LOCAL_BLOB_PREFIX}stand-in`, size: 1, type: 'text/plain' })) }); + useAuthStore.setState({ client: client as never, activeAccountId: 'acc', getClientForAccount: () => client } as never); + try { + await expect(getOutboxHooks()!.resolveHeldAttachments(client as never, [{ blobId }])).rejects.toMatchObject({ name: 'SendNotStartedError' }); + } finally { + useAuthStore.setState({ getClientForAccount: useRealClient, client: null }); + } + }); + + it('doesn\'t send a stand-in for a file the upload held again', async () => { + writeEnabled(['acc']); + const { blobId } = await holdFile('acc', new File(['x'], 'x.txt', { type: 'text/plain' }), 'x.txt'); + await queueSend('acc', { subject: 'Hello', to: [] }, { kind: 'email', args: [undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, [{ blobId, name: 'x.txt', type: 'text/plain', size: 1 }]] }); + const sendEmail = vi.fn(async () => ({ scheduled: false, emailId: 'e', emailSubmissionId: 's' })); + const uploadBlob = vi.fn(async () => ({ blobId: `${LOCAL_BLOB_PREFIX}stand-in`, size: 1, type: 'text/plain' })); + await flushQueue('acc', fakeClient('https://mail.example.org', { sendEmail, uploadBlob }) as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(sendEmail).not.toHaveBeenCalled(); + expect((await readQueue('acc')).outbox[0].status).toBe('waiting'); + }); +}); + +describe('waiting mail after a normal start', () => { + const realFetchPolicy = usePolicyStore.getState().fetchPolicy; + + beforeEach(() => { + vi.useFakeTimers(); + // rawPolicy: what the policy is worked out from again when the subject changes. + usePolicyStore.setState({ fetchPolicy: vi.fn(async () => {}), source: 'server', rawPolicy: usePolicyStore.getState().policy } as never); + useOfflineStore.setState({ downServers: [] }); + writeEnabled(['acc']); + }); + + afterEach(() => { + vi.useRealTimers(); + usePolicyStore.setState({ fetchPolicy: realFetchPolicy }); + useOfflineStore.setState({ downServers: [] }); + }); + + it('goes when the client becomes real after the flush at the start', async () => { + await queueSend('acc', { subject: 'Hello', to: [] }, { kind: 'email', args: [] }); + stop = startOfflineMode(); + // The flush at the start finds no client yet. + await vi.advanceTimersByTimeAsync(5_000); + const client = fakeClient(); + useAuthStore.setState({ getClientForAccount: () => client, client: client as never } as never); + try { + await vi.advanceTimersByTimeAsync(1_500); + expect(client.sendEmail).toHaveBeenCalledTimes(1); + } finally { + useAuthStore.setState({ getClientForAccount: useRealClient, client: null }); + } + }); + + it('comes back to a send a closed tab left behind', async () => { + const id = await queueSend('acc', { subject: 'Hello', to: [] }, { kind: 'email', args: [] }); + await claimForSending('acc', id); + stop = startOfflineMode(); + await vi.advanceTimersByTimeAsync(5_000); + expect((await readQueue('acc')).outbox[0].status).toBe('sending'); + await vi.advanceTimersByTimeAsync(SENDING_STALE_MS); + expect((await readQueue('acc')).outbox[0].status).toBe('uncertain'); + }); + + it('asks a server that was down, on an interval, and flushes when it answers', async () => { + const fetcher = vi.fn(async (_url: string) => new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetcher); + try { + stop = startOfflineMode(); + await vi.advanceTimersByTimeAsync(5_000); + setServerDown('https://mail.example.org', true); + await vi.advanceTimersByTimeAsync(30_000); + expect(fetcher.mock.calls.map(([url]) => String(url))).toContain('https://mail.example.org/.well-known/jmap'); + expect(isOfflineNow('https://mail.example.org')).toBe(false); + } finally { + vi.unstubAllGlobals(); + } + }); +}); + +describe('old files', () => { + it('go after a flush, once a day, not a file that\'s still recent', async () => { + forgetOrphanCheck(); + writeEnabled(['acc']); + const { blobId } = await holdFile('acc', new File(['x'], 'x.txt', { type: 'text/plain' }), 'x.txt'); + const client = fakeClient() as unknown as JMAPClient; + const flush = () => flushQueue('acc', client, { notify: vi.fn(), allowed: () => true }); + await flush(); + expect(await store.area('acc', 'outbox.heldFile', { blobId })).not.toBeNull(); + + const area = vi.spyOn(store, 'area'); + await flush(); + expect(area.mock.calls.some(([, name]) => name === 'outbox.dropOrphans')).toBe(false); + + const later = Date.now() + ORPHAN_FILE_AGE_MS + 1000; + const clock = vi.spyOn(Date, 'now').mockReturnValue(later); + try { + await flush(); + expect(area.mock.calls.some(([, name]) => name === 'outbox.dropOrphans')).toBe(true); + expect(await store.area('acc', 'outbox.heldFile', { blobId })).toBeNull(); + } finally { + clock.mockRestore(); + area.mockRestore(); + } + }); +}); + +describe('a browser that can\'t run the store', () => { + it('says so when the old database holds unsent mail', async () => { + await new Promise((resolve) => { + const request = indexedDB.open('bulwark-offline', 1); + request.onupgradeneeded = () => { request.result.createObjectStore('queue', { keyPath: 'id' }); }; + request.onsuccess = () => { + const put = request.result.transaction('queue', 'readwrite').objectStore('queue'); + put.put({ id: 1, kind: 'patch' }); + put.put({ id: 2, kind: 'send' }); + put.transaction.oncomplete = () => { request.result.close(); resolve(); }; + }; + }); + expect(await strandedLegacyMessages()).toBe(1); + store.isSupported = false; + const notify = vi.fn(); + setOfflineNotifier(notify); + stop = startOfflineMode(); + await vi.waitFor(() => expect(notify).toHaveBeenCalledWith({ kind: 'legacy-stranded' })); + store.isSupported = true; + await new Promise((resolve) => { + const remove = indexedDB.deleteDatabase('bulwark-offline'); + remove.onsuccess = () => resolve(); + }); + expect(await strandedLegacyMessages()).toBe(0); + }); + + it('turns the switch off in a window that refuses the private file system, and wipes nothing', async () => { + writeEnabled(['acc']); + Object.defineProperty(window.navigator, 'storage', { configurable: true, value: { getDirectory: async () => { throw new DOMException('The operation is insecure.', 'SecurityError'); } } }); + try { + stop = startOfflineMode(); + await vi.waitFor(() => expect(useOfflineStore.getState().storageBlocked).toBe(true)); + expect(offlineSupported()).toBe(false); + enforcePolicy(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(useOfflineStore.getState().enabled).toEqual(['acc']); + } finally { + delete (window.navigator as { storage?: unknown }).storage; + useOfflineStore.setState({ storageBlocked: false }); + } + }); + + it('doesn\'t retry the install where the browser refuses it', () => { + expect(installMayPass(new DOMException('The operation is insecure.', 'SecurityError'))).toBe(false); + expect(installMayPass(new DOMException('Handle is busy', 'NoModificationAllowedError'))).toBe(true); + }); +}); + +describe('the wording of an outage', () => { + it('tells a proxy with no server behind it from a missing connection', () => { + const gateway = Object.assign(new Error('Request failed: 502'), { name: 'GatewayError' }); + expect(isServerOutage(gateway)).toBe(true); + expect(isServerOutage(new SendNotStartedError(gateway))).toBe(true); + expect(isServerOutage(new SendNotStartedError(new TypeError('Failed to fetch')))).toBe(false); + expect(isServerOutage(new TypeError('Failed to fetch'))).toBe(false); + }); +}); + +describe('looking up a send that may have gone', () => { + it('waits until about 30 seconds after the last try', async () => { + writeEnabled(['acc']); + const id = await queueSend('acc', { subject: 'Hello', to: [] }, { kind: 'email', args: [] }, undefined, 'fixed@example.org'); + await claimForSending('acc', id); + await updateOutboxEntry({ account: 'acc', id }, { status: 'uncertain' }); + const requestNetworkOnly = vi.fn(async () => { throw new TypeError('Failed to fetch'); }); + const client = fakeClient('https://mail.example.org', { requestNetworkOnly, resolveSendAccountId: async () => 'j1', submissionAccountFor: () => 'j1' }) as unknown as JMAPClient; + const retryLater = vi.fn(); + await flushQueue('acc', client, { notify: vi.fn(), allowed: () => true, retryLater }); + expect(requestNetworkOnly).not.toHaveBeenCalled(); + expect(retryLater).toHaveBeenCalled(); + const later = Date.now() + LOOKUP_WAIT_MS + 1000; + const clock = vi.spyOn(Date, 'now').mockReturnValue(later); + try { + await flushQueue('acc', client, { notify: vi.fn(), allowed: () => true, retryLater }); + expect(requestNetworkOnly).toHaveBeenCalled(); + } finally { + clock.mockRestore(); + } + }); +}); + +describe('what the store is told to keep', () => { + it('follows what the account chose, on top of the defaults', () => { + expect(offlineScopeFor('acc')).toEqual(DEFAULT_SCOPE); + localStorage.setItem(OFFLINE_SCOPE_KEY, JSON.stringify({ acc: { attachments: false, days: 7 } })); + expect(offlineScopeFor('acc')).toEqual({ ...DEFAULT_SCOPE, attachments: false, days: 7 }); + expect(offlineScopeFor('other')).toEqual(DEFAULT_SCOPE); + }); +}); + +describe('a send on the local copy', () => { + it('queues, before the swap too, while the probes say the server is up', () => { + writeEnabled(['acc']); + stop = startOfflineMode(); + useAuthStore.setState({ client: fakeClient('https://mail.example.org', { isOfflineOnly: () => true }) as never }); + const client = useAuthStore.getState().client as never; + expect(isOfflineNow('https://mail.example.org')).toBe(false); + expect(getOutboxHooks()?.shouldQueue(client)).toBe(true); + useAuthStore.setState({ client: fakeClient() as never }); + expect(getOutboxHooks()?.shouldQueue(useAuthStore.getState().client as never)).toBe(false); + }); }); describe('12: one server down doesn’t hold another’s mail', () => { @@ -281,10 +682,13 @@ describe('12: one server down doesn’t hold another’s mail', () => { }); describe('13: file names stay sealed', () => { - it('keeps no name or type of a held file readable', async () => { - await holdFile('acc', new File(['x'], 'merger-plan.pdf', { type: 'application/pdf' }), 'merger-plan.pdf'); - const records = await idbGetAll>('uploads'); - expect(JSON.stringify(records.map((r) => ({ ...r, data: undefined, iv: undefined, meta: undefined })))).not.toMatch(/merger|pdf/); + it('keeps the name and type of a held file only in the encrypted store', async () => { + const held = await holdFile('acc', new File(['x'], 'merger-plan.pdf', { type: 'application/pdf' }), 'merger-plan.pdf'); + expect(held.blobId).not.toMatch(/merger|pdf/); + const db = store.dbs.get('acc')!; + expect(db.get<{ name: string }>('SELECT name FROM upload')?.name).toBe('merger-plan.pdf'); + const everythingElse = JSON.stringify({ local: { ...localStorage }, session: { ...sessionStorage }, dbs: await indexedDB.databases() }); + expect(everythingElse).not.toMatch(/merger/); }); }); @@ -318,25 +722,3 @@ describe('15: only the plugin running the composer’s send counts', () => { expect(completesComposeTakeover('smime', 'i1')).toBe(false); }); }); - -describe('recording: what the cache takes', () => { - it('keeps recent mail of the Inbox and opened folders, refreshes kept messages, skips the rest', async () => { - const scope = { inbox: { j1: 'inbox' }, opened: { j1: ['projects'] } }; - expect(inScope(scope, 'j1', { id: 'a', mailboxIds: { inbox: true }, receivedAt: recent() })).toBe(true); - expect(inScope(scope, 'j1', { id: 'b', mailboxIds: { projects: true }, receivedAt: recent() })).toBe(true); - expect(inScope(scope, 'j1', { id: 'c', mailboxIds: { archive: true }, receivedAt: recent() })).toBe(false); - expect(inScope(scope, 'j1', { id: 'd', mailboxIds: { inbox: true }, receivedAt: new Date(Date.now() - 40 * day).toISOString() })).toBe(false); - - await putMisc('acc', 'mailboxes', { j1: [{ id: 'inbox', role: 'inbox', name: 'Inbox' }] }); - await recordAnswer('acc', 'j1', [['Email/get', { ids: ['a', 'c'] }, '0'], ['Mailbox/get', { properties: ['id', 'role'] }, '1']], { - methodResponses: [ - ['Email/get', { accountId: 'j1', list: [{ id: 'a', mailboxIds: { inbox: true }, receivedAt: recent() }, { id: 'c', mailboxIds: { archive: true }, receivedAt: recent() }] }, '0'], - ['Mailbox/get', { accountId: 'j1', list: [{ id: 'inbox', role: 'inbox' }] }, '1'], - ], - }); - const index = await emailIndex('acc'); - expect([...index.values()].map((e) => e.id)).toEqual(['a']); - // A partial folder list (id and role only) doesn't replace the full one. - expect((await scopeOf('acc')).inbox).toEqual({ j1: 'inbox' }); - }); -}); diff --git a/lib/offline/cache.ts b/lib/offline/cache.ts deleted file mode 100644 index 0411a5bde..000000000 --- a/lib/offline/cache.ts +++ /dev/null @@ -1,310 +0,0 @@ -/** - * The mail kept on this device for offline mode (#523), per Bulwark account - * (`account`, the local account id) and JMAP account (`jmap`). - * - * Messages are kept in two parts: what the list shows (emails) and the text - * and structure of the message (bodies), so a folder list never decrypts - * the bodies. Attachments and inline images are blobs. Every value is - * encrypted (crypto.ts); only the ledger (size, date) and the keys (ids) are - * not. - * - * The cache holds at most CACHE_CAP_BYTES on the device, across accounts; - * the oldest messages go first, with their bodies and attachments. The - * outbox (queue.ts) doesn't count and never goes. - */ - -import { idbDeleteAccount, idbGet, idbGetAll, idbGetAllByAccount, idbWrite, type StoreName } from './db'; -import { openBytes, openJson, sealBytes, sealJson, type Sealed } from './crypto'; - -/** Most bytes the cache keeps on the device (messages, bodies, attachments). */ -export const CACHE_CAP_BYTES = 250 * 1024 * 1024; -/** How far back the Inbox (and the folders the user opened) are kept. */ -export const CACHE_DAYS = 30; -export const CACHE_WINDOW_MS = CACHE_DAYS * 24 * 60 * 60 * 1000; - -/** Where the metadata keeps the blob ids of a message's parts (never handed out). */ -export const BLOBS = '_bulwarkBlobs'; - -/** The parts of a message that only the reader needs. */ -const BODY_KEYS = ['bodyValues', 'textBody', 'htmlBody', 'bodyStructure', 'headers', 'attachments'] as const; - -export type CachedEmail = Record & { - id: string; - threadId?: string; - mailboxIds?: Record; - keywords?: Record; - receivedAt?: string; -}; - -interface SealedRecord extends Sealed { - key: string; - account: string; -} - -interface LedgerEntry { - key: string; - account: string; - store: StoreName; - ref: string; - /** The message's date (ms), for oldest-first eviction. */ - at: number; - bytes: number; - /** The message a body or blob belongs to. */ - emailKey?: string; -} - -const join = (...parts: string[]) => parts.join('\u0001'); -export const emailKey = (account: string, jmap: string, id: string) => join(account, jmap, id); -const blobKey = (account: string, jmap: string, blobId: string) => join(account, jmap, 'blob', blobId); -const miscKey = (account: string, name: string) => join(account, 'misc', name); -const ledgerKey = (store: StoreName, key: string) => `${store}:${key}`; - -const dateOf = (email: { receivedAt?: unknown }) => { - const at = typeof email.receivedAt === 'string' ? Date.parse(email.receivedAt) : NaN; - return Number.isFinite(at) ? at : 0; -}; - -const byteSize = (sealed: Sealed) => sealed.data.byteLength + 64; - -// Decrypted message metadata per account, loaded on first use. -const indexes = new Map>>(); - -/** The decrypted metadata of every cached message of an account, by emailKey. */ -export function emailIndex(account: string): Promise> { - let index = indexes.get(account); - if (!index) { - index = (async () => { - const map = new Map(); - const records = await idbGetAllByAccount('emails', account); - for (const record of records) { - try { - map.set(record.key, await openJson(record)); - } catch { - // Sealed with a key that is gone: dropped at the next write. - } - } - return map; - })(); - index.catch(() => indexes.delete(account)); - indexes.set(account, index); - } - return index; -} - -/** Forget what is held in memory (after a wipe). */ -export function forgetIndexes(account?: string): void { - if (account) indexes.delete(account); - else indexes.clear(); -} - -function splitBody(email: CachedEmail): { meta: CachedEmail; body: Record | null } { - const meta: CachedEmail = { ...email }; - let body: Record | null = null; - for (const key of BODY_KEYS) { - if (key in meta) { - body ??= {}; - body[key] = meta[key]; - delete meta[key]; - } - } - // Only a fetch with the text counts as the message's body. - return { meta, body: body && 'bodyValues' in body ? body : null }; -} - -/** Keep messages (as the server sent them, any properties) and their bodies when they came with one. */ -export async function putEmails(account: string, jmap: string, emails: CachedEmail[]): Promise { - if (emails.length === 0) return; - const index = await emailIndex(account); - const ops: Array<{ store: StoreName; put?: unknown; del?: IDBValidKey }> = []; - for (const email of emails) { - if (!email || typeof email.id !== 'string') continue; - const key = emailKey(account, jmap, email.id); - const { meta, body } = splitBody(email); - const merged = { ...(index.get(key) ?? {}), ...meta } as CachedEmail; - // The attachments and inline images of the message: only these blobs are kept (blobsOf). - if (body && Array.isArray(body.attachments)) { - merged[BLOBS] = (body.attachments as Array<{ blobId?: unknown }>).map((a) => a.blobId).filter((id): id is string => typeof id === 'string'); - } - index.set(key, merged); - const at = dateOf(merged); - const sealed = await sealJson(merged); - ops.push({ store: 'emails', put: { key, account, ...sealed } }); - ops.push({ store: 'ledger', put: { key: ledgerKey('emails', key), account, store: 'emails', ref: key, at, bytes: byteSize(sealed), emailKey: key } satisfies LedgerEntry }); - if (body) { - const sealedBody = await sealJson(body); - ops.push({ store: 'bodies', put: { key, account, ...sealedBody } }); - ops.push({ store: 'ledger', put: { key: ledgerKey('bodies', key), account, store: 'bodies', ref: key, at, bytes: byteSize(sealedBody), emailKey: key } satisfies LedgerEntry }); - } - } - await idbWrite(ops); -} - -/** Apply a change made to messages (keywords, folders), here or on the server. */ -export async function patchEmails(account: string, jmap: string, patches: Record>): Promise { - const index = await emailIndex(account); - const changed: CachedEmail[] = []; - for (const [id, patch] of Object.entries(patches)) { - const current = index.get(emailKey(account, jmap, id)); - if (!current) continue; - changed.push(applyPatch(current, patch)); - } - if (changed.length) await putEmails(account, jmap, changed); -} - -/** A JMAP PatchObject (RFC 8620 §5.3) applied to a message, for the properties offline mode changes. */ -export function applyPatch(email: CachedEmail, patch: Record): CachedEmail { - const next: CachedEmail = { ...email, keywords: { ...(email.keywords ?? {}) }, mailboxIds: { ...(email.mailboxIds ?? {}) } }; - for (const [path, value] of Object.entries(patch)) { - const [prop, ...rest] = path.split('/'); - if (rest.length === 0) { - (next as Record)[prop] = value; - continue; - } - if (prop !== 'keywords' && prop !== 'mailboxIds') continue; - const name = rest.join('/').replace(/~1/g, '/').replace(/~0/g, '~'); - const map = next[prop] as Record; - if (value === null || value === false) delete map[name]; - else map[name] = true; - } - return next; -} - -export async function removeEmails(account: string, keys: string[]): Promise { - if (keys.length === 0) return; - const index = await emailIndex(account); - const ops: Array<{ store: StoreName; del: IDBValidKey }> = []; - const gone = new Set(keys); - for (const key of keys) { - index.delete(key); - ops.push({ store: 'emails', del: key }, { store: 'bodies', del: key }); - ops.push({ store: 'ledger', del: ledgerKey('emails', key) }, { store: 'ledger', del: ledgerKey('bodies', key) }); - } - // Their attachments go with them. - for (const entry of await idbGetAllByAccount('ledger', account)) { - if (entry.store === 'blobs' && entry.emailKey && gone.has(entry.emailKey)) { - ops.push({ store: 'blobs', del: entry.ref }, { store: 'ledger', del: entry.key }); - } - } - await idbWrite(ops); -} - -/** The kept message a blob (attachment, inline image) belongs to, with its date. */ -export async function messageOfBlob(account: string, jmap: string, blobId: string): Promise<{ key: string; at: number } | null> { - const index = await emailIndex(account); - const prefix = `${account}\u0001${jmap}\u0001`; - for (const [key, email] of index) { - if (!key.startsWith(prefix)) continue; - const blobs = email[BLOBS]; - if (Array.isArray(blobs) && blobs.includes(blobId)) return { key, at: dateOf(email) }; - } - return null; -} - -/** A message's body parts, or null when only its list entry is kept. */ -export async function getBody(account: string, jmap: string, id: string): Promise | null> { - const record = await idbGet('bodies', emailKey(account, jmap, id)); - if (!record) return null; - try { - return await openJson>(record); - } catch { - return null; - } -} - -export async function hasBody(account: string, jmap: string, id: string): Promise { - return (await idbGet('bodies', emailKey(account, jmap, id))) !== undefined; -} - -export async function putBlob(account: string, jmap: string, blobId: string, blob: Blob, at: number, ofEmail?: string): Promise { - const key = blobKey(account, jmap, blobId); - const sealed = await sealBytes(await blob.arrayBuffer()); - // The media type too says something about the message: sealed as well. - const meta = await sealJson({ type: blob.type }); - await idbWrite([ - { store: 'blobs', put: { key, account, meta, ...sealed } }, - { store: 'ledger', put: { key: ledgerKey('blobs', key), account, store: 'blobs', ref: key, at, bytes: byteSize(sealed), emailKey: ofEmail } satisfies LedgerEntry }, - ]); -} - -export async function getBlob(account: string, jmap: string, blobId: string): Promise { - const record = await idbGet('blobs', blobKey(account, jmap, blobId)); - if (!record) return null; - try { - const meta = record.meta ? await openJson<{ type?: string }>(record.meta) : {}; - return new Blob([await openBytes(record)], { type: meta.type || 'application/octet-stream' }); - } catch { - return null; - } -} - -export async function hasBlob(account: string, jmap: string, blobId: string): Promise { - return (await idbGet('blobs', blobKey(account, jmap, blobId))) !== undefined; -} - -/** Small named values of an account (session, folders, identities, opened folders). */ -export async function putMisc(account: string, name: string, value: unknown): Promise { - const key = miscKey(account, name); - await idbWrite([{ store: 'misc', put: { key, account, ...(await sealJson(value)) } }]); -} - -export async function getMisc(account: string, name: string): Promise { - const record = await idbGet('misc', miscKey(account, name)); - if (!record) return null; - try { - return await openJson(record); - } catch { - return null; - } -} - -/** Bytes the cache uses on this device, and per account. */ -export async function cacheUsage(): Promise<{ total: number; byAccount: Record }> { - const entries = await idbGetAll('ledger'); - const byAccount: Record = {}; - let total = 0; - for (const entry of entries) { - total += entry.bytes; - byAccount[entry.account] = (byAccount[entry.account] ?? 0) + entry.bytes; - } - return { total, byAccount }; -} - -/** - * Keeps the cache under `cap`: the oldest messages go first, each with its - * body and attachments. Returns the bytes freed. - */ -export async function enforceCap(cap: number = CACHE_CAP_BYTES): Promise { - const entries = await idbGetAll('ledger'); - let total = entries.reduce((sum, entry) => sum + entry.bytes, 0); - if (total <= cap) return 0; - // Group by message: a message and everything that belongs to it go together. - const groups = new Map(); - for (const entry of entries) { - const id = entry.emailKey ?? entry.key; - const group = groups.get(id) ?? { at: entry.at, bytes: 0, account: entry.account, entries: [] }; - group.at = Math.max(group.at, entry.at); - group.bytes += entry.bytes; - group.entries.push(entry); - groups.set(id, group); - } - const oldestFirst = [...groups.values()].sort((a, b) => a.at - b.at); - let freed = 0; - const ops: Array<{ store: StoreName; del: IDBValidKey }> = []; - for (const group of oldestFirst) { - if (total <= cap) break; - for (const entry of group.entries) { - ops.push({ store: entry.store, del: entry.ref }, { store: 'ledger', del: entry.key }); - if (entry.store === 'emails') (await indexes.get(entry.account))?.delete(entry.ref); - } - total -= group.bytes; - freed += group.bytes; - } - await idbWrite(ops); - return freed; -} - -/** Everything cached for one account (not its outbox). */ -export async function wipeAccountCache(account: string): Promise { - forgetIndexes(account); - await idbDeleteAccount(account, ['emails', 'bodies', 'blobs', 'misc', 'ledger']); -} diff --git a/lib/offline/credentials-rejected.ts b/lib/offline/credentials-rejected.ts new file mode 100644 index 000000000..a42282998 --- /dev/null +++ b/lib/offline/credentials-rejected.ts @@ -0,0 +1,26 @@ +/** + * The server really refused this login (`invalid_grant`), which the auth store + * follows with a sign-out that wipes the account's local data. An outage or any + * other refusal doesn't come here, and keeps the data. HOOK: lib/store/wipe + * registers here for what else a permanent rejection should do. Free of the + * stores, so the auth store can call it. + */ + +type Listener = (accountId: string) => void; + +const listeners = new Set(); + +export function onCredentialsRejected(listener: Listener): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function noteCredentialsRejected(accountId: string): void { + for (const listener of [...listeners]) { + try { + listener(accountId); + } catch { + // A sign-out must go on whatever a listener does. + } + } +} diff --git a/lib/offline/crypto.ts b/lib/offline/crypto.ts deleted file mode 100644 index 13b53b01c..000000000 --- a/lib/offline/crypto.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Encryption at rest for offline mode (#523): AES-GCM with a 256-bit key - * the browser generates as non-extractable and keeps in the offline - * database. Script on this origin can't read the key's bytes, only use it. - * - * What it protects against: someone who copies the browser profile's files - * (a backup, a stolen disk image) without being able to run this site in - * that profile. What it doesn't: someone using the unlocked browser profile, - * or a script running on this origin, can decrypt everything. The docs say - * so and recommend disk encryption. - */ - -import { idbGet, idbWrite } from './db'; - -export interface Sealed { - iv: Uint8Array; - data: ArrayBuffer; -} - -let keyPromise: Promise | null = null; - -async function loadOrCreateKey(): Promise { - const stored = await idbGet<{ id: string; key: CryptoKey }>('keys', 'aes'); - if (stored?.key) return stored.key; - const key = await crypto.subtle.generateKey({ name: 'AES-GCM', length: 256 }, false, ['encrypt', 'decrypt']); - await idbWrite([{ store: 'keys', put: { id: 'aes', key } }]); - return key; -} - -function deviceKey(): Promise { - if (!keyPromise) { - keyPromise = loadOrCreateKey(); - keyPromise.catch(() => { keyPromise = null; }); - } - return keyPromise; -} - -/** Forget the key in memory (the database with it was deleted). */ -export function forgetDeviceKey(): void { - keyPromise = null; -} - -export async function sealBytes(bytes: ArrayBuffer | Uint8Array): Promise { - const iv = crypto.getRandomValues(new Uint8Array(12)); - const data = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, await deviceKey(), bytes as BufferSource); - return { iv, data }; -} - -export async function openBytes(sealed: Sealed): Promise { - return crypto.subtle.decrypt({ name: 'AES-GCM', iv: sealed.iv as BufferSource }, await deviceKey(), sealed.data); -} - -export async function sealJson(value: unknown): Promise { - return sealBytes(new TextEncoder().encode(JSON.stringify(value))); -} - -export async function openJson(sealed: Sealed): Promise { - return JSON.parse(new TextDecoder().decode(await openBytes(sealed))) as T; -} diff --git a/lib/offline/db.ts b/lib/offline/db.ts deleted file mode 100644 index 066910a49..000000000 --- a/lib/offline/db.ts +++ /dev/null @@ -1,219 +0,0 @@ -/** - * The IndexedDB database of offline mode (#523): one per browser profile, - * deleted as a whole at a full sign-out (lib/sign-out-cleanup.ts) and when - * the last account switches offline mode off. - * - * Stores (every value but the key and the ledger is encrypted, see crypto.ts): - * - keys: the AES-GCM key, non-extractable. - * - emails: message metadata (what the list shows), per account and JMAP account. - * - bodies: the text of a message (bodyValues and the parts that say how to show it). - * - blobs: attachments and inline images. - * - misc: the session, folders, identities, the folders the user opened. - * - ledger: size and date of each cached record, unencrypted, so the cache - * can be kept under its cap without decrypting everything. - * - queue: changes and messages waiting for the network (never evicted). - * - uploads: files attached while offline, until their message is sent. - */ - -export const OFFLINE_DB_NAME = 'bulwark-offline'; -const VERSION = 1; - -export const STORES = ['keys', 'emails', 'bodies', 'blobs', 'misc', 'ledger', 'queue', 'uploads'] as const; -export type StoreName = (typeof STORES)[number]; - -let dbPromise: Promise | null = null; -/** - * Only switching offline mode on may create the database. Everything else - * opens it only when it exists: another tab may have deleted it (offline - * mode switched off, a sign-out), and opening it would create it anew. - */ -let mayCreate = false; - -export function allowOfflineDbCreation(): void { - mayCreate = true; -} - -async function offlineDbExists(): Promise { - try { - if (typeof indexedDB.databases !== 'function') return true; - return (await indexedDB.databases()).some((d) => d.name === OFFLINE_DB_NAME); - } catch { - return true; - } -} - -/** Thrown when the database was deleted (or never made) and may not be created here. */ -export class OfflineDbMissingError extends Error { - constructor() { - super('The offline database is not set up'); - this.name = 'OfflineDbMissingError'; - } -} - -export function hasIndexedDb(): boolean { - try { - return typeof indexedDB !== 'undefined' && indexedDB !== null; - } catch { - return false; - } -} - -export function openOfflineDb(): Promise { - if (dbPromise) return dbPromise; - const opening = (async () => { - if (!mayCreate && !(await offlineDbExists())) throw new OfflineDbMissingError(); - return openNow(); - })(); - dbPromise = opening; - opening.catch(() => { - if (dbPromise === opening) dbPromise = null; - }); - return opening; -} - -function openNow(): Promise { - return new Promise((resolve, reject) => { - const request = indexedDB.open(OFFLINE_DB_NAME, VERSION); - request.onupgradeneeded = () => { - const db = request.result; - for (const name of STORES) { - if (db.objectStoreNames.contains(name)) continue; - if (name === 'queue') { - db.createObjectStore(name, { keyPath: 'id', autoIncrement: true }).createIndex('account', 'account'); - } else if (name === 'keys') { - db.createObjectStore(name, { keyPath: 'id' }); - } else { - db.createObjectStore(name, { keyPath: 'key' }).createIndex('account', 'account'); - } - } - }; - request.onsuccess = () => { - const db = request.result; - // A sign-out in this or another tab deletes the database: let it. - db.onversionchange = () => { - db.close(); - dbPromise = null; - mayCreate = false; - }; - db.onclose = () => { - dbPromise = null; - }; - resolve(db); - }; - request.onerror = () => reject(request.error); - request.onblocked = () => reject(new Error('offline database blocked')); - }); -} - -function done(tx: IDBTransaction): Promise { - return new Promise((resolve, reject) => { - tx.oncomplete = () => resolve(); - tx.onerror = () => reject(tx.error); - tx.onabort = () => reject(tx.error ?? new Error('transaction aborted')); - }); -} - -function result(request: IDBRequest): Promise { - return new Promise((resolve, reject) => { - request.onsuccess = () => resolve(request.result); - request.onerror = () => reject(request.error); - }); -} - -export async function idbGet(store: StoreName, key: IDBValidKey): Promise { - const db = await openOfflineDb(); - return result(db.transaction(store, 'readonly').objectStore(store).get(key)) as Promise; -} - -export async function idbGetAllByAccount(store: StoreName, account: string): Promise { - const db = await openOfflineDb(); - return result(db.transaction(store, 'readonly').objectStore(store).index('account').getAll(account)) as Promise; -} - -export async function idbGetAll(store: StoreName): Promise { - const db = await openOfflineDb(); - return result(db.transaction(store, 'readonly').objectStore(store).getAll()) as Promise; -} - -/** Writes and deletes in several stores at once. */ -export async function idbWrite(ops: Array<{ store: StoreName; put?: unknown; del?: IDBValidKey }>): Promise { - if (ops.length === 0) return; - const db = await openOfflineDb(); - const names = [...new Set(ops.map((op) => op.store))]; - const tx = db.transaction(names, 'readwrite'); - for (const op of ops) { - const store = tx.objectStore(op.store); - if (op.put !== undefined) store.put(op.put); - else if (op.del !== undefined) store.delete(op.del); - } - await done(tx); -} - -/** - * Reads a record and writes what `change` makes of it in one transaction; - * `change` returns undefined to leave it. True when it wrote. - */ -export async function idbUpdate(store: StoreName, key: IDBValidKey, change: (value: T | undefined) => T | undefined): Promise { - const db = await openOfflineDb(); - const tx = db.transaction(store, 'readwrite'); - const objects = tx.objectStore(store); - let wrote = false; - const read = objects.get(key); - read.onsuccess = () => { - const next = change(read.result as T | undefined); - if (next !== undefined) { - objects.put(next); - wrote = true; - } - }; - await done(tx); - return wrote; -} - -export async function idbAdd(store: StoreName, value: unknown): Promise { - const db = await openOfflineDb(); - const tx = db.transaction(store, 'readwrite'); - const key = result(tx.objectStore(store).add(value)); - await done(tx); - return key; -} - -/** Deletes every record of one account in the given stores. */ -export async function idbDeleteAccount(account: string, stores: readonly StoreName[]): Promise { - const db = await openOfflineDb(); - const tx = db.transaction([...stores], 'readwrite'); - for (const name of stores) { - const index = tx.objectStore(name).index('account'); - const request = index.openKeyCursor(IDBKeyRange.only(account)); - request.onsuccess = () => { - const cursor = request.result; - if (!cursor) return; - tx.objectStore(name).delete(cursor.primaryKey); - cursor.continue(); - }; - } - await done(tx); -} - -/** Deletes the whole database (sign-out, offline mode off everywhere). */ -export async function deleteOfflineDb(): Promise { - if (!hasIndexedDb()) return; - mayCreate = false; - const open = dbPromise; - dbPromise = null; - try { - (await open)?.close(); - } catch { - /* it never opened */ - } - await new Promise((resolve) => { - try { - const request = indexedDB.deleteDatabase(OFFLINE_DB_NAME); - request.onsuccess = () => resolve(); - request.onerror = () => resolve(); - request.onblocked = () => resolve(); - } catch { - resolve(); - } - }); -} diff --git a/lib/offline/flush.ts b/lib/offline/flush.ts index 347c0998d..3aa05b411 100644 --- a/lib/offline/flush.ts +++ b/lib/offline/flush.ts @@ -1,22 +1,25 @@ /** * Sends what waited on this device (#523), once the server answers again: - * first the changes (read, flagged, tags, moves) in the order they were - * made, then the outbox, oldest first. + * changes (read, flagged, tags, moves, deletes) and messages, in the order + * they were made. * * - One tab at a time per account (Web Locks), and each message is taken - * for sending in one database transaction (claimForSending), so two tabs - * never send it twice. - * - Changes and messages go in the order they were made. - * - A change the server refuses (the message is gone, moved elsewhere) is - * dropped: the server's state wins. + * for sending in one transaction (claimForSending), so two tabs never send + * it twice. + * - A change the server refuses, or that meets a record changed since, goes + * to the user as a conflict (lib/store/outbox/flush-patches.ts). * - A message is sent as it was finished at Send (queue.ts); files attached * offline are uploaded first. A hold for undo-send that has passed is * dropped, so the message goes at once. - * - Only a failure before the submission request (SendNotStartedError, a - * file upload on a dead network) puts a message back to waiting. A send - * whose connection broke after that is marked uncertain and never sent - * again by itself: it may have gone out. A refusal by the server is marked - * failed, with the reason. + * - A failure before the submission request (SendNotStartedError, a file + * upload on a dead network), a rate limit, a login that needs renewing and + * trouble with this device's store put a message back to waiting. A send whose + * connection broke, or that got a 5xx, is marked uncertain. A refusal by the + * server is marked failed, with the reason. + * - An uncertain message is looked up when the server is back, by the + * Message-ID fixed when it was queued (lib/store/outbox/dedupe.ts): it's + * done, its copy gets submitted, or it goes again. Only when the lookup + * can't tell does it stay uncertain, for the user to decide. * - A message taken for sending by a tab that was closed on the way is * uncertain after a minute. * - A message finished under rules the organisation has since made stricter @@ -26,17 +29,22 @@ import type { JMAPClient } from '@/lib/jmap/client'; import { isNetworkFailure } from '@/lib/jmap/offline-hooks'; import type { SendEmailResult } from '@/lib/jmap/types'; +import { getStore } from '@/lib/store/client/api'; +import { isStorageFailure } from '@/lib/store/storage'; +import type { MethodCall } from '@/lib/store/engine/transport'; +import { decideRetry, type LookupIo, type RetryDecision } from '@/lib/store/outbox/dedupe'; +import { flushPatches, type PatchIo, type PatchOutcome, type PendingPatch } from '@/lib/store/outbox/flush-patches'; import { claimForSending, heldFile, isLocalBlobId, readQueue, removeOutboxEntry, - removeQueued, updateOutboxEntry, type OutboxEntry, type OutboxError, } from './queue'; +import { visibleAccounts } from './state'; import type { OfflineNotice } from './offline-mode'; export interface FlushDeps { @@ -46,8 +54,13 @@ export interface FlushDeps { stricterNow?: (entry: OutboxEntry) => boolean; /** A held send went out (an undo window still open). */ onScheduled?: (entry: OutboxEntry, result: SendEmailResult) => void; + /** A send's connection broke though the server may be fine: look it up again soon. */ + retryLater?: () => void; } +/** A message that just went may not show in a lookup yet, which would send it twice. */ +export const LOOKUP_WAIT_MS = 30_000; + /** A send is taken for at most this long before a closed tab's send counts as uncertain. */ export const SENDING_STALE_MS = 60_000; @@ -64,6 +77,23 @@ class OutboxPrepError extends Error { } } +/** + * Not the server's word about this message: it's rate limiting, the login needs + * renewing, or this device's store is in the way. The next try goes as usual. + */ +function waitsForNextTry(error: unknown): boolean { + if (!(error instanceof Error)) return false; + if (error.name === 'RateLimitError' || error.name === 'StoreOwnerGoneError' || error.name === 'StoreKeyError') return true; + if (isStorageFailure(error)) return true; + return /^Request failed: (401|408|429)\b/.test(error.message); +} + +/** A server error can come after the message went through: looked up before it goes again. */ +function mayHaveArrived(error: unknown): boolean { + if (isNetworkFailure(error)) return true; + return error instanceof Error && (error.name === 'SendUnconfirmedError' || /^Request failed: 5\d\d\b/.test(error.message)); +} + function notStartedError(message: string): Error { return Object.assign(new Error(message), { name: 'SendNotStartedError' }); } @@ -81,12 +111,16 @@ function notStarted(error: unknown): boolean { async function uploadHeld(account: string, client: JMAPClient, blobId: string): Promise { const file = await heldFile(account, blobId); if (!file) throw new OutboxPrepError('file_missing'); + let uploaded: string; try { - return (await client.uploadBlob(file)).blobId; + uploaded = (await client.uploadBlob(file)).blobId; } catch (error) { if (isNetworkFailure(error)) throw notStartedError('upload failed'); throw error; } + // The server went away just now, and offline mode held the file again. + if (isLocalBlobId(uploaded)) throw notStartedError('upload held'); + return uploaded; } async function sendEntry(account: string, client: JMAPClient, entry: OutboxEntry): Promise { @@ -95,7 +129,7 @@ async function sendEntry(account: string, client: JMAPClient, entry: OutboxEntry const file = await heldFile(account, item.blobId); if (!file) throw new OutboxPrepError('file_missing'); const delayed = item.delayedUntil && Date.parse(item.delayedUntil) - Date.now() > MIN_HOLD_MS ? item.delayedUntil : undefined; - if (item.submitOnly) return client.submitRawEmail(file, item.identityId, delayed, item.envelopeRecipients); + if (item.submitOnly) return client.submitRawEmail(file, item.identityId, delayed, item.envelopeRecipients, entry.copyId); let mailboxes; try { mailboxes = await client.getMailboxes(); @@ -106,11 +140,12 @@ async function sendEntry(account: string, client: JMAPClient, entry: OutboxEntry const sent = mailboxes.find((m) => m.role === 'sent'); if (!sent) throw new OutboxPrepError('no_sent_mailbox'); const drafts = mailboxes.find((m) => m.role === 'drafts'); - return client.sendRawEmail(file, item.identityId, sent.id, drafts?.id, delayed, item.envelopeRecipients); + return client.sendRawEmail(file, item.identityId, sent.id, drafts?.id, delayed, item.envelopeRecipients, entry.copyId); } const args = [...item.args]; const attachments = args[10]; - if (Array.isArray(attachments)) { + // A copy on the server has its files already. + if (Array.isArray(attachments) && !entry.copyId) { const resolved = []; for (const attachment of attachments as Array<{ blobId: string }>) { resolved.push(isLocalBlobId(attachment.blobId) ? { ...attachment, blobId: await uploadHeld(account, client, attachment.blobId) } : attachment); @@ -119,6 +154,13 @@ async function sendEntry(account: string, client: JMAPClient, entry: OutboxEntry } const delayedUntil = args[13] as string | undefined; if (delayedUntil && Date.parse(delayedUntil) - Date.now() <= MIN_HOLD_MS) args[13] = undefined; + if (entry.messageId || entry.copyId) { + args[15] = { + ...((args[15] as Record | undefined) ?? {}), + ...(entry.messageId ? { messageId: entry.messageId } : {}), + ...(entry.copyId ? { existingCopyId: entry.copyId } : {}), + }; + } const send = client.sendEmail.bind(client) as (...a: unknown[]) => Promise; return send(...args); } @@ -134,14 +176,106 @@ export async function flushQueue(account: string, client: JMAPClient, deps: Flus await run(); } +/** A file attached offline to something that was then discarded goes after this long. */ +export const ORPHAN_FILE_AGE_MS = 7 * 24 * 60 * 60 * 1000; +const ORPHAN_CHECK_MS = 24 * 60 * 60 * 1000; +const orphansCheckedAt = new Map(); + +async function dropOrphanFiles(account: string): Promise { + const now = Date.now(); + if (now - (orphansCheckedAt.get(account) ?? 0) < ORPHAN_CHECK_MS) return; + orphansCheckedAt.set(account, now); + await getStore().area(account, 'outbox.dropOrphans', { olderThan: now - ORPHAN_FILE_AGE_MS }).catch(() => {}); +} + +/** Tests only. */ +export function forgetOrphanCheck(): void { + orphansCheckedAt.clear(); +} + /** Marks sends of closed tabs uncertain (they may have gone out). */ export async function settleStaleSends(account?: string, now = Date.now()): Promise { + const accounts = account ? [account] : visibleAccounts(); + // One account's store in trouble doesn't hold up another's. + await Promise.allSettled(accounts.map((id) => getStore().area(id, 'outbox.settleStale', { before: now - SENDING_STALE_MS }))); +} + +/** The changes queued before `upToSeq` (all, with null). False when the run stopped on an outage. */ +async function sendPatches(account: string, client: JMAPClient, deps: FlushDeps, upToSeq: number | null): Promise { + const pending = await getStore().area(account, 'outbox.pendingPatches', { upToSeq }); + if (pending.length === 0) return true; + const io: PatchIo = { + request: (calls: MethodCall[], using: string[]) => client.requestNetworkOnly(calls, using), + }; + return flushPatches(io, pending, async (outcomes: PatchOutcome[]) => { + const conflicts = await getStore().area(account, 'outbox.patchResults', { outcomes }); + if (conflicts > 0) deps.notify({ kind: 'conflict', count: conflicts }); + }); +} + +async function lookUp(client: JMAPClient, entry: OutboxEntry): Promise { + const io: LookupIo = { request: (calls, using) => client.requestNetworkOnly(calls, using) }; + const fromEmail = entry.item.kind === 'email' ? entry.item.args[6] as string | undefined : undefined; + const jmap = await client.resolveSendAccountId(fromEmail); + const options = entry.item.kind === 'email' ? entry.item.args[15] as { noSentCopy?: boolean } | undefined : undefined; + return decideRetry(io, { + jmap, + submissionJmap: client.submissionAccountFor(jmap), + messageId: entry.messageId, + attemptAt: entry.firstAttemptAt ?? entry.createdAt, + now: Date.now(), + copyKept: entry.item.kind === 'raw' ? !entry.item.submitOnly : !options?.noSentCopy, + }); +} + +/** The draft the send was meant to replace: the lost response left it behind. Best effort. */ +async function dropOldDraft(client: JMAPClient, entry: OutboxEntry): Promise { + if (entry.item.kind !== 'email' || typeof entry.item.args[7] !== 'string') return; + const options = entry.item.args[15] as { draftAccountId?: string } | undefined; + try { + await client.requestNetworkOnly([['Email/set', { accountId: options?.draftAccountId || client.getAccountId(), destroy: [entry.item.args[7]] }, '0']]); + } catch { + // An orphan in Drafts, nothing worse. + } +} + +/** Uncertain messages, looked up now that the server answers. False when the run stopped on an outage. */ +async function settleUncertain(account: string, client: JMAPClient, deps: FlushDeps): Promise { const { outbox } = await readQueue(account); for (const entry of outbox) { - if (entry.status === 'sending' && (entry.sendingSince ?? 0) < now - SENDING_STALE_MS) { - await updateOutboxEntry(entry, { status: 'uncertain' }); + if (entry.status !== 'uncertain') continue; + if (stillOffline() || !deps.allowed()) return false; + if (entry.lastAttemptAt && Date.now() - entry.lastAttemptAt < LOOKUP_WAIT_MS) { + deps.retryLater?.(); + continue; + } + let decision: RetryDecision; + try { + decision = await lookUp(client, entry); + } catch (error) { + // The lookup failed: it stays uncertain, for this time. + if (isNetworkFailure(error)) return false; + continue; + } + switch (decision.action) { + case 'done': + await removeOutboxEntry(entry); + await dropOldDraft(client, entry); + void client.pruneEmailSubmissions(); + // A canceled one wasn't sent: someone decided that, and it's in Drafts. + if (decision.why !== 'canceled') deps.notify({ kind: 'sent', subject: entry.summary.subject }); + break; + case 'submit': + await updateOutboxEntry(entry, { status: 'waiting', copyId: decision.emailId }); + break; + case 'send': + await updateOutboxEntry(entry, { status: 'waiting', copyId: null }); + break; + case 'undecided': + break; } } + return true; } async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps): Promise { @@ -149,36 +283,21 @@ async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps) flushing.add(account); try { await settleStaleSends(account); - const { patches, outbox } = await readQueue(account); + if (!(await settleUncertain(account, client, deps))) return; // In the order they were made: a reply's $answered, queued after the // reply, goes after it. - const items = [ - ...patches.map((patch) => ({ id: patch.id, patch })), - ...outbox.map((entry) => ({ id: entry.id, entry })), - ].sort((x, y) => x.id - y.id); - - for (const item of items) { - if ('patch' in item && item.patch) { - if (stillOffline()) return; - try { - await client.applyEmailPatches(item.patch.item.jmap, item.patch.item.update); - } catch (error) { - // Still no server: try again later. Anything else: the server's state wins. - if (isNetworkFailure(error)) return; - } - await removeQueued([item.patch.id]); - continue; - } - const entry = (item as { entry: OutboxEntry }).entry; + const { outbox } = await readQueue(account); + for (const entry of outbox) { if (entry.status !== 'waiting') continue; if (stillOffline() || !deps.allowed()) return; + if (!(await sendPatches(account, client, deps, entry.seq))) return; if (deps.stricterNow?.(entry)) { await updateOutboxEntry(entry, { status: 'held' }); deps.notify({ kind: 'held', subject: entry.summary.subject }); continue; } // Another tab took it, or it was deleted meanwhile. - if (!(await claimForSending(entry.id))) continue; + if (!(await claimForSending(account, entry.id))) continue; let result: SendEmailResult; try { result = await sendEntry(account, client, entry); @@ -190,17 +309,24 @@ async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps) } if (error instanceof Error && error.name === 'OutboxPrepError') { const code = (error as OutboxPrepError).code; - await updateOutboxEntry(entry, { status: 'failed', error: { code } }); + await updateOutboxEntry(entry, { status: 'failed', error: { code }, copyId: null }); deps.notify({ kind: 'send-failed', subject: entry.summary.subject, error: { code } }); continue; } - if (isNetworkFailure(error) || (error instanceof Error && (error.name === 'RequestTimeoutError' || error.name === 'SendUnconfirmedError'))) { - // It may have reached the server: never sent again by itself. + if (waitsForNextTry(error)) { + // Nothing the user can do about it, and the copy on the server stays useful. + await updateOutboxEntry(entry, { status: 'waiting' }); + deps.retryLater?.(); + return; + } + if (mayHaveArrived(error)) { await updateOutboxEntry(entry, { status: 'uncertain' }); + deps.retryLater?.(); return; } const text = error instanceof Error ? error.message : String(error); - await updateOutboxEntry(entry, { status: 'failed', error: { text } }); + // The client took back the copy of a refused send. + await updateOutboxEntry(entry, { status: 'failed', error: { text }, copyId: null }); deps.notify({ kind: 'send-failed', subject: entry.summary.subject, error: { text } }); continue; } @@ -209,21 +335,22 @@ async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps) if (result.scheduled) deps.onScheduled?.(entry, result); deps.notify({ kind: 'sent', subject: entry.summary.subject }); } + if (stillOffline() || !deps.allowed()) return; + await sendPatches(account, client, deps, null); + await dropOrphanFiles(account); } finally { flushing.delete(account); } } /** "Send again" for a failed or uncertain message (a held one waits for its rules). */ -export async function retryOutboxEntry(id: number): Promise { - const { outbox } = await readQueue(); +export async function retryOutboxEntry(account: string, id: number): Promise { + const { outbox } = await readQueue(account); const entry = outbox.find((e) => e.id === id); if (entry && (entry.status === 'failed' || entry.status === 'uncertain')) await updateOutboxEntry(entry, { status: 'waiting' }); } /** "Delete" in the outbox. */ -export async function deleteOutboxEntry(id: number): Promise { - const { outbox } = await readQueue(); - const entry = outbox.find((e) => e.id === id); - if (entry) await removeOutboxEntry(entry); +export async function deleteOutboxEntry(account: string, id: number): Promise { + await removeOutboxEntry({ account, id }); } diff --git a/lib/offline/migrate-legacy.ts b/lib/offline/migrate-legacy.ts new file mode 100644 index 000000000..12877bc98 --- /dev/null +++ b/lib/offline/migrate-legacy.ts @@ -0,0 +1,211 @@ +/** + * One time, on the first start after the update: what can't be fetched again + * moves from the old IndexedDB `bulwark-offline` into the local store. That's + * the outbox, the changes that wait, the files they hold and the session kept + * for a start without the server. The mail itself syncs again. + * + * The old database holds its own key, so it goes only after the store has + * committed every account. A move that fails leaves it in place and the next + * start tries again; the store takes each account's part only once. + */ + +import { generateMessageId } from '@/lib/jmap/client'; +import { getStore } from '@/lib/store/client/api'; +import type { StoreSession } from '@/lib/store/engine/transport'; +import type { LegacyImport } from '@/lib/store/outbox/calls'; +import type { OutboxError, OutboxStatus, OutboxSummary, PolicyFingerprint, SendItem } from '@/lib/store/outbox/items'; +import { useOfflineStore } from './state'; + +export const LEGACY_DB_NAME = 'bulwark-offline'; + +interface Sealed { + iv: Uint8Array; + data: ArrayBuffer; +} + +interface QueueRecord extends Sealed { + id: number; + account: string; + kind: 'patch' | 'send'; + createdAt: number; + status?: string; +} + +interface UploadRecord extends Sealed { + key: string; + account: string; + meta: Sealed; +} + +interface SendPayload { + summary: OutboxSummary; + item: SendItem; + fingerprint?: PolicyFingerprint; + error?: OutboxError; +} + +function result(request: IDBRequest): Promise { + return new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); +} + +/** Opens the old database only if it exists: opening it must never create it. */ +function openLegacy(): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(LEGACY_DB_NAME); + let creating = false; + request.onupgradeneeded = () => { + creating = true; + request.transaction?.abort(); + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => { + if (creating) resolve(null); + else reject(request.error); + }; + request.onblocked = () => resolve(null); + }); +} + +function deleteLegacy(): Promise { + return new Promise((resolve) => { + const request = indexedDB.deleteDatabase(LEGACY_DB_NAME); + request.onsuccess = () => resolve(); + request.onerror = () => resolve(); + // An old tab still has it open: it goes when that tab does. + request.onblocked = () => resolve(); + }); +} + +function storeNamesHold(db: IDBDatabase, ...names: string[]): boolean { + return names.every((name) => db.objectStoreNames.contains(name)); +} + +async function open(key: CryptoKey, sealed: Sealed): Promise { + return crypto.subtle.decrypt({ name: 'AES-GCM', iv: sealed.iv as BufferSource }, key, sealed.data); +} + +async function openJson(key: CryptoKey, sealed: Sealed): Promise { + return JSON.parse(new TextDecoder().decode(await open(key, sealed))) as T; +} + +/** The Message-ID of a finished raw message, from its head. */ +function rawMessageId(bytes: Uint8Array): string | undefined { + const head = new TextDecoder().decode(bytes.subarray(0, 64 * 1024)); + const end = head.search(/\r?\n\r?\n/); + const block = (end >= 0 ? head.slice(0, end) : head).replace(/\r?\n[ \t]+/g, ' '); + return block.match(/^message-id:\s*<([^>\s]+)>/im)?.[1]; +} + +interface AccountPart extends LegacyImport { + session: StoreSession | null; +} + +async function readAccount(db: IDBDatabase, key: CryptoKey | null, account: string): Promise { + const part: AccountPart = { entries: [], patches: [], files: [], session: null }; + if (!key) return part; + const stores = db.transaction(['queue', 'uploads', 'misc'], 'readonly'); + const [queue, uploads, session] = await Promise.all([ + result(stores.objectStore('queue').index('account').getAll(account)) as Promise, + result(stores.objectStore('uploads').index('account').getAll(account)) as Promise, + result(stores.objectStore('misc').get(`${account}\u0001misc\u0001session`)) as Promise, + ]); + if (session) part.session = await openJson(key, session).catch(() => null); + + const rawIds = new Map(); + for (const record of uploads) { + // Sealed with a key that is gone, as the old code skipped it. + try { + const meta = await openJson<{ name?: string; type?: string }>(key, record.meta); + const data = new Uint8Array(await open(key, record)); + const blobId = record.key.split('\u0001')[1]; + part.files.push({ blobId, name: meta.name || 'attachment', type: meta.type || 'application/octet-stream', data }); + rawIds.set(blobId, rawMessageId(data)); + } catch { + continue; + } + } + + // In the order they were made. + for (const record of [...queue].sort((a, b) => a.id - b.id)) { + try { + if (record.kind === 'patch') { + const item = await openJson<{ jmap: string; update: Record> }>(key, record); + part.patches.push({ jmap: item.jmap, update: item.update }); + continue; + } + const payload = await openJson(key, record); + // Sent by a tab that's gone: it may have left, so it's looked up or asked about. + const status = (record.status === 'sending' ? 'uncertain' : record.status ?? 'waiting') as OutboxStatus; + const item = payload.item; + let messageId: string | undefined; + if (item.kind === 'raw') messageId = rawIds.get(item.blobId); + // Not sent yet: it can still get a fixed id. One that left under the old + // code had another id, which nothing here knows. + // Only its domain is used, and the sender may be written "Name
    ". + else if (status === 'waiting' && typeof item.args[6] === 'string') messageId = generateMessageId(item.args[6].replace(/>.*$/, '')); + part.entries.push({ + summary: payload.summary, + item, + fingerprint: payload.fingerprint, + error: payload.error, + status, + createdAt: record.createdAt, + messageId, + }); + } catch { + continue; + } + } + return part; +} + +let moving: Promise | null = null; + +async function moveNow(): Promise { + if (typeof indexedDB === 'undefined') return; + const db = await openLegacy(); + if (!db) return; + try { + if (!storeNamesHold(db, 'keys', 'queue', 'uploads', 'misc')) return; + const keyRecord = await result(db.transaction('keys', 'readonly').objectStore('keys').get('aes')) as { key: CryptoKey } | undefined; + const key = keyRecord?.key ?? null; + for (const account of useOfflineStore.getState().enabled) { + const part = await readAccount(db, key, account); + const { session, ...content } = part; + if (session) await getStore().ensureOpen(account, session); + if (content.entries.length + content.patches.length + content.files.length === 0) continue; + await getStore().area(account, 'outbox.importLegacy', content); + } + } finally { + db.close(); + } + await deleteLegacy(); +} + +/** Runs the move once per page. A failure is kept for nobody to see: the old database stays and the next start tries again. */ +export function moveLegacyOutbox(): Promise { + moving ??= moveNow().catch(() => {}); + return moving; +} + +/** Messages an earlier version left in its database. For a browser the store can't run in, which can't move them. */ +export async function strandedLegacyMessages(): Promise { + if (typeof indexedDB === 'undefined') return 0; + const db = await openLegacy().catch(() => null); + if (!db) return 0; + try { + if (!storeNamesHold(db, 'queue')) return 0; + const records = await result(db.transaction('queue', 'readonly').objectStore('queue').getAll()) as Array<{ kind?: string }>; + return records.filter((record) => record.kind !== 'patch').length; + } finally { + db.close(); + } +} + +/** Tests only: forget that this page already ran it. */ +export function forgetLegacyMove(): void { + moving = null; +} diff --git a/lib/offline/offline-mode.ts b/lib/offline/offline-mode.ts index 356bf131c..836e1286b 100644 --- a/lib/offline/offline-mode.ts +++ b/lib/offline/offline-mode.ts @@ -4,19 +4,22 @@ * - Opt-in per account and per browser (state.ts), off by default, hidden * and wiped when the organisation forbids it (policy.json * `organisation.devices.noOffline`, per account) and in Lite. - * - While on, the JMAP client hands every answer of the mail server to the - * cache (cache.ts) and asks this device when the network is gone - * (responder.ts); a background sync keeps the last 30 days of the Inbox - * and of the folders the user opened, with bodies and attachments, under - * a cap (sync.ts). - * - Changes and sends made offline wait in the queue (queue.ts) and go out - * in order once the server answers again and the organisation's policy - * was fetched afresh (flush.ts). - * - Everything is wiped at a full sign-out and the automatic sign-outs - * (lib/sign-out-cleanup.ts deletes the database), for one account when it - * signs out or switches offline mode off, and when the organisation - * forbids it (checked on every policy fetch). Unsent messages a sign-out - * deletes without the user's word are named (lost.ts). + * - While on, the local store (lib/store, a database in a worker) keeps the + * account's mail in sync by itself, and the JMAP client asks it when the + * network is gone. This file is the orchestration around it: who may use + * it, when it syncs and flushes, the wipes. + * - Changes and sends made offline wait in the store's outbox and go out in + * order once the server answers again and the organisation's policy was + * fetched afresh (flush.ts). + * - Everything is wiped at a full sign-out, at the session limits and when the + * server really refuses the login (`invalid_grant`), by + * lib/sign-out-cleanup.ts; for one account when it signs out, is removed or + * switches offline mode off, and when the organisation forbids it (checked on + * every policy fetch). An account the app drops by itself because it couldn't + * confirm the login (an outage, an unknown refusal, a session cookie that ran + * out) keeps its data for 30 days or the next sign-in, hidden from everyone + * else. Unsent messages a sign-out deletes without the user's word are named + * (lost.ts). */ import { IS_LITE } from '@/lib/lite'; @@ -25,52 +28,36 @@ import { devicesForAccount } from '@/lib/device-policy'; import { accountSubjectPending, useAccountSubjects } from '@/lib/org/account-subjects'; import { labelsPolicyFor } from '@/lib/outgoing/classification'; import { getSendGuardSource, guardState, guardsSending } from '@/lib/plugin-send-guards'; -import { useAccountStore } from '@/stores/account-store'; +import { emitPushStateChange } from '@/lib/push-events'; +import { requestOutboxSync } from '@/lib/pwa-notify/outbox-sync'; +import { STORAGE_ERROR_AREA } from '@/lib/store/area'; +import { configureStore, getStore, type StorePageHooks } from '@/lib/store/client/api'; +import { purgeStoreFiles } from '@/lib/store/client/purge'; +import type { StoreSession } from '@/lib/store/engine/transport'; +import { onAccountRemoved, useAccountStore } from '@/stores/account-store'; import { useAuthStore } from '@/stores/auth-store'; import { useEmailStore } from '@/stores/email-store'; import { usePolicyStore, whenPolicySettled } from '@/stores/policy-store'; import { onBeforeSignOut, onFullSignOut } from '@/lib/sign-out-cleanup'; -import { setOfflineHooks, type OfflineClientRef, type OfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks'; +import { generateMessageId } from '@/lib/jmap/client'; +import { SendNotStartedError, setOfflineHooks, type OfflineClientRef, type OfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks'; import type { SendEmailResult } from '@/lib/jmap/types'; import { setOutboxHooks, type OutboxHooks } from './outbox-hooks'; -import { answerOffline } from './responder'; -import { - CACHE_WINDOW_MS, - cacheUsage, - emailIndex, - enforceCap, - forgetIndexes, - getBlob, - getMisc, - messageOfBlob, - patchEmails, - putBlob, - putEmails, - putMisc, - removeEmails, - wipeAccountCache, - emailKey, - type CachedEmail, -} from './cache'; -import { allowOfflineDbCreation, deleteOfflineDb, hasIndexedDb } from './db'; -import { forgetDeviceKey } from './crypto'; import { heldFile, holdFile, isLocalBlobId, - queuePatch, queueSend, readQueue, - wipeAccountQueue, type OutboxEntry, type OutboxError, type OutboxSummary, type PolicyFingerprint, } from './queue'; import { clearHandoffs } from '@/lib/launch-handoff'; -import { isOfflineNow, OFFLINE_ACCOUNTS_KEY, readEnabledFrom, serverKey, setServerDown, useOfflineStore, writeEnabled, type OutboxRow } from './state'; -import { flushQueue, settleStaleSends } from './flush'; -import { syncAccount } from './sync'; +import { clearEvicted, EVICTED_KEEP_MS, forgetMailSnapshot, isEvicted, offlineScopeFor, isOfflineNow, OFFLINE_ACCOUNTS_KEY, readEnabledFrom, readEvicted, visibleAccounts, serverKey, setServerDown, useOfflineStore, writeEnabled, type OutboxRow } from './state'; +import { flushQueue, LOOKUP_WAIT_MS, SENDING_STALE_MS, settleStaleSends } from './flush'; +import { LEGACY_DB_NAME, moveLegacyOutbox, strandedLegacyMessages } from './migrate-legacy'; import { deleteShellCaches, tellServiceWorker } from './shell'; import { rememberLost } from './lost'; @@ -80,6 +67,11 @@ export type OfflineNotice = | { kind: 'send-failed'; subject: string; error: OutboxError } | { kind: 'held'; subject: string } | { kind: 'lost'; subjects: string[] } + | { kind: 'conflict'; count: number } + | { kind: 'storage-failed' } + | { kind: 'store-unavailable' } + | { kind: 'store-reset' } + | { kind: 'legacy-stranded' } | { kind: 'refused-delete' } | { kind: 'refused-other' } | { kind: 'forbidden' } @@ -98,11 +90,20 @@ export function offlineNotify(notice: OfflineNotice): void { // ---- Who may use it ---- -/** Offline mode is possible here at all: not in Lite, with IndexedDB and WebCrypto. */ +function hasIndexedDb(): boolean { + try { + return typeof indexedDB !== 'undefined' && indexedDB !== null; + } catch { + return false; + } +} + +/** Offline mode is possible here at all: not in Lite, with what the store needs. */ export function offlineSupported(): boolean { if (IS_LITE || !hasIndexedDb()) return false; + if (useOfflineStore.getState().storageBlocked) return false; try { - return typeof crypto !== 'undefined' && !!crypto.subtle; + return typeof crypto !== 'undefined' && !!crypto.subtle && getStore().supported(); } catch { return false; } @@ -139,179 +140,88 @@ function onlineClientFor(account: string) { return client; } -function serverOf(account: string): string | undefined { - return useAccountStore.getState().accounts.find((a) => a.id === account)?.serverUrl; +// ---- The store's way to this tab ---- + +/** A full disk stays full: once in a while is enough to say so. */ +const STORAGE_NOTICE_MS = 30 * 60_000; +let lastStorageNotice = 0; +let lastUnavailableNotice = 0; + +/** The store won't open: what waits in it is out of reach, and the user should know why. */ +function noteStoreUnavailable(): void { + if (useOfflineStore.getState().enabled.length === 0) return; + const now = Date.now(); + if (now - lastUnavailableNotice < STORAGE_NOTICE_MS) return; + lastUnavailableNotice = now; + notify({ kind: 'store-unavailable' }); } -// ---- What the server sends, into the cache ---- - -export interface Scope { - inbox: Record; - opened: Record; -} - -const MAX_OPENED_FOLDERS = 10; - -/** The Inbox and the opened folders that still exist, per JMAP account. */ -export async function scopeOf(account: string): Promise { - const mailboxes = (await getMisc>>(account, 'mailboxes')) ?? {}; - const inbox: Record = {}; - for (const [jmap, list] of Object.entries(mailboxes)) { - const box = list.find((m) => m.role === 'inbox'); - if (box) inbox[jmap] = box.id; - } - const stored = (await getMisc>(account, 'opened')) ?? {}; - // A folder deleted or moved away is no longer opened. - const opened: Record = {}; - let pruned = false; - for (const [jmap, ids] of Object.entries(stored)) { - const list = mailboxes[jmap]; - const kept = list ? ids.filter((id) => list.some((m) => m.id === id)) : ids; - if (kept.length !== ids.length) pruned = true; - opened[jmap] = kept; - } - if (pruned) await putMisc(account, 'opened', opened); - return { inbox, opened }; -} - -export async function scopeMailboxes(account: string): Promise> { - const scope = await scopeOf(account); - const out: Array<{ jmap: string; mailboxId: string; isInbox: boolean }> = []; - for (const [jmap, id] of Object.entries(scope.inbox)) out.push({ jmap, mailboxId: id, isInbox: true }); - for (const [jmap, ids] of Object.entries(scope.opened)) { - for (const id of ids) if (scope.inbox[jmap] !== id) out.push({ jmap, mailboxId: id, isInbox: false }); - } - return out; -} - -function keptFolders(scope: Scope): Record { - const out: Record = {}; - for (const [jmap, id] of Object.entries(scope.inbox)) out[jmap] = [id]; - for (const [jmap, ids] of Object.entries(scope.opened)) out[jmap] = [...(out[jmap] ?? []), ...ids]; - return out; -} - -/** A message the cache keeps: received in the last 30 days, in the Inbox or an opened folder. */ -export function inScope(scope: Scope, jmap: string, email: CachedEmail, now = Date.now()): boolean { - const at = Date.parse(String(email.receivedAt ?? '')); - if (Number.isFinite(at) && at < now - CACHE_WINDOW_MS) return false; - const boxes = Object.keys(email.mailboxIds ?? {}); - const wanted = new Set([...(scope.opened[jmap] ?? []), ...(scope.inbox[jmap] ? [scope.inbox[jmap]] : [])]); - return boxes.some((id) => wanted.has(id)); -} - -/** The folder a list query opened, if it is one. */ -function openedFolder(args: Record): string | null { - if (typeof args.limit !== 'number') return null; - const filter = args.filter as Record | undefined; - if (!filter) return null; - if (typeof filter.inMailbox === 'string' && Object.keys(filter).every((k) => k === 'inMailbox' || k === 'hasKeyword')) return filter.inMailbox; - if (filter.operator === 'AND' && Array.isArray(filter.conditions)) { - for (const c of filter.conditions as Array>) if (typeof c.inMailbox === 'string' && Object.keys(c).length === 1) return c.inMailbox; - } - return null; -} - -async function noteOpenedFolder(account: string, jmap: string, mailboxId: string): Promise { - const opened = (await getMisc>(account, 'opened')) ?? {}; - const list = opened[jmap] ?? []; - if (list[0] === mailboxId) return; - opened[jmap] = [mailboxId, ...list.filter((id) => id !== mailboxId)].slice(0, MAX_OPENED_FOLDERS); - await putMisc(account, 'opened', opened); - scheduleSync(account, 5_000); -} - -async function putList(account: string, name: 'mailboxes' | 'identities', jmap: string, list: unknown[]): Promise { - const all = (await getMisc>(account, name)) ?? {}; - all[jmap] = list; - await putMisc(account, name, all); -} - -/** The methods whose answers the cache takes. */ -const RECORDED = new Set(['Mailbox/get', 'Identity/get', 'Email/get', 'Email/query', 'Email/set']); - -/** What the server answered, into the cache. */ -export async function recordAnswer(account: string, defaultJmap: string, calls: OfflineMethodCall[], response: OfflineResponse): Promise { - const scope = await scopeOf(account); - for (const [name, result, callId] of response.methodResponses) { - if (!result || typeof result !== 'object') continue; - const call = calls.find(([n, , id]) => id === callId && n === name); - const args = call?.[1] ?? {}; - const jmap = (result.accountId as string) || (args.accountId as string) || defaultJmap; - const properties = args.properties as string[] | undefined; - switch (name) { - case 'Mailbox/get': - if (!args.ids && (!properties || properties.includes('name')) && Array.isArray(result.list)) { - await putList(account, 'mailboxes', jmap, result.list); - } - break; - case 'Identity/get': - if (!args.ids && Array.isArray(result.list)) await putList(account, 'identities', jmap, result.list); - break; - case 'Email/get': { - const list = (Array.isArray(result.list) ? result.list : []) as CachedEmail[]; - // Fresh state for every message already kept; new ones only within the scope. - const index = await emailIndex(account); - const keep = list.filter((email) => index.has(emailKey(account, jmap, email.id)) || inScope(scope, jmap, email)); - await putEmails(account, jmap, keep); - if (keep.length) scheduleCapCheck(); - break; - } - case 'Email/query': { - const folder = openedFolder(args); - if (folder) await noteOpenedFolder(account, jmap, folder); - break; - } - case 'Email/set': { - const update = (args.update ?? {}) as Record>; - const updated = Object.keys((result.updated ?? {}) as object); - if (updated.length) await patchEmails(account, jmap, Object.fromEntries(updated.filter((id) => update[id]).map((id) => [id, update[id]]))); - const destroyed = (Array.isArray(result.destroyed) ? result.destroyed : []) as string[]; - if (destroyed.length) await removeEmails(account, destroyed.map((id) => emailKey(account, jmap, id))); - break; - } - default: - break; +const storeHooks: StorePageHooks = { + inputs(account) { + const client = onlineClientFor(account); + return { + session: (client?.getSessionSnapshot() as StoreSession | undefined) ?? null, + auth: client?.getAuthHeader() ?? null, + invertedKeywordSort: client?.keywordSortInverted() ?? null, + scope: offlineScopeFor(account), + }; + }, + async refreshAuth(account) { + const client = onlineClientFor(account); + if (!client) return null; + // Through the client: a 401 there runs its token renewal or re-auth. + await client.requestNetworkOnly([['Core/echo', {}, '0']]); + return client.getAuthHeader(); + }, + onStoreLost(account) { + // Whoever is here wasn't signed in to a dropped account: nothing to tell them. + if (isEvicted(account)) return; + useOfflineStore.setState((s) => ({ storeLost: [...new Set([...s.storeLost, account])] })); + notify({ kind: 'store-reset' }); + }, + onEvent(event) { + if (event.type === 'synced') { + useOfflineStore.setState((s) => ({ lastSyncedAt: { ...s.lastSyncedAt, [event.account]: event.at } })); + forgetMailSnapshot(event.account); + void refreshUsage(); } - } + if (event.type === 'error' && event.area === STORAGE_ERROR_AREA) { + const now = Date.now(); + if (now - lastStorageNotice < STORAGE_NOTICE_MS) return; + lastStorageNotice = now; + notify({ kind: 'storage-failed' }); + } + }, +}; + +// ---- What the client asks and tells ---- + +/** Parts cut short for a list view aren't what the user opened. */ +function hasWholeText(email: { bodyValues?: Record }): boolean { + const values = Object.values(email.bodyValues ?? {}); + return values.length > 0 && !values.some((value) => value?.isTruncated); } -// One write after the other per account, so a list and its refresh land in order. -const recording = new Map>(); - -function enqueueRecord(account: string, task: () => Promise): void { - const previous = recording.get(account) ?? Promise.resolve(); - const next = previous.then(task).catch(() => {}); - recording.set(account, next); -} - -let capTimer: ReturnType | null = null; - -/** The cap holds between syncs too: checked shortly after writes. */ -function scheduleCapCheck(delayMs = 5_000): void { - if (capTimer) return; - capTimer = setTimeout(() => { - capTimer = null; - if (useOfflineStore.getState().enabled.length === 0) return; - void enforceCap().then(() => refreshUsage()).catch(() => {}); - }, delayMs); -} - -/** Mutating calls the device can't take: a send, an import, calendar and contact changes. */ +/** Mutating calls the device can't take: an import, calendar and contact changes. */ const NEEDS_SERVER = /^(EmailSubmission|Email\/import|CalendarEvent\/set|ContactCard\/set|AddressBook\/set|Calendar\/set|Mailbox\/set|FileNode\/set|SieveScript\/set|VacationResponse\/set|Identity\/set)/; let lastRefusalNotice = 0; -function noticeRefused(methodCalls: OfflineMethodCall[]): void { - if (!methodCalls.some(([name]) => NEEDS_SERVER.test(name))) return; +function noticeRefused(answered: OfflineResponse | null, methodCalls: OfflineMethodCall[]): void { + const refusedCall = methodCalls.some(([name, , id]) => NEEDS_SERVER.test(name) + && (!answered || answered.methodResponses.some(([responseName, , responseId]) => responseId === id && responseName === 'error'))); + const refusedChange = answered?.methodResponses.some(([name, result]) => name === 'Email/set' && result?.notUpdated) ?? false; + if (!refusedCall && !refusedChange) return; const now = Date.now(); if (now - lastRefusalNotice < 5_000) return; lastRefusalNotice = now; - notify({ kind: 'refused-other' }); + const emptied = answered?.methodResponses.some(([name, result]) => name === 'Email/set' + && Object.values((result?.notUpdated ?? {}) as Record).some((error) => error.type === 'invalidProperties')); + notify({ kind: emptied ? 'refused-delete' : 'refused-other' }); } -// ---- Hooks for the JMAP client ---- - const probes = new Map>(); +/** A black-holed connection never answers the probe either. */ +const PROBE_TIMEOUT_MS = 5_000; /** * Whether a mail server is gone. Browsers report a refusal without CORS @@ -326,7 +236,7 @@ export function probeServer(serverUrl: string): Promise { if (running) return running; const probe = (async () => { try { - await fetch(`${desktopJmapBase(serverUrl.replace(/\/+$/, ''))}/.well-known/jmap`, { mode: 'no-cors', cache: 'no-store', credentials: 'omit' }); + await fetch(`${desktopJmapBase(serverUrl.replace(/\/+$/, ''))}/.well-known/jmap`, { mode: 'no-cors', cache: 'no-store', credentials: 'omit', signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) }); if (setServerDown(serverUrl, false)) scheduleFlushAll(500); return false; } catch { @@ -346,32 +256,53 @@ const hooks: OfflineHooks = { async answer(client, methodCalls) { const account = accountForClient(client); if (!offlineActive(account) || !account) return null; - const scope = await scopeOf(account).catch(() => null); - const answered = await answerOffline({ - account, - defaultJmap: client.getAccountId(), - keptFolders: scope ? keptFolders(scope) : undefined, - queuePatch: async (jmap, update) => { - await queuePatch(account, jmap, update); - void refreshQueueState(); - }, - onRefused: (what) => notify({ kind: what === 'destroy' ? 'refused-delete' : 'refused-other' }), - }, methodCalls); - if (!answered) noticeRefused(methodCalls); + let answered: OfflineResponse | null = null; + try { + answered = await getStore().answer(account, methodCalls); + } catch { + // The store couldn't start or lost its owner: as if nothing were kept. + } + noticeRefused(answered, methodCalls); + if (methodCalls.some(([name]) => name === 'Email/set') && answered) { + requestOutboxSync(); + void refreshQueueState(); + } return answered; }, - record(client, methodCalls, response) { + async resolveStandIns(client, ids) { + const account = accountForClient(client); + if (!offlineActive(account) || !account) return {}; + return getStore().area>(account, 'calendar.resolveStandIns', { ids }).catch(() => ({})); + }, + fileOpened(client, accountId, id) { const account = accountForClient(client); if (!offlineActive(account) || !account) return; - // Only the answers the cache takes; copied now, because callers change - // the answer's objects after this returns. - const wanted = response.methodResponses.filter(([name]) => RECORDED.has(name)); - if (wanted.length === 0) return; - const ids = new Set(wanted.map(([, , id]) => id)); - const copy = JSON.parse(JSON.stringify({ methodResponses: wanted })) as OfflineResponse; - const calls = JSON.parse(JSON.stringify(methodCalls.filter(([, , id]) => ids.has(id)))) as OfflineMethodCall[]; - const jmap = client.getAccountId(); - enqueueRecord(account, () => recordAnswer(account, jmap, calls, copy)); + void getStore().area(account, 'files.noteOpened', { jmap: accountId, id }).catch(() => {}); + }, + async searchLocal(client, methodCalls) { + const account = accountForClient(client); + if (!offlineActive(account) || !account) return null; + try { + return await getStore().answer(account, methodCalls); + } catch { + return null; + } + }, + record(client, methodCalls, response) { + // The store syncs by itself. What it may not have is a message the user + // opened from outside the chosen window, so only those come through here. + const account = accountForClient(client); + if (!offlineActive(account) || !account) return; + for (const [name, result, callId] of response.methodResponses) { + if (name !== 'Email/get' || !Array.isArray(result?.list)) continue; + const opened = (result.list as Array<{ bodyValues?: Record }>).filter(hasWholeText); + if (opened.length === 0) continue; + const args = methodCalls.find(([n, , id]) => id === callId && n === name)?.[1]; + const jmap = (result.accountId as string) || (args?.accountId as string) || client.getAccountId(); + // Copied now: callers change the answer's objects after this returns. + const emails = JSON.parse(JSON.stringify(opened)) as unknown[]; + void getStore().area(account, 'mail.recordOpened', { jmap, emails }).catch(() => {}); + } }, noteNetwork(client, reachable) { if (reachable) { @@ -380,36 +311,48 @@ const hooks: OfflineHooks = { void probeServer(client.getServerUrl()); } }, - async serverGone(client) { + async serverGone(client, error) { if (!offlineActive(accountForClient(client))) return false; if (typeof navigator !== 'undefined' && navigator.onLine === false) return true; + // The proxy answered, so a probe would pass; the server behind it is what's gone. + if (error instanceof Error && error.name === 'GatewayError') { + setServerDown(client.getServerUrl(), true); + return true; + } return probeServer(client.getServerUrl()); }, + stateChange(client, change) { + const account = accountForClient(client); + if (!offlineActive(account) || !account || !change.changed) return; + void getStore().noteStateChange(account, change.changed).catch(() => {}); + }, async blob(client, blobId, jmap) { const account = accountForClient(client); if (!account || !offlineActive(account)) return null; if (isLocalBlobId(blobId)) return heldFile(account, blobId); - return getBlob(account, jmap, blobId); + // An attachment, or a file from the Files area. + const kept = await getStore().area(account, 'mail.getBlob', { jmap, blobId }).catch(() => null) + ?? await getStore().area(account, 'files.blob', { jmap, blobId }).catch(() => null); + return kept ? new Blob([kept.data as BlobPart], { type: kept.type || 'application/octet-stream' }) : null; }, recordBlob(client, blobId, jmap, blob) { const account = accountForClient(client); if (!account || !offlineActive(account) || blob.size > MAX_RECORDED_BLOB_BYTES) return; - // Only a part of a kept message (an attachment, an inline image), filed - // with it: never a whole message an export reads, never a stray file. - enqueueRecord(account, async () => { - const owner = await messageOfBlob(account, jmap, blobId); - if (!owner) return; - await putBlob(account, jmap, blobId, blob, owner.at, owner.key); - scheduleCapCheck(); - }); + // Filed under its kept message by the store: never a whole message an + // export reads, never a stray file. + void blob.arrayBuffer() + .then((buffer) => getStore().area(account, 'mail.recordBlob', { jmap, blobId, type: blob.type || null, data: new Uint8Array(buffer) })) + .catch(() => {}); }, - async holdUpload(client, file) { + async holdUpload(client, file, accountId) { const account = accountForClient(client); if (!account || !offlineActive(account)) return null; - return holdFile(account, file, file.name); + return holdFile(account, file, file.name, accountId); }, }; +interface KeptBlob { type: string | null; data: Uint8Array } + /** Attachments and images opened while online are kept up to this size. */ const MAX_RECORDED_BLOB_BYTES = 5 * 1024 * 1024; @@ -472,6 +415,8 @@ const outbox: OutboxHooks = { shouldQueue(client) { const account = accountForClient(client); if (!account || !offlineActive(account)) return false; + // A local copy has no way to the server, whatever the probes last said. + if (client.isOfflineOnly?.()) return true; return isOfflineNow(client.getServerUrl()) || (typeof navigator !== 'undefined' && navigator.onLine === false); }, async queueEmail(client, args) { @@ -483,7 +428,11 @@ const outbox: OutboxHooks = { const stored = [...args.slice(0, 15), options]; const summary = summaryOf(stored); const labelled = !!options.labelHeaders && Object.keys(options.labelHeaders as object).length > 0; - const id = await queueSend(account, summary, { kind: 'email', args: stored }, policyFingerprint(account, labelled)); + // Fixed now and kept across retries: how a lost response is told from a send that never arrived. + // Only its domain is used, and the sender may be written "Name
    ". + const sender = ((args[6] as string | undefined) || client.getUsername()).replace(/>.*$/, ''); + const messageId = generateMessageId(sender); + const id = await queueSend(account, summary, { kind: 'email', args: stored }, policyFingerprint(account, labelled), messageId); await refreshQueueState(); notify({ kind: 'queued', subject: summary.subject }); return queuedResult(id); @@ -493,8 +442,8 @@ const outbox: OutboxHooks = { if (!account) throw new Error('No account for this send'); if (pluginsNotRunning()) throw new OfflinePluginsDownError(); const held = await holdFile(account, raw, 'message.eml'); - const { summary, labelled } = await rawSummary(raw); - const id = await queueSend(account, summary, { kind: 'raw', blobId: held.blobId, ...options }, policyFingerprint(account, labelled)); + const { summary, labelled, messageId } = await rawSummary(raw); + const id = await queueSend(account, summary, { kind: 'raw', blobId: held.blobId, ...options }, policyFingerprint(account, labelled), messageId); await refreshQueueState(); notify({ kind: 'queued', subject: summary.subject }); return queuedResult(id); @@ -510,20 +459,26 @@ const outbox: OutboxHooks = { const file = await heldFile(account, attachment.blobId); if (!file) throw Object.assign(new Error('A file attached offline is no longer on this device'), { name: 'OfflineFileMissingError' }); const { blobId } = await real.uploadBlob(file); + // The server went away just now, and offline mode held the file again. + if (isLocalBlobId(blobId)) throw new SendNotStartedError(); out.push({ ...attachment, blobId }); } return out; }, }; -/** Subject, recipients and label of a raw message, for the outbox list. */ -async function rawSummary(raw: Blob): Promise<{ summary: OutboxSummary; labelled: boolean }> { +/** Subject, recipients, label and Message-ID of a raw message, for the outbox. */ +async function rawSummary(raw: Blob): Promise<{ summary: OutboxSummary; labelled: boolean; messageId?: string }> { const head = await raw.slice(0, 64 * 1024).text(); const end = head.search(/\r?\n\r?\n/); const block = (end >= 0 ? head.slice(0, end) : head).replace(/\r?\n[ \t]+/g, ' '); const header = (name: string) => block.split(/\r?\n/).find((line) => line.toLowerCase().startsWith(`${name}:`))?.slice(name.length + 1).trim() ?? ''; const to = ['to', 'cc'].flatMap((name) => header(name).split(',')).map((s) => s.trim()).filter(Boolean); - return { summary: { subject: decodeMimeWords(header('subject')), to }, labelled: header('sio-label') !== '' }; + return { + summary: { subject: decodeMimeWords(header('subject')), to }, + labelled: header('sio-label') !== '', + messageId: header('message-id').match(/<([^>\s]+)>/)?.[1], + }; } function decodeMimeWords(value: string): string { @@ -543,15 +498,16 @@ function decodeMimeWords(value: string): string { export async function refreshQueueState(): Promise { if (!offlineSupported()) return; // Nothing is kept for an account that hasn't it on; and opening the - // database would create it (on the sign-in page, after a sign-out). - if (useOfflineStore.getState().enabled.length === 0) { - useOfflineStore.setState({ outbox: [], pendingChanges: 0 }); + // store would create it (on the sign-in page, after a sign-out). + if (visibleAccounts().length === 0) { + useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [] }); return; } try { - // Sends of a tab closed on the way: uncertain, so they can be retried or deleted. + // Sends of a tab closed on the way: uncertain, so they can be looked up. await settleStaleSends(); - const { patches, outbox: entries } = await readQueue(); + const { failed, patches, outbox: entries, conflicts } = await readQueue(); + if (failed.length > 0) noteStoreUnavailable(); const rows: OutboxRow[] = entries.map((entry) => ({ id: entry.id, account: entry.account, @@ -561,16 +517,21 @@ export async function refreshQueueState(): Promise { status: entry.status, error: entry.error, })); - useOfflineStore.setState({ outbox: rows, pendingChanges: patches.length }); + // What an account's store can't tell right now is still there. + const unread = useOfflineStore.getState().outbox.filter((row) => failed.includes(row.account)); + useOfflineStore.setState({ outbox: [...unread, ...rows], pendingChanges: patches, conflicts }); + if (rows.some((row) => row.status === 'sending')) checkStaleSendsLater(); } catch { - // The database is being deleted. + // The store is being deleted, or can't start. + noteStoreUnavailable(); } } export async function refreshUsage(): Promise { - if (!offlineSupported() || useOfflineStore.getState().enabled.length === 0) return; + if (!offlineSupported() || visibleAccounts().length === 0) return; try { - useOfflineStore.setState({ usageBytes: (await cacheUsage()).total }); + const usage = await Promise.all(visibleAccounts().map((account) => getStore().area<{ bytes: number }>(account, 'mail.usage'))); + useOfflineStore.setState({ usageBytes: usage.reduce((sum, { bytes }) => sum + bytes, 0) }); } catch { /* deleted meanwhile */ } @@ -599,12 +560,9 @@ export async function runSync(account: string): Promise { syncsRunning.add(account); useOfflineStore.setState((s) => ({ syncing: [...new Set([...s.syncing, account])] })); try { - await syncAccount(account, client, { - stillActive: () => offlineActive(account), - recorded: async () => { await recording.get(account); }, - scopeMailboxes, - }); - useOfflineStore.setState((s) => ({ lastSyncedAt: { ...s.lastSyncedAt, [account]: Date.now() } })); + await getStore().ensureOpen(account); + getStore().refresh(account); + await getStore().sync(account); } catch { // Tried again at the next change or in 15 minutes. } finally { @@ -626,6 +584,8 @@ export function scheduleFlushAll(delayMs = 1_000): void { /** How long after a failed policy fetch the outbox asks again. */ const POLICY_RETRY_MS = 60_000; +/** A lost response with the server still there: nothing else would start the lookup. */ +const LOOKUP_AGAIN_MS = LOOKUP_WAIT_MS + 1_000; /** A send held by the server for undo, from the outbox: the undo toast as after any send. */ function offerUndo(entry: OutboxEntry, result: SendEmailResult): void { @@ -644,13 +604,37 @@ function offerUndo(entry: OutboxEntry, result: SendEmailResult): void { }); } +/** + * What the other areas queued. The page may hold stand-in ids for what just + * went out, so it reads again, the way a push makes it. Only the types go by + * that way: the "state" isn't one, and nothing stores it. + */ +const AREA_FLUSHES = [ + { call: 'calendar.flush', types: ['Calendar', 'CalendarEvent'] }, + { call: 'contacts.flush', types: ['AddressBook', 'ContactCard'] }, +]; + +async function flushAreas(account: string, client: NonNullable>): Promise { + for (const { call, types } of AREA_FLUSHES) { + // Contacts may not be there in this build: nothing to flush then. + const tally = await getStore().area<{ sent?: number; conflicts?: number | unknown[] } | null>(account, call).catch(() => null); + if (!tally) continue; + const conflicts = Array.isArray(tally.conflicts) ? tally.conflicts.length : tally.conflicts ?? 0; + if (!tally.sent && !conflicts) continue; + const role = (useAuthStore.getState().client as unknown) === client ? 'active' : 'background'; + const changed = { [client.getAccountId()]: Object.fromEntries(types.map((type) => [type, 'flushed'])) }; + emitPushStateChange({ '@type': 'StateChange', changed } as never, client, role); + } +} + export async function flushAll(): Promise { if (useOfflineStore.getState().startedOffline) { void leaveOfflineStart(); return; } - const { outbox: rows, pendingChanges } = useOfflineStore.getState(); - if (rows.length === 0 && pendingChanges === 0) return; + // Calendar and contact edits wait in their own areas, so what waits isn't known here. + const { outbox: rows, enabled } = useOfflineStore.getState(); + if (rows.length === 0 && enabled.length === 0) return; // The organisation's current word before anything goes: a label now // required, a send check now required or offline mode now forbidden must // stop what waited under the old rules. @@ -660,8 +644,7 @@ export async function flushAll(): Promise { return; } enforcePolicy(); - const accounts = new Set(rows.map((r) => r.account)); - if (pendingChanges > 0) for (const id of useOfflineStore.getState().enabled) accounts.add(id); + const accounts = new Set([...rows.map((r) => r.account), ...useOfflineStore.getState().enabled]); for (const account of accounts) { const client = onlineClientFor(account); if (!client || !offlineActive(account)) continue; @@ -670,7 +653,9 @@ export async function flushAll(): Promise { allowed: () => offlineAllowed(account), stricterNow, onScheduled: offerUndo, - }); + retryLater: () => scheduleFlushAll(LOOKUP_AGAIN_MS), + }).catch(() => {}); + await flushAreas(account, client); } await refreshQueueState(); for (const account of useOfflineStore.getState().enabled) scheduleSync(account, 3_000); @@ -680,7 +665,6 @@ export async function flushAll(): Promise { export async function enableOffline(account: string): Promise { if (!offlineAllowed(account)) return; - allowOfflineDbCreation(); const enabled = useOfflineStore.getState().enabled; if (!enabled.includes(account)) writeEnabled([...enabled, account]); // Ask the browser not to clear it when space runs low (it may say no). @@ -689,8 +673,7 @@ export async function enableOffline(account: string): Promise { } catch { /* no Storage API */ } - const client = onlineClientFor(account); - if (client) await putMisc(account, 'session', client.getSessionSnapshot()); + await getStore().ensureOpen(account).catch(() => {}); await tellServiceWorker(true); scheduleSync(account, 0); } @@ -717,16 +700,40 @@ export function confirmLoss(scope: string): void { async function unsentSubjects(scope: string): Promise { let rows = useOfflineStore.getState().outbox; + // Reading an account's store would create it if it never had one. + if (scope !== 'all' && !useOfflineStore.getState().enabled.includes(scope)) return rows.filter((row) => row.account === scope).map((row) => row.subject); try { rows = (await readQueue(scope === 'all' ? undefined : scope)).outbox.map((entry) => ({ id: entry.id, account: entry.account, createdAt: entry.createdAt, subject: entry.summary.subject, to: entry.summary.to, status: entry.status, })); } catch { - // No database: what the store last saw. + // No store: what the state last saw. } return rows.filter((row) => scope === 'all' || row.account === scope).map((row) => row.subject); } +/** How long a sign-out waits for the store's word on what would be lost. */ +const UNSENT_READ_MS = 2_000; + +/** What a sign-out of `scope` would delete: messages, and edits the server hasn't got. */ +export async function unsentCount(scope: string, waitMs = UNSENT_READ_MS): Promise<{ messages: number; changes: number }> { + const shown = useOfflineStore.getState(); + const known = { messages: shown.outbox.filter((row) => scope === 'all' || row.account === scope).length, changes: shown.pendingChanges }; + const accounts = (scope === 'all' ? visibleAccounts() : [scope]).filter((account) => shown.enabled.includes(account)); + if (accounts.length === 0) return known; + const pendingIn = (account: string, area: string) => getStore().area(account, `${area}.pending`).catch(() => 0); + const read = (async () => { + const { failed, outbox, patches } = await readQueue(scope === 'all' ? undefined : scope); + const edits = await Promise.all(accounts.flatMap((account) => [pendingIn(account, 'calendar'), pendingIn(account, 'contacts')])); + return { + // An account that can't answer counts with what it last said. + messages: outbox.length + shown.outbox.filter((row) => failed.includes(row.account)).length, + changes: patches + edits.reduce((sum, count) => sum + count, 0), + }; + })().catch(() => known); + return Promise.race([read, new Promise((resolve) => setTimeout(() => resolve(known), waitMs))]); +} + /** Names the unsent messages a wipe of `scope` deletes, unless the user agreed. */ async function nameLost(scope: string, stillSignedIn: boolean): Promise { if (named.has(scope) || named.has('all')) return; @@ -748,25 +755,26 @@ export async function wipeAccount(account: string): Promise { const timer = syncTimers.get(account); if (timer) clearTimeout(timer); syncTimers.delete(account); - await recording.get(account)?.catch(() => {}); + // Waited for a sign-in that never came: whoever is here now mustn't read its subjects. + if (isEvicted(account)) named.add(account); if (useOfflineStore.getState().enabled.includes(account) || useOfflineStore.getState().outbox.some((row) => row.account === account)) { await nameLost(account, useAuthStore.getState().isAuthenticated); } named.delete(account); + clearEvicted(account); if (useOfflineStore.getState().enabled.includes(account)) { writeEnabled(useOfflineStore.getState().enabled.filter((id) => id !== account)); } - // Nobody keeps mail here any more: the whole database goes (and isn't - // opened again, which would create it anew). + // Nobody keeps mail here any more: everything goes (and isn't opened + // again, which would create it anew). if (useOfflineStore.getState().enabled.length === 0) { await wipeEverything(); return; } try { - await wipeAccountCache(account); - await wipeAccountQueue(account); + await getStore().remove(account); } catch { - // No database: nothing to remove. + // No store: nothing to remove. The key is gone in any case. } await refreshQueueState(); await refreshUsage(); @@ -777,19 +785,25 @@ export async function wipeEverything(): Promise { for (const timer of syncTimers.values()) clearTimeout(timer); syncTimers.clear(); writeEnabled([]); - forgetIndexes(); - forgetDeviceKey(); - await deleteOfflineDb(); + getStore().stop(); + await purgeStoreFiles(); + try { + indexedDB.deleteDatabase(LEGACY_DB_NAME); + } catch { + /* no IndexedDB */ + } await tellServiceWorker(false); await deleteShellCaches(); clearHandoffs(); confirmed.clear(); named.clear(); - useOfflineStore.setState({ outbox: [], pendingChanges: 0, usageBytes: 0, syncing: [], lastSyncedAt: {} }); + useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [], usageBytes: 0, syncing: [], lastSyncedAt: {} }); } /** Whether the policy of an account is known well enough to decide on it. */ function policyDecided(account: string): boolean { + // Dropped, with no groups to wait for: what applies to everyone reaches it. + if (isEvicted(account)) return true; const auth = useAuthStore.getState(); if (account === auth.activeAccountId || !auth.activeAccountId) return !usePolicyStore.getState().subjectPending; // Another signed-in account: its groups have to be known first (a group @@ -804,10 +818,14 @@ export function enforcePolicy(): void { if (!loaded) return; for (const account of useOfflineStore.getState().enabled) { if (!policyDecided(account)) continue; - if (!offlineAllowed(account)) { + // What the organisation says, not whether this window can open the store: + // a blocked store says nothing against the data in it. + if (devicesForAccount(account).noOffline) { + // Whoever is here now wasn't signed in to a dropped account. + const someoneElses = isEvicted(account); clearHandoffs(); void wipeAccount(account); - notify({ kind: 'forbidden' }); + if (!someoneElses) notify({ kind: 'forbidden' }); } } } @@ -852,18 +870,35 @@ async function leaveOfflineStart(): Promise { let started = false; const cleanups: Array<() => void> = []; -/** Runs offline mode in this tab. Safe to call more than once. */ -export function startOfflineMode(): () => void { - if (started || !offlineSupported()) return () => {}; - started = true; +/** + * The JMAP client's hooks. Separate from startOfflineMode because the auth + * store needs them before it restores accounts: a start from the local copy + * asks the store long before the host component mounts. + */ +export function registerOfflineHooks(): void { + if (!offlineSupported()) return; + configureStore(storeHooks); setOfflineHooks(hooks); setOutboxHooks(outbox); + void moveLegacyOutbox(); +} + +/** Runs offline mode in this tab. Safe to call more than once. */ +export function startOfflineMode(): () => void { + if (!started && !offlineSupported()) void noteStrandedMail(); + if (started || !offlineSupported()) return () => {}; + started = true; + registerOfflineHooks(); const onOnline = () => { useOfflineStore.setState({ browserOffline: false }); + void getStore().setOnline(true).catch(() => {}); scheduleFlushAll(500); }; - const onOffline = () => useOfflineStore.setState({ browserOffline: true }); + const onOffline = () => { + useOfflineStore.setState({ browserOffline: true }); + void getStore().setOnline(false).catch(() => {}); + }; window.addEventListener('online', onOnline); window.addEventListener('offline', onOffline); cleanups.push(() => { @@ -871,8 +906,17 @@ export function startOfflineMode(): () => void { window.removeEventListener('offline', onOffline); }); + // Back in front: catch up now rather than at the next interval. + const onVisible = () => { + if (document.visibilityState !== 'visible') return; + for (const account of useOfflineStore.getState().enabled) scheduleSync(account, 0); + }; + document.addEventListener('visibilitychange', onVisible); + cleanups.push(() => document.removeEventListener('visibilitychange', onVisible)); + // Another tab switched it off for an account (or everyone signed out - // there): stop here too, and don't write anything for that account. + // there): stop here too, and don't write anything for that account. One + // switched on there has to be opened here if this tab owns the store. const onStorage = (event: StorageEvent) => { if (event.key !== OFFLINE_ACCOUNTS_KEY && event.key !== null) return; const next = readEnabledFrom(event.key === null ? null : event.newValue); @@ -882,13 +926,9 @@ export function startOfflineMode(): () => void { const timer = syncTimers.get(id); if (timer) clearTimeout(timer); syncTimers.delete(id); - forgetIndexes(id); - } - if (next.length === 0) { - forgetIndexes(); - forgetDeviceKey(); } useOfflineStore.setState({ enabled: next }); + for (const id of next) if (!before.includes(id)) void getStore().ensureOpen(id).catch(() => {}); void refreshQueueState(); }; window.addEventListener('storage', onStorage); @@ -902,22 +942,32 @@ export function startOfflineMode(): () => void { cleanups.push(useAccountSubjects.subscribe(() => enforcePolicy())); enforcePolicy(); - // An account that signs out (or is removed) takes its offline data with it. - let known = new Set(useAccountStore.getState().accounts.map((a) => a.id)); - cleanups.push(useAccountStore.subscribe((state) => { - const now = new Set(state.accounts.map((a) => a.id)); - for (const id of known) if (!now.has(id)) void wipeAccount(id); - known = now; + // An account signed out or removed takes its data with it. One the app + // dropped by itself doesn't: its session ran out, or the server can't + // confirm it right now, and that says nothing against the mail on this device. + cleanups.push(onAccountRemoved((id, reason) => { + if (reason === 'explicit') void wipeAccount(id); })); + void sweepAccountsGone(); + void checkStorageAccess(); // Session, then a sync, for every account signed in with offline mode on. + // A client that just became real is what the waiting mail was waiting for: + // the flush at the start may have run before it was connected. + const flushedFor = new WeakSet(); const syncSignedIn = () => { + let connected = false; for (const account of useOfflineStore.getState().enabled) { const client = onlineClientFor(account); if (!client || !offlineActive(account)) continue; - void putMisc(account, 'session', client.getSessionSnapshot()).catch(() => {}); + void getStore().ensureOpen(account).then(() => getStore().refresh(account)).catch(() => {}); scheduleSync(account, 3_000); + if (!flushedFor.has(client)) { + flushedFor.add(client); + if (!isOfflineNow(client.getServerUrl())) connected = true; + } } + if (connected) scheduleFlushAll(1_000); }; cleanups.push(useAuthStore.subscribe((state, prev) => { if (state.client !== prev.client || state.connectedAccountsRevision !== prev.connectedAccountsRevision) syncSignedIn(); @@ -927,15 +977,22 @@ export function startOfflineMode(): () => void { const interval = setInterval(() => { for (const account of useOfflineStore.getState().enabled) scheduleSync(account, 0); }, SYNC_INTERVAL_MS); - const reconnect = setInterval(() => { void leaveOfflineStart(); }, RECONNECT_MS); + const reconnect = setInterval(() => { + void leaveOfflineStart(); + // Nothing else tells a server that was down is back: a client that thinks + // it's down has no reason to ask it. A passing probe starts the flush. + for (const server of useOfflineStore.getState().downServers) void probeServer(server); + }, RECONNECT_MS); cleanups.push(() => { clearInterval(interval); clearInterval(reconnect); + if (staleCheck) clearTimeout(staleCheck); + staleCheck = null; }); if (useOfflineStore.getState().enabled.length > 0) void tellServiceWorker(true); void (async () => { - if (useOfflineStore.getState().enabled.length > 0) await settleStaleSends().catch(() => {}); + await moveLegacyOutbox(); await refreshQueueState(); if (!isOfflineNow()) scheduleFlushAll(2_000); })(); @@ -945,28 +1002,75 @@ export function startOfflineMode(): () => void { for (const cleanup of cleanups.splice(0)) cleanup(); setOfflineHooks(null); setOutboxHooks(null); + configureStore(null); started = false; }; } -/** How often the cache catches up while the tab is open. */ +/** + * Data no signed-in account vouches for. A dropped account's waits 30 days. One + * that left the registry on purpose before this ran (a login without "remember + * me" ends at a reload, before the host mounts) goes now, naming lost mail. + */ +async function sweepAccountsGone(now = Date.now()): Promise { + const evicted = readEvicted(); + const accounts = useAccountStore.getState().accounts; + for (const account of useOfflineStore.getState().enabled) { + const at = evicted[account]; + const gone = at !== undefined ? now - at > EVICTED_KEEP_MS : !accounts.some((entry) => entry.id === account); + if (gone) await wipeAccount(account).catch(() => {}); + } +} + +let staleCheck: ReturnType | null = null; + +/** A send taken by a tab that closed on the way: nothing else comes back to it. */ +function checkStaleSendsLater(): void { + if (staleCheck) return; + staleCheck = setTimeout(() => { + staleCheck = null; + void refreshQueueState().then(() => scheduleFlushAll(0)); + }, SENDING_STALE_MS + 1_000); +} + +/** Firefox's private windows refuse this at once, where the worker would try for half a minute. */ +async function checkStorageAccess(): Promise { + try { + await navigator.storage.getDirectory(); + } catch (error) { + if ((error as Error)?.name === 'SecurityError') useOfflineStore.setState({ storageBlocked: true }); + } +} + +let strandedNoted = false; + +/** An earlier version's unsent mail, in a browser that can't run what would move it. */ +async function noteStrandedMail(): Promise { + if (strandedNoted || IS_LITE || !hasIndexedDb()) return; + if ((await strandedLegacyMessages().catch(() => 0)) === 0) return; + strandedNoted = true; + notify({ kind: 'legacy-stranded' }); +} + +/** How often the store catches up while the tab is open, on top of pushes. */ export const SYNC_INTERVAL_MS = 15 * 60 * 1000; -// A full sign-out deletes the database (lib/sign-out-cleanup.ts); forget -// what is held in memory as well. -onFullSignOut(() => { +// A full sign-out deletes the store (lib/sign-out-cleanup.ts); forget what +// is held in memory as well. +onFullSignOut(({ keepLocalStore }) => { for (const timer of syncTimers.values()) clearTimeout(timer); syncTimers.clear(); - forgetIndexes(); - forgetDeviceKey(); + if (!keepLocalStore) getStore().stop(); confirmed.clear(); named.clear(); - useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, usageBytes: 0, syncing: [], lastSyncedAt: {}, startedOffline: false }); + useOfflineStore.setState((s) => ({ enabled: keepLocalStore ? s.enabled : [], outbox: [], pendingChanges: 0, conflicts: [], usageBytes: 0, syncing: [], lastSyncedAt: {}, startedOffline: false })); }); // Every sign-out the user didn't agree to lose mail at (the session limits, // a refused token renewal, a removed account) names the unsent messages. -onBeforeSignOut(async (accountId) => { +onBeforeSignOut(async (accountId, { keepLocalStore }) => { + // Nothing is lost: it waits for the next sign-in. + if (keepLocalStore) return; const stillSignedIn = accountId !== 'all' && useAccountStore.getState().accounts.some((a) => a.id !== accountId); await nameLost(accountId, stillSignedIn); }); @@ -977,16 +1081,35 @@ export async function serverUnreachable(serverUrl: string): Promise { return probeServer(serverUrl); } -/** The session kept for an account, for starting without the server. */ -export async function offlineSessionFor(account: string): Promise { +/** How long a start waits for the store before it signs in the usual way. */ +const SESSION_READ_TIMEOUT_MS = 5_000; + +/** The session kept for an account, for starting from the local copy. */ +/** A start doesn't hang on a black-holed fetch or a stuck database. */ +const BOOT_WAIT_MS = 3_000; + +function withinBootWait(work: Promise): Promise { + return Promise.race([work.catch(() => {}), new Promise((resolve) => setTimeout(resolve, BOOT_WAIT_MS))]); +} + +export async function offlineSessionFor(account: string, options: { withFolders?: boolean } = {}): Promise { // The organisation's last word: the policy kept from the last fetch (the // fetch itself fails without a network), never the defaults. const policy = usePolicyStore.getState(); - if (!policy.loaded && !policy.fetching) await policy.fetchPolicy(); - else await whenPolicySettled(3000); - if (!offlineActive(account)) return null; + await withinBootWait(!policy.loaded && !policy.fetching ? policy.fetchPolicy() : whenPolicySettled(BOOT_WAIT_MS)); + // Still nothing to go on: the usual start, which has its own errors. + if (!usePolicyStore.getState().loaded || !offlineActive(account)) return null; + registerOfflineHooks(); try { - return await getMisc(account, 'session'); + // What's in the old database goes on moving behind the start. + await withinBootWait(moveLegacyOutbox()); + return await Promise.race([ + (async () => { + if (options.withFolders && !(await getStore().area(account, 'mail.hasFolders'))) return null; + return getStore().area(account, 'mail.session'); + })(), + new Promise((resolve) => setTimeout(() => resolve(null), SESSION_READ_TIMEOUT_MS)), + ]); } catch { return null; } diff --git a/lib/offline/queue.ts b/lib/offline/queue.ts index 4da2d5bf0..b9e0232be 100644 --- a/lib/offline/queue.ts +++ b/lib/offline/queue.ts @@ -1,254 +1,96 @@ /** - * What waits on this device for the network (#523): changes made offline - * (read, flagged, tags, moves) and the outbox. Both are kept encrypted in - * the offline database and sent in the order they were made. - * - * An outbox entry is a finished message: everything a send does up to the - * transport ran when the user clicked Send (the organisation's checks, - * the classification label, signatures and disclaimers, plugin send checks, - * S/MIME or PGP with the user present). What is kept is the arguments of the - * final send, with the files that couldn't be uploaded yet, or the finished - * raw message of a plugin's send. Sending it later can't skip any of that. - * - * The state of an outbox entry (waiting, sending, ...) is kept unencrypted - * next to it, so that taking an entry for sending is one database - * transaction: two tabs can't both take it (claimForSending). + * What waits on this device for the network (#523): the page's view of the + * outbox in the local store (lib/store/outbox). Changes made offline wait there + * merged per record; messages wait as finished sends. Everything is kept + * encrypted in the store, and taking an entry for sending is one SQLite + * transaction, so two tabs can't both take it (claimForSending). */ -import { idbAdd, idbDeleteAccount, idbGet, idbGetAll, idbGetAllByAccount, idbUpdate, idbWrite } from './db'; -import { openBytes, openJson, sealBytes, sealJson, type Sealed } from './crypto'; -import { generateUUID } from '@/lib/utils'; import { requestOutboxSync } from '@/lib/pwa-notify/outbox-sync'; +import { getStore } from '@/lib/store/client/api'; +import type { ConflictSummary, OutboxError, OutboxStatus, OutboxSummary, PolicyFingerprint, SendItem, StoredConflict, StoredOutboxEntry } from '@/lib/store/outbox/items'; +import { visibleAccounts } from './state'; -export interface PatchItem { - kind: 'patch'; - jmap: string; - update: Record>; +export { LOCAL_BLOB_PREFIX, heldBlobIds, isLocalBlobId } from '@/lib/store/outbox/items'; +export type { EmailSendItem, OutboxError, OutboxStatus, OutboxSummary, PolicyFingerprint, RawSendItem, SendItem } from '@/lib/store/outbox/items'; + +export interface OutboxEntry extends StoredOutboxEntry { + account: string; } -/** The arguments of useEmailStore.sendEmail after the client (positional, as the client takes them). */ -export interface EmailSendItem { - kind: 'email'; - args: unknown[]; -} - -export interface RawSendItem { - kind: 'raw'; - /** The finished message, kept as an upload until it is sent. */ - blobId: string; - identityId: string; - delayedUntil?: string; - envelopeRecipients?: string[]; - /** submitRaw: no copy in Sent. */ - submitOnly?: boolean; -} - -export type SendItem = EmailSendItem | RawSendItem; - -export interface OutboxSummary { - subject: string; - to: string[]; -} - -/** - * - waiting: goes out by itself when the server answers; - * - sending: on its way (a tab took it); - * - failed: the server refused it; the error says why; - * - uncertain: the connection broke after it may have reached the server; - * never sent again by itself; - * - held: the organisation's rules got stricter while it waited. - */ -export type OutboxStatus = 'waiting' | 'sending' | 'failed' | 'uncertain' | 'held'; - -/** Why an entry failed: one of ours (a code the UI translates) or the server's own words. */ -export interface OutboxError { - code?: 'file_missing' | 'no_sent_mailbox' | 'no_drafts_mailbox'; - text?: string; -} - -/** What the organisation required when the message was finished (see offline-mode.ts policyFingerprint). */ -export interface PolicyFingerprint { - labelRequired: boolean; - hasLabel: boolean; - guards: string[]; -} - -export interface OutboxEntry { +export interface ConflictEntry { id: number; account: string; + recordId: string; createdAt: number; - summary: OutboxSummary; - status: OutboxStatus; - /** When a tab took it for sending (ms). */ - sendingSince?: number; - error?: OutboxError; - item: SendItem; - fingerprint?: PolicyFingerprint; + summary: ConflictSummary; } -interface QueueRecord extends Sealed { - id?: number; - account: string; - kind: 'patch' | 'send'; - createdAt: number; - status?: OutboxStatus; - sendingSince?: number; -} - -interface SendPayload { - summary: OutboxSummary; - error?: OutboxError; - item: SendItem; - fingerprint?: PolicyFingerprint; -} - -export async function queuePatch(account: string, jmap: string, update: PatchItem['update']): Promise { - const item: PatchItem = { kind: 'patch', jmap, update }; - await idbAdd('queue', { account, kind: 'patch', createdAt: Date.now(), ...(await sealJson(item)) } satisfies QueueRecord); - requestOutboxSync(); -} - -export async function queueSend(account: string, summary: OutboxSummary, item: SendItem, fingerprint?: PolicyFingerprint): Promise { - const payload: SendPayload = { summary, item, fingerprint }; - const id = Number(await idbAdd('queue', { account, kind: 'send', createdAt: Date.now(), status: 'waiting', ...(await sealJson(payload)) } satisfies QueueRecord)); +export async function queueSend(account: string, summary: OutboxSummary, item: SendItem, fingerprint?: PolicyFingerprint, messageId?: string): Promise { + const id = await getStore().area(account, 'outbox.queueSend', { summary, item, fingerprint, messageId }); requestOutboxSync(); return id; } -export interface QueuedPatch { id: number; account: string; item: PatchItem } - -/** Everything waiting for one account (or all), oldest first. */ -export async function readQueue(account?: string): Promise<{ patches: QueuedPatch[]; outbox: OutboxEntry[] }> { - const records = account ? await idbGetAllByAccount('queue', account) : await idbGetAll('queue'); - records.sort((a, b) => (a.id ?? 0) - (b.id ?? 0)); - const patches: QueuedPatch[] = []; - const outbox: OutboxEntry[] = []; - for (const record of records) { - try { - if (record.kind === 'patch') { - patches.push({ id: record.id!, account: record.account, item: await openJson(record) }); - } else { - const payload = await openJson(record); - outbox.push({ - id: record.id!, - account: record.account, - createdAt: record.createdAt, - status: record.status ?? 'waiting', - sendingSince: record.sendingSince, - ...payload, - }); - } - } catch { - // Sealed with a key that is gone; removed with the account. - } - } - return { patches, outbox }; +interface StoredQueue { + patches: number; + entries: StoredOutboxEntry[]; + conflicts: StoredConflict[]; } -/** - * Takes a waiting entry for sending, in one transaction: false when it - * isn't waiting any more (another tab took it, the user deleted it). - */ -export async function claimForSending(id: number): Promise { - return idbUpdate('queue', id, (record) => { - if (!record || record.kind !== 'send' || (record.status ?? 'waiting') !== 'waiting') return undefined; - return { ...record, status: 'sending', sendingSince: Date.now() }; - }); -} - -/** Sets an entry's state; with an error or a new after-send list, its payload too. */ -export async function updateOutboxEntry( - entry: OutboxEntry, - patch: { status: OutboxStatus; error?: OutboxError }, -): Promise { - const payload: SendPayload = { - summary: entry.summary, - item: entry.item, - fingerprint: entry.fingerprint, - error: patch.error, +/** Everything waiting for one account (or every account with it on), oldest first. */ +export async function readQueue(account?: string): Promise<{ failed: string[]; patches: number; outbox: OutboxEntry[]; conflicts: ConflictEntry[] }> { + const accounts = account ? [account] : visibleAccounts(); + const settled = await Promise.allSettled(accounts.map(async (id) => ({ id, queue: await getStore().area(id, 'outbox.read') }))); + const read = settled.flatMap((result) => (result.status === 'fulfilled' ? [result.value] : [])); + const failed = accounts.filter((_, index) => settled[index].status === 'rejected'); + // Nothing could be read: that isn't an empty queue. + if (read.length === 0 && failed.length > 0) throw (settled[0] as PromiseRejectedResult).reason; + return { + failed, + patches: read.reduce((sum, { queue }) => sum + queue.patches, 0), + outbox: read.flatMap(({ id, queue }) => queue.entries.map((entry) => ({ ...entry, account: id }))), + conflicts: read.flatMap(({ id, queue }) => queue.conflicts.map((conflict) => ({ + id: conflict.id, + account: id, + recordId: conflict.recordId, + createdAt: conflict.createdAt, + summary: conflict.summary, + }))), }; - const sealed = await sealJson(payload); - await idbUpdate('queue', entry.id, (record) => { - // Deleted meanwhile: stays deleted. - if (!record) return undefined; - return { ...record, status: patch.status, sendingSince: patch.status === 'sending' ? Date.now() : undefined, ...sealed }; - }); } -export async function removeQueued(ids: number[]): Promise { - await idbWrite(ids.map((id) => ({ store: 'queue' as const, del: id }))); +/** Takes a waiting entry for sending: false when it isn't waiting any more (another tab took it, the user deleted it). */ +export function claimForSending(account: string, id: number): Promise { + return getStore().area(account, 'outbox.claim', { id }); +} + +export function updateOutboxEntry( + entry: Pick, + patch: { status: OutboxStatus; error?: OutboxError; copyId?: string | null }, +): Promise { + return getStore().area(entry.account, 'outbox.update', { id: entry.id, ...patch }); } /** Removes an outbox entry and the files it held. */ -export async function removeOutboxEntry(entry: OutboxEntry): Promise { - const blobIds = heldBlobIds(entry.item); - await idbWrite([ - { store: 'queue', del: entry.id }, - ...blobIds.map((blobId) => ({ store: 'uploads' as const, del: uploadKey(entry.account, blobId) })), - ]); +export function removeOutboxEntry(entry: Pick): Promise { + return getStore().area(entry.account, 'outbox.remove', { id: entry.id }); +} + +export function dismissConflict(account: string, id: number): Promise { + return getStore().area(account, 'outbox.dismissConflict', { id }); } // ---- Files attached while offline ---- -/** Stand-in blob ids of files kept on this device start with this. */ -export const LOCAL_BLOB_PREFIX = 'bulwark-local-'; - -const uploadKey = (account: string, blobId: string) => `${account}\u0001${blobId}`; - -export function isLocalBlobId(blobId: unknown): blobId is string { - return typeof blobId === 'string' && blobId.startsWith(LOCAL_BLOB_PREFIX); -} - -interface UploadRecord extends Sealed { - key: string; - account: string; - createdAt: number; - /** The file's name and type, sealed: names say a lot about a message. */ - meta: Sealed; -} - /** Keeps a file under a stand-in blob id until its message is sent. */ -export async function holdFile(account: string, file: Blob, name: string): Promise<{ blobId: string; size: number; type: string }> { - const blobId = `${LOCAL_BLOB_PREFIX}${generateUUID()}`; - const sealed = await sealBytes(await file.arrayBuffer()); - const meta = await sealJson({ name, type: file.type }); - await idbWrite([{ store: 'uploads', put: { key: uploadKey(account, blobId), account, createdAt: Date.now(), meta, ...sealed } satisfies UploadRecord }]); - return { blobId, size: file.size, type: file.type || 'application/octet-stream' }; +export async function holdFile(account: string, file: Blob, name: string, jmap?: string): Promise<{ blobId: string; size: number; type: string }> { + const data = new Uint8Array(await file.arrayBuffer()); + return getStore().area(account, 'outbox.holdFile', { jmap, name, type: file.type, data }); } export async function heldFile(account: string, blobId: string): Promise { - const record = await idbGet('uploads', uploadKey(account, blobId)); - if (!record) return null; - try { - const meta = await openJson<{ name?: string; type?: string }>(record.meta); - return new File([await openBytes(record)], meta.name || 'attachment', { type: meta.type || 'application/octet-stream' }); - } catch { - return null; - } -} - -/** The stand-in blob ids an outbox entry needs uploaded first. */ -export function heldBlobIds(item: SendItem): string[] { - if (item.kind === 'raw') return isLocalBlobId(item.blobId) ? [item.blobId] : []; - const attachments = item.args[10]; - if (!Array.isArray(attachments)) return []; - return attachments.map((a) => (a as { blobId?: unknown }).blobId).filter(isLocalBlobId); -} - -/** Files held for no outbox entry (attached offline to a message that was then discarded). */ -export async function dropOrphanFiles(account: string, olderThanMs = 24 * 60 * 60 * 1000): Promise { - const { outbox } = await readQueue(account); - const needed = new Set(outbox.flatMap((entry) => heldBlobIds(entry.item))); - const held = await idbGetAllByAccount<{ key: string; createdAt?: number }>('uploads', account); - const cutoff = Date.now() - olderThanMs; - const stale = held.filter((record) => { - const blobId = record.key.split('\u0001')[1]; - return !needed.has(blobId) && (record.createdAt ?? 0) < cutoff; - }); - await idbWrite(stale.map((record) => ({ store: 'uploads' as const, del: record.key }))); -} - -/** The queue and held files of one account. */ -export async function wipeAccountQueue(account: string): Promise { - await idbDeleteAccount(account, ['queue', 'uploads']); + const held = await getStore().area<{ name: string; type: string; data: Uint8Array } | null>(account, 'outbox.heldFile', { blobId }); + if (!held) return null; + return new File([held.data as BlobPart], held.name, { type: held.type }); } diff --git a/lib/offline/responder.ts b/lib/offline/responder.ts deleted file mode 100644 index 9495ee9a2..000000000 --- a/lib/offline/responder.ts +++ /dev/null @@ -1,394 +0,0 @@ -/** - * Answers JMAP requests from the mail kept on this device while the server - * can't be reached (#523), so the app works as usual: the folder list, the - * message list of a cached folder, opening a cached message, threads. - * - * Changes are queued (queue.ts) and applied to the cache at once: keywords - * (read, flagged, any tag) and folders (move, delete to Trash). Deleting for - * good, emptying a folder and everything else that would change the server - * is refused, as are calls the cache can't answer (search, changes since a - * state): the whole request then fails as it would without offline mode. - * - * Answers carry no collection state strings: a made-up one would end up in an - * Email/changes once the server is back, which refuses it. - */ - -import { BLOBS, emailIndex, getBody, getMisc, patchEmails, putMisc, type CachedEmail } from './cache'; -import type { OfflineMethodCall, OfflineResponse } from '@/lib/jmap/offline-hooks'; - -export type MethodResult = Record; - -export interface ResponderContext { - account: string; - /** The client's own JMAP account (calls without accountId use it). */ - defaultJmap: string; - /** Queue changes to messages; resolves once they are kept. */ - queuePatch: (jmap: string, update: Record>) => Promise; - /** A change offline mode refuses (delete for good); the caller tells the user. */ - onRefused: (what: 'destroy' | 'other') => void; - /** - * The folders whose mail is kept here, per JMAP account. A list of any - * other folder isn't answered (it would look empty while its count says - * otherwise). Without it, every folder is answered. - */ - keptFolders?: Record; -} - -/** The folders a filter lists, as far as it names them. */ -function filterFolders(filter: Filter | undefined | null, out: string[] = []): string[] { - if (!filter) return out; - if (typeof filter.inMailbox === 'string') out.push(filter.inMailbox); - if (Array.isArray(filter.conditions)) for (const c of filter.conditions as Filter[]) filterFolders(c, out); - return out; -} - -class Unanswerable extends Error {} - -/** RFC 8620 §3.7 result references, with the `*` array step. */ -function resolvePointer(value: unknown, path: string): unknown { - const steps = path.split('/').slice(1).map((s) => s.replace(/~1/g, '/').replace(/~0/g, '~')); - const walk = (node: unknown, i: number): unknown => { - if (i === steps.length) return node; - const step = steps[i]; - if (step === '*') { - if (!Array.isArray(node)) throw new Unanswerable(); - return node.flatMap((item) => { - const out = walk(item, i + 1); - return Array.isArray(out) ? out : [out]; - }); - } - if (node === null || typeof node !== 'object') throw new Unanswerable(); - return walk((node as Record)[step], i + 1); - }; - return walk(value, 0); -} - -function resolveArgs(args: Record, done: Array<[string, MethodResult, string]>): Record { - const out: Record = {}; - for (const [key, value] of Object.entries(args)) { - if (!key.startsWith('#')) { - out[key] = value; - continue; - } - const ref = value as { resultOf?: string; name?: string; path?: string }; - const source = done.find(([name, , id]) => id === ref.resultOf && name === ref.name); - if (!source || typeof ref.path !== 'string') throw new Unanswerable(); - out[key.slice(1)] = resolvePointer(source[1], ref.path); - } - return out; -} - -// ---- Email/query filters and sort ---- - -type Filter = Record; - -function matches(email: CachedEmail, filter: Filter | undefined | null): boolean { - if (!filter || Object.keys(filter).length === 0) return true; - if (typeof filter.operator === 'string') { - const conditions = Array.isArray(filter.conditions) ? (filter.conditions as Filter[]) : []; - if (filter.operator === 'AND') return conditions.every((c) => matches(email, c)); - if (filter.operator === 'OR') return conditions.some((c) => matches(email, c)); - if (filter.operator === 'NOT') return !conditions.some((c) => matches(email, c)); - throw new Unanswerable(); - } - for (const [key, value] of Object.entries(filter)) { - switch (key) { - case 'inMailbox': - if (!email.mailboxIds?.[value as string]) return false; - break; - case 'inMailboxOtherThan': - if (!Object.keys(email.mailboxIds ?? {}).some((id) => !(value as string[]).includes(id))) return false; - break; - case 'hasKeyword': - if (!email.keywords?.[value as string]) return false; - break; - case 'notKeyword': - if (email.keywords?.[value as string]) return false; - break; - case 'after': - if (Date.parse(String(email.receivedAt)) < Date.parse(String(value))) return false; - break; - case 'before': - if (Date.parse(String(email.receivedAt)) >= Date.parse(String(value))) return false; - break; - case 'hasAttachment': - if (Boolean(email.hasAttachment) !== Boolean(value)) return false; - break; - default: - // Text search and the rest need the server. - throw new Unanswerable(); - } - } - return true; -} - -interface Comparator { property?: string; isAscending?: boolean; keyword?: string } - -function firstAddress(value: unknown): string { - const first = Array.isArray(value) ? (value[0] as { name?: string; email?: string } | undefined) : undefined; - return (first?.name || first?.email || '').toLowerCase(); -} - -function compareBy(sort: Comparator[]): (a: CachedEmail, b: CachedEmail) => number { - const levels = sort.length > 0 ? sort : [{ property: 'receivedAt', isAscending: false }]; - return (a, b) => { - for (const level of levels) { - let diff = 0; - switch (level.property) { - case 'receivedAt': - case 'sentAt': - diff = (Date.parse(String(a[level.property] ?? a.receivedAt)) || 0) - (Date.parse(String(b[level.property] ?? b.receivedAt)) || 0); - break; - case 'size': - diff = Number(a.size ?? 0) - Number(b.size ?? 0); - break; - case 'subject': - diff = String(a.subject ?? '').localeCompare(String(b.subject ?? '')); - break; - case 'from': - diff = firstAddress(a.from).localeCompare(firstAddress(b.from)); - break; - case 'to': - diff = firstAddress(a.to).localeCompare(firstAddress(b.to)); - break; - case 'hasKeyword': - diff = Number(Boolean(a.keywords?.[level.keyword ?? ''])) - Number(Boolean(b.keywords?.[level.keyword ?? ''])); - break; - default: - diff = 0; - } - if (diff !== 0) return level.isAscending === false ? -diff : diff; - } - return 0; - }; -} - -async function emailsOf(ctx: ResponderContext, jmap: string): Promise { - const index = await emailIndex(ctx.account); - const prefix = `${ctx.account}\u0001${jmap}\u0001`; - const out: CachedEmail[] = []; - for (const [key, email] of index) if (key.startsWith(prefix)) out.push(email); - return out; -} - -async function emailQuery(ctx: ResponderContext, args: Record): Promise { - const jmap = (args.accountId as string) || ctx.defaultJmap; - if (args.anchor !== undefined) throw new Unanswerable(); - if (ctx.keptFolders) { - const kept = ctx.keptFolders[jmap] ?? []; - if (filterFolders(args.filter as Filter).some((id) => !kept.includes(id))) throw new Unanswerable(); - } - let list = (await emailsOf(ctx, jmap)).filter((email) => matches(email, args.filter as Filter)); - list.sort(compareBy((args.sort as Comparator[]) ?? [])); - if (args.collapseThreads) { - const seen = new Set(); - list = list.filter((email) => { - const thread = email.threadId ?? email.id; - if (seen.has(thread)) return false; - seen.add(thread); - return true; - }); - } - const position = Math.max(0, Number(args.position ?? 0)); - const limit = typeof args.limit === 'number' ? args.limit : list.length; - return { - accountId: jmap, - canCalculateChanges: false, - position, - ids: list.slice(position, position + limit).map((email) => email.id), - total: list.length, - }; -} - -function pick(email: Record, properties: unknown): Record { - if (!Array.isArray(properties)) { - const all = { ...email }; - delete all[BLOBS]; - return all; - } - const out: Record = { id: email.id }; - for (const prop of properties as string[]) if (prop in email) out[prop] = email[prop]; - return out; -} - -const NEEDS_BODY = new Set(['bodyValues', 'textBody', 'htmlBody', 'bodyStructure', 'headers', 'attachments']); - -async function emailGet(ctx: ResponderContext, args: Record): Promise { - const jmap = (args.accountId as string) || ctx.defaultJmap; - const index = await emailIndex(ctx.account); - const ids = Array.isArray(args.ids) ? (args.ids as string[]) : null; - if (!ids) throw new Unanswerable(); - const properties = args.properties as string[] | undefined; - const wantsBody = args.fetchTextBodyValues || args.fetchHTMLBodyValues || args.fetchAllBodyValues - || !properties || properties.some((p) => NEEDS_BODY.has(p)); - const list: Record[] = []; - const notFound: string[] = []; - for (const id of ids) { - const meta = index.get(`${ctx.account}\u0001${jmap}\u0001${id}`); - if (!meta) { notFound.push(id); continue; } - if (!wantsBody) { list.push(pick(meta, properties)); continue; } - const body = await getBody(ctx.account, jmap, id); - // Asked for the text and it isn't here: as if the message weren't - // there, rather than an empty message. - if (!body && (args.fetchTextBodyValues || args.fetchHTMLBodyValues || args.fetchAllBodyValues || properties?.includes('bodyValues'))) { - notFound.push(id); - continue; - } - list.push(pick({ ...(body ?? {}), ...meta }, properties)); - } - return { accountId: jmap, list, notFound }; -} - -async function threadGet(ctx: ResponderContext, args: Record): Promise { - const jmap = (args.accountId as string) || ctx.defaultJmap; - const ids = Array.isArray(args.ids) ? (args.ids as string[]) : []; - const emails = await emailsOf(ctx, jmap); - const list = [...new Set(ids)].map((id) => ({ - id, - emailIds: emails - .filter((email) => email.threadId === id) - .sort((a, b) => (Date.parse(String(a.receivedAt)) || 0) - (Date.parse(String(b.receivedAt)) || 0)) - .map((email) => email.id), - })); - return { accountId: jmap, list: list.filter((t) => t.emailIds.length > 0), notFound: list.filter((t) => t.emailIds.length === 0).map((t) => t.id) }; -} - -async function listGet(ctx: ResponderContext, args: Record, name: 'mailboxes' | 'identities'): Promise { - const jmap = (args.accountId as string) || ctx.defaultJmap; - const all = await getMisc[]>>(ctx.account, name); - const list = all?.[jmap]; - if (!list) throw new Unanswerable(); - const ids = Array.isArray(args.ids) ? new Set(args.ids as string[]) : null; - const chosen = ids ? list.filter((item) => ids.has(item.id as string)) : list; - return { - accountId: jmap, - list: chosen.map((item) => pick(item, args.properties)), - notFound: ids ? [...ids].filter((id) => !list.some((item) => item.id === id)) : [], - }; -} - -/** Unfiltered mailbox pages used to load a complete folder list offline. */ -async function mailboxQuery(ctx: ResponderContext, args: Record): Promise { - if (args.filter || args.sort || args.anchor || args.sortAsTree) throw new Unanswerable(); - const { list } = await listGet(ctx, { accountId: args.accountId }, 'mailboxes'); - const mailboxes = list as Record[]; - const position = typeof args.position === 'number' ? args.position : 0; - const limit = typeof args.limit === 'number' ? args.limit : mailboxes.length; - return { - accountId: (args.accountId as string) || ctx.defaultJmap, - // A query token only, never a Mailbox collection state for /changes. - queryState: JSON.stringify(mailboxes), - position, - ids: mailboxes.slice(position, position + limit).map((mailbox) => mailbox.id), - ...(args.calculateTotal ? { total: mailboxes.length } : {}), - }; -} - -/** Whether a PatchObject only changes what offline mode may queue. */ -function queueablePatch(patch: Record): boolean { - return Object.keys(patch).length > 0 && Object.keys(patch).every((path) => - path === 'keywords' || path === 'mailboxIds' || path.startsWith('keywords/') || path.startsWith('mailboxIds/')); -} - -async function emailSet(ctx: ResponderContext, args: Record): Promise { - const jmap = (args.accountId as string) || ctx.defaultJmap; - // A new message (a draft saved while offline) needs the server; the - // composer says the draft wasn't saved, so no notice here. - if (args.create && Object.keys(args.create as object).length > 0) throw new Unanswerable(); - if (Array.isArray(args.destroy) && args.destroy.length > 0) { ctx.onRefused('destroy'); throw new Unanswerable(); } - const update = (args.update ?? {}) as Record>; - if (!Object.values(update).every(queueablePatch)) { ctx.onRefused('other'); throw new Unanswerable(); } - // A message moved out of every folder would be gone: that is a delete. - const index = await emailIndex(ctx.account); - for (const [id, patch] of Object.entries(update)) { - if ('mailboxIds' in patch && Object.keys((patch.mailboxIds as object) ?? {}).length === 0) { ctx.onRefused('destroy'); throw new Unanswerable(); } - const current = index.get(`${ctx.account}\u0001${jmap}\u0001${id}`); - if (current && Object.keys(patch).some((p) => p.startsWith('mailboxIds/'))) { - const after = new Set(Object.keys(current.mailboxIds ?? {})); - for (const [path, value] of Object.entries(patch)) { - if (!path.startsWith('mailboxIds/')) continue; - const box = path.slice('mailboxIds/'.length); - if (value) after.add(box); else after.delete(box); - } - if (after.size === 0) { ctx.onRefused('destroy'); throw new Unanswerable(); } - } - } - await ctx.queuePatch(jmap, update); - const before = new Map(Object.keys(update).map((id) => [id, index.get(`${ctx.account}\u0001${jmap}\u0001${id}`)])); - await patchEmails(ctx.account, jmap, update); - await adjustFolderCounts(ctx, jmap, before, index); - return { - accountId: jmap, - updated: Object.fromEntries(Object.keys(update).map((id) => [id, null])), - }; -} - -/** The folder counts kept with the folder list follow a change made offline (read state, moves). */ -async function adjustFolderCounts( - ctx: ResponderContext, - jmap: string, - before: Map, - index: Map, -): Promise { - const delta = new Map(); - const bump = (box: string, total: number, unread: number) => { - const d = delta.get(box) ?? { total: 0, unread: 0 }; - d.total += total; - d.unread += unread; - delta.set(box, d); - }; - for (const [id, old] of before) { - const now = index.get(`${ctx.account}\u0001${jmap}\u0001${id}`); - if (!old || !now) continue; - const wasUnread = !old.keywords?.$seen ? 1 : 0; - const isUnread = !now.keywords?.$seen ? 1 : 0; - const oldBoxes = new Set(Object.keys(old.mailboxIds ?? {})); - const newBoxes = new Set(Object.keys(now.mailboxIds ?? {})); - for (const box of oldBoxes) if (!newBoxes.has(box)) bump(box, -1, -wasUnread); - for (const box of newBoxes) { - if (!oldBoxes.has(box)) bump(box, 1, isUnread); - else if (wasUnread !== isUnread) bump(box, 0, isUnread - wasUnread); - } - } - if (delta.size === 0) return; - const all = await getMisc>>>(ctx.account, 'mailboxes'); - const list = all?.[jmap]; - if (!all || !list) return; - all[jmap] = list.map((box) => { - const d = delta.get(box.id as string); - if (!d) return box; - const count = (key: string, by: number) => (typeof box[key] === 'number' ? Math.max(0, (box[key] as number) + by) : box[key]); - // Messages only: how the conversations change can't be told from here. - return { ...box, totalEmails: count('totalEmails', d.total), unreadEmails: count('unreadEmails', d.unread) }; - }); - await putMisc(ctx.account, 'mailboxes', all); -} - -/** - * The answer to a whole request from this device, or null when any call in - * it needs the server. - */ -export async function answerOffline(ctx: ResponderContext, methodCalls: OfflineMethodCall[]): Promise { - const done: Array<[string, MethodResult, string]> = []; - try { - for (const [name, rawArgs, callId] of methodCalls) { - const args = resolveArgs(rawArgs, done); - let result: MethodResult; - switch (name) { - case 'Email/query': result = await emailQuery(ctx, args); break; - case 'Email/get': result = await emailGet(ctx, args); break; - case 'Thread/get': result = await threadGet(ctx, args); break; - case 'Mailbox/query': result = await mailboxQuery(ctx, args); break; - case 'Mailbox/get': result = await listGet(ctx, args, 'mailboxes'); break; - case 'Identity/get': result = await listGet(ctx, args, 'identities'); break; - case 'Email/set': result = await emailSet(ctx, args); break; - default: throw new Unanswerable(); - } - done.push([name, result, callId]); - } - } catch (error) { - if (error instanceof Unanswerable) return null; - throw error; - } - return { methodResponses: done }; -} diff --git a/lib/offline/state.ts b/lib/offline/state.ts index 14f64d32d..cfff48f59 100644 --- a/lib/offline/state.ts +++ b/lib/offline/state.ts @@ -1,13 +1,14 @@ /** * What offline mode (#523) shows: which accounts keep mail on this device, * whether the browser is offline, what waits in the outbox, how much the - * cache uses. The switch itself ("Keep mail on this device") is per account + * local store uses. The switch itself ("Keep mail on this device") is per account * and per browser, in localStorage, never synced: it is off until the user * turns it on here. */ import { create } from 'zustand'; -import type { OutboxEntry, OutboxError } from './queue'; +import { DEFAULT_SCOPE, type StoreScope } from '@/lib/store/area'; +import type { ConflictEntry, OutboxEntry, OutboxError } from './queue'; /** localStorage: the local account ids that keep mail on this device. */ export const OFFLINE_ACCOUNTS_KEY = 'bulwark-offline-accounts'; @@ -31,8 +32,17 @@ interface OfflineState { downServers: string[]; /** The app started without the server, from the mail on this device. */ startedOffline: boolean; + /** The browser won't hand out the private file system here: a private window, say. */ + storageBlocked: boolean; + /** + * Accounts whose store came back empty since this page loaded: the browser + * cleared their key or file. Whatever warns about eviction can go by this. + */ + storeLost: string[]; outbox: OutboxRow[]; pendingChanges: number; + /** Changes the server refused, or that met a message changed on another device. */ + conflicts: ConflictEntry[]; usageBytes: number; /** Accounts whose mail is being brought onto the device right now. */ syncing: string[]; @@ -54,8 +64,11 @@ export const useOfflineStore = create()(() => ({ browserOffline: typeof navigator !== 'undefined' && navigator.onLine === false, downServers: [], startedOffline: false, + storeLost: [], + storageBlocked: false, outbox: [], pendingChanges: 0, + conflicts: [], usageBytes: 0, syncing: [], lastSyncedAt: {}, @@ -71,6 +84,78 @@ export function writeEnabled(enabled: string[]): void { } } +/** localStorage: what each account chose to keep offline, on top of the defaults. */ +export const OFFLINE_SCOPE_KEY = 'bulwark-offline-scope'; + +export function offlineScopeFor(account: string): StoreScope { + try { + const chosen = (JSON.parse(localStorage.getItem(OFFLINE_SCOPE_KEY) ?? '{}') as Record>)[account]; + return { ...DEFAULT_SCOPE, ...chosen }; + } catch { + return DEFAULT_SCOPE; + } +} + +/** localStorage: accounts the app dropped by itself, and when. Their data waits for a new sign-in. */ +export const OFFLINE_EVICTED_KEY = 'bulwark-offline-evicted'; +/** How long that data waits. */ +export const EVICTED_KEEP_MS = 30 * 24 * 60 * 60 * 1000; + +export function readEvicted(): Record { + try { + const parsed = JSON.parse(localStorage.getItem(OFFLINE_EVICTED_KEY) ?? '{}') as unknown; + return parsed && typeof parsed === 'object' ? parsed as Record : {}; + } catch { + return {}; + } +} + +function writeEvicted(evicted: Record): void { + try { + if (Object.keys(evicted).length > 0) localStorage.setItem(OFFLINE_EVICTED_KEY, JSON.stringify(evicted)); + else localStorage.removeItem(OFFLINE_EVICTED_KEY); + } catch { + /* storage unavailable */ + } +} + +/** Only an account that keeps mail here has anything to wait for. */ +export function markEvicted(account: string, now = Date.now()): void { + if (useOfflineStore.getState().enabled.includes(account)) writeEvicted({ ...readEvicted(), [account]: now }); +} + +export function clearEvicted(account: string): void { + const evicted = readEvicted(); + if (!(account in evicted)) return; + delete evicted[account]; + writeEvicted(evicted); +} + +export function isEvicted(account: string): boolean { + return account in readEvicted(); +} + +/** The accounts whose data may show here: not those waiting for a sign-in, which may be someone else's. */ +export function visibleAccounts(): string[] { + const evicted = readEvicted(); + return useOfflineStore.getState().enabled.filter((account) => !(account in evicted)); +} + +/** + * Drops the plain-text list kept in localStorage for a quick start (email-store) + * once the store can do that job. Until an account's first sync finishes it's + * what stays on screen, so turning the store on alone doesn't remove it. + */ +export function forgetMailSnapshot(account: string): void { + try { + const raw = localStorage.getItem('email-snapshot'); + if (raw && (JSON.parse(raw) as { account?: unknown } | null)?.account !== account) return; + localStorage.removeItem('email-snapshot'); + } catch { + localStorage.removeItem('email-snapshot'); + } +} + export function serverKey(url: string): string { return url.replace(/\/+$/, '').toLowerCase(); } diff --git a/lib/offline/sync.ts b/lib/offline/sync.ts deleted file mode 100644 index c998aef45..000000000 --- a/lib/offline/sync.ts +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Brings an account's recent mail onto the device (#523): the last 30 days - * of the Inbox and of the folders the user opened (at most INBOX_LIMIT and - * FOLDER_LIMIT messages), with bodies, and the attachments and inline - * images of those messages up to ATTACHMENT_MAX_BYTES each. Messages that - * left that window or the folders are removed; then the cache is brought - * under its cap, oldest first. - * - * Everything goes through the JMAP client, whose answers the cache records - * (offline-mode.ts), so the sync only asks. - */ - -import type { JMAPClient } from '@/lib/jmap/client'; -import { CACHE_WINDOW_MS, emailIndex, emailKey, enforceCap, getBody, hasBlob, hasBody, putBlob, putMisc, removeEmails, type CachedEmail } from './cache'; - -export const INBOX_LIMIT = 500; -export const FOLDER_LIMIT = 200; -export const ATTACHMENT_MAX_BYTES = 5 * 1024 * 1024; -/** Attachment bytes one sync run fetches at most. */ -const ATTACHMENT_BUDGET_BYTES = 100 * 1024 * 1024; - -interface SyncDeps { - stillActive: () => boolean; - /** Resolves once the cache has written every answer the client recorded so far. */ - recorded: () => Promise; - scopeMailboxes: (account: string) => Promise>; -} - -interface AttachmentPart { - blobId?: string; - name?: string | null; - type?: string; - size?: number; -} - -/** Once more after a second: Stalwart refuses requests beyond its limit of parallel ones. */ -async function twice(run: () => Promise): Promise { - try { - return await run(); - } catch { - await new Promise((r) => setTimeout(r, 1000)); - return run(); - } -} - -export async function syncAccount(account: string, client: JMAPClient, deps: SyncDeps): Promise { - await putMisc(account, 'session', client.getSessionSnapshot()); - // Folders and identities are recorded from these answers. - await twice(() => client.getAllMailboxes()).catch(() => client.getMailboxes()); - await twice(() => client.getIdentities()).catch(() => []); - // The folder list says which folder is the Inbox. - await deps.recorded(); - if (!deps.stillActive()) return; - - const since = new Date(Date.now() - CACHE_WINDOW_MS).toISOString(); - const wanted = new Map>(); - const scopes = await deps.scopeMailboxes(account); - // Per folder: which ones were asked, and which couldn't be. - const scopeFolders = new Map>(); - const failedFolders = new Map>(); - const add = (map: Map>, jmap: string, id: string) => { - const set = map.get(jmap) ?? new Set(); - set.add(id); - map.set(jmap, set); - }; - for (const { jmap, mailboxId, isInbox } of scopes) { - add(scopeFolders, jmap, mailboxId); - let ids: string[]; - try { - ids = await twice(() => client.getRecentEmailIds(mailboxId, jmap, since, isInbox ? INBOX_LIMIT : FOLDER_LIMIT)); - } catch { - add(failedFolders, jmap, mailboxId); - continue; - } - const set = wanted.get(jmap) ?? new Set(); - ids.forEach((id) => set.add(id)); - wanted.set(jmap, set); - } - if (!deps.stillActive()) return; - - for (const [jmap, idSet] of wanted) { - const ids = [...idSet]; - // Fresh keywords and folders for all of them (cheap: list properties). - if (ids.length) await twice(() => client.getSomeEmails(ids, jmap === client.getAccountId() ? undefined : jmap)).catch(() => []); - // Bodies for the ones that don't have one yet, a batch at a time. - const missing: string[] = []; - for (const id of ids) if (!(await hasBody(account, jmap, id))) missing.push(id); - for (let i = 0; i < missing.length; i += 20) { - // Fetched again at the next sync. - await twice(() => client.getFullEmails(missing.slice(i, i + 20), jmap)).catch(() => {}); - if (!deps.stillActive()) return; - } - } - await deps.recorded(); - - // What left the window or the folders goes; a folder that couldn't be - // asked keeps what it had, the others are brought up to date. - const index = await emailIndex(account); - const stale: string[] = []; - if (scopes.length > 0) { - for (const [key, email] of index) { - const jmap = key.split('\u0001')[1]; - if (wanted.get(jmap)?.has(email.id)) continue; - const folders = Object.keys(email.mailboxIds ?? {}).filter((id) => scopeFolders.get(jmap)?.has(id)); - if (folders.some((id) => failedFolders.get(jmap)?.has(id))) continue; - stale.push(key); - } - } - await removeEmails(account, stale); - - // Attachments and inline images, newest messages first, within the budget. - let budget = ATTACHMENT_BUDGET_BYTES; - const newestFirst = [...index.entries()].sort(([, a], [, b]) => (Date.parse(String(b.receivedAt)) || 0) - (Date.parse(String(a.receivedAt)) || 0)); - for (const [key, email] of newestFirst) { - if (budget <= 0 || !deps.stillActive()) break; - const jmap = key.split('\u0001')[1]; - for (const part of await attachmentsOf(account, email, jmap)) { - if (!part.blobId || !part.size || part.size > ATTACHMENT_MAX_BYTES || part.size > budget) continue; - if (await hasBlob(account, jmap, part.blobId)) continue; - try { - const blob = await client.fetchBlob(part.blobId, part.name ?? undefined, part.type, jmap); - await putBlob(account, jmap, part.blobId, blob, Date.parse(String(email.receivedAt)) || Date.now(), emailKey(account, jmap, email.id)); - budget -= blob.size; - } catch { - // Fetched again at the next sync. - } - } - } - - await enforceCap(); -} - -async function attachmentsOf(account: string, email: CachedEmail, jmap: string): Promise { - const body = await getBody(account, jmap, email.id); - const parts = body?.attachments; - return Array.isArray(parts) ? (parts as AttachmentPart[]) : []; -} diff --git a/lib/sign-out-cleanup.ts b/lib/sign-out-cleanup.ts index af06ead2b..531c99f7f 100644 --- a/lib/sign-out-cleanup.ts +++ b/lib/sign-out-cleanup.ts @@ -2,6 +2,7 @@ import { fileStorage } from '@/lib/plugin-storage'; import { clearHandoffs } from '@/lib/launch-handoff'; import { clearAllCheckpoints } from '@/lib/mail-import/checkpoints'; import { sweepPrivateExports } from '@/lib/mail-export/sinks'; +import { purgeStoreFiles } from '@/lib/store/client/purge'; /** * What a full sign-out removes from this browser beyond the stores that @@ -35,26 +36,40 @@ const SIGNED_OUT_STORAGE_KEYS = [ // Offline mode's "Keep mail on this device" (lib/offline/state.ts): the // next person signing in here starts with it off. 'bulwark-offline-accounts', + // Which accounts' data waits for a new sign-in, and until when. + 'bulwark-offline-evicted', + 'bulwark-store-opened', ]; +const LOCAL_STORE_KEYS = new Set(['bulwark-offline-accounts', 'bulwark-offline-evicted', 'bulwark-store-opened']); + const SIGNED_OUT_SIGNAL = 'bulwark:signed-out'; export { SIGNED_OUT_SIGNAL as SIGNED_OUT_KEY }; // Stores the auth store can't import (they import it) clear themselves here. -const signedOutCleanups = new Set<() => void>(); +const signedOutCleanups = new Set<(options: SignOutCleanupOptions) => void>(); + +export interface SignOutCleanupOptions { + /** + * Keep the local store and its keys. For a sign-out the app decided on + * because it couldn't confirm the login (an outage, an unknown refusal), or + * when an account that was dropped that way still has data here. + */ + keepLocalStore?: boolean; +} /** Run `cleanup` when everyone signs out in this tab. */ -export function onFullSignOut(cleanup: () => void): void { +export function onFullSignOut(cleanup: (options: SignOutCleanupOptions) => void): void { signedOutCleanups.add(cleanup); } // Work that has to reach the server before an account signs out (changes // still queued for it). The auth store waits for it within its sign-out // budget, next to the plugins' onBeforeLogout. -const beforeSignOutTasks = new Set<(accountId: string) => Promise>(); +const beforeSignOutTasks = new Set<(accountId: string, options: SignOutCleanupOptions) => Promise>(); /** Run `task` before an account signs out; `accountId` is 'all' when every account does. */ -export function onBeforeSignOut(task: (accountId: string) => Promise): () => void { +export function onBeforeSignOut(task: (accountId: string, options: SignOutCleanupOptions) => Promise): () => void { beforeSignOutTasks.add(task); return () => beforeSignOutTasks.delete(task); } @@ -81,14 +96,15 @@ export async function passSignOutGates(accountId: string): Promise { return true; } -export async function runBeforeSignOut(accountId: string): Promise { - await Promise.all([...beforeSignOutTasks].map((task) => task(accountId).catch(() => {}))); +export async function runBeforeSignOut(accountId: string, options: SignOutCleanupOptions = {}): Promise { + await Promise.all([...beforeSignOutTasks].map((task) => task(accountId, options).catch(() => {}))); } /** Remove the accounts' leftovers from browser storage. */ -export function purgeSignedOutData(): void { +export function purgeSignedOutData(options: SignOutCleanupOptions = {}): void { if (typeof window === 'undefined') return; for (const key of SIGNED_OUT_STORAGE_KEYS) { + if (options.keepLocalStore && LOCAL_STORE_KEYS.has(key)) continue; try { window.localStorage.removeItem(key); } catch { @@ -108,19 +124,23 @@ export function purgeSignedOutData(): void { void clearAllCheckpoints().catch(() => {}); // Mail exports written to the browser's private files and not deleted yet. void sweepPrivateExports({ force: true }).catch(() => {}); - // Offline mode (#523): the mail kept on this device, the outbox, the key - // they were encrypted with, and the app shell the service worker kept. - try { - window.indexedDB?.deleteDatabase('bulwark-offline'); - } catch { - /* storage unavailable */ + // Offline mode (#523): the local store with the mail and the outbox, the keys + // it was encrypted with, and the app shell the service worker kept. Also the + // database of older versions, in case it wasn't moved yet. + if (!options.keepLocalStore) { + void purgeStoreFiles(); + try { + window.indexedDB?.deleteDatabase('bulwark-offline'); + } catch { + /* storage unavailable */ + } + forgetOfflineShell(); } - forgetOfflineShell(); // Files and links on their way into a draft or an import. clearHandoffs(); for (const cleanup of signedOutCleanups) { try { - cleanup(); + cleanup(options); } catch { /* one store failing must not keep the others' data */ } diff --git a/lib/store/__tests__/helpers/fake-jmap-mail.ts b/lib/store/__tests__/helpers/fake-jmap-mail.ts new file mode 100644 index 000000000..b09683999 --- /dev/null +++ b/lib/store/__tests__/helpers/fake-jmap-mail.ts @@ -0,0 +1,170 @@ +/** + * FakeJmap with what the mail answers and the outbox talk to: identities, + * quotas, Email/set and submissions. Follows what was measured on Stalwart + * (stalwart-changes-prf.md section 1): `header` and `text` filters find no + * Message-ID, a second create or submission of the same message is accepted, + * a canceled submission still shows in EmailSubmission/query. + */ + +import { FakeJmap, type FakeEmail } from './fake-jmap'; + +export interface FakeSubmission { + id: string; + emailId: string; + identityId?: string; + undoStatus: 'pending' | 'final' | 'canceled'; +} + +type Filter = Record; + +export class FakeJmapMail extends FakeJmap { + readonly identities = new Map & { id: string }>(); + readonly quotas: Array & { id: string }> = []; + readonly submissions = new Map(); + /** Email/set update or destroy of these ids is refused with this error. */ + readonly refuse = new Map(); + /** A method answered with a per-call error once. */ + readonly failNext = new Map(); + identityState = 0; + nextSubmission = 1; + + private matches(email: FakeEmail, filter: Filter | undefined): boolean { + if (!filter) return true; + if (filter.operator === 'AND') return (filter.conditions as Filter[]).every((c) => this.matches(email, c)); + if (filter.operator === 'OR') return (filter.conditions as Filter[]).some((c) => this.matches(email, c)); + if (filter.operator === 'NOT') return !(filter.conditions as Filter[]).some((c) => this.matches(email, c)); + for (const [key, value] of Object.entries(filter)) { + switch (key) { + case 'inMailbox': + if (!(email.mailboxIds as Record)[value as string]) return false; + break; + case 'after': + if (Date.parse(String(email.receivedAt)) < Date.parse(String(value))) return false; + break; + // The filters that find nothing on Stalwart for a Message-ID. + case 'header': + case 'text': + return false; + default: + throw new Error(`FakeJmapMail: filter ${key} isn't modelled`); + } + } + return true; + } + + private applyPatch(email: FakeEmail, patch: Record): void { + const keywords = { ...(email.keywords as Record) }; + const boxes = { ...(email.mailboxIds as Record) }; + let wholeKeywords = keywords; + let wholeBoxes = boxes; + for (const [path, value] of Object.entries(patch)) { + if (path === 'keywords') wholeKeywords = { ...(value as Record) }; + else if (path === 'mailboxIds') wholeBoxes = { ...(value as Record) }; + else if (path.startsWith('keywords/')) { + const name = path.slice('keywords/'.length).replace(/~1/g, '/').replace(/~0/g, '~'); + if (value) wholeKeywords[name] = true; + else delete wholeKeywords[name]; + } else if (path.startsWith('mailboxIds/')) { + const name = path.slice('mailboxIds/'.length); + if (value) wholeBoxes[name] = true; + else delete wholeBoxes[name]; + } + } + this.updateEmail(email.id, { keywords: wholeKeywords, mailboxIds: wholeBoxes }); + } + + protected override handle(name: string, args: Record): Record { + const failure = this.failNext.get(name); + if (failure) { + this.failNext.delete(name); + throw Object.assign(new Error(failure), { jmapError: failure }); + } + switch (name) { + case 'Identity/get': { + const ids = args.ids as string[] | null; + const all = [...this.identities.values()]; + return { accountId: args.accountId, state: `s${this.identityState}`, list: ids ? all.filter((i) => ids.includes(i.id)) : all, notFound: [] }; + } + case 'Identity/changes': + return { + accountId: args.accountId, + oldState: args.sinceState, + newState: `s${this.identityState}`, + hasMoreChanges: false, + created: args.sinceState === `s${this.identityState}` ? [] : [...this.identities.keys()], + updated: [], + destroyed: [], + }; + case 'Quota/get': + return { accountId: args.accountId, list: this.quotas, notFound: [] }; + case 'Email/query': { + const matched = [...this.emails.values()] + .filter((email) => this.matches(email, args.filter as Filter | undefined)) + .sort((a, b) => Date.parse(String(b.receivedAt)) - Date.parse(String(a.receivedAt))); + return { + accountId: args.accountId, + queryState: this.coll.Email.state(), + canCalculateChanges: true, + ...this.pageOf(matched.map((email) => email.id), args), + ...(args.calculateTotal ? { total: matched.length } : {}), + }; + } + case 'Email/set': { + const out: Record = { accountId: args.accountId }; + if (args.update) { + const updated: Record = {}; + const notUpdated: Record = {}; + for (const [id, patch] of Object.entries(args.update as Record>)) { + const email = this.emails.get(id); + if (this.refuse.has(id)) notUpdated[id] = this.refuse.get(id); + else if (!email) notUpdated[id] = { type: 'notFound' }; + else { + this.applyPatch(email, patch); + updated[id] = null; + } + } + out.updated = updated; + if (Object.keys(notUpdated).length > 0) out.notUpdated = notUpdated; + } + if (args.destroy) { + const destroyed: string[] = []; + const notDestroyed: Record = {}; + for (const id of args.destroy as string[]) { + if (this.refuse.has(id)) notDestroyed[id] = this.refuse.get(id); + else if (!this.emails.has(id)) notDestroyed[id] = { type: 'notFound' }; + else { + this.destroyEmail(id); + destroyed.push(id); + } + } + out.destroyed = destroyed; + if (Object.keys(notDestroyed).length > 0) out.notDestroyed = notDestroyed; + } + return out; + } + case 'EmailSubmission/query': { + const emailIds = ((args.filter as Filter | undefined)?.emailIds ?? []) as string[]; + const ids = [...this.submissions.values()].filter((s) => emailIds.includes(s.emailId)).map((s) => s.id); + return { accountId: args.accountId, queryState: 'q', canCalculateChanges: false, position: 0, ids }; + } + case 'EmailSubmission/get': { + const ids = (args.ids ?? []) as string[]; + return { + accountId: args.accountId, + state: 'q', + list: ids.flatMap((id) => (this.submissions.has(id) ? [this.submissions.get(id)] : [])), + notFound: ids.filter((id) => !this.submissions.has(id)), + }; + } + default: + return super.handle(name, args); + } + } + + /** A submission record, as if a send got that far. */ + addSubmission(emailId: string, undoStatus: FakeSubmission['undoStatus'] = 'pending'): FakeSubmission { + const submission: FakeSubmission = { id: `sub${this.nextSubmission++}`, emailId, undoStatus }; + this.submissions.set(submission.id, submission); + return submission; + } +} diff --git a/lib/store/__tests__/helpers/fake-jmap.ts b/lib/store/__tests__/helpers/fake-jmap.ts index bac572400..b218138f8 100644 --- a/lib/store/__tests__/helpers/fake-jmap.ts +++ b/lib/store/__tests__/helpers/fake-jmap.ts @@ -82,6 +82,8 @@ export class FakeJmap implements JmapTransport { offline = false; /** Fails the next N requests as busy. */ busyNext = 0; + /** Ids Email/query leaves out while Email/get still has them: a pass that missed a message. */ + readonly hiddenFromQuery = new Set(); /** Called before each request; can change data while a sync is running. */ beforeRequest: ((n: number) => void) | null = null; nextId = 1; @@ -160,6 +162,18 @@ export class FakeJmap implements JmapTransport { this.coll[type].truncatedBelow = this.coll[type].log.length; } + /** The ids of a query's page, by position or by anchor, as Stalwart answers. */ + protected pageOf(ids: string[], args: Record): { position: number; ids: string[] } { + const limit = Math.min(Number(args.limit ?? 5000) || 5000, 5000); + let position = Number(args.position ?? 0); + if (args.anchor !== undefined) { + const at = ids.indexOf(String(args.anchor)); + if (at < 0) throw Object.assign(new Error('anchor gone'), { jmapError: 'anchorNotFound' }); + position = Math.max(0, at + Number(args.anchorOffset ?? 0)); + } + return { position, ids: ids.slice(position, position + limit) }; + } + private queryIds(args: Record): string[] { const filter = args.filter as { after?: string; inMailbox?: string } | undefined; let list = [...this.emails.values()]; @@ -169,7 +183,7 @@ export class FakeJmap implements JmapTransport { return list.map((e) => e.id); } - private handle(name: string, args: Record): Record { + protected handle(name: string, args: Record): Record { switch (name) { case 'Mailbox/get': { const ids = args.ids as string[] | null; @@ -178,15 +192,12 @@ export class FakeJmap implements JmapTransport { return { accountId: args.accountId, state: this.coll.Mailbox.state(), list, notFound: [] }; } case 'Email/query': { - const ids = this.queryIds(args); - const position = Number(args.position ?? 0); - const limit = Number(args.limit ?? 5000) || 5000; + const ids = this.queryIds(args).filter((id) => !this.hiddenFromQuery.has(id)); return { accountId: args.accountId, queryState: this.coll.Email.state(), canCalculateChanges: true, - position, - ids: ids.slice(position, position + Math.min(limit, 5000)), + ...this.pageOf(ids, args), ...(args.calculateTotal ? { total: ids.length } : {}), }; } diff --git a/lib/store/__tests__/helpers/in-process-store.ts b/lib/store/__tests__/helpers/in-process-store.ts new file mode 100644 index 000000000..81699f3a3 --- /dev/null +++ b/lib/store/__tests__/helpers/in-process-store.ts @@ -0,0 +1,104 @@ +/** + * The local store for tests of lib/offline: the real engine on node:sqlite in + * the same process, behind the seam the page uses. Fetches nothing unless a + * test hands it a transport. + */ + +import { DEFAULT_SCOPE } from '../../area'; +import { STORE_AREAS } from '../../areas'; +import type { StoreBackend } from '../../client/api'; +import { AccountEngine } from '../../engine/account-engine'; +import type { JmapTransport, MethodCall, StoreSession } from '../../engine/transport'; +import type { StorageAdapter } from '../../storage'; +import { FAKE_SESSION } from './fake-jmap'; +import { openMemoryStore } from './node-adapter'; + +const NO_SERVER: JmapTransport = { + request: async () => { throw new Error('This test has no server for the store'); }, + download: async () => new Uint8Array(), + upload: async () => ({ blobId: 'x', type: '', size: 0 }), +}; + +export class InProcessStore implements StoreBackend { + readonly engines = new Map(); + readonly dbs = new Map(); + /** What a test wants a failing store to do. */ + failArea: ((name: string) => boolean) | null = null; + removed: string[] = []; + stopped = 0; + isSupported = true; + + constructor(private readonly transport: JmapTransport = NO_SERVER, private readonly session: StoreSession = FAKE_SESSION) {} + + supported(): boolean { + return this.isSupported; + } + + engine(account: string, session?: StoreSession | null): AccountEngine { + let engine = this.engines.get(account); + if (!engine) { + const db = openMemoryStore(); + this.dbs.set(account, db); + engine = new AccountEngine({ + account, + db, + session: session ?? this.session, + transport: this.transport, + areas: STORE_AREAS, + scope: { ...DEFAULT_SCOPE, days: null }, + sleep: async () => {}, + }); + this.engines.set(account, engine); + } + return engine; + } + + has(account: string): boolean { + return this.engines.has(account); + } + + async ensureOpen(account: string, session?: StoreSession | null): Promise { + this.engine(account, session); + } + + refresh(): void {} + + async answer(account: string, calls: MethodCall[]) { + const answer = await this.engine(account).answer(calls); + // Across the worker boundary answers are copies. + return answer ? structuredClone(answer) : null; + } + + async area(account: string, name: string, params?: unknown): Promise { + if (this.failArea?.(name)) throw new Error(`${name} failed`); + const result = await this.engine(account).areaCall(name, structuredClone(params)); + return (result === undefined ? undefined : structuredClone(result)) as T; + } + + async sync(account: string): Promise { + await this.engine(account).requestSync(); + } + + async noteStateChange(account: string, changed: Record>): Promise { + this.engine(account).noteStateChange(changed); + } + + async setOnline(online: boolean): Promise { + for (const engine of this.engines.values()) engine.setOnline(online); + } + + async remove(account: string): Promise { + this.removed.push(account); + await this.engines.get(account)?.close(); + this.dbs.get(account)?.close(); + this.engines.delete(account); + this.dbs.delete(account); + } + + stop(): void { + this.stopped += 1; + for (const engine of this.engines.values()) void engine.close(); + this.engines.clear(); + this.dbs.clear(); + } +} diff --git a/lib/store/__tests__/helpers/node-adapter.ts b/lib/store/__tests__/helpers/node-adapter.ts index 421324cfa..7c662687e 100644 --- a/lib/store/__tests__/helpers/node-adapter.ts +++ b/lib/store/__tests__/helpers/node-adapter.ts @@ -1,14 +1,17 @@ /** The storage adapter for tests: real SQLite with FTS5 through node:sqlite. */ -import { DatabaseSync, type SQLInputValue } from 'node:sqlite'; +import type { DatabaseSync, SQLInputValue } from 'node:sqlite'; import { runTransaction, type SqlRow, type SqlValue, type StorageAdapter } from '../../storage'; +// Not an import: Vite refuses node:sqlite in the jsdom tests of lib/offline. +const sqlite = process.getBuiltinModule('node:sqlite'); + export function openMemoryStore(): StorageAdapter { - return adapterFor(new DatabaseSync(':memory:')); + return adapterFor(new sqlite.DatabaseSync(':memory:')); } export function openFileStore(file: string): StorageAdapter { - return adapterFor(new DatabaseSync(file)); + return adapterFor(new sqlite.DatabaseSync(file)); } function adapterFor(db: DatabaseSync): StorageAdapter { diff --git a/lib/store/__tests__/mail-answers.test.ts b/lib/store/__tests__/mail-answers.test.ts new file mode 100644 index 000000000..980b6ea12 --- /dev/null +++ b/lib/store/__tests__/mail-answers.test.ts @@ -0,0 +1,571 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { DEFAULT_SCOPE } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError, type MethodCall, type StoreSession } from '../engine/transport'; +import { mailArea } from '../mail'; +import { headerValue, parseHeaderProperty } from '../mail/headers'; +import { FAKE_SESSION, JMAP_ACCOUNT } from './helpers/fake-jmap'; +import { FakeJmapMail } from './helpers/fake-jmap-mail'; +import { openMemoryStore } from './helpers/node-adapter'; + +const EVERYTHING = { ...DEFAULT_SCOPE, days: null }; + +const FULL_SESSION: StoreSession = { + ...FAKE_SESSION, + accounts: { + [JMAP_ACCOUNT]: { + name: 'a@example.org', + isPersonal: true, + accountCapabilities: { + 'urn:ietf:params:jmap:mail': {}, + 'urn:ietf:params:jmap:submission': {}, + 'urn:ietf:params:jmap:quota': {}, + }, + }, + }, +}; + +function engineFor(server: FakeJmapMail, scope = EVERYTHING, session = FULL_SESSION) { + const db = openMemoryStore(); + const engine = new AccountEngine({ + account: 'a@fake.test', + db, + session, + transport: server, + areas: [mailArea], + scope, + now: () => Date.UTC(2026, 10, 3), + sleep: async () => {}, + }); + return { engine, db }; +} + +async function one(engine: AccountEngine, call: MethodCall) { + const response = await engine.answer([call]); + return response?.methodResponses[0]; +} + +const HEADERS = [ + { name: 'List-Id', value: ' Old list ' }, + { name: 'List-Id', value: ' =?UTF-8?B?VGVzdA==?= list\r\n ' }, + { name: 'From', value: ' "Doe, Jane" , Bob ' }, + { name: 'To', value: ' Team: a@example.org, b@example.org;, c@example.org' }, + { name: 'Message-ID', value: ' ' }, + { name: 'Date', value: ' Tue, 03 Nov 2026 10:00:00 +0100' }, + { name: 'List-Unsubscribe', value: ' , ' }, +]; + +describe('Email/get property forms', () => { + async function seeded() { + const server = new FakeJmapMail(); + server.addEmail({ id: 'm', headers: HEADERS, cc: [{ name: 'Carol', email: 'carol@example.org' }] }); + const { engine } = engineFor(server); + await engine.requestSync(); + return engine; + } + + it('answers every header form from the raw headers', async () => { + const engine = await seeded(); + const [, result] = (await one(engine, ['Email/get', { + ids: ['m'], + properties: [ + 'header:List-Id', + 'header:List-Id:asText', + 'header:list-id:asText:all', + 'header:List-Id:all', + 'header:From:asAddresses', + 'header:To:asGroupedAddresses', + 'header:Message-ID:asMessageIds', + 'header:Date:asDate', + 'header:List-Unsubscribe:asURLs', + 'header:Nope:asText', + 'header:Nope:all', + ], + }, 'g']))!; + const got = result.list[0]; + expect(got['header:List-Id']).toBe(' =?UTF-8?B?VGVzdA==?= list\r\n '); + expect(got['header:List-Id:asText']).toBe('Test list '); + expect(got['header:list-id:asText:all']).toEqual(['Old list ', 'Test list ']); + expect(got['header:List-Id:all']).toHaveLength(2); + expect(got['header:From:asAddresses']).toEqual([{ name: 'Doe, Jane', email: 'jane@example.org' }, { name: 'Bob', email: 'bob@example.org' }]); + expect(got['header:To:asGroupedAddresses']).toEqual([ + { name: 'Team', addresses: [{ name: null, email: 'a@example.org' }, { name: null, email: 'b@example.org' }] }, + { name: null, addresses: [{ name: null, email: 'c@example.org' }] }, + ]); + expect(got['header:Message-ID:asMessageIds']).toEqual(['abc@example.org']); + expect(got['header:Date:asDate']).toBe('2026-11-03T09:00:00.000Z'); + expect(got['header:List-Unsubscribe:asURLs']).toEqual(['https://example.org/u', 'mailto:u@example.org']); + expect(got['header:Nope:asText']).toBeNull(); + expect(got['header:Nope:all']).toEqual([]); + }); + + it('prefers a form the list already stored, and has no header values for a message without a body', async () => { + const server = new FakeJmapMail(); + server.addEmail({ id: 'm', headers: HEADERS, 'header:Authentication-Results:all': ['dkim=pass'] }); + const { engine, db } = engineFor(server); + await engine.requestSync(); + db.run('DELETE FROM email_body'); + const [, result] = (await one(engine, ['Email/get', { + ids: ['m'], + properties: ['id', 'header:Authentication-Results:all', 'header:List-Id:asText'], + }, 'g']))!; + expect(result.list).toEqual([{ id: 'm', 'header:Authentication-Results:all': ['dkim=pass'] }]); + }); + + it('cuts body parts to bodyProperties, in the structure too', async () => { + const server = new FakeJmapMail(); + server.addEmail({ + id: 'm', + attachments: [{ partId: '2', blobId: 'b', name: 'a.pdf', type: 'application/pdf', size: 3, language: ['de'] }], + bodyStructure: { type: 'multipart/mixed', subParts: [{ partId: '1', type: 'text/plain', language: ['de'] }] }, + }); + const { engine } = engineFor(server, { ...EVERYTHING, attachments: false }); + await engine.requestSync(); + const [, result] = (await one(engine, ['Email/get', { + ids: ['m'], + properties: ['attachments', 'bodyStructure', 'textBody'], + bodyProperties: ['partId', 'blobId', 'size', 'name', 'type'], + }, 'g']))!; + expect(result.list[0].attachments).toEqual([{ partId: '2', blobId: 'b', name: 'a.pdf', type: 'application/pdf', size: 3 }]); + expect(result.list[0].bodyStructure).toEqual({ type: 'multipart/mixed', subParts: [{ partId: '1', type: 'text/plain' }] }); + }); + + it('gives the whole text with no truncation mark, whatever maxBodyValueBytes says', async () => { + const long = 'x'.repeat(5000); + const server = new FakeJmapMail(); + server.addEmail({ id: 'm', bodyValues: { 1: { value: long, isTruncated: false, isEncodingProblem: false } } }); + const { engine } = engineFor(server); + await engine.requestSync(); + const [, result] = (await one(engine, ['Email/get', { ids: ['m'], properties: ['bodyValues'], fetchAllBodyValues: true, maxBodyValueBytes: 100 }, 'g']))!; + expect(result.list[0].bodyValues['1'].value).toBe(long); + expect(result.list[0].bodyValues['1'].isTruncated).toBe(false); + }); + + it('returns only the body values asked for', async () => { + const server = new FakeJmapMail(); + server.addEmail({ + id: 'm', + textBody: [{ partId: 't', type: 'text/plain' }], + htmlBody: [{ partId: 'h', type: 'text/html' }], + bodyValues: { t: { value: 'text' }, h: { value: '

    html

    ' } }, + }); + const { engine } = engineFor(server); + await engine.requestSync(); + const values = async (flags: Record) => (await one(engine, ['Email/get', { ids: ['m'], properties: ['bodyValues'], ...flags }, 'g']))![1].list[0].bodyValues; + expect(Object.keys(await values({ fetchTextBodyValues: true }))).toEqual(['t']); + expect(Object.keys(await values({ fetchHTMLBodyValues: true }))).toEqual(['h']); + expect(Object.keys(await values({ fetchAllBodyValues: true })).sort()).toEqual(['h', 't']); + expect(await values({})).toEqual({}); + }); + + it('keeps refusing ids: null, because the server gives only the first 500 there', async () => { + const engine = await seeded(); + const response = await engine.answer([['Email/get', { ids: null }, 'g'], ['Core/echo', {}, 'e']]); + expect(response?.methodResponses[0]).toEqual(['error', expect.objectContaining({ type: 'requestTooLarge' }), 'g']); + }); +}); + +describe('header property parsing', () => { + it('leaves out what is not a header form', () => { + expect(parseHeaderProperty('subject')).toBeNull(); + expect(parseHeaderProperty('header:')).toBeNull(); + expect(parseHeaderProperty('header:X:asWhat')).toBeNull(); + expect(parseHeaderProperty('header:X-Label:asText')).toEqual({ name: 'X-Label', form: 'asText', all: false }); + expect(headerValue({ name: 'x', form: 'asRaw', all: false }, [{ name: 'X', value: ' 1' }, { name: 'x', value: ' 2' }])).toBe(' 2'); + }); +}); + +describe('Email/query filters and sorts', () => { + async function seeded() { + const server = new FakeJmapMail({ + mailboxes: [ + { id: 'inbox', name: 'Inbox', role: 'inbox' }, + { id: 'trash', name: 'Trash', role: 'trash' }, + { id: 'work', name: 'Work' }, + ], + }); + server.addEmail({ + id: 'a', subject: 'Rechnung', from: [{ name: 'Anna', email: 'anna@example.org' }], to: [{ email: 'x@example.org' }], + cc: [{ name: 'Carol', email: 'carol@example.org' }], keywords: { $seen: true, '$label:work': true }, size: 500, + receivedAt: '2026-10-01T00:00:00Z', hasAttachment: true, headers: [{ name: 'X-Tag', value: ' blue' }], bcc: [{ email: 'hidden@example.org' }], + }); + server.addEmail({ + id: 'b', subject: 'Hello', from: [{ name: 'Bert', email: 'bert@example.org' }], to: [{ email: 'y@example.org' }], + keywords: { $flagged: true }, size: 100, receivedAt: '2026-10-02T00:00:00Z', mailboxIds: { inbox: true, work: true }, + }); + server.addEmail({ + id: 'c', subject: 'Trash mail', from: [{ name: 'Cora', email: 'cora@example.org' }], to: [{ email: 'y@example.org' }], + keywords: {}, size: 2000, receivedAt: '2026-10-03T00:00:00Z', mailboxIds: { trash: true }, + }); + const { engine } = engineFor(server); + await engine.requestSync(); + return engine; + } + + const ids = async (engine: AccountEngine, args: Record) => (await one(engine, ['Email/query', args, 'q']))![1].ids; + + it('answers every shape the UI sends', async () => { + const engine = await seeded(); + const newest = [{ property: 'receivedAt', isAscending: false }]; + const cases: Array<[Record, string[]]> = [ + [{ inMailbox: 'inbox' }, ['b', 'a']], + [{ hasKeyword: '$seen' }, ['a']], + [{ notKeyword: '$seen' }, ['c', 'b']], + [{ operator: 'AND', conditions: [{ inMailbox: 'inbox' }, { notKeyword: '$seen' }] }, ['b']], + [{ operator: 'NOT', conditions: [{ hasKeyword: '$label:work' }] }, ['c', 'b']], + [{ operator: 'OR', conditions: [{ inMailbox: 'trash' }, { inMailbox: 'work' }] }, ['c', 'b']], + [{ inMailboxOtherThan: ['trash'] }, ['b', 'a']], + [{ text: 'rechnung' }, ['a']], + [{ subject: 'hello' }, ['b']], + [{ from: 'anna' }, ['a']], + [{ to: 'y@example.org' }, ['c', 'b']], + [{ cc: 'carol' }, ['a']], + [{ bcc: 'hidden' }, ['a']], + [{ header: ['X-Tag'] }, ['a']], + [{ header: ['X-Tag', 'blue'] }, ['a']], + [{ header: ['X-Tag', 'red'] }, []], + [{ hasAttachment: true }, ['a']], + [{ after: '2026-10-02T00:00:00Z' }, ['c', 'b']], + [{ before: '2026-10-02T00:00:00Z' }, ['a']], + [{ minSize: 500 }, ['c', 'a']], + [{ maxSize: 500 }, ['b']], + [{ operator: 'AND', conditions: [{ inMailbox: 'inbox' }, { operator: 'OR', conditions: [{ to: 'x@example.org' }, { cc: 'nobody' }] }] }, ['a']], + ]; + for (const [filter, want] of cases) expect([filter, await ids(engine, { filter, sort: newest })]).toEqual([filter, want]); + expect(await ids(engine, { sort: newest })).toEqual(['c', 'b', 'a']); + }); + + it('sorts by every level the list offers', async () => { + const engine = await seeded(); + const sorted = (sort: unknown[]) => ids(engine, { sort }); + expect(await sorted([{ property: 'size', isAscending: true }])).toEqual(['b', 'a', 'c']); + expect(await sorted([{ property: 'subject', isAscending: true }])).toEqual(['b', 'a', 'c']); + expect(await sorted([{ property: 'from', isAscending: false }])).toEqual(['c', 'b', 'a']); + expect(await sorted([{ property: 'sentAt', isAscending: true }])).toHaveLength(3); + expect(await sorted([{ property: 'hasKeyword', keyword: '$flagged', isAscending: false }, { property: 'receivedAt', isAscending: false }])).toEqual(['b', 'c', 'a']); + }); + + it('gives a per-call error for a filter it can\'t answer, and no made-up state', async () => { + const engine = await seeded(); + const response = await engine.answer([['Email/query', { filter: { allInThreadHaveKeyword: '$seen' } }, 'q'], ['Core/echo', {}, 'e']]); + expect(response?.methodResponses[0][0]).toBe('error'); + expect(response?.methodResponses[0][1].type).toBe('unsupportedFilter'); + expect(await engine.answer([['Email/query', { filter: { allInThreadHaveKeyword: '$seen' } }, 'q']])).toBeNull(); + }); + + it('says total only when asked, and pages by position', async () => { + const engine = await seeded(); + const [, plain] = (await one(engine, ['Email/query', {}, 'q']))!; + expect(plain).not.toHaveProperty('total'); + const [, page] = (await one(engine, ['Email/query', { position: 1, limit: 1, calculateTotal: true, sort: [{ property: 'receivedAt', isAscending: false }] }, 'q']))!; + expect(page).toMatchObject({ ids: ['b'], total: 3, position: 1 }); + }); +}); + +describe('a store without its mail yet', () => { + it('says so rather than answering an empty list, until the first sync has finished', async () => { + const server = new FakeJmapMail(); + server.addEmail({ id: 'one' }); + const { engine } = engineFor(server); + const inbox = Object.keys((server.emails.get('one')!.mailboxIds) as Record)[0]; + const query = () => one(engine, ['Email/query', { filter: { inMailbox: inbox } }, 'q']); + // No answer: the client reports it can't tell, and the list on screen stays. + expect(await query()).toBeUndefined(); + await engine.requestSync(); + expect((await query())![1].ids).toEqual(['one']); + // After a finished sync, an empty folder really is empty. + server.destroyEmail('one'); + await engine.requestSync(); + expect((await query())![1].ids).toEqual([]); + }); + + it('tells a start whether there are folders to show', async () => { + const server = new FakeJmapMail(); + const { engine } = engineFor(server); + expect(await engine.areaCall('mail.hasFolders', {})).toBe(false); + await engine.requestSync(); + expect(await engine.areaCall('mail.hasFolders', {})).toBe(true); + }); +}); + +describe('keyword case, as measured on both Stalwart versions', () => { + it('keeps two keywords differing only in case, and sync doesn\'t stop on them', async () => { + const server = new FakeJmapMail(); + server.addEmail({ id: 'both', keywords: { Tag: true, tag: true, $seen: true } }); + server.addEmail({ id: 'upper', keywords: { Tag: true, $SEEN: true, $seen: true } }); + server.addEmail({ id: 'none', keywords: {} }); + const { engine, db } = engineFor(server); + await engine.requestSync(); + expect(db.get<{ n: number }>('SELECT COUNT(*) AS n FROM email')?.n).toBe(3); + const has = async (keyword: string) => (await one(engine, ['Email/query', { filter: { hasKeyword: keyword }, sort: [{ property: 'receivedAt', isAscending: false }] }, 'q']))![1].ids; + expect(await has('Tag')).toEqual(['upper', 'both']); + expect(await has('tag')).toEqual(['both']); + expect(await has('TAG')).toEqual([]); + expect(await has('$SEEN')).toEqual(['upper', 'both']); + const [, counts] = (await one(engine, ['Keyword/get', { ids: ['Tag', 'tag', '$SEEN'] }, 'k']))!; + expect(counts.list).toEqual([{ id: 'Tag', total: 2, unread: 0 }, { id: 'tag', total: 1, unread: 0 }, { id: '$SEEN', total: 2, unread: 0 }]); + }); +}); + +describe('the rest of the mail methods', () => { + it('answers Core/echo with what it was given', async () => { + const { engine } = engineFor(new FakeJmapMail()); + expect(await one(engine, ['Core/echo', { ping: 'pong' }, 'e'])).toEqual(['Core/echo', { ping: 'pong' }, 'e']); + }); + + it('answers /changes with an error, so the client does a full fetch', async () => { + const server = new FakeJmapMail(); + server.addEmail(); + const { engine } = engineFor(server); + await engine.requestSync(); + const response = await engine.answer([ + ['Email/changes', { sinceState: 's1' }, 'e'], + ['Mailbox/changes', { sinceState: 's1' }, 'm'], + ['Core/echo', {}, 'x'], + ]); + expect(response?.methodResponses[0]).toEqual(['error', expect.objectContaining({ type: 'cannotCalculateChanges' }), 'e']); + expect(response?.methodResponses[1]).toEqual(['error', expect.objectContaining({ type: 'cannotCalculateChanges' }), 'm']); + }); + + it('answers a search with its snippets in one request', async () => { + const server = new FakeJmapMail(); + server.addEmail({ subject: 'Findme' }); + const { engine } = engineFor(server); + await engine.requestSync(); + const response = await engine.answer([ + ['Email/query', { filter: { text: 'findme' } }, '0'], + ['SearchSnippet/get', { filter: { text: 'findme' }, '#emailIds': { resultOf: '0', name: 'Email/query', path: '/ids' } }, 'snippets'], + ]); + expect(response?.methodResponses[0][1].ids).toHaveLength(1); + expect(response?.methodResponses[1][0]).toBe('SearchSnippet/get'); + }); + + it('counts keywords for Keyword/get', async () => { + const server = new FakeJmapMail(); + server.addEmail({ keywords: { '$label:a': true, $seen: true } }); + server.addEmail({ keywords: { '$label:a': true } }); + const { engine } = engineFor(server); + await engine.requestSync(); + const [, wanted] = (await one(engine, ['Keyword/get', { ids: ['$label:a', '$label:none'] }, 'k']))!; + expect(wanted.list).toEqual([{ id: '$label:a', total: 2, unread: 1 }, { id: '$label:none', total: 0, unread: 0 }]); + const [, all] = (await one(engine, ['Keyword/get', {}, 'k']))!; + expect(all.totalEmails).toBe(2); + expect(all.list.map((k: { id: string }) => k.id).sort()).toEqual(['$label:a', '$seen']); + }); + + it('keeps identities through a sync and answers them, with the real state', async () => { + const server = new FakeJmapMail(); + server.identities.set('i1', { id: 'i1', email: 'a@example.org', name: 'A' }); + const { engine } = engineFor(server); + expect((await one(engine, ['Identity/get', {}, 'i']))).toBeUndefined(); + await engine.requestSync(); + const [, result] = (await one(engine, ['Identity/get', {}, 'i']))!; + expect(result.list).toEqual([{ id: 'i1', email: 'a@example.org', name: 'A' }]); + expect(result.state).toBe('s0'); + + server.identities.set('i2', { id: 'i2', email: 'b@example.org', name: 'B' }); + server.identityState = 1; + await engine.requestSync(); + expect((await one(engine, ['Identity/get', { ids: ['i2'] }, 'i']))![1].list).toEqual([{ id: 'i2', email: 'b@example.org', name: 'B' }]); + }); + + it('keeps the last quota', async () => { + const server = new FakeJmapMail(); + server.quotas.push({ id: 'q', used: 10, hardLimit: 100 }); + const { engine } = engineFor(server); + await engine.requestSync(); + server.offline = true; + expect((await one(engine, ['Quota/get', {}, 'q']))![1].list).toEqual([{ id: 'q', used: 10, hardLimit: 100 }]); + }); + + it('keeps going when one account has no submission capability', async () => { + const server = new FakeJmapMail(); + server.addEmail(); + const { engine, db } = engineFor(server, EVERYTHING, FAKE_SESSION); + await engine.requestSync(); + expect(db.get('SELECT 1 AS x FROM email')).toBeTruthy(); + expect(server.calls.some(([name]) => name === 'Identity/get')).toBe(false); + }); + + it('removes a folder deleted on the server, and answers what is left', async () => { + const server = new FakeJmapMail({ mailboxes: [{ id: 'inbox', name: 'Inbox', role: 'inbox' }, { id: 'old', name: 'Old' }] }); + const { engine } = engineFor(server); + await engine.requestSync(); + server.mailboxes.delete('old'); + server.coll.Mailbox.push('destroyed', 'old'); + await engine.requestSync(); + const [, result] = (await one(engine, ['Mailbox/get', {}, 'm']))!; + expect(result.list.map((box: { id: string }) => box.id)).toEqual(['inbox']); + const [, queried] = (await one(engine, ['Mailbox/query', { filter: { role: 'inbox' } }, 'q']))!; + expect(queried).toMatchObject({ ids: ['inbox'], queryState: server.coll.Mailbox.state() }); + }); +}); + +describe('attachments', () => { + class Files extends FakeJmapMail { + downloads: string[] = []; + missing = new Set(); + override async download(_jmap?: string, blobId?: string): Promise { + this.downloads.push(String(blobId)); + if (this.offline) throw new TransportError('network', 'Failed to fetch'); + if (this.missing.has(String(blobId))) throw new TransportError('http', 'Gone', 404); + return new Uint8Array([1, 2, 3]); + } + } + + const withFile = (server: Files, id: string, blobId: string, size = 3) => server.addEmail({ + id, + attachments: [{ partId: '2', blobId, name: `${blobId}.pdf`, type: 'application/pdf', size }], + }); + + it('keeps the files of kept messages within the chosen size, and deletes them with the message', async () => { + const server = new Files(); + withFile(server, 'small', 'b-small'); + withFile(server, 'big', 'b-big', 10_000); + const { engine, db } = engineFor(server, { ...EVERYTHING, maxAttachmentBytes: 1000 }); + await engine.requestSync(); + expect(server.downloads).toEqual(['b-small']); + const stored = await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-small' }) as { type: string; data: Uint8Array }; + expect(stored.type).toBe('application/pdf'); + expect([...stored.data]).toEqual([1, 2, 3]); + + server.destroyEmail('small'); + await engine.requestSync(); + expect(await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-small' })).toBeNull(); + expect(db.get('SELECT 1 AS x FROM email_part WHERE blob_id = ?', ['b-small'])).toBeUndefined(); + }); + + it('keeps a shared file until the last message using it goes', async () => { + const server = new Files(); + withFile(server, 'one', 'b-same'); + withFile(server, 'two', 'b-same'); + const { engine } = engineFor(server); + await engine.requestSync(); + server.destroyEmail('one'); + await engine.requestSync(); + expect(await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-same' })).not.toBeNull(); + server.destroyEmail('two'); + await engine.requestSync(); + expect(await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-same' })).toBeNull(); + }); + + it('keeps none when the user chose so, stops at the total, and doesn\'t ask again for a file the server refused', async () => { + const off = new Files(); + withFile(off, 'm', 'b-1'); + await engineFor(off, { ...EVERYTHING, attachments: false }).engine.requestSync(); + expect(off.downloads).toEqual([]); + + const capped = new Files(); + withFile(capped, 'old', 'b-old', 3); + withFile(capped, 'new', 'b-new', 3); + await engineFor(capped, { ...EVERYTHING, maxTotalBytes: 4 }).engine.requestSync(); + expect(capped.downloads).toEqual(['b-new']); + + const refused = new Files(); + refused.missing.add('b-gone'); + withFile(refused, 'm', 'b-gone'); + const { engine } = engineFor(refused); + await engine.requestSync(); + await engine.requestSync(); + expect(refused.downloads).toEqual(['b-gone']); + }); + + it.each([500, 503, 429, 401])('tries a file again after a %i, which says nothing about the file', async (status) => { + class Failing extends Files { + fails = true; + override async download(jmap?: string, blobId?: string): Promise { + if (this.fails) throw new TransportError('http', 'Nope', status); + return super.download(jmap, blobId); + } + } + const server = new Failing(); + server.addEmail({ id: 'm', attachments: [{ blobId: 'b', size: 10, type: 'image/png' }] }); + const { engine, db } = engineFor(server); + await engine.requestSync(); + expect(db.get<{ refused: number }>('SELECT refused FROM email_part')?.refused).toBe(0); + server.fails = false; + server.addEmail({ id: 'n' }); + await engine.requestSync(); + expect(await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b' })).not.toBeNull(); + }); + + it.each([403, 404, 410])('gives up on a file after a %i', async (status) => { + class Refusing extends Files { + override async download(): Promise { + throw new TransportError('http', 'Nope', status); + } + } + const server = new Refusing(); + server.addEmail({ id: 'm', attachments: [{ blobId: 'b', size: 10, type: 'image/png' }] }); + const { engine, db } = engineFor(server); + await engine.requestSync(); + expect(db.get<{ refused: number }>('SELECT refused FROM email_part')?.refused).toBe(1); + }); + + it('files a blob the user opened only as far as the chosen scope allows', async () => { + const server = new Files(); + server.addEmail({ id: 'm', attachments: [{ blobId: 'b-kept', size: 10, type: 'image/png' }] }); + const data = new Uint8Array(10); + const record = { jmap: JMAP_ACCOUNT, blobId: 'b-kept', type: 'image/png', data }; + + const none = engineFor(server, { ...EVERYTHING, attachments: false }); + await none.engine.requestSync(); + expect(await none.engine.areaCall('mail.recordBlob', record)).toBe(false); + expect(await none.engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-kept' })).toBeNull(); + + const small = engineFor(server, { ...EVERYTHING, attachments: true, maxAttachmentBytes: 9 }); + await small.engine.requestSync(); + expect(await small.engine.areaCall('mail.recordBlob', record)).toBe(false); + + const fits = engineFor(server, { ...EVERYTHING, attachments: true, maxAttachmentBytes: 10 }); + await fits.engine.requestSync(); + expect(await fits.engine.areaCall('mail.recordBlob', record)).toBe(true); + }); + + it('files a blob the user opened only under a kept message: never a stray file', async () => { + const server = new Files(); + server.addEmail({ id: 'm', attachments: [{ blobId: 'b-kept', size: 99_999_999, type: 'image/png' }] }); + const { engine } = engineFor(server); + await engine.requestSync(); + expect(await engine.areaCall('mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'export-of-a-message', type: null, data: new Uint8Array([1]) })).toBe(false); + expect(await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'export-of-a-message' })).toBeNull(); + expect(await engine.areaCall('mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-kept', type: 'image/png', data: new Uint8Array([7]) })).toBe(true); + expect(await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-kept' })).toMatchObject({ type: 'image/png' }); + }); +}); + +describe('what the user opened online', () => { + it('keeps a message from outside the window with its text, and leaves kept ones alone', async () => { + const server = new FakeJmapMail(); + server.addEmail({ id: 'kept', subject: 'Kept' }); + const { engine } = engineFor(server); + await engine.requestSync(); + const opened = { + id: 'old', threadId: 'told', mailboxIds: { inbox: true }, keywords: {}, receivedAt: '2019-01-01T00:00:00Z', subject: 'Old', + textBody: [{ partId: '1', type: 'text/plain' }], htmlBody: [], attachments: [], bodyValues: { 1: { value: 'Old text' } }, headers: [{ name: 'X', value: ' y' }], + }; + await engine.areaCall('mail.recordOpened', { + jmap: JMAP_ACCOUNT, + emails: [ + opened, + { id: 'kept', subject: 'Changed?', bodyValues: { 1: { value: 'other' } }, mailboxIds: {}, receivedAt: '2019-01-01T00:00:00Z' }, + { id: 'cut', mailboxIds: { inbox: true }, receivedAt: '2019-01-01T00:00:00Z', bodyValues: { 1: { value: 'par', isTruncated: true } } }, + { id: 'nobody', mailboxIds: { inbox: true }, receivedAt: '2019-01-01T00:00:00Z' }, + ], + }); + const [, result] = (await one(engine, ['Email/get', { ids: ['old', 'kept', 'cut', 'nobody'], properties: ['subject', 'bodyValues', 'header:X:asText'], fetchAllBodyValues: true }, 'g']))!; + expect(result.notFound).toEqual(['cut', 'nobody']); + expect(result.list[0]).toMatchObject({ id: 'old', subject: 'Old', 'header:X:asText': 'y' }); + expect(result.list[0].bodyValues['1'].value).toBe('Old text'); + expect(result.list[1].subject).toBe('Kept'); + }); +}); + +describe('usage and session', () => { + it('reports the bytes of the file and the kept session', async () => { + const { engine } = engineFor(new FakeJmapMail()); + expect((await engine.areaCall('mail.usage', {}) as { bytes: number }).bytes).toBeGreaterThan(0); + expect(await engine.areaCall('mail.session', {})).toEqual(FULL_SESSION); + }); +}); diff --git a/lib/store/__tests__/mail-speed.test.ts b/lib/store/__tests__/mail-speed.test.ts new file mode 100644 index 000000000..3e9212e9a --- /dev/null +++ b/lib/store/__tests__/mail-speed.test.ts @@ -0,0 +1,276 @@ +// @vitest-environment node +import { beforeEach, describe, expect, it } from 'vitest'; +import { DEFAULT_SCOPE, STORAGE_ERROR_AREA, type StoreEvent } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError } from '../engine/transport'; +import { mailArea } from '../mail'; +import { outboxArea } from '../outbox'; +import { isStorageFailure, type SqlValue, type StorageAdapter } from '../storage'; +import { FAKE_SESSION } from './helpers/fake-jmap'; +import { FakeJmapMail } from './helpers/fake-jmap-mail'; +import { openMemoryStore } from './helpers/node-adapter'; + +/** Records the SQL of a query, so its plan can be read. */ +function spying(db: StorageAdapter) { + const seen: Array<{ sql: string; params: readonly SqlValue[] }> = []; + const spied: StorageAdapter = { + exec: (sql) => db.exec(sql), + run: (sql, params) => db.run(sql, params), + all: (sql: string, params?: readonly SqlValue[]) => { + seen.push({ sql, params: params ?? [] }); + return db.all(sql, params); + }, + get: (sql: string, params?: readonly SqlValue[]) => { + seen.push({ sql, params: params ?? [] }); + return db.get(sql, params); + }, + transaction: (fn) => db.transaction(fn), + close: () => db.close(), + }; + return { spied, seen }; +} + +function setup(emails: Array<{ id: string; keywords?: Record; mailboxIds?: Record }>) { + const server = new FakeJmapMail({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'other' }] }); + emails.forEach((email, i) => server.addEmail({ + keywords: {}, + mailboxIds: { inbox: true }, + receivedAt: new Date(Date.UTC(2026, 9, 1) + i * 60_000).toISOString(), + ...email, + })); + const raw = openMemoryStore(); + const { spied, seen } = spying(raw); + const engine = new AccountEngine({ + account: 'a', db: spied, session: FAKE_SESSION, transport: server, areas: [mailArea, outboxArea], + scope: { ...DEFAULT_SCOPE, days: null, attachments: false }, sleep: async () => {}, + }); + return { server, raw, seen, engine }; +} + +const NEWEST = { property: 'receivedAt', isAscending: false }; +const PINNED = { property: 'hasKeyword', keyword: '$pinned', isAscending: false }; +const UNREAD_FIRST = { property: 'hasKeyword', keyword: '$seen', isAscending: true }; +const IN_INBOX = { inMailbox: 'inbox' }; + +async function ids(engine: AccountEngine, args: Record): Promise { + const answer = await engine.answer([['Email/query', args, 'q']]); + return answer?.methodResponses[0][1].ids; +} + +async function plans(t: ReturnType, args: Record) { + t.seen.length = 0; + await t.engine.answer([['Email/query', args, 'q']]); + const reads = t.seen.filter(({ sql }) => /^SELECT (e\.id|COUNT)/.test(sql) && !sql.includes('sync_state')); + return reads.map(({ sql, params }) => ({ + sql, + plan: t.raw.all<{ detail: string }>(`EXPLAIN QUERY PLAN ${sql}`, params).map((row) => row.detail).join('; '), + })); +} + +describe('pinned and unread first', () => { + const MAIL: Array<{ id: string; keywords?: Record; mailboxIds?: Record }> = [ + { id: 'old-read', keywords: { $seen: true } }, + { id: 'old-unread' }, + { id: 'pin-read', keywords: { $seen: true, $pinned: true } }, + { id: 'mid-read', keywords: { $seen: true } }, + { id: 'pin-unread', keywords: { $pinned: true } }, + { id: 'new-unread' }, + { id: 'new-read', keywords: { $seen: true } }, + { id: 'elsewhere', keywords: { $pinned: true }, mailboxIds: { other: true } }, + ]; + + let t: ReturnType; + + beforeEach(async () => { + t = setup(MAIL); + await t.engine.requestSync(); + }); + + it('comes out the same as sorting the whole folder would', async () => { + expect(await ids(t.engine, { filter: IN_INBOX, sort: [PINNED, NEWEST] })) + .toEqual(['pin-unread', 'pin-read', 'new-read', 'new-unread', 'mid-read', 'old-unread', 'old-read']); + expect(await ids(t.engine, { filter: IN_INBOX, sort: [UNREAD_FIRST, NEWEST] })) + .toEqual(['new-unread', 'pin-unread', 'old-unread', 'new-read', 'mid-read', 'pin-read', 'old-read']); + expect(await ids(t.engine, { filter: IN_INBOX, sort: [PINNED, UNREAD_FIRST, NEWEST] })) + .toEqual(['pin-unread', 'pin-read', 'new-unread', 'old-unread', 'new-read', 'mid-read', 'old-read']); + // No date level at all: the same. + expect(await ids(t.engine, { filter: IN_INBOX, sort: [PINNED] })) + .toEqual(['pin-unread', 'pin-read', 'new-read', 'new-unread', 'mid-read', 'old-unread', 'old-read']); + }); + + it('pages across the groups', async () => { + const whole = await ids(t.engine, { filter: IN_INBOX, sort: [PINNED, NEWEST] }); + for (let position = 0; position < whole.length + 2; position += 1) { + for (const limit of [1, 2, 3, 50]) { + expect(await ids(t.engine, { filter: IN_INBOX, sort: [PINNED, NEWEST], position, limit })).toEqual(whole.slice(position, position + limit)); + } + } + }); + + it('keeps conditions, across the whole account too', async () => { + expect(await ids(t.engine, { filter: { operator: 'AND', conditions: [IN_INBOX, { notKeyword: '$seen' }] }, sort: [PINNED, NEWEST] })) + .toEqual(['pin-unread', 'new-unread', 'old-unread']); + expect(await ids(t.engine, { sort: [PINNED, NEWEST], limit: 2 })).toEqual(['elsewhere', 'pin-unread']); + expect(await ids(t.engine, { filter: { text: 'nothing-matches-this' }, sort: [PINNED, NEWEST] })).toEqual([]); + }); + + it('reads the comparator the way the client sends it for a server that flips it', async () => { + t.engine.setInvertedKeywordSort(true); + expect((await ids(t.engine, { filter: IN_INBOX, sort: [{ ...PINNED, isAscending: true }, NEWEST], limit: 2 }))).toEqual(['pin-unread', 'pin-read']); + expect((await ids(t.engine, { filter: IN_INBOX, sort: [{ ...PINNED, isAscending: false }, NEWEST], limit: 1 }))).toEqual(['new-read']); + }); + + it('answers the client\'s polarity probe the way a server would', async () => { + const asc = await ids(t.engine, { sort: [{ property: 'hasKeyword', keyword: '$seen', isAscending: true }, NEWEST], limit: 1 }); + const desc = await ids(t.engine, { sort: [{ property: 'hasKeyword', keyword: '$seen', isAscending: false }, NEWEST], limit: 1 }); + expect(asc).toEqual(['elsewhere']); + expect(desc).toEqual(['new-read']); + }); + + it('never sorts the folder', async () => { + for (const sort of [[PINNED, NEWEST], [UNREAD_FIRST, NEWEST], [PINNED, UNREAD_FIRST, NEWEST]]) { + const reads = await plans(t, { filter: IN_INBOX, sort, limit: 5 }); + expect(reads.length).toBeGreaterThan(0); + for (const { plan, sql } of reads) { + // A rare keyword's own rows may be sorted. The folder's never are. + if (sql.includes('email_keyword r')) expect(plan).toContain('email_keyword_lookup'); + else expect(plan).not.toContain('TEMP B-TREE'); + } + } + }); + + it('reads a rare keyword off its index first, and the rest off the folder\'s', async () => { + const reads = await plans(t, { filter: IN_INBOX, sort: [PINNED, NEWEST], limit: 3 }); + const pages = reads.filter(({ sql }) => sql.startsWith('SELECT e.id')); + expect(pages[0].plan).toContain('email_keyword_lookup'); + expect(pages[1].plan).toContain('email_mailbox_list'); + expect(pages[1].plan).not.toContain('TEMP B-TREE'); + }); + + it('counts what it skips without reading a message', async () => { + const reads = await plans(t, { filter: IN_INBOX, sort: [PINNED, NEWEST], position: 3, limit: 1 }); + const counts = reads.filter(({ sql }) => sql.startsWith('SELECT COUNT')); + expect(counts.length).toBeGreaterThan(0); + for (const { plan } of counts) expect(plan).not.toMatch(/\bSCAN e\b|SEARCH e\b/); + }); + + it('leaves other orders to a full sort, and they still work', async () => { + expect(await ids(t.engine, { filter: IN_INBOX, sort: [PINNED, { property: 'size', isAscending: true }, NEWEST], limit: 2 })).toHaveLength(2); + expect(await ids(t.engine, { filter: IN_INBOX, sort: [NEWEST], collapseThreads: true, limit: 2 })).toHaveLength(2); + }); +}); + +describe('calculateTotal', () => { + it('counts folder conditions off the index and the keyword rows, reading no message', async () => { + const t = setup([{ id: 'a' }, { id: 'b', keywords: { $seen: true } }]); + await t.engine.requestSync(); + t.seen.length = 0; + const answer = await t.engine.answer([['Email/query', { + filter: { operator: 'AND', conditions: [IN_INBOX, { notKeyword: '$seen' }, { after: '2020-01-01T00:00:00Z' }] }, + sort: [NEWEST], + limit: 1, + calculateTotal: true, + }, 'q']]); + expect(answer?.methodResponses[0][1].total).toBe(1); + const count = t.seen.find(({ sql }) => sql.startsWith('SELECT COUNT'))!; + const plan = t.raw.all<{ detail: string }>(`EXPLAIN QUERY PLAN ${count.sql}`, count.params).map((row) => row.detail).join('; '); + expect(plan).toContain('email_mailbox_list'); + expect(plan).not.toMatch(/\bSCAN e\b|SEARCH e\b/); + }); + + it('still reads the message for a condition only it can answer', async () => { + const t = setup([{ id: 'a', size: 5 } as never, { id: 'b', size: 500 } as never]); + await t.engine.requestSync(); + const answer = await t.engine.answer([['Email/query', { filter: { operator: 'AND', conditions: [IN_INBOX, { minSize: 100 }] }, calculateTotal: true }, 'q']]); + expect(answer?.methodResponses[0][1].total).toBe(1); + }); +}); + +describe('a failed write', () => { + function failing(now: { at: number }) { + const server = new FakeJmapMail(); + for (let i = 0; i < 400; i += 1) server.addEmail({ bodyValues: { 1: { value: 'x'.repeat(2000) } } }); + const db = openMemoryStore(); + const events: StoreEvent[] = []; + const engine = new AccountEngine({ + account: 'a', db, session: FAKE_SESSION, transport: server, areas: [mailArea, outboxArea], + scope: { ...DEFAULT_SCOPE, days: null, attachments: false }, sleep: async () => {}, + emit: (event) => events.push(event), now: () => now.at, + }); + // A real full disk, as SQLite reports it. + const pages = Number(db.get<{ page_count: number }>('PRAGMA page_count')?.page_count); + db.exec(`PRAGMA max_page_count = ${pages + 40}`); + return { server, db, events, engine }; + } + + const storageErrors = (events: StoreEvent[]) => events.filter((event) => event.type === 'error' && event.area === STORAGE_ERROR_AREA); + + it('tells the page, says no "synced", and doesn\'t try again on every push', async () => { + const now = { at: 1_000_000 }; + const t = failing(now); + await t.engine.requestSync(); + expect(storageErrors(t.events)).toHaveLength(1); + expect(t.events.some((event) => event.type === 'synced')).toBe(false); + + const requests = t.server.requests; + t.engine.noteStateChange({ acc1: { Email: 'whatever' } }); + await t.engine.requestSync(); + await t.engine.requestSync(); + expect(t.server.requests).toBe(requests); + expect(storageErrors(t.events)).toHaveLength(1); + }); + + it('tries again after a while, longer each time, and carries on once there\'s room', async () => { + const now = { at: 1_000_000 }; + const t = failing(now); + await t.engine.requestSync(); + + now.at += 61_000; + await t.engine.requestSync(); + expect(storageErrors(t.events)).toHaveLength(2); + + // Now it waits two minutes. + now.at += 61_000; + const requests = t.server.requests; + await t.engine.requestSync(); + expect(t.server.requests).toBe(requests); + + now.at += 61_000; + t.db.exec('PRAGMA max_page_count = 1000000'); + await t.engine.requestSync(); + expect(t.events.some((event) => event.type === 'synced')).toBe(true); + expect(t.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM email')?.n).toBe(400); + }); + + it('lets the user make room by keeping less, without waiting', async () => { + const now = { at: 1_000_000 }; + const t = failing(now); + await t.engine.requestSync(); + const requests = t.server.requests; + t.engine.setScope({ ...DEFAULT_SCOPE, days: 1 }); + await t.engine.requestSync(); + expect(t.server.requests).toBeGreaterThan(requests); + }); + + it('reports a change that couldn\'t be kept, and fails it rather than pretending', async () => { + const now = { at: 1_000_000 }; + const t = failing(now); + t.db.exec('PRAGMA max_page_count = 1000000'); + await t.engine.requestSync(); + const pages = Number(t.db.get<{ page_count: number }>('PRAGMA page_count')?.page_count); + t.db.exec(`PRAGMA max_page_count = ${pages}`); + const big = 'x'.repeat(5_000_000); + await expect(t.engine.areaCall('outbox.queueSend', { summary: { subject: big, to: [] }, item: { kind: 'email', args: [] } })).rejects.toThrow(/full/); + expect(storageErrors(t.events)).toHaveLength(1); + t.db.exec('PRAGMA max_page_count = 1000000'); + expect((await t.engine.areaCall('outbox.read', {}) as { entries: unknown[] }).entries).toEqual([]); + }); + + it('knows these errors from others', () => { + expect(isStorageFailure(new Error('database or disk is full'))).toBe(true); + expect(isStorageFailure(new Error('SQLITE_FULL: sqlite3 result code 13'))).toBe(true); + expect(isStorageFailure(Object.assign(new Error('x'), { name: 'QuotaExceededError' }))).toBe(true); + expect(isStorageFailure(new Error('no such table: x'))).toBe(false); + expect(isStorageFailure(new TransportError('network', 'Failed to fetch'))).toBe(false); + }); +}); diff --git a/lib/store/__tests__/mail-sync.test.ts b/lib/store/__tests__/mail-sync.test.ts index a5ddae149..13d438c6c 100644 --- a/lib/store/__tests__/mail-sync.test.ts +++ b/lib/store/__tests__/mail-sync.test.ts @@ -130,11 +130,116 @@ describe('persisted state', () => { await engineFor(server, second).requestSync(); expect(count(second, 'email')).toBe(4500); const firstQuery = callsNamed(server, 'Email/query')[0]; - expect(firstQuery[1].position).toBe(listed); + expect(firstQuery[1]).toMatchObject({ anchorOffset: 1 }); + expect(second.get<{ id: string }>('SELECT id FROM email WHERE id = ?', [firstQuery[1].anchor as string])).toBeTruthy(); + expect(firstQuery[1]).not.toHaveProperty('position'); second.close(); } finally { rmSync(dir, { recursive: true, force: true }); } + }, 30_000); +}); + +describe('a full pass while the server changes', () => { + // Newest first, four pages of 500 to a request: the second request starts at 2000. + function bigServer() { + const server = new FakeJmap(); + for (let i = 0; i < 2600; i += 1) server.addEmail({ id: `m${i}` }); + return server; + } + + it('misses no message when something before its place is deleted, which would shift positions', async () => { + const server = bigServer(); + const db = openMemoryStore(); + let deleted = false; + server.beforeRequest = () => { + // Once the first request of the list has been answered: the newest + // message, long passed by the pass. + if (!deleted && callsNamed(server, 'Email/query').length >= 4) { + deleted = true; + server.destroyEmail('m2599'); + } + }; + await engineFor(server, db).requestSync(); + expect(deleted).toBe(true); + expect(count(db, 'email')).toBe(2599); + for (let i = 0; i < 2599; i += 1) expect(db.get('SELECT 1 AS x FROM email WHERE id = ?', [`m${i}`])).toBeTruthy(); + }); + + it('goes back to positions, with an overlap, when the anchor itself is deleted', async () => { + const server = bigServer(); + const db = openMemoryStore(); + let deleted = false; + server.beforeRequest = () => { + if (deleted || server.calls.filter(([n]) => n === 'Email/query').length < 4) return; + deleted = true; + // The last id of the first request, and one before it. + const newestFirst = [...server.emails.keys()].reverse(); + server.destroyEmail(newestFirst[1999]); + server.destroyEmail(newestFirst[0]); + }; + await engineFor(server, db).requestSync(); + expect(deleted).toBe(true); + expect(count(db, 'email')).toBe(2598); + expect(callsNamed(server, 'Email/query').some(([, args]) => args.anchor === undefined && Number(args.position) > 0)).toBe(true); + expect(server.emails.size).toBe(2598); + for (const id of server.emails.keys()) expect(db.get('SELECT 1 AS x FROM email WHERE id = ?', [id])).toBeTruthy(); + }); + + it('keeps what a pass missed but the server still has, and drops only what is gone or out of the window', async () => { + const server = new FakeJmap(); + server.addEmail({ id: 'stays', subject: 'Before', receivedAt: '2026-11-01T00:00:00Z' }); + server.addEmail({ id: 'gone', receivedAt: '2026-11-01T00:00:01Z' }); + server.addEmail({ id: 'old', receivedAt: '2026-10-01T00:00:00Z' }); + const db = openMemoryStore(); + const engine = new AccountEngine({ + account: 'a@fake.test', db, session: FAKE_SESSION, transport: server, areas: [mailArea], + scope: { ...DEFAULT_SCOPE, days: null }, now: () => Date.UTC(2026, 10, 3), sleep: async () => {}, + }); + await engine.requestSync(); + expect(count(db, 'email')).toBe(3); + + // A start over, in a narrower window, where paging misses `stays`. + engine.setScope({ ...DEFAULT_SCOPE, days: 10 }); + server.hiddenFromQuery.add('stays'); + server.updateEmail('stays', { subject: 'After' }); + server.destroyEmail('gone'); + server.truncateLog('Email'); + server.calls.length = 0; + await engine.requestSync(); + expect(db.all('SELECT id FROM email').map((row) => row.id)).toEqual(['stays']); + expect(JSON.parse(String(db.get<{ json: string }>('SELECT json FROM email WHERE id = ?', ['stays'])?.json)).subject).toBe('After'); + // Its text wasn't thrown away either. + expect(count(db, 'email_body')).toBe(1); + expect(callsNamed(server, 'Email/get').some(([, args]) => (args.ids as string[] | undefined)?.includes('gone'))).toBe(true); + }); + + it('picks the check up again after a stop, and asks 500 at a time', async () => { + const server = new FakeJmap(); + for (let i = 0; i < 700; i += 1) server.addEmail({ id: `m${i}` }); + const db = openMemoryStore(); + const engine = engineFor(server, db); + await engine.requestSync(); + for (let i = 0; i < 700; i += 1) server.hiddenFromQuery.add(`m${i}`); + server.truncateLog('Email'); + server.calls.length = 0; + let gets = 0; + server.beforeRequest = () => { + if (callsNamed(server, 'Email/get').filter(([, args]) => Array.isArray(args.ids) && (args.ids as string[]).length > 0).length > gets) gets += 1; + if (gets === 1) server.offline = true; + }; + await engine.requestSync(); + expect(count(db, 'email')).toBe(700); + server.offline = false; + server.beforeRequest = null; + await engine.requestSync(); + expect(count(db, 'email')).toBe(700); + const sizes = callsNamed(server, 'Email/get').map(([, args]) => (args.ids as string[] | undefined)?.length ?? 0); + expect(Math.max(...sizes)).toBeLessThanOrEqual(500); + // The pass finished: back to /changes. + server.calls.length = 0; + await engine.requestSync(); + expect(callsNamed(server, 'Email/query')).toHaveLength(0); }); }); @@ -338,7 +443,7 @@ describe('query plans', () => { }, }; const answer = await answerRequest( - { account: 'a', db: spied as typeof db, session: FAKE_SESSION, scope: EVERYTHING, now: Date.now, emit: () => {}, defaultJmap: JMAP_ACCOUNT, invertedKeywordSort: false }, + { account: 'a', db: spied as typeof db, session: FAKE_SESSION, scope: EVERYTHING, now: Date.now, emit: () => {}, overlay: (_jmap, _type, records) => records, defaultJmap: JMAP_ACCOUNT, invertedKeywordSort: false }, buildMethodTable([mailArea]), [['Email/query', { filter: INBOX, sort: [{ property: 'receivedAt', isAscending: false }], limit: 5 }, 'q']], ); diff --git a/lib/store/__tests__/outbox.test.ts b/lib/store/__tests__/outbox.test.ts new file mode 100644 index 000000000..d820ab2d5 --- /dev/null +++ b/lib/store/__tests__/outbox.test.ts @@ -0,0 +1,665 @@ +// @vitest-environment node +import { beforeEach, describe, expect, it } from 'vitest'; +import { DEFAULT_SCOPE } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError, type MethodCall } from '../engine/transport'; +import { mailArea } from '../mail'; +import { outboxArea } from '../outbox'; +import { CLOCK_SKEW_MARGIN_MS, decideRetry, LOOKUP_TRUSTED_MS, type LookupArgs } from '../outbox/dedupe'; +import { addHeldFileSource } from '../outbox/calls'; +import { flushPatches, type PatchOutcome, type PendingPatch } from '../outbox/flush-patches'; +import type { StoredConflict, StoredOutboxEntry } from '../outbox/items'; +import { mergePatches, patchConflicts } from '../outbox/patches'; +import { FAKE_SESSION, JMAP_ACCOUNT, type FakeEmail } from './helpers/fake-jmap'; +import { FakeJmapMail } from './helpers/fake-jmap-mail'; +import { openMemoryStore } from './helpers/node-adapter'; + +const NOW = Date.UTC(2026, 10, 3); +const BOXES = [ + { id: 'inbox', name: 'Inbox', role: 'inbox', totalEmails: 2, unreadEmails: 1 }, + { id: 'trash', name: 'Trash', role: 'trash', totalEmails: 1, unreadEmails: 0 }, + { id: 'drafts', name: 'Drafts', role: 'drafts', totalEmails: 0, unreadEmails: 0 }, + { id: 'sent', name: 'Sent', role: 'sent', totalEmails: 0, unreadEmails: 0 }, +]; + +function setup() { + const server = new FakeJmapMail({ mailboxes: BOXES.map((box) => ({ ...box })) }); + server.addEmail({ id: 'a', threadId: 't1', subject: 'Aye', keywords: {}, mailboxIds: { inbox: true } }); + server.addEmail({ id: 'b', threadId: 't1', subject: 'Bee', keywords: { $seen: true }, mailboxIds: { inbox: true } }); + server.addEmail({ id: 'c', threadId: 't2', subject: 'Sea', keywords: { $seen: true }, mailboxIds: { trash: true } }); + const db = openMemoryStore(); + const engine = new AccountEngine({ + account: 'a@fake.test', + db, + session: FAKE_SESSION, + transport: server, + areas: [mailArea, outboxArea], + scope: { ...DEFAULT_SCOPE, days: null }, + now: () => clock, + sleep: async () => {}, + }); + return { server, db, engine }; +} + +let t: ReturnType; + +beforeEach(async () => { + clock = NOW; + t = setup(); + await t.engine.requestSync(); +}); + +const set = (args: Record) => t.engine.answer([['Email/set', args, 's']]); +const update = (patches: Record>) => set({ update: patches }); +const pending = () => t.engine.areaCall('outbox.pendingPatches', {}) as Promise; +const read = () => t.engine.areaCall('outbox.read', {}) as Promise<{ patches: number; entries: StoredOutboxEntry[]; conflicts: StoredConflict[] }>; + +async function local(id: string) { + const answer = await t.engine.answer([['Email/get', { ids: [id], properties: ['keywords', 'mailboxIds', 'subject'] }, 'g']]); + return answer?.methodResponses[0][1].list[0]; +} + +let clock = NOW; + +async function boxes() { + const answer = await t.engine.answer([['Mailbox/get', {}, 'm']]); + return Object.fromEntries((answer?.methodResponses[0][1].list as Array<{ id: string; totalEmails: number; unreadEmails: number }>).map((box) => [box.id, [box.totalEmails, box.unreadEmails]])); +} + +const io = (server: FakeJmapMail = t.server) => ({ + request: (calls: MethodCall[]) => server.request(calls), +}); + +async function flush(ioOverride = io()) { + const outcomes: PatchOutcome[] = []; + const finished = await flushPatches(ioOverride, await pending(), async (batch) => { + outcomes.push(...batch); + await t.engine.areaCall('outbox.patchResults', { outcomes: batch }); + }); + return { outcomes, finished }; +} + +describe('merging patches per record', () => { + it('keeps one row per record, with later paths on top of earlier ones', async () => { + await update({ a: { 'keywords/$seen': true } }); + await update({ a: { 'keywords/$flagged': true } }); + await update({ a: { 'keywords/$seen': null } }); + const rows = await pending(); + expect(rows).toHaveLength(1); + expect(rows[0].patch).toEqual({ 'keywords/$seen': null, 'keywords/$flagged': true }); + expect((await local('a')).keywords).toEqual({ $flagged: true }); + }); + + it('lets a whole map replace earlier paths into it, and takes later paths into the map', () => { + expect(mergePatches({ 'keywords/$seen': true, 'mailboxIds/x': true }, { keywords: { $flagged: true } })) + .toEqual({ keywords: { $flagged: true }, 'mailboxIds/x': true }); + expect(mergePatches({ keywords: { $flagged: true } }, { 'keywords/$seen': true })) + .toEqual({ keywords: { $flagged: true, $seen: true } }); + expect(mergePatches({ keywords: { $flagged: true } }, { 'keywords/$flagged': null })).toEqual({ keywords: {} }); + expect(mergePatches({ mailboxIds: { a: true } }, { mailboxIds: { b: true } })).toEqual({ mailboxIds: { b: true } }); + }); + + it('reads escaped keyword names', () => { + expect(mergePatches({ keywords: {} }, { 'keywords/a~1b': true })).toEqual({ keywords: { 'a/b': true } }); + }); + + it('queues a change to a message it doesn\'t keep, and answers it as done', async () => { + const answer = await update({ unknown: { 'keywords/$seen': true } }); + expect(answer?.methodResponses[0][1].updated).toEqual({ unknown: null }); + expect((await pending()).map((row) => [row.recordId, row.base])).toEqual([['unknown', null]]); + }); + + it('refuses per record what can\'t wait: other properties, a new folder, no folder at all', async () => { + const answer = await update({ + a: { subject: 'x' }, + b: { 'mailboxIds/#new': true }, + c: { mailboxIds: {} }, + }); + const result = answer?.methodResponses[0][1]; + expect(Object.keys(result.notUpdated).sort()).toEqual(['a', 'b', 'c']); + expect(result.notUpdated.a.type).toBe('forbidden'); + expect(result.notUpdated.c.type).toBe('invalidProperties'); + expect(await pending()).toEqual([]); + const emptied = await update({ c: { 'mailboxIds/trash': null } }); + expect(emptied?.methodResponses[0][1].notUpdated.c.type).toBe('invalidProperties'); + expect(await pending()).toEqual([]); + expect((await local('c')).mailboxIds).toEqual({ trash: true }); + }); + + it('answers a create per message, so the composer\'s "not saved" path runs (seam for the drafts agent)', async () => { + const answer = await set({ create: { draft: { subject: 'x' } } }); + expect(answer?.methodResponses[0][0]).toBe('Email/set'); + expect(answer?.methodResponses[0][1].notCreated.draft.type).toBe('serverUnavailable'); + }); +}); + +describe('showing changes at once', () => { + it('moves a message and keeps the folder counts right', async () => { + expect(await boxes()).toMatchObject({ inbox: [2, 1], trash: [1, 0] }); + await update({ a: { 'mailboxIds/inbox': null, 'mailboxIds/trash': true } }); + expect(await boxes()).toMatchObject({ inbox: [1, 0], trash: [2, 1] }); + const answer = await t.engine.answer([['Email/query', { filter: { inMailbox: 'trash' }, sort: [{ property: 'receivedAt', isAscending: false }] }, 'q']]); + expect(answer?.methodResponses[0][1].ids).toEqual(['c', 'a']); + }); + + it('counts a read change once, also after later changes to the same record', async () => { + await update({ a: { 'keywords/$seen': true } }); + expect((await boxes()).inbox).toEqual([2, 0]); + await update({ a: { 'keywords/$flagged': true } }); + expect((await boxes()).inbox).toEqual([2, 0]); + await update({ a: { 'keywords/$seen': null } }); + expect((await boxes()).inbox).toEqual([2, 1]); + }); + + it('whole-map patches count the same', async () => { + await update({ a: { keywords: { $seen: true } } }); + expect((await boxes()).inbox).toEqual([2, 0]); + await update({ b: { mailboxIds: { trash: true } } }); + expect(await boxes()).toMatchObject({ inbox: [1, 0], trash: [2, 0] }); + }); + + it('deletes for good right away, from lists, threads and counts', async () => { + const answer = await set({ destroy: ['a'] }); + expect(answer?.methodResponses[0][1].destroyed).toEqual(['a']); + expect(await boxes()).toMatchObject({ inbox: [1, 0] }); + const check = await t.engine.answer([ + ['Email/get', { ids: ['a'], properties: ['id'] }, 'g'], + ['Thread/get', { ids: ['t1'] }, 't'], + ]); + expect(check?.methodResponses[0][1].notFound).toEqual(['a']); + expect(check?.methodResponses[1][1].list).toEqual([{ id: 't1', emailIds: ['b'] }]); + expect((await pending()).map((row) => row.op)).toEqual(['destroy']); + }); + + it('empties a folder through the back-reference the client sends', async () => { + const answer = await t.engine.answer([ + ['Email/query', { filter: { inMailbox: 'trash' }, limit: 500 }, '0'], + ['Email/set', { '#destroy': { resultOf: '0', name: 'Email/query', path: '/ids' } }, '1'], + ]); + expect(answer?.methodResponses[1][1].destroyed).toEqual(['c']); + expect((await boxes()).trash).toEqual([0, 0]); + const again = await t.engine.answer([['Email/query', { filter: { inMailbox: 'trash' } }, '0']]); + expect(again?.methodResponses[0][1].ids).toEqual([]); + await flush(); + expect(t.server.emails.has('c')).toBe(false); + expect(await pending()).toEqual([]); + }); + + it('a change to a message waiting to be deleted is not found', async () => { + await set({ destroy: ['a'] }); + const answer = await update({ a: { 'keywords/$seen': true } }); + expect(answer?.methodResponses[0][1].notUpdated.a.type).toBe('notFound'); + expect((await pending())[0].op).toBe('destroy'); + }); +}); + +describe('sync and waiting changes', () => { + it('reapplies a waiting change over what the server sends', async () => { + await update({ a: { 'keywords/$flagged': true } }); + t.server.updateEmail('a', { subject: 'Renamed', keywords: { $seen: true } }); + await t.engine.requestSync(); + expect(await local('a')).toMatchObject({ subject: 'Renamed', keywords: { $seen: true, $flagged: true } }); + expect(await pending()).toHaveLength(1); + }); + + it('survives a full resync, and keeps a pending delete away', async () => { + await update({ a: { 'keywords/$flagged': true } }); + await set({ destroy: ['b'] }); + t.server.truncateLog('Email'); + t.server.addEmail({ id: 'new' }); + await t.engine.requestSync(); + expect((await local('a')).keywords).toEqual({ $flagged: true }); + expect(await local('b')).toBeUndefined(); + expect(await local('new')).toBeTruthy(); + }); + + it('doesn\'t bring back a message waiting to be deleted when the server changes it', async () => { + await set({ destroy: ['a'] }); + t.server.updateEmail('a', { keywords: { $flagged: true } }); + await t.engine.requestSync(); + expect(await local('a')).toBeUndefined(); + }); + + it('follows the server\'s new counts without counting a waiting change twice', async () => { + await update({ a: { 'keywords/$seen': true } }); + expect((await boxes()).inbox).toEqual([2, 0]); + t.server.mailboxes.get('inbox')!.totalEmails = 3; + t.server.mailboxes.get('inbox')!.unreadEmails = 2; + t.server.coll.Mailbox.push('updated', 'inbox'); + t.server.addEmail({ id: 'new', keywords: {} }); + await t.engine.requestSync(); + expect((await boxes()).inbox).toEqual([3, 1]); + }); + + it('drops the patch once the server took it, and the next run brings the truth', async () => { + await update({ a: { 'keywords/$seen': true } }); + const { outcomes, finished } = await flush(); + expect(finished).toBe(true); + expect(outcomes).toEqual([expect.objectContaining({ recordId: 'a', outcome: 'done' })]); + expect(await pending()).toEqual([]); + expect((t.server.emails.get('a') as FakeEmail).keywords).toEqual({ $seen: true }); + t.server.updateEmail('a', { keywords: { $seen: true, $answered: true } }); + await t.engine.requestSync(); + expect((await local('a')).keywords).toEqual({ $seen: true, $answered: true }); + }); +}); + +describe('conflicts', () => { + it('holds back a change to what the server changed since, and shows the server\'s state', async () => { + await update({ a: { 'keywords/$flagged': true } }); + t.server.updateEmail('a', { keywords: { $flagged: true, $seen: true } }); + // The same outcome as wanted: not a conflict. + let run = await flush(); + expect(run.outcomes[0]).toMatchObject({ outcome: 'done' }); + + await update({ b: { 'keywords/$seen': null } }); + t.server.updateEmail('b', { keywords: { $seen: true, $flagged: true } }); + // Another keyword moved: this one still stands. + run = await flush(); + expect(run.outcomes[0]).toMatchObject({ outcome: 'done' }); + + await update({ c: { 'mailboxIds/trash': null, 'mailboxIds/inbox': true } }); + t.server.updateEmail('c', { mailboxIds: { sent: true } }); + run = await flush(); + expect(run.outcomes[0]).toMatchObject({ outcome: 'conflict', reason: 'changed' }); + const { conflicts, patches } = await read(); + expect(patches).toBe(0); + expect(conflicts).toEqual([expect.objectContaining({ recordId: 'c', jmap: JMAP_ACCOUNT, summary: { subject: 'Sea', op: 'update', reason: 'changed' } })]); + expect((t.server.emails.get('c') as FakeEmail).mailboxIds).toEqual({ sent: true }); + expect((await local('c')).mailboxIds).toEqual({ sent: true }); + }); + + it('never sends a conflicting change, and sends the rest of the batch', async () => { + await update({ a: { 'mailboxIds/inbox': null, 'mailboxIds/trash': true }, b: { 'keywords/$flagged': true } }); + t.server.updateEmail('a', { mailboxIds: { sent: true } }); + await flush(); + expect((t.server.emails.get('a') as FakeEmail).mailboxIds).toEqual({ sent: true }); + expect((t.server.emails.get('b') as FakeEmail).keywords).toEqual({ $seen: true, $flagged: true }); + }); + + it('shows what the server refused', async () => { + await update({ a: { 'keywords/$flagged': true } }); + t.server.refuse.set('a', { type: 'forbidden', description: 'Read only' }); + await flush(); + const { conflicts } = await read(); + expect(conflicts[0].summary).toEqual({ subject: 'Aye', op: 'update', reason: 'refused', detail: 'Read only' }); + expect((await local('a')).keywords).toEqual({}); + await t.engine.areaCall('outbox.dismissConflict', { id: conflicts[0].id }); + expect((await read()).conflicts).toEqual([]); + }); + + it('shows a change to a message gone from the server, and removes the local copy', async () => { + await update({ a: { 'keywords/$flagged': true } }); + t.server.destroyEmail('a'); + await flush(); + expect((await read()).conflicts[0].summary).toMatchObject({ reason: 'gone', subject: 'Aye' }); + expect(await local('a')).toBeUndefined(); + }); + + it('a delete of what\'s already gone is simply done', async () => { + await set({ destroy: ['a'] }); + t.server.destroyEmail('a'); + const { outcomes } = await flush(); + expect(outcomes[0]).toMatchObject({ outcome: 'done' }); + expect((await read()).conflicts).toEqual([]); + }); + + it('holds back a delete of what changed on the server, and the message comes back', async () => { + await set({ destroy: ['a'] }); + t.server.updateEmail('a', { keywords: { $flagged: true } }); + await t.engine.requestSync(); + await flush(); + expect(t.server.emails.has('a')).toBe(true); + expect((await read()).conflicts[0].summary).toMatchObject({ op: 'destroy', reason: 'changed' }); + expect((await local('a')).keywords).toEqual({ $flagged: true }); + }); + + it('shows a refused delete', async () => { + await set({ destroy: ['a'] }); + t.server.refuse.set('a', { type: 'forbidden' }); + await flush(); + expect((await read()).conflicts[0].summary).toMatchObject({ op: 'destroy', reason: 'refused', detail: 'forbidden' }); + expect((await local('a'))?.subject).toBe('Aye'); + }); + + it('compares per property, keywords the way the server does', () => { + const base = { keywords: { '$SEEN': true, '$label:A': true }, mailboxIds: { x: true } }; + // System flags fold. + expect(patchConflicts({ 'keywords/$seen': null }, base, { keywords: { $seen: true, '$label:A': true }, mailboxIds: { x: true } })).toBe(false); + // Custom keywords are case-sensitive: the server now has another one. + expect(patchConflicts({ 'keywords/$label:A': null }, base, { keywords: { '$label:a': true }, mailboxIds: { x: true } })).toBe(false); + expect(patchConflicts({ 'keywords/$label:a': true }, { keywords: {}, mailboxIds: {} }, { keywords: { '$label:A': true }, mailboxIds: {} })).toBe(false); + expect(patchConflicts({ 'keywords/$label:A': true }, { keywords: { '$label:A': true }, mailboxIds: {} }, { keywords: { '$label:a': true }, mailboxIds: {} })).toBe(true); + }); + + it('compares per property', () => { + const base = { keywords: { '$label:a': true }, mailboxIds: { x: true } }; + expect(patchConflicts({ 'keywords/$seen': true }, base, { keywords: {}, mailboxIds: { x: true } })).toBe(false); + expect(patchConflicts({ keywords: { $seen: true } }, base, { keywords: {}, mailboxIds: { x: true } })).toBe(true); + expect(patchConflicts({ keywords: { $seen: true } }, base, { keywords: { $seen: true }, mailboxIds: { x: true } })).toBe(false); + expect(patchConflicts({ keywords: { $seen: true } }, null, { keywords: {}, mailboxIds: {} })).toBe(false); + }); +}); + +describe('when the server can\'t be reached during a flush', () => { + it('stops and keeps every change, without calling anything a conflict', async () => { + await update({ a: { 'keywords/$seen': true } }); + t.server.offline = true; + const { outcomes, finished } = await flush(); + expect(finished).toBe(false); + expect(outcomes).toEqual([]); + expect(await pending()).toHaveLength(1); + expect((await read()).conflicts).toEqual([]); + expect((await local('a')).keywords).toEqual({ $seen: true }); + }); + + it('stops on "not now" from the server too', async () => { + await update({ a: { 'keywords/$seen': true } }); + t.server.failNext.set('Email/set', 'serverUnavailable'); + const { finished } = await flush(); + expect(finished).toBe(false); + expect(await pending()).toHaveLength(1); + }); + + /** Refuses the Email/set with this answer, and reads go through. */ + const refusingSet = (message: string) => ({ + request: async (calls: MethodCall[]) => { + if (calls[0][0] === 'Email/set') throw new Error(message); + return t.server.request(calls); + }, + }); + const PROBLEM = 'Request failed: 400 - {"type":"urn:ietf:params:jmap:error:notRequest","status":400}'; + + it('turns a request the server refuses for good into a conflict, and shows what the server has', async () => { + await update({ a: { 'keywords/$seen': true } }); + const { outcomes, finished } = await flush(refusingSet(PROBLEM)); + expect(finished).toBe(true); + expect(outcomes[0]).toMatchObject({ outcome: 'conflict', reason: 'refused' }); + expect(await pending()).toEqual([]); + // The change is gone, and so is its effect here. + const answer = await t.engine.answer([['Email/get', { ids: ['a'], properties: ['keywords'] }, 'g']]); + expect(answer?.methodResponses[0][1].list[0].keywords).toEqual({}); + expect((await read()).conflicts).toHaveLength(1); + }); + + it.each([ + ['the parallel-request limit', 'Request failed: 400 - {"type":"urn:ietf:params:jmap:error:limit","limit":"maxConcurrentRequests"}'], + ['a proxy\'s error page', 'Request failed: 403 - Forbidden'], + ['a body that\'s no JMAP problem', 'Request failed: 413 - Payload too large'], + ['a server error', 'Request failed: 500 - oops'], + ])('keeps the change for later on %s', async (_name, message) => { + await update({ a: { 'keywords/$seen': true } }); + const { outcomes, finished } = await flush(refusingSet(message)); + expect(finished).toBe(false); + expect(outcomes).toEqual([]); + expect(await pending()).toHaveLength(1); + expect((await read()).conflicts).toEqual([]); + }); + + it('keeps the change for later when the read after a refusal fails', async () => { + await update({ a: { 'keywords/$seen': true } }); + let calls = 0; + const { finished } = await flush({ + request: async (batch: MethodCall[]) => { + calls += 1; + if (calls === 1) return t.server.request(batch); + if (batch[0][0] === 'Email/set') throw new Error(PROBLEM); + throw new TypeError('Failed to fetch'); + }, + }); + expect(finished).toBe(false); + expect(await pending()).toHaveLength(1); + }); + + it('keeps a change made while the flush ran, and doesn\'t compare it with the state the flush caused', async () => { + await update({ a: { 'keywords/$seen': true } }); + const rows = await pending(); + await update({ a: { 'keywords/$flagged': true } }); + await flushPatches(io(), rows, async (batch) => { await t.engine.areaCall('outbox.patchResults', { outcomes: batch }); }); + const left = await pending(); + expect(left).toHaveLength(1); + expect(left[0].base).toBeNull(); + await flush(); + expect((t.server.emails.get('a') as FakeEmail).keywords).toEqual({ $seen: true, $flagged: true }); + expect((await read()).conflicts).toEqual([]); + }); +}); + +describe('sends', () => { + const summary = { subject: 'S', to: ['b@example.org'] }; + const item = { kind: 'email', args: [['b@example.org'], 'S', 'x'] }; + + it('takes an entry for sending once: a second claim loses', async () => { + const id = await t.engine.areaCall('outbox.queueSend', { summary, item, messageId: 'fixed@example.org' }) as number; + expect(await t.engine.areaCall('outbox.claim', { id })).toBe(true); + expect(await t.engine.areaCall('outbox.claim', { id })).toBe(false); + const entry = (await read()).entries[0]; + expect(entry).toMatchObject({ status: 'sending', sendingSince: NOW, lastAttemptAt: NOW, messageId: 'fixed@example.org', item }); + }); + + it('marks what a closed tab left behind uncertain', async () => { + const id = await t.engine.areaCall('outbox.queueSend', { summary, item }) as number; + await t.engine.areaCall('outbox.claim', { id }); + await t.engine.areaCall('outbox.settleStale', { before: NOW }); + expect((await read()).entries[0].status).toBe('sending'); + await t.engine.areaCall('outbox.settleStale', { before: NOW + 1 }); + expect((await read()).entries[0]).toMatchObject({ status: 'uncertain', firstAttemptAt: NOW }); + }); + + it('keeps the first attempt time across retries, so the lookup covers the first copy', async () => { + const id = await t.engine.areaCall('outbox.queueSend', { summary, item }) as number; + await t.engine.areaCall('outbox.claim', { id }); + await t.engine.areaCall('outbox.update', { id, status: 'uncertain' }); + expect((await read()).entries[0].firstAttemptAt).toBe(NOW); + clock = NOW + 60_000; + await t.engine.areaCall('outbox.update', { id, status: 'waiting' }); + await t.engine.areaCall('outbox.claim', { id }); + await t.engine.areaCall('outbox.update', { id, status: 'uncertain' }); + const entry = (await read()).entries[0]; + expect(entry.firstAttemptAt).toBe(NOW); + expect(entry.lastAttemptAt).toBe(NOW + 60_000); + }); + + it('doesn\'t call a try that never started an attempt', async () => { + const id = await t.engine.areaCall('outbox.queueSend', { summary, item }) as number; + for (let i = 0; i < 2; i += 1) { + await t.engine.areaCall('outbox.claim', { id }); + await t.engine.areaCall('outbox.update', { id, status: 'waiting' }); + } + expect((await read()).entries[0].firstAttemptAt).toBeUndefined(); + }); + + it('keeps order across changes and sends, and a later merge moves a change later', async () => { + await update({ a: { 'keywords/$seen': true } }); + await t.engine.areaCall('outbox.queueSend', { summary, item }); + await update({ b: { 'keywords/$flagged': true } }); + const { entries } = await read(); + const before = await t.engine.areaCall('outbox.pendingPatches', { upToSeq: entries[0].seq }) as PendingPatch[]; + expect(before.map((row) => row.recordId)).toEqual(['a']); + await update({ a: { 'keywords/$flagged': true } }); + expect(await t.engine.areaCall('outbox.pendingPatches', { upToSeq: entries[0].seq })).toEqual([]); + }); + + it('removes the files of a removed entry, and keeps an update from bringing a deleted one back', async () => { + const data = new TextEncoder().encode('hello'); + const held = await t.engine.areaCall('outbox.holdFile', { name: 'note.txt', type: 'text/plain', data }) as { blobId: string; size: number }; + expect(held).toMatchObject({ size: 5, type: 'text/plain' }); + expect(held.blobId).toMatch(/^bulwark-local-/); + const file = await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId }) as { name: string; data: Uint8Array }; + expect(file.name).toBe('note.txt'); + expect(new TextDecoder().decode(file.data)).toBe('hello'); + const id = await t.engine.areaCall('outbox.queueSend', { summary, item: { kind: 'raw', blobId: held.blobId, identityId: 'i' } }) as number; + await t.engine.areaCall('outbox.remove', { id }); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId })).toBeNull(); + await t.engine.areaCall('outbox.update', { id, status: 'failed' }); + expect((await read()).entries).toEqual([]); + }); + + it('leaves a file alone that another area holds', async () => { + const held = await t.engine.areaCall('outbox.holdFile', { name: 'draft.txt', type: 'text/plain', data: new Uint8Array(1) }) as { blobId: string }; + const stale = await t.engine.areaCall('outbox.holdFile', { name: 'old.txt', type: 'text/plain', data: new Uint8Array(1) }) as { blobId: string }; + addHeldFileSource(() => [held.blobId]); + expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW + 1 })).toBe(1); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId })).not.toBeNull(); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: stale.blobId })).toBeNull(); + }); + + it('drops old files nothing needs', async () => { + const data = new Uint8Array([1]); + const orphan = await t.engine.areaCall('outbox.holdFile', { name: 'a', type: '', data }) as { blobId: string }; + const used = await t.engine.areaCall('outbox.holdFile', { name: 'b', type: '', data }) as { blobId: string }; + await t.engine.areaCall('outbox.queueSend', { summary, item: { kind: 'email', args: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, [{ blobId: used.blobId }]] } }); + expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW })).toBe(0); + expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW + 1 })).toBe(1); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: orphan.blobId })).toBeNull(); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: used.blobId })).not.toBeNull(); + }); +}); + +describe('looking a send up before it goes again', () => { + const MESSAGE_ID = 'k3x.11111111@example.org'; + const args = (over: Partial = {}): LookupArgs => ({ + jmap: JMAP_ACCOUNT, + submissionJmap: JMAP_ACCOUNT, + messageId: MESSAGE_ID, + attemptAt: NOW - 60_000, + now: NOW, + copyKept: true, + ...over, + }); + + function server() { + return new FakeJmapMail({ mailboxes: BOXES.map((box) => ({ ...box })) }); + } + + const inDrafts = (s: FakeJmapMail, over: Partial = {}) => s.addEmail({ + id: 'copy', + messageId: [MESSAGE_ID], + mailboxIds: { drafts: true }, + keywords: { $seen: true, $draft: true }, + receivedAt: new Date(NOW - 30_000).toISOString(), + ...over, + }); + + // The five cases of decision 4. + it('request never arrived: send again', async () => { + const s = server(); + // A message from before, and another with its own id. + s.addEmail({ id: 'other', messageId: ['other@example.org'], mailboxIds: { sent: true }, receivedAt: new Date(NOW - 10_000).toISOString() }); + expect(await decideRetry(io(s), args())).toEqual({ action: 'send' }); + }); + + it('draft arrived, submission didn\'t: submit that copy, don\'t create another', async () => { + const s = server(); + inDrafts(s); + expect(await decideRetry(io(s), args())).toEqual({ action: 'submit', emailId: 'copy' }); + }); + + it('both arrived: done', async () => { + const s = server(); + inDrafts(s, { mailboxIds: { sent: true }, keywords: { $seen: true } }); + s.addSubmission('copy', 'final'); + expect(await decideRetry(io(s), args())).toEqual({ action: 'done', emailId: 'copy', why: 'sent' }); + // A record Stalwart already removed doesn't change that. + s.submissions.clear(); + expect(await decideRetry(io(s), args())).toEqual({ action: 'done', emailId: 'copy', why: 'sent' }); + }); + + it('scheduled and waiting: done, leave it', async () => { + const s = server(); + inDrafts(s); + s.addSubmission('copy', 'pending'); + expect(await decideRetry(io(s), args())).toEqual({ action: 'done', emailId: 'copy', why: 'scheduled' }); + }); + + it('canceled: someone decided against it, so it doesn\'t go', async () => { + const s = server(); + inDrafts(s); + s.addSubmission('copy', 'canceled'); + expect(await decideRetry(io(s), args())).toEqual({ action: 'done', emailId: 'copy', why: 'canceled' }); + }); + + it('never uses a header or text filter, which find nothing on either version', async () => { + const s = server(); + inDrafts(s, { mailboxIds: { sent: true }, keywords: {} }); + await decideRetry(io(s), args()); + const filters = JSON.stringify(s.calls.filter(([name]) => name === 'Email/query').map(([, a]) => a.filter)); + expect(filters).not.toMatch(/header|text/); + expect(filters).toMatch(/inMailbox/); + }); + + it('finds a copy whose server time is behind this device\'s clock, within the margin', async () => { + const s = server(); + const skew = CLOCK_SKEW_MARGIN_MS - 60_000; + inDrafts(s, { receivedAt: new Date(NOW - 60_000 - skew).toISOString() }); + expect((await decideRetry(io(s), args())).action).toBe('submit'); + }); + + it('matches with or without angle brackets', async () => { + const s = server(); + inDrafts(s); + expect((await decideRetry(io(s), args({ messageId: `<${MESSAGE_ID}>` }))).action).toBe('submit'); + }); + + it('can\'t tell without a Message-ID (a send from before they were fixed)', async () => { + expect(await decideRetry(io(server()), args({ messageId: undefined }))).toEqual({ action: 'undecided', why: 'no-message-id' }); + }); + + it('can\'t tell without the folders', async () => { + const s = new FakeJmapMail(); + expect(await decideRetry(io(s), args())).toEqual({ action: 'undecided', why: 'no-folders' }); + }); + + it('can\'t tell when nothing stays after a good send, and nothing is found', async () => { + expect(await decideRetry(io(server()), args({ copyKept: false }))).toEqual({ action: 'undecided', why: 'no-copy-kept' }); + }); + + it('can\'t tell once the server may have forgotten the submission', async () => { + const old = args({ attemptAt: NOW - LOOKUP_TRUSTED_MS - 1000 }); + expect(await decideRetry(io(server()), old)).toEqual({ action: 'undecided', why: 'too-old' }); + const s = server(); + inDrafts(s, { receivedAt: new Date(old.attemptAt).toISOString() }); + expect(await decideRetry(io(s), old)).toEqual({ action: 'undecided', why: 'too-old' }); + }); + + it('can\'t tell when more mail arrived than it reads, unless the copy is among it', async () => { + const s = server(); + for (let i = 0; i < 2500; i += 1) s.addEmail({ mailboxIds: { sent: true }, receivedAt: new Date(NOW - 1000).toISOString() }); + expect(await decideRetry(io(s), args())).toEqual({ action: 'undecided', why: 'too-many' }); + }, 30_000); + + it('lets a failed lookup fail, so the caller leaves the entry alone', async () => { + const s = server(); + s.offline = true; + await expect(decideRetry(io(s), args())).rejects.toBeInstanceOf(TransportError); + }); +}); + +describe('moving the old outbox in', () => { + const legacy = { + entries: [ + { summary: { subject: 'Old', to: [] }, item: { kind: 'email', args: [] }, status: 'uncertain', createdAt: 5 }, + { summary: { subject: 'Older file', to: [] }, item: { kind: 'raw', blobId: 'bulwark-local-x', identityId: 'i' }, status: 'waiting', createdAt: 6 }, + ], + patches: [{ jmap: JMAP_ACCOUNT, update: { a: { 'keywords/$seen': true } } }], + files: [{ blobId: 'bulwark-local-x', name: 'x.eml', type: 'message/rfc822', data: new Uint8Array([1, 2]) }], + }; + + it('takes all of it once, keeping stand-in ids and order', async () => { + expect(await t.engine.areaCall('outbox.importLegacy', legacy)).toBe(true); + expect(await t.engine.areaCall('outbox.importLegacy', legacy)).toBe(false); + const { entries, patches } = await read(); + expect(entries.map((e) => [e.summary.subject, e.status, e.createdAt])).toEqual([['Old', 'uncertain', 5], ['Older file', 'waiting', 6]]); + expect(patches).toBe(1); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: 'bulwark-local-x' })).toMatchObject({ name: 'x.eml' }); + expect((await local('a')).keywords).toEqual({ $seen: true }); + }); + + it('takes nothing when a part of it fails, and can run again', async () => { + const broken = { ...legacy, files: [{ blobId: 'x', name: null, type: null, data: null }] }; + await expect(t.engine.areaCall('outbox.importLegacy', broken)).rejects.toThrow(); + expect((await read()).entries).toEqual([]); + expect(await pending()).toEqual([]); + expect(await t.engine.areaCall('outbox.importLegacy', legacy)).toBe(true); + }); +}); diff --git a/lib/store/__tests__/page-side.test.ts b/lib/store/__tests__/page-side.test.ts new file mode 100644 index 000000000..fbd336b69 --- /dev/null +++ b/lib/store/__tests__/page-side.test.ts @@ -0,0 +1,406 @@ +import 'fake-indexeddb/auto'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { configureStore, getStore, setStoreBackend, STORE_OPENED_KEY } from '../client/api'; +import { forgetAccountKey, getAccountRecord } from '../client/keys'; +import { purgeStoreFiles } from '../client/purge'; +import { StoreClient, StoreOwnerGoneError } from '../client/store-client'; +import { storeFilesToPrune } from '../worker/host'; +import { FAKE_SESSION } from './helpers/fake-jmap'; + +/** Everything on one page: a BroadcastChannel that also reaches other objects here, as in a browser. */ +class FakeChannel { + static open = new Set(); + onmessage: ((event: MessageEvent) => void) | null = null; + constructor(readonly name: string) { + FakeChannel.open.add(this); + } + postMessage(data: unknown): void { + for (const channel of FakeChannel.open) { + if (channel !== this && channel.name === this.name) channel.onmessage?.({ data } as MessageEvent); + } + } + close(): void { + FakeChannel.open.delete(this); + } +} + +class FakeWorker { + static all: FakeWorker[] = []; + static openResult: unknown; + onmessage: ((event: MessageEvent) => void) | null = null; + posted: Array> = []; + terminated = false; + constructor(readonly url: string, readonly options: unknown) { + FakeWorker.all.push(this); + } + postMessage(message: Record): void { + this.posted.push(message); + if (message.kind === 'open' || message.kind === 'prune-files') queueMicrotask(() => this.onmessage?.({ data: { kind: 'result', id: message.id, result: message.kind === 'open' ? FakeWorker.openResult : undefined } } as MessageEvent)); + } + terminate(): void { + this.terminated = true; + } + becomeOwner(): void { + this.onmessage?.({ data: { kind: 'owner' } } as MessageEvent); + } +} + +beforeEach(() => { + FakeChannel.open.clear(); + FakeWorker.all = []; + vi.stubGlobal('BroadcastChannel', FakeChannel); + vi.stubGlobal('Worker', FakeWorker); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + setStoreBackend(null); + configureStore(null); +}); + +describe('whether the store can run here', () => { + afterEach(() => { + delete (window.navigator as { storage?: unknown }).storage; + delete (window.navigator as { locks?: unknown }).locks; + }); + + it('doesn’t ask the page for what only exists in a worker', () => { + expect(StoreClient.supported()).toBe(false); + Object.defineProperty(window.navigator, 'storage', { configurable: true, value: { getDirectory: async () => ({}) } }); + Object.defineProperty(window.navigator, 'locks', { configurable: true, value: { request: async () => {} } }); + // As on a real page: the class is there, without createSyncAccessHandle on it. + vi.stubGlobal('FileSystemFileHandle', class {}); + expect(StoreClient.supported()).toBe(true); + }); +}); + +describe('a tab that starts after the owner', () => { + it('asks who owns the store, so its calls don\'t wait for an announcement that never comes', async () => { + const owner = new FakeChannel('bulwark-store-rpc'); + const asked: unknown[] = []; + owner.onmessage = (event) => { + asked.push(event.data); + if ((event.data as { kind: string }).kind === 'who-owns') owner.postMessage({ kind: 'owner-changed', owner: 'other-tab' }); + if ((event.data as { kind: string }).kind === 'rpc') { + const { id, from } = event.data as { id: string; from: string }; + owner.postMessage({ kind: 'reply', id, to: from, result: { answered: true } }); + } + }; + const client = new StoreClient(); + client.start(); + expect(asked).toEqual([{ kind: 'who-owns' }]); + await expect(client.area('a', 'x.y')).resolves.toEqual({ answered: true }); + client.halt(); + }); + + it('passes the error\'s name over the channel, so a caller can wait for the account to open', async () => { + const owner = new FakeChannel('bulwark-store-rpc'); + owner.onmessage = (event) => { + if ((event.data as { kind: string }).kind === 'who-owns') owner.postMessage({ kind: 'owner-changed', owner: 'other-tab' }); + if ((event.data as { kind: string }).kind === 'rpc') { + const { id, from } = event.data as { id: string; from: string }; + owner.postMessage({ kind: 'reply', id, to: from, error: 'Not open', errorName: 'StoreAccountClosedError' }); + } + }; + const client = new StoreClient(); + client.start(); + await expect(client.area('a', 'x.y')).rejects.toMatchObject({ name: 'StoreAccountClosedError' }); + client.halt(); + }); +}); + +describe('an owner that stopped answering', () => { + afterEach(() => vi.useRealTimers()); + + async function clientWithOwner(answers: () => boolean) { + vi.useFakeTimers(); + const owner = new FakeChannel('bulwark-store-rpc'); + owner.onmessage = (event) => { + const data = event.data as { kind: string }; + if (data.kind === 'who-owns' && answers()) owner.postMessage({ kind: 'owner-changed', owner: 'elsewhere' }); + }; + const client = new StoreClient({ basePath: '' }); + client.start(); + await vi.advanceTimersByTimeAsync(0); + return client; + } + + it('stops its calls waiting, long before their own time limit', async () => { + let answering = true; + const client = await clientWithOwner(() => answering); + const call = client.sync('a@x'); + const failed = vi.fn(); + void call.catch(failed); + await vi.advanceTimersByTimeAsync(0); + answering = false; + await vi.advanceTimersByTimeAsync(39_000); + expect(failed).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(11_000); + expect(failed.mock.calls[0][0]).toBeInstanceOf(StoreOwnerGoneError); + client.halt(); + }); + + it('leaves calls alone while the owner answers', async () => { + const client = await clientWithOwner(() => true); + const failed = vi.fn(); + void client.sync('a@x').catch(failed); + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(failed).not.toHaveBeenCalled(); + client.halt(); + }); +}); + +describe('the owner tab calling its own worker', () => { + it('waits for the worker on the channel, since a call posted before that is lost', async () => { + const worker = new FakeChannel('bulwark-store-rpc'); + const heard: Array<{ kind: string; id?: string; from?: string }> = []; + worker.onmessage = (event) => heard.push(event.data as { kind: string }); + const client = new StoreClient(); + client.start(); + FakeWorker.all[0].becomeOwner(); + expect(client.ownsStore).toBe(true); + + const call = client.area('a', 'x.y'); + await new Promise((resolve) => setTimeout(resolve, 5)); + expect(heard.some(({ kind }) => kind === 'rpc')).toBe(false); + // It asked again, for when the first announcement was missed. + expect(heard.filter(({ kind }) => kind === 'who-owns')).toHaveLength(2); + + worker.postMessage({ kind: 'owner-changed', owner: client.clientId }); + await vi.waitFor(() => expect(heard.some(({ kind }) => kind === 'rpc')).toBe(true)); + const rpc = heard.find(({ kind }) => kind === 'rpc')!; + worker.postMessage({ kind: 'reply', id: rpc.id, to: rpc.from, result: 'done' }); + await expect(call).resolves.toBe('done'); + client.halt(); + }); +}); + +describe('a sign-out in any tab', () => { + it('stops this tab\'s worker and rejects what was waiting', async () => { + const halted = vi.fn(); + const client = new StoreClient({ onHalted: halted }); + client.start(); + const owner = new FakeChannel('bulwark-store-rpc'); + owner.postMessage({ kind: 'owner-changed', owner: 'x' }); + const pending = client.area('a', 'x.y'); + await Promise.resolve(); + owner.postMessage({ kind: 'shutdown' }); + await expect(pending).rejects.toMatchObject({ name: 'StoreOwnerGoneError' }); + expect(FakeWorker.all[0].terminated).toBe(true); + expect(halted).toHaveBeenCalled(); + }); +}); + +describe('purgeStoreFiles', () => { + function storage(removeEntry: (name: string, options: unknown) => Promise) { + Object.defineProperty(window.navigator, 'storage', { configurable: true, value: { getDirectory: async () => ({ removeEntry }) } }); + } + + afterEach(() => { + delete (window.navigator as { storage?: unknown }).storage; + delete (window.navigator as { locks?: unknown }).locks; + }); + + it('forgets the keys first, tells every tab, and deletes the pool holding the owner lock', async () => { + const seen: unknown[] = []; + const tab = new FakeChannel('bulwark-store-rpc'); + tab.onmessage = (event) => seen.push(event.data); + const order: string[] = []; + Object.defineProperty(window.navigator, 'locks', { + configurable: true, + value: { request: async (name: string, _options: unknown, run: () => Promise) => { order.push(`lock ${name}`); await run(); order.push('unlock'); } }, + }); + storage(async (name, options) => { + order.push(`remove ${name} ${JSON.stringify(options)}`); + }); + await purgeStoreFiles(); + expect(seen).toEqual([{ kind: 'shutdown' }]); + expect(order).toEqual(['lock bulwark-store-owner', 'remove .bulwark-store {"recursive":true}', 'unlock']); + expect((await indexedDB.databases()).map((d) => d.name)).not.toContain('bulwark-store'); + }); + + it('deletes what it can when another tab never lets go of the owner lock', async () => { + const remove = vi.fn(async () => {}); + storage(remove); + let waited: AbortSignal | undefined; + Object.defineProperty(window.navigator, 'locks', { + configurable: true, + value: { + // The browser gives up when the signal says so. + request: async (_name: string, options: { signal: AbortSignal }) => { + waited = options.signal; + throw new DOMException('Took too long', 'TimeoutError'); + }, + }, + }); + await purgeStoreFiles(); + expect(waited).toBeInstanceOf(AbortSignal); + expect(remove).toHaveBeenCalledTimes(1); + }); + + it('waits out a worker that still holds the files, and shrugs at a pool that isn\'t there', async () => { + const remove = vi.fn() + .mockRejectedValueOnce(Object.assign(new Error('busy'), { name: 'NoModificationAllowedError' })) + .mockResolvedValue(undefined); + storage(remove); + await purgeStoreFiles(); + expect(remove).toHaveBeenCalledTimes(2); + + storage(async () => { throw Object.assign(new Error('none'), { name: 'NotFoundError' }); }); + await expect(purgeStoreFiles()).resolves.toBeUndefined(); + }); +}); + +describe('which files have no key', () => { + it('names store files with no key, and leaves anything else alone', () => { + expect(storeFilesToPrune(['/a01.db', '/a02.db', '/a02.db-journal', '/other', '/ab.DB'], ['/a02.db'])).toEqual(['/a01.db']); + }); +}); + +describe('the page\'s store', () => { + function allowEngine() { + configureStore({ + inputs: () => ({ session: FAKE_SESSION, auth: 'Basic x', invertedKeywordSort: true }), + refreshAuth: async () => 'Bearer fresh', + onEvent: () => {}, + }); + } + + it('opens an account only as the owner, with the key to its own worker', async () => { + allowEngine(); + const store = getStore(); + await store.ensureOpen('a@x'); + const worker = FakeWorker.all[0]; + expect(worker.posted.some((m) => m.kind === 'open')).toBe(false); + worker.becomeOwner(); + await vi.waitFor(() => expect(worker.posted.some((m) => m.kind === 'open')).toBe(true)); + const open = worker.posted.find((m) => m.kind === 'open') as { params: { account: string; keyHex: string; file: string; auth: string; invertedKeywordSort: boolean } }; + expect(open.params).toMatchObject({ account: 'a@x', auth: 'Basic x', invertedKeywordSort: true }); + expect(open.params.keyHex).toMatch(/^[0-9a-f]{64}$/); + expect(open.params.file).toMatch(/^\/a[0-9a-f]+\.db$/); + store.stop(); + expect(worker.terminated).toBe(true); + }); + + describe('when the key or the file of a store is gone', () => { + const lost = vi.fn(); + const open = async () => { + configureStore({ + inputs: () => ({ session: FAKE_SESSION, auth: 'Basic x', invertedKeywordSort: true }), + refreshAuth: async () => null, + onEvent: () => {}, + onStoreLost: lost, + }); + const store = getStore(); + await store.ensureOpen('lost@x'); + const worker = FakeWorker.all[FakeWorker.all.length - 1]; + worker.becomeOwner(); + await vi.waitFor(() => expect(worker.posted.some((m) => m.kind === 'open')).toBe(true)); + await vi.waitFor(() => expect(localStorage.getItem(STORE_OPENED_KEY)).toContain('lost@x')); + store.stop(); + return worker; + }; + + beforeEach(() => { + lost.mockClear(); + localStorage.clear(); + }); + + afterEach(async () => { + await forgetAccountKey('lost@x'); + }); + + it('stays quiet about a first open, and removes nothing a key still names', async () => { + const worker = await open(); + expect(lost).not.toHaveBeenCalled(); + const prune = worker.posted.find((m) => m.kind === 'prune-files') as unknown as { keep: string[] }; + const record = await getAccountRecord('lost@x'); + expect(prune.keep).toContain(record!.file); + }); + + it('deletes the orphan and tells the page when the key went', async () => { + await open(); + const before = await getAccountRecord('lost@x'); + await forgetAccountKey('lost@x'); + const worker = await open(); + const after = await getAccountRecord('lost@x'); + expect(after!.file).not.toBe(before!.file); + const prune = worker.posted.find((m) => m.kind === 'prune-files') as unknown as { keep: string[] }; + expect(prune.keep).toContain(after!.file); + expect(prune.keep).not.toContain(before!.file); + expect(lost).toHaveBeenCalledWith('lost@x'); + }); + + it('stays quiet on a normal reopen', async () => { + await open(); + const worker = await open(); + expect(lost).not.toHaveBeenCalled(); + expect(worker.posted.some((m) => m.kind === 'prune-files')).toBe(false); + }); + + it('tells the page when the file went and the key stayed', async () => { + await open(); + FakeWorker.openResult = { created: true }; + try { + await open(); + } finally { + FakeWorker.openResult = undefined; + } + expect(lost).toHaveBeenCalledWith('lost@x'); + }); + + it('doesn\'t remember an account that was removed', async () => { + await open(); + vi.spyOn(StoreClient.prototype, 'removeAccount').mockResolvedValue(true as never); + await getStore().remove('lost@x'); + getStore().stop(); + vi.restoreAllMocks(); + await open(); + expect(lost).not.toHaveBeenCalled(); + }); + }); + + it('runs the client\'s refresh when the worker needs auth, and hands the result over', async () => { + allowEngine(); + const store = getStore(); + await store.ensureOpen('a@x'); + const worker = FakeWorker.all[0]; + worker.onmessage?.({ data: { kind: 'need-auth', account: 'a@x' } } as MessageEvent); + await vi.waitFor(() => expect(worker.posted).toContainEqual({ kind: 'set-auth', account: 'a@x', auth: 'Bearer fresh' })); + store.stop(); + }); + + it('forgets the key when an account is removed, also in a tab that isn\'t the owner', async () => { + allowEngine(); + const store = getStore(); + await store.ensureOpen('a@x'); + const worker = FakeWorker.all[0]; + worker.becomeOwner(); + await vi.waitFor(() => expect(worker.posted.some((m) => m.kind === 'open')).toBe(true)); + expect(await getAccountRecord('a@x')).not.toBeNull(); + + const owner = new FakeChannel('bulwark-store-rpc'); + owner.postMessage({ kind: 'owner-changed', owner: 'worker' }); + const removed: unknown[] = []; + owner.onmessage = (event) => { + const message = event.data as { kind: string; id: string; from: string; method: string; params: unknown }; + if (message.kind !== 'rpc') return; + removed.push([message.method, message.params]); + owner.postMessage({ kind: 'reply', id: message.id, to: message.from, result: true }); + }; + const record = await getAccountRecord('a@x'); + await store.remove('a@x'); + expect(await getAccountRecord('a@x')).toBeNull(); + expect(removed).toEqual([['removeAccount', { account: 'a@x', file: record?.file }]]); + store.stop(); + }); + + it('stops waiting when the engine couldn\'t start in this browser', async () => { + allowEngine(); + const store = getStore(); + await store.ensureOpen('a@x'); + FakeWorker.all[0].onmessage?.({ data: { kind: 'event', event: { type: 'error', account: '', area: 'store', message: 'no wasm' } } } as MessageEvent); + await expect(store.answer('a@x', [])).rejects.toThrow('could not start'); + store.stop(); + }); +}); diff --git a/lib/store/adapters/wasm.ts b/lib/store/adapters/wasm.ts index 2af1373fe..d9ec6387d 100644 --- a/lib/store/adapters/wasm.ts +++ b/lib/store/adapters/wasm.ts @@ -12,11 +12,11 @@ */ import { runTransaction, type SqlRow, type SqlValue, type StorageAdapter } from '../storage'; +import { POOL_DIRECTORY } from '../worker/protocol'; const VFS = 'multipleciphers-opfs-sahpool'; const INIT_TIMEOUT_MS = 30_000; const REQUIRED_PAGE_SIZE = 8192; -const POOL_DIRECTORY = '.bulwark-store'; const INITIAL_CAPACITY = 16; /** A database and its journal, plus room for one more. */ const SLOTS_PER_DATABASE = 3; @@ -117,6 +117,10 @@ class WasmAdapter implements StorageAdapter { } } +export function installMayPass(error: unknown): boolean { + return (error as Error)?.name !== 'SecurityError'; +} + /** The module and the pool, once. Only the owner gets here. */ export class WasmStorage { private constructor(private readonly sqlite3: any, private readonly pool: Pool) {} @@ -141,6 +145,8 @@ export class WasmStorage { }) as Pool; return new WasmStorage(sqlite3, pool); } catch (error) { + // The browser won't allow it here at all, a private window: waiting won't help. + if (!installMayPass(error)) throw error; lastError = error; await new Promise((resolve) => setTimeout(resolve, 100 + attempt * 50)); } diff --git a/lib/store/area.ts b/lib/store/area.ts index b73740b5b..c435a4bce 100644 --- a/lib/store/area.ts +++ b/lib/store/area.ts @@ -28,6 +28,9 @@ export const DEFAULT_SCOPE: StoreScope = { maxTotalBytes: 2 * 1024 * 1024 * 1024, }; +/** The `area` of an error event for a write that failed: a full disk, the browser's quota. */ +export const STORAGE_ERROR_AREA = 'storage'; + export interface SyncStateRow { state: string | null; /** The newest state the server told us about, to know when a run is caught up. */ @@ -52,8 +55,18 @@ export interface StoreContext { scope: StoreScope; now(): number; emit(event: StoreEvent): void; + /** Records of one type as this device shows them: see RecordOverlay. */ + overlay(jmap: string, type: string, records: T[]): T[]; } +/** + * What an area changes about records another area stores: the outbox lays + * changes that wait for the server over what the server sent. Runs inside the + * caller's transaction. Returns the records to use; a record left out is one + * this device already treats as gone. + */ +export type RecordOverlay = (ctx: Pick, jmap: string, type: string, records: Array<{ id: string }>) => Array<{ id: string }>; + export interface SyncContext extends StoreContext { transport: JmapTransport; /** True once the engine is closing or lost the network: stop at the next batch. */ @@ -107,4 +120,5 @@ export interface StoreArea { methods: Readonly>; syncers: readonly Syncer[]; calls?: Readonly>; + overlay?: RecordOverlay; } diff --git a/lib/store/areas.ts b/lib/store/areas.ts index 30a6541ff..08fe2c607 100644 --- a/lib/store/areas.ts +++ b/lib/store/areas.ts @@ -3,5 +3,6 @@ import type { StoreArea } from './area'; import { calendarArea } from './calendar'; import { mailArea } from './mail'; +import { outboxArea } from './outbox'; -export const STORE_AREAS: readonly StoreArea[] = [mailArea, calendarArea]; +export const STORE_AREAS: readonly StoreArea[] = [mailArea, outboxArea, calendarArea]; diff --git a/lib/store/calendar/index.ts b/lib/store/calendar/index.ts index 12d908660..1401c050e 100644 --- a/lib/store/calendar/index.ts +++ b/lib/store/calendar/index.ts @@ -11,6 +11,8 @@ export const calendarArea: StoreArea = { syncers: [calendarSyncer], calls: { 'calendar.flush': (ctx) => flushCalendar(ctx), + /** Edits the server hasn't got, for the question before a sign-out. */ + 'calendar.pending': (ctx) => Number(ctx.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM calendar_pending')?.n ?? 0), 'calendar.conflicts': (ctx) => listConflicts(ctx.db), 'calendar.resolveStandIns': (ctx, params: { ids: string[] }) => resolveStandIns(ctx.db, params.ids), 'calendar.dismissConflict': (ctx, params: { id: number }) => { diff --git a/lib/store/client/api.ts b/lib/store/client/api.ts new file mode 100644 index 000000000..83cb83655 --- /dev/null +++ b/lib/store/client/api.ts @@ -0,0 +1,246 @@ +/** + * The page's way to the local store: one StoreClient per tab, accounts opened + * while this tab is the owner, sessions and auth handed over. Knows nothing of + * the app's stores: lib/offline/offline-mode.ts wires those in with + * configureStore, so the worker bundle stays free of them. + */ + +import { DEFAULT_SCOPE, type StoreEvent, type StoreScope } from '../area'; +import type { JmapResponse, MethodCall, StoreSession } from '../engine/transport'; +import { forgetAccountKey, getAccountRecord, knownStoreFiles, openAccountKey, type WrapLayer } from './keys'; +import { StoreClient } from './store-client'; + +export interface StoreAccountInputs { + session: StoreSession | null; + auth: string | null; + scope?: StoreScope; + invertedKeywordSort: boolean | null; +} + +export interface StorePageHooks { + /** What this tab knows about an account right now. */ + inputs(account: string): StoreAccountInputs; + /** Runs the client's refresh path; the fresh Authorization header, or null. */ + refreshAuth(account: string): Promise; + onEvent(event: StoreEvent): void; + /** Wrap layers on top of the WebCrypto one (desktop keychain, app lock). */ + keyLayers?(): readonly WrapLayer[]; + /** An account this device kept mail for has an empty store now: its key or its file went. */ + onStoreLost?(account: string): void; +} + +/** What lib/offline uses. A seam, so its tests run the real engine on node:sqlite. */ +export interface StoreBackend { + supported(): boolean; + /** Asks for the account to be open. `session` is for a first open with no client yet. */ + ensureOpen(account: string, session?: StoreSession | null): Promise; + /** Hands the owner what this tab now knows: session, auth, sort polarity. */ + refresh(account: string): void; + answer(account: string, calls: MethodCall[]): Promise; + area(account: string, name: string, params?: unknown): Promise; + sync(account: string): Promise; + noteStateChange(account: string, changed: Record>): Promise; + setOnline(online: boolean): Promise; + /** Closes the account, forgets its key and deletes its file. */ + remove(account: string): Promise; + /** Stops this tab's worker at once (a sign-out; the files go separately). */ + stop(): void; +} + +/** How long a call waits for the owner to have opened an account. */ +const OPEN_RETRIES = 25; +const OPEN_RETRY_MS = 200; + +/** + * localStorage: accounts that have had a store here. Holds nothing but the + * account's id, and it outlives what a browser may clear of the rest. + */ +export const STORE_OPENED_KEY = 'bulwark-store-opened'; + +function readOpenedAccounts(): string[] { + try { + const parsed = JSON.parse(localStorage.getItem(STORE_OPENED_KEY) ?? '[]') as unknown; + return Array.isArray(parsed) ? parsed as string[] : []; + } catch { + return []; + } +} + +function rememberOpened(account: string, opened: boolean): void { + try { + const others = readOpenedAccounts().filter((id) => id !== account); + localStorage.setItem(STORE_OPENED_KEY, JSON.stringify(opened ? [...others, account] : others)); + } catch { + // Without it a lost store goes unnoticed, and the store still works. + } +} + +class PageStore implements StoreBackend { + private client: StoreClient | null = null; + private readonly wanted = new Map(); + private readonly opening = new Map>(); + /** The engine can't start in this browser: don't keep every caller waiting for an owner. */ + private broken = false; + /** What the owner's engine was last told to keep. */ + private readonly sentScope = new Map(); + + constructor(private readonly hooks: () => StorePageHooks | null) {} + + supported(): boolean { + return StoreClient.supported(); + } + + private started(): StoreClient { + if (this.client) return this.client; + this.broken = false; + const client = new StoreClient({ + basePath: (process.env.NEXT_PUBLIC_BASE_PATH ?? '').replace(/\/+$/, ''), + buildId: process.env.NEXT_PUBLIC_BUILD_ID, + onEvent: (event) => { + if (event.type === 'error' && event.area === 'store' && !event.account) this.broken = true; + this.hooks()?.onEvent(event); + }, + onOwner: () => { + this.opening.clear(); + for (const account of this.wanted.keys()) void this.openHere(account).catch(() => {}); + }, + onNeedAuth: (account) => { + void (this.hooks()?.refreshAuth(account) ?? Promise.resolve(null)) + .catch(() => null) + .then((auth) => this.client?.setAuth(account, auth)); + }, + onHalted: () => this.reset(), + }); + client.start(); + this.client = client; + return client; + } + + private reset(): void { + this.client = null; + this.sentScope.clear(); + this.wanted.clear(); + this.opening.clear(); + this.broken = false; + } + + /** Only the owner may open: the key goes to this tab's own worker, never over the channel. */ + private openHere(account: string): Promise { + const running = this.opening.get(account); + if (running) return running; + const run = (async () => { + const client = this.started(); + const { keyHex, record, fresh } = await openAccountKey(account, this.hooks()?.keyLayers?.() ?? []); + // The old file, if the key alone went, can never be read again. + if (fresh) await client.pruneFiles(await knownStoreFiles()).catch(() => {}); + const inputs = this.hooks()?.inputs(account) ?? { session: null, auth: null, invertedKeywordSort: null }; + if (inputs.scope) this.sentScope.set(account, JSON.stringify(inputs.scope)); + const opened = await client.openAccount({ + account, + file: record.file, + keyHex, + session: inputs.session ?? this.wanted.get(account) ?? null, + scope: inputs.scope ?? DEFAULT_SCOPE, + invertedKeywordSort: inputs.invertedKeywordSort, + auth: inputs.auth, + }); + if (fresh || opened?.created) { + if (readOpenedAccounts().includes(account)) this.hooks()?.onStoreLost?.(account); + } + rememberOpened(account, true); + })(); + this.opening.set(account, run); + run.catch(() => this.opening.delete(account)); + return run; + } + + async ensureOpen(account: string, session?: StoreSession | null): Promise { + this.wanted.set(account, session ?? this.wanted.get(account) ?? null); + const client = this.started(); + if (client.ownsStore) await this.openHere(account); + // Otherwise the owner opens it: calls below wait for that. + } + + refresh(account: string): void { + const client = this.client; + if (!client?.ownsStore) return; + const inputs = this.hooks()?.inputs(account); + if (!inputs) return; + if (inputs.session) client.setSession(account, inputs.session); + client.setAuth(account, inputs.auth); + // Only a real change: setting it starts over what a full disk made the store give up on. + const scope = inputs.scope ? JSON.stringify(inputs.scope) : null; + if (inputs.scope && scope !== this.sentScope.get(account)) { + if (this.sentScope.has(account)) client.setScope(account, inputs.scope); + this.sentScope.set(account, scope!); + } + if (inputs.invertedKeywordSort !== null) client.setInvertedKeywordSort(account, inputs.invertedKeywordSort); + } + + /** A call for an account: the owner may still be opening it, in this tab or another. */ + private async whenOpen(account: string, call: (client: StoreClient) => Promise): Promise { + if (this.broken) throw new Error('The local store could not start'); + await this.ensureOpen(account); + for (let attempt = 0; ; attempt += 1) { + try { + return await call(this.started()); + } catch (error) { + if ((error as Error)?.name !== 'StoreAccountClosedError' || attempt >= OPEN_RETRIES) throw error; + await new Promise((resolve) => setTimeout(resolve, OPEN_RETRY_MS)); + } + } + } + + answer(account: string, calls: MethodCall[]): Promise { + return this.whenOpen(account, (client) => client.answer(account, calls)); + } + + area(account: string, name: string, params?: unknown): Promise { + return this.whenOpen(account, (client) => client.area(account, name, params)); + } + + sync(account: string): Promise { + return this.whenOpen(account, (client) => client.sync(account)); + } + + noteStateChange(account: string, changed: Record>): Promise { + return this.whenOpen(account, (client) => client.noteStateChange(account, changed)); + } + + async setOnline(online: boolean): Promise { + if (!this.client || this.broken) return; + await this.client.setOnline(online); + } + + async remove(account: string): Promise { + const record = await getAccountRecord(account); + await forgetAccountKey(account); + rememberOpened(account, false); + this.wanted.delete(account); + this.opening.delete(account); + if (record && !this.broken) await this.started().removeAccount(account, record.file); + } + + stop(): void { + this.client?.halt(); + this.reset(); + } +} + +let hooks: StorePageHooks | null = null; +let backend: StoreBackend | null = null; + +/** Registers what the store needs from the app. */ +export function configureStore(next: StorePageHooks | null): void { + hooks = next; +} + +export function getStore(): StoreBackend { + backend ??= new PageStore(() => hooks); + return backend; +} + +/** Tests run the real engine in-process. */ +export function setStoreBackend(next: StoreBackend | null): void { + backend = next; +} diff --git a/lib/store/client/keys.ts b/lib/store/client/keys.ts index ecd4fe4c2..9723c6710 100644 --- a/lib/store/client/keys.ts +++ b/lib/store/client/keys.ts @@ -83,6 +83,14 @@ export function newStoreFileName(): string { return `/a${toHex(crypto.getRandomValues(new Uint8Array(8)))}.db`; } +/** The file of every account with a key. Any other file in the pool can't be read by anyone. */ +export async function knownStoreFiles(): Promise { + return withDb(async (db) => { + const records = await promisify(db.transaction(ACCOUNTS, 'readonly').objectStore(ACCOUNTS).getAll()) as AccountRecord[]; + return records.map((record) => record.file); + }); +} + export async function getAccountRecord(account: string): Promise { return withDb(async (db) => { const found = await promisify(db.transaction(ACCOUNTS, 'readonly').objectStore(ACCOUNTS).get(account)) as AccountRecord | undefined; @@ -95,7 +103,7 @@ export async function listAccountRecords(): Promise { } /** The account's key in hex, created on first use. Returns the record too, for the file name. */ -export async function openAccountKey(account: string, layers: readonly WrapLayer[] = []): Promise<{ keyHex: string; record: AccountRecord }> { +export async function openAccountKey(account: string, layers: readonly WrapLayer[] = []): Promise<{ keyHex: string; record: AccountRecord; fresh: boolean }> { return withDb(async (db) => { const wrapping = await wrapKey(db); const found = await promisify(db.transaction(ACCOUNTS, 'readonly').objectStore(ACCOUNTS).get(account)) as AccountRecord | undefined; @@ -109,7 +117,7 @@ export async function openAccountKey(account: string, layers: readonly WrapLayer const raw = new Uint8Array(await crypto.subtle.decrypt({ name: 'AES-GCM', iv: asBuffer(found.iv) }, wrapping, asBuffer(data))); const keyHex = toHex(raw); raw.fill(0); - return { keyHex, record: found }; + return { keyHex, record: found, fresh: false }; } const raw = crypto.getRandomValues(new Uint8Array(32)); const iv = crypto.getRandomValues(new Uint8Array(12)); @@ -126,7 +134,7 @@ export async function openAccountKey(account: string, layers: readonly WrapLayer await promisify(db.transaction(ACCOUNTS, 'readwrite').objectStore(ACCOUNTS).put(record)); const keyHex = toHex(raw); raw.fill(0); - return { keyHex, record }; + return { keyHex, record, fresh: true }; }); } diff --git a/lib/store/client/purge.ts b/lib/store/client/purge.ts new file mode 100644 index 000000000..10fb04dfb --- /dev/null +++ b/lib/store/client/purge.ts @@ -0,0 +1,64 @@ +/** + * Deletes the local store from the page: the keys and the OPFS pool, with no + * worker running and no wasm loaded, also on the sign-in page. Imports nothing + * heavy, because lib/sign-out-cleanup.ts is in every page's bundle. + */ + +import { OWNER_LOCK, POOL_DIRECTORY, RPC_CHANNEL, type ChannelMessage } from '../worker/protocol'; +import { deleteKeyDatabase } from './keys'; + +/** A worker just stopped may hold the files for a moment longer. */ +const REMOVE_TRIES = 10; +const REMOVE_WAIT_MS = 200; +/** A sign-out doesn't wait for a tab that never lets go of the owner lock. */ +const POOL_LOCK_WAIT_MS = 5_000; + +async function removePool(): Promise { + const root = await navigator.storage.getDirectory(); + for (let attempt = 0; ; attempt += 1) { + try { + await root.removeEntry(POOL_DIRECTORY, { recursive: true }); + return; + } catch (error) { + if ((error as Error)?.name === 'NotFoundError') return; + if (attempt >= REMOVE_TRIES) throw error; + await new Promise((resolve) => setTimeout(resolve, REMOVE_WAIT_MS)); + } + } +} + +/** Deletes the pool folder, holding the owner lock so no worker can take the files meanwhile. */ +export async function deleteStorePool(): Promise { + if (typeof navigator === 'undefined' || typeof navigator.storage?.getDirectory !== 'function') return; + if (!navigator.locks) { + await removePool(); + return; + } + try { + await navigator.locks.request(OWNER_LOCK, { signal: AbortSignal.timeout(POOL_LOCK_WAIT_MS) }, removePool); + } catch (error) { + if ((error as Error)?.name !== 'TimeoutError' && (error as Error)?.name !== 'AbortError') throw error; + // A tab that won't let go, stuck or asleep: the key is gone, so what can be + // deleted goes now, and what it still holds can't be read anyway. + await removePool(); + } +} + +/** + * Forgets every key first, which makes the files unreadable even if deleting + * them fails, then has every tab stop its worker and deletes the files. + * Started synchronously: the sign-out may navigate away right after. + */ +export function purgeStoreFiles(): Promise { + deleteKeyDatabase(); + try { + const channel = new BroadcastChannel(RPC_CHANNEL); + channel.postMessage({ kind: 'shutdown' } satisfies ChannelMessage); + channel.close(); + } catch { + // No BroadcastChannel: no store either. + } + return deleteStorePool().catch(() => { + // The key is gone, so what's left can't be read. + }); +} diff --git a/lib/store/client/store-client.ts b/lib/store/client/store-client.ts index 5ae621748..66e7c3b92 100644 --- a/lib/store/client/store-client.ts +++ b/lib/store/client/store-client.ts @@ -21,6 +21,11 @@ import { const CALL_TIMEOUT_MS = 60_000; const OWNER_WAIT_MS = 15_000; +/** Calls waiting on the owner ask it this often whether it's still there. */ +const OWNER_PING_MS = 10_000; +const OWNER_PING_MISSES = 3; +/** A first sync of a large account runs a long time, and its promise is what ends "syncing". */ +const SYNC_TIMEOUT_MS = 30 * 60_000; export class StoreOwnerGoneError extends Error { constructor() { @@ -44,6 +49,8 @@ export interface StoreClientOptions { onOwner?(): void; /** The worker asks for a fresh Authorization header. */ onNeedAuth?(account: string): void; + /** Another tab signed everyone out: this tab's worker is gone. */ + onHalted?(): void; } export class StoreClient { @@ -54,6 +61,9 @@ export class StoreClient { private isOwner = false; private readonly pending = new Map(); private readonly ownerWaiters: Array<() => void> = []; + private ownerWatch: ReturnType | null = null; + private ownerAnswered: (() => void) | null = null; + private lifecycleBound = false; constructor(private readonly options: StoreClientOptions = {}) {} @@ -63,8 +73,10 @@ export class StoreClient { && typeof navigator !== 'undefined' && Boolean(navigator.locks) && typeof navigator.storage?.getDirectory === 'function' - && typeof FileSystemFileHandle !== 'undefined' - && 'createSyncAccessHandle' in FileSystemFileHandle.prototype; + // createSyncAccessHandle itself only exists inside a dedicated worker, + // so the page can't check for it. A browser without it fails at the + // worker's start, which the page hears as an error event. + && typeof FileSystemFileHandle !== 'undefined'; } start(): void { @@ -80,10 +92,14 @@ export class StoreClient { clientId: this.clientId, moduleUrl: new URL(`${base}/${SQLITE3MC_PUBLIC_DIR}/sqlite3.mjs`, location.origin).href, }); + // An owner from before this tab won't announce itself again. + this.channel.postMessage({ kind: 'who-owns' } satisfies ChannelMessage); this.bindLifecycle(); } private bindLifecycle(): void { + if (this.lifecycleBound) return; + this.lifecycleBound = true; // A clean handover when the tab goes. If this doesn't get to run, the lock // and the handles die with the worker and the next owner retries. addEventListener('pagehide', (event) => { @@ -99,8 +115,10 @@ export class StoreClient { switch (message.kind) { case 'owner': this.isOwner = true; - this.owner = this.clientId; - this.flushOwnerWaiters(); + // Not an owner to call yet. This message can overtake the worker's + // channel getting registered, and a call posted then never reaches it, + // so calls wait for the worker's answer on the channel itself. + this.channel?.postMessage({ kind: 'who-owns' } satisfies ChannelMessage); this.options.onOwner?.(); break; case 'result': { @@ -137,8 +155,13 @@ export class StoreClient { break; case 'owner-changed': this.owner = message.owner; + if (message.owner) this.ownerAnswered?.(); if (message.owner) this.flushOwnerWaiters(); break; + case 'shutdown': + this.halt(); + this.options.onHalted?.(); + break; default: break; } @@ -149,8 +172,48 @@ export class StoreClient { if (!pending) return; this.pending.delete(message.id); clearTimeout(pending.timer); - if (message.error) pending.reject(new Error(message.error)); - else pending.resolve(message.result); + if (message.error) { + const error = new Error(message.error); + if (message.errorName) error.name = message.errorName; + pending.reject(error); + } else { + pending.resolve(message.result); + } + } + + /** + * A tab that's frozen or stuck keeps the lock, and nothing else would tell. + * While calls wait, the owner is asked to answer; when it doesn't for a while, + * they stop waiting instead of running into their long time limits. + */ + private watchOwner(): void { + if (this.ownerWatch) return; + let unanswered = 0; + this.ownerAnswered = () => { unanswered = 0; }; + this.ownerWatch = setInterval(() => { + if (this.pending.size === 0) { + this.stopWatchingOwner(); + return; + } + if (unanswered >= OWNER_PING_MISSES) { + this.owner = null; + this.stopWatchingOwner(); + for (const [id, pending] of this.pending) { + this.pending.delete(id); + clearTimeout(pending.timer); + pending.reject(new StoreOwnerGoneError()); + } + return; + } + unanswered += 1; + this.channel?.postMessage({ kind: 'who-owns' } satisfies ChannelMessage); + }, OWNER_PING_MS); + } + + private stopWatchingOwner(): void { + if (this.ownerWatch) clearInterval(this.ownerWatch); + this.ownerWatch = null; + this.ownerAnswered = null; } private flushOwnerWaiters(): void { @@ -178,10 +241,11 @@ export class StoreClient { }); } - private async rpc(method: K, params: Parameters[0]): Promise> { + private async rpc(method: K, params: Parameters[0], timeoutMs = CALL_TIMEOUT_MS): Promise> { await this.waitForOwner(); + this.watchOwner(); const id = crypto.randomUUID(); - const result = this.track>(id, CALL_TIMEOUT_MS); + const result = this.track>(id, timeoutMs); this.channel?.postMessage({ kind: 'rpc', id, from: this.clientId, method, params } as ChannelMessage); return result; } @@ -198,18 +262,20 @@ export class StoreClient { return this.isOwner; } - openAccount(params: OpenAccountParams): Promise { + /** `created`: there was no database under this name. */ + openAccount(params: OpenAccountParams): Promise<{ created?: boolean } | undefined> { return this.direct({ kind: 'open', id: crypto.randomUUID(), params }); } + /** Deletes every store file but these. */ + pruneFiles(keep: string[]): Promise { + return this.direct({ kind: 'prune-files', id: crypto.randomUUID(), keep }); + } + closeAccount(account: string): Promise { return this.direct({ kind: 'close-account', id: crypto.randomUUID(), account }); } - removeAccount(account: string, file: string): Promise { - return this.direct({ kind: 'remove-account', id: crypto.randomUUID(), account, file }); - } - setAuth(account: string, auth: string | null): void { this.post({ kind: 'set-auth', account, auth }); } @@ -222,19 +288,38 @@ export class StoreClient { this.post({ kind: 'set-scope', account, scope }); } + setInvertedKeywordSort(account: string, inverted: boolean): void { + this.post({ kind: 'set-inverted', account, inverted }); + } + /** Hands the store over: the worker closes everything, then lets go of the lock. */ async release(): Promise { if (!this.worker) return; - const worker = this.worker; if (this.isOwner) { await this.direct({ kind: 'release', id: crypto.randomUUID() }); this.isOwner = false; this.owner = null; } - worker.terminate(); + this.halt(); + } + + /** + * Stops at once, with no handover. For a sign-out: the data goes anyway, and + * the worker's end frees the lock and the OPFS handles. + */ + halt(): void { + this.worker?.terminate(); this.worker = null; this.channel?.close(); this.channel = null; + this.isOwner = false; + this.owner = null; + this.stopWatchingOwner(); + for (const [id, pending] of this.pending) { + this.pending.delete(id); + clearTimeout(pending.timer); + pending.reject(new StoreOwnerGoneError()); + } } // ---- Any tab ---- @@ -244,7 +329,11 @@ export class StoreClient { } sync(account: string): Promise { - return this.rpc('sync', { account }); + return this.rpc('sync', { account }, SYNC_TIMEOUT_MS); + } + + removeAccount(account: string, file: string): Promise { + return this.rpc('removeAccount', { account, file }); } noteStateChange(account: string, changed: Record>): Promise { diff --git a/lib/store/engine/account-engine.ts b/lib/store/engine/account-engine.ts index ab3a2b9b5..df9452248 100644 --- a/lib/store/engine/account-engine.ts +++ b/lib/store/engine/account-engine.ts @@ -4,8 +4,8 @@ * the UI. Nothing here touches the DOM, so tests run it on node:sqlite. */ -import { MethodError, type AreaCall, type MethodHandler, type StoreArea, type StoreContext, type StoreEvent, type StoreScope, type SyncContext, type SyncStateRow, type Syncer } from '../area'; -import { migrate, type StorageAdapter } from '../storage'; +import { MethodError, STORAGE_ERROR_AREA, type AreaCall, type MethodHandler, type RecordOverlay, type StoreArea, type StoreContext, type StoreEvent, type StoreScope, type SyncContext, type SyncStateRow, type Syncer } from '../area'; +import { isStorageFailure, migrate, type StorageAdapter } from '../storage'; import { answerRequest, buildMethodTable } from './responder'; import { TransportError, type JmapResponse, type JmapTransport, type MethodCall, type StoreSession } from './transport'; import { CORE_MIGRATIONS, metaGet, metaSet } from './core-schema'; @@ -26,6 +26,9 @@ export interface AccountEngineOptions { const RATE_RETRIES = 5; const RATE_BACKOFF_MS = 2_000; +/** After a failed write: the first wait, and the longest. */ +const STORAGE_BACKOFF_MS = 60_000; +const STORAGE_BACKOFF_MAX_MS = 60 * 60_000; export class AccountEngine { readonly account: string; @@ -36,6 +39,7 @@ export class AccountEngine { private readonly syncers: readonly Syncer[]; private readonly methods: ReadonlyMap; private readonly calls = new Map(); + private readonly overlays: readonly RecordOverlay[]; private readonly emitter: (event: StoreEvent) => void; private readonly now: () => number; private readonly sleep: (ms: number) => Promise; @@ -44,6 +48,9 @@ export class AccountEngine { private online = true; private running: Promise | null = null; private again = false; + /** A write failed (full disk, quota): no sync before this time, however much the server pushes. */ + private storageBlockedUntil = 0; + private storageFailures = 0; constructor(options: AccountEngineOptions) { this.account = options.account; @@ -53,6 +60,7 @@ export class AccountEngine { this.transport = options.transport; this.syncers = options.areas.flatMap((area) => area.syncers); this.methods = buildMethodTable(options.areas); + this.overlays = options.areas.flatMap((area) => (area.overlay ? [area.overlay] : [])); for (const area of options.areas) { for (const [name, call] of Object.entries(area.calls ?? {})) { if (this.calls.has(name)) throw new Error(`Two store areas define ${name}`); @@ -62,9 +70,11 @@ export class AccountEngine { this.emitter = options.emit ?? (() => {}); this.now = options.now ?? Date.now; this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); - this.invertedKeywordSort = options.invertedKeywordSort ?? false; migrate(this.db, [...CORE_MIGRATIONS, ...options.areas.flatMap((area) => area.migrations)]); metaSet(this.db, 'session', JSON.stringify(options.session)); + // An offline start has no way to probe the server, so the last answer stays. + this.invertedKeywordSort = options.invertedKeywordSort ?? metaGet(this.db, 'inverted_keyword_sort') === '1'; + if (options.invertedKeywordSort !== undefined) metaSet(this.db, 'inverted_keyword_sort', options.invertedKeywordSort ? '1' : '0'); } /** The session kept from the last time, for a start without a network. */ @@ -81,11 +91,20 @@ export class AccountEngine { setScope(scope: StoreScope): void { this.scope = scope; + // The user may just have made room by keeping less. + this.storageBlockedUntil = 0; metaSet(this.db, 'scope', JSON.stringify(scope)); } setInvertedKeywordSort(inverted: boolean): void { this.invertedKeywordSort = inverted; + metaSet(this.db, 'inverted_keyword_sort', inverted ? '1' : '0'); + } + + private overlay(jmap: string, type: string, records: T[]): T[] { + let out: Array<{ id: string }> = records; + for (const overlay of this.overlays) out = overlay({ db: this.db, now: this.now }, jmap, type, out); + return out as T[]; } setOnline(online: boolean): void { @@ -128,7 +147,7 @@ export class AccountEngine { /** Starts a run, or asks the running one to go again when it's done. */ requestSync(): Promise { - if (this.closed) return Promise.resolve(); + if (this.closed || this.now() < this.storageBlockedUntil) return Promise.resolve(); if (this.running) { this.again = true; return this.running; @@ -143,7 +162,7 @@ export class AccountEngine { do { this.again = false; await this.runOnce(); - } while (this.again && !this.closed && this.online); + } while (this.again && !this.closed && this.online && this.now() >= this.storageBlockedUntil); } private async runOnce(): Promise { @@ -158,6 +177,7 @@ export class AccountEngine { if (!(await this.runSyncer(ctx, syncer, jmap))) return; } } + this.storageFailures = 0; if (!this.closed && this.online) this.emitter({ type: 'synced', account: this.account, at: this.now() }); } @@ -175,6 +195,11 @@ export class AccountEngine { continue; } } + // The others would fail on the same disk: the run ends, and sync waits. + if (isStorageFailure(error)) { + this.noteStorageFailure(error); + return false; + } // One part failing doesn't hold up the others. It runs again next time. this.emitter({ type: 'error', @@ -187,6 +212,18 @@ export class AccountEngine { } } + private noteStorageFailure(error: unknown): void { + this.storageFailures += 1; + const wait = Math.min(STORAGE_BACKOFF_MS * 2 ** (this.storageFailures - 1), STORAGE_BACKOFF_MAX_MS); + this.storageBlockedUntil = this.now() + wait; + this.emitter({ + type: 'error', + account: this.account, + area: STORAGE_ERROR_AREA, + message: error instanceof Error ? error.message : String(error), + }); + } + private syncContext(): SyncContext { return { account: this.account, @@ -199,6 +236,7 @@ export class AccountEngine { stopped: () => this.closed || !this.online, syncState: (jmap, type) => this.syncState(jmap, type), setSyncState: (jmap, type, row) => this.setSyncState(jmap, type, row), + overlay: (jmap, type, records) => this.overlay(jmap, type, records), }; } @@ -210,6 +248,7 @@ export class AccountEngine { scope: this.scope, now: this.now, emit: this.emitter, + overlay: (jmap, type, records) => this.overlay(jmap, type, records), }; } @@ -217,22 +256,32 @@ export class AccountEngine { async areaCall(name: string, params: unknown): Promise { const call = this.calls.get(name); if (!call) throw new Error(`Unknown store call ${name}`); - return call(this.syncContext(), params as never); + try { + return await call(this.syncContext(), params as never); + } catch (error) { + if (isStorageFailure(error)) this.noteStorageFailure(error); + throw error; + } } /** An answer from the device. Null when nothing in the request could be answered. */ async answer(calls: MethodCall[]): Promise { if (this.closed) throw new MethodError('serverUnavailable', 'The local store is closed'); const primary = this.session.primaryAccounts['urn:ietf:params:jmap:mail'] ?? Object.keys(this.session.accounts)[0]; - return answerRequest( - { - ...this.storeContext(), - defaultJmap: primary, - invertedKeywordSort: this.invertedKeywordSort, - }, - this.methods, - calls, - ); + try { + return await answerRequest( + { + ...this.storeContext(), + defaultJmap: primary, + invertedKeywordSort: this.invertedKeywordSort, + }, + this.methods, + calls, + ); + } catch (error) { + if (isStorageFailure(error)) this.noteStorageFailure(error); + throw error; + } } /** Stops syncing and waits for the current batch. The database stays open for the caller to close. */ diff --git a/lib/store/mail/calls.ts b/lib/store/mail/calls.ts new file mode 100644 index 000000000..713975810 --- /dev/null +++ b/lib/store/mail/calls.ts @@ -0,0 +1,65 @@ +/** What the page asks the mail part of the store outside JMAP. */ + +import type { AreaCall, StoreContext } from '../area'; +import { putBlob, storeBody, upsertEmail, type EmailObject } from './records'; + +/** + * A message the user opened online: kept with its text if the store doesn't + * have that yet, for instance because it's older than the window the user + * chose. Parts cut short for a list view aren't kept. + */ +function recordOpened(ctx: StoreContext, params: { jmap: string; emails: EmailObject[] }): void { + ctx.db.transaction(() => { + for (const email of ctx.overlay(params.jmap, 'Email', params.emails)) { + const values = (email.bodyValues ?? {}) as Record; + if (Object.keys(values).length === 0 || Object.values(values).some((value) => value?.isTruncated)) continue; + const kept = ctx.db.get<{ rowid: number; json: string; body_state: number }>( + 'SELECT rowid, json, body_state FROM email WHERE jmap = ? AND id = ?', + [params.jmap, email.id], + ); + if (kept && Number(kept.body_state) === 1) continue; + if (kept) { + storeBody(ctx.db, Number(kept.rowid), String(kept.json), email); + continue; + } + // A row needs at least what lists sort and filter on. + if (typeof email.receivedAt !== 'string' || !email.mailboxIds) continue; + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM email WHERE jmap = ?', [params.jmap])?.gen ?? 0); + const { bodyValues, textBody, htmlBody, attachments, bodyStructure, headers, bcc, replyTo, messageId, inReplyTo, references, sender, ...list } = email; + const rowid = upsertEmail(ctx.db, params.jmap, list as EmailObject, gen); + storeBody(ctx.db, rowid, JSON.stringify(list), { bodyValues, textBody, htmlBody, attachments, bodyStructure, headers, bcc, replyTo, messageId, inReplyTo, references, sender }); + } + }); +} + +/** A file of a kept message the user opened. Nothing else is kept: not an export, not a stray file. */ +function recordBlob(ctx: StoreContext, params: { jmap: string; blobId: string; type: string | null; data: Uint8Array }): boolean { + if (!ctx.scope.attachments || params.data.byteLength > ctx.scope.maxAttachmentBytes) return false; + const owner = ctx.db.get<{ received_at: number }>('SELECT received_at FROM email_part WHERE jmap = ? AND blob_id = ?', [params.jmap, params.blobId]); + if (!owner) return false; + putBlob(ctx.db, params.jmap, params.blobId, params.type, params.data, Number(owner.received_at)); + return true; +} + +function getBlob(ctx: StoreContext, params: { jmap: string; blobId: string }): { type: string | null; data: Uint8Array } | null { + const row = ctx.db.get<{ type: string | null; data: Uint8Array }>('SELECT type, data FROM blob WHERE jmap = ? AND blob_id = ?', [params.jmap, params.blobId]); + return row ? { type: row.type, data: row.data } : null; +} + +/** How much of the disk this account's file takes. */ +function usage(ctx: StoreContext): { bytes: number } { + const pages = Number(ctx.db.get<{ page_count: number }>('PRAGMA page_count')?.page_count ?? 0); + const size = Number(ctx.db.get<{ page_size: number }>('PRAGMA page_size')?.page_size ?? 0); + return { bytes: pages * size }; +} + +export const MAIL_CALLS: Readonly> = { + 'mail.recordOpened': recordOpened, + 'mail.recordBlob': recordBlob, + 'mail.getBlob': getBlob, + 'mail.usage': usage, + /** The session kept with the account, for a start from the local copy. */ + 'mail.session': (ctx) => ctx.session, + /** Whether a start may show the store: it has a folder list from the server. */ + 'mail.hasFolders': (ctx) => ctx.db.get("SELECT 1 AS yes FROM sync_state WHERE type = 'Mailbox' AND state IS NOT NULL") !== undefined, +}; diff --git a/lib/store/mail/headers.ts b/lib/store/mail/headers.ts new file mode 100644 index 000000000..a35211cc5 --- /dev/null +++ b/lib/store/mail/headers.ts @@ -0,0 +1,181 @@ +/** + * The `header:[:
    ][:all]` properties of Email/get (RFC 8621 + * section 4.1.2), worked out from the raw `headers` a kept message stores. + * Callers read these for mailing list ids, classification labels and rules. + */ + +export interface RawHeader { + name: string; + value: string; +} + +export interface HeaderForm { + name: string; + form: 'asRaw' | 'asText' | 'asAddresses' | 'asGroupedAddresses' | 'asMessageIds' | 'asDate' | 'asURLs'; + all: boolean; +} + +const FORMS = new Set(['asRaw', 'asText', 'asAddresses', 'asGroupedAddresses', 'asMessageIds', 'asDate', 'asURLs']); + +export function parseHeaderProperty(property: string): HeaderForm | null { + if (!property.startsWith('header:')) return null; + const parts = property.slice('header:'.length).split(':'); + const name = parts.shift(); + if (!name) return null; + let all = false; + if (parts[parts.length - 1] === 'all') { + all = true; + parts.pop(); + } + if (parts.length > 1) return null; + const form = (parts[0] ?? 'asRaw') as HeaderForm['form']; + return FORMS.has(form) ? { name, form, all } : null; +} + +function unfold(value: string): string { + return value.replace(/\r?\n(?=[ \t])/g, '').trim(); +} + +function decodeBytes(charset: string, bytes: Uint8Array): string { + try { + return new TextDecoder(charset).decode(bytes); + } catch { + return new TextDecoder().decode(bytes); + } +} + +/** RFC 2047 encoded words. Whitespace between two of them goes. */ +function decodeWords(value: string): string { + return value + .replace(/(=\?[^?\s]+\?[bBqQ]\?[^?\s]*\?=)\s+(?==\?[^?\s]+\?[bBqQ]\?)/g, '$1') + .replace(/=\?([^?\s]+)\?([bBqQ])\?([^?\s]*)\?=/g, (whole, charset: string, encoding: string, text: string) => { + try { + if (encoding.toUpperCase() === 'B') { + return decodeBytes(charset, Uint8Array.from(atob(text), (c) => c.charCodeAt(0))); + } + const raw = text.replace(/_/g, ' ').replace(/=([0-9A-Fa-f]{2})/g, (_, hex: string) => String.fromCharCode(parseInt(hex, 16))); + return decodeBytes(charset, Uint8Array.from(raw, (c) => c.charCodeAt(0))); + } catch { + return whole; + } + }); +} + +interface Address { name: string | null; email: string } +interface Group { name: string | null; addresses: Address[] } + +/** Splits on a separator outside quotes, comments and angle brackets. */ +function splitOutside(value: string, separators: string): string[] { + const parts: string[] = []; + let current = ''; + let quoted = false; + let depth = 0; + let angle = false; + for (let i = 0; i < value.length; i += 1) { + const c = value[i]; + if (quoted) { + current += c; + if (c === '\\') { + current += value[i + 1] ?? ''; + i += 1; + } else if (c === '"') quoted = false; + continue; + } + if (c === '"') quoted = true; + else if (c === '(') depth += 1; + else if (c === ')') depth = Math.max(0, depth - 1); + else if (c === '<') angle = true; + else if (c === '>') angle = false; + if (depth === 0 && !angle && separators.includes(c)) { + parts.push(current); + if (c !== ',') parts.push(c); + current = ''; + continue; + } + current += c; + } + parts.push(current); + return parts; +} + +function parseAddress(text: string): Address | null { + const trimmed = text.replace(/\((?:[^()\\]|\\.)*\)/g, ' ').trim(); + if (!trimmed) return null; + const angle = trimmed.match(/^([\s\S]*)<([^<>]*)>\s*$/); + const email = (angle ? angle[2] : trimmed).trim(); + if (!angle && !email.includes('@')) return null; + const rawName = angle ? angle[1].trim() : ''; + const name = rawName.startsWith('"') && rawName.endsWith('"') && rawName.length >= 2 + ? rawName.slice(1, -1).replace(/\\(.)/g, '$1') + : rawName; + return { name: name ? decodeWords(name) : null, email }; +} + +function parseGroups(value: string): Group[] { + const groups: Group[] = []; + let current: Group | null = null; + let loose: Group | null = null; + const tokens = splitOutside(unfold(value), ',:;'); + for (let i = 0; i < tokens.length; i += 1) { + const token = tokens[i]; + if (tokens[i + 1] === ':') { + current = { name: decodeWords(token.trim().replace(/^"|"$/g, '')) || null, addresses: [] }; + groups.push(current); + loose = null; + i += 1; + continue; + } + if (token === ';') { + current = null; + continue; + } + if (token === ':') continue; + const address = parseAddress(token); + if (!address) continue; + if (current) current.addresses.push(address); + else { + if (!loose) { + loose = { name: null, addresses: [] }; + groups.push(loose); + } + loose.addresses.push(address); + } + } + return groups; +} + +function parseMessageIds(value: string): string[] | null { + const ids = [...unfold(value).matchAll(/<([^<>\s]+)>/g)].map((match) => match[1]); + return ids.length > 0 ? ids : null; +} + +function parseDate(value: string): string | null { + const ms = Date.parse(unfold(value).replace(/\((?:[^()\\]|\\.)*\)/g, ' ')); + return Number.isFinite(ms) ? new Date(ms).toISOString() : null; +} + +function parseUrls(value: string): string[] | null { + const urls = [...unfold(value).matchAll(/<([^<>\s]+)>/g)].map((match) => match[1]); + return urls.length > 0 ? urls : null; +} + +function convert(value: string, form: HeaderForm['form']): unknown { + switch (form) { + case 'asRaw': return value; + case 'asText': return decodeWords(unfold(value)); + case 'asAddresses': return parseGroups(value).flatMap((group) => group.addresses); + case 'asGroupedAddresses': return parseGroups(value); + case 'asMessageIds': return parseMessageIds(value); + case 'asDate': return parseDate(value); + case 'asURLs': return parseUrls(value); + } +} + +/** The value of one header property. Without `:all`, the last header of that name, or null. */ +export function headerValue(form: HeaderForm, headers: readonly RawHeader[]): unknown { + const wanted = form.name.toLowerCase(); + const found = headers.filter((header) => header.name.toLowerCase() === wanted); + if (form.all) return found.map((header) => convert(header.value, form.form)); + const last = found[found.length - 1]; + return last ? convert(last.value, form.form) : null; +} diff --git a/lib/store/mail/index.ts b/lib/store/mail/index.ts index 0c7749864..aa0ba6e6b 100644 --- a/lib/store/mail/index.ts +++ b/lib/store/mail/index.ts @@ -1,12 +1,14 @@ import type { StoreArea } from '../area'; +import { MAIL_CALLS } from './calls'; import { MAIL_METHODS } from './methods'; import { MAIL_MIGRATIONS } from './schema'; import { SEARCH_MIGRATIONS } from './search-schema'; -import { mailSyncer } from './sync'; +import { identitySyncer, mailSyncer, quotaSyncer } from './sync'; export const mailArea: StoreArea = { name: 'mail', migrations: [...MAIL_MIGRATIONS, ...SEARCH_MIGRATIONS], methods: MAIL_METHODS, - syncers: [mailSyncer], + syncers: [mailSyncer, identitySyncer, quotaSyncer], + calls: MAIL_CALLS, }; diff --git a/lib/store/mail/methods.ts b/lib/store/mail/methods.ts index 1b5c954d9..8462563e4 100644 --- a/lib/store/mail/methods.ts +++ b/lib/store/mail/methods.ts @@ -1,8 +1,14 @@ -/** Mail reads answered from the store. */ +/** + * Mail reads answered from the store. What an answer must look like is in + * local-data/desktop-offline-facts-parts/offline-jmap.md sections 1 and 4: it + * follows what the calling method in lib/jmap/client.ts reads. + */ import { MethodError, type MethodContext, type MethodHandler, type MethodResult } from '../area'; import { listObjects } from '../engine/core-schema'; import type { SqlValue } from '../storage'; +import { headerValue, parseHeaderProperty, type RawHeader } from './headers'; +import { keywordKey } from './records'; import { limitTextConditions, searchSnippetGet, textCondition, TEXT_FIELDS, type TextField } from './search'; type Filter = Record; @@ -13,14 +19,26 @@ function jmapOf(ctx: MethodContext, args: Record): string { return (args.accountId as string | undefined) ?? ctx.defaultJmap; } +function lastState(ctx: MethodContext, jmap: string, type: string): string | null { + return ctx.db.get<{ state: string | null }>('SELECT state FROM sync_state WHERE jmap = ? AND type = ?', [jmap, type])?.state ?? null; +} + +function likeTerm(value: unknown): string { + return `%${String(value).replace(/[\\%_]/g, (c) => `\\${c}`)}%`; +} + /** * `drivenBy` is the folder the query already starts from: repeating that - * condition would look the membership up again for every row. + * condition would look the membership up again for every row. With a folder row + * (`d`) at hand, conditions read it instead of the message's own row, which + * saves reading the message at all when nothing else needs it. */ -function filterSql(filter: Filter, jmap: string, drivenBy: string | null = null): Sql { +function filterSql(filter: Filter, jmap: string, drivenBy: string | null = null, folderRow = drivenBy !== null): Sql { + const rowid = folderRow ? 'd.email_rowid' : 'e.rowid'; + const receivedAt = folderRow ? 'd.received_at' : 'e.received_at'; if (typeof filter.operator === 'string') { const conditions = Array.isArray(filter.conditions) ? filter.conditions as Filter[] : []; - const parts = conditions.map((c) => filterSql(c, jmap, filter.operator === 'AND' ? drivenBy : null)); + const parts = conditions.map((c) => filterSql(c, jmap, filter.operator === 'AND' ? drivenBy : null, folderRow)); const params = parts.flatMap((p) => p.params); const joined = (joiner: string) => parts.map((p) => `(${p.sql})`).join(joiner); if (filter.operator === 'AND') return { sql: parts.length ? joined(' AND ') : '1', params }; @@ -34,29 +52,29 @@ function filterSql(filter: Filter, jmap: string, drivenBy: string | null = null) switch (key) { case 'inMailbox': if (value === drivenBy) break; - conditions.push('EXISTS (SELECT 1 FROM email_mailbox m WHERE m.email_rowid = e.rowid AND m.mailbox_id = ?)'); + conditions.push(`EXISTS (SELECT 1 FROM email_mailbox m WHERE m.email_rowid = ${rowid} AND m.mailbox_id = ?)`); params.push(String(value)); break; case 'inMailboxOtherThan': { const ids = (value as string[]).map(String); - conditions.push(`EXISTS (SELECT 1 FROM email_mailbox m WHERE m.email_rowid = e.rowid AND m.mailbox_id NOT IN (${ids.map(() => '?').join(',') || "''"}))`); + conditions.push(`EXISTS (SELECT 1 FROM email_mailbox m WHERE m.email_rowid = ${rowid} AND m.mailbox_id NOT IN (${ids.map(() => '?').join(',') || "''"}))`); params.push(...ids); break; } case 'hasKeyword': - conditions.push('EXISTS (SELECT 1 FROM email_keyword k WHERE k.email_rowid = e.rowid AND k.keyword = ?)'); - params.push(String(value).toLowerCase()); + conditions.push(`EXISTS (SELECT 1 FROM email_keyword k WHERE k.email_rowid = ${rowid} AND k.keyword = ?)`); + params.push(keywordKey(String(value))); break; case 'notKeyword': - conditions.push('NOT EXISTS (SELECT 1 FROM email_keyword k WHERE k.email_rowid = e.rowid AND k.keyword = ?)'); - params.push(String(value).toLowerCase()); + conditions.push(`NOT EXISTS (SELECT 1 FROM email_keyword k WHERE k.email_rowid = ${rowid} AND k.keyword = ?)`); + params.push(keywordKey(String(value))); break; case 'after': - conditions.push('e.received_at >= ?'); + conditions.push(`${receivedAt} >= ?`); params.push(Date.parse(String(value)) || 0); break; case 'before': - conditions.push('e.received_at < ?'); + conditions.push(`${receivedAt} < ?`); params.push(Date.parse(String(value)) || 0); break; case 'minSize': @@ -74,6 +92,16 @@ function filterSql(filter: Filter, jmap: string, drivenBy: string | null = null) conditions.push(`e.id IN (${(value as string[]).map(() => '?').join(',') || "''"})`); params.push(...(value as string[]).map(String)); break; + case 'header': { + // Stalwart finds nothing for this on either version, plugins may still + // ask. Answered from the raw headers of the messages kept with a body. + const [name, wanted] = value as [string, string | undefined]; + conditions.push(`EXISTS (SELECT 1 FROM email_body b, json_each(b.json, '$.headers') h + WHERE b.email_rowid = e.rowid AND lower(json_extract(h.value, '$.name')) = ?${wanted === undefined ? '' : " AND json_extract(h.value, '$.value') LIKE ? ESCAPE '\\'"})`); + params.push(String(name).toLowerCase()); + if (wanted !== undefined) params.push(likeTerm(wanted)); + break; + } default: if ((TEXT_FIELDS as readonly string[]).includes(key)) { const text = textCondition(key as TextField, String(value)); @@ -81,7 +109,7 @@ function filterSql(filter: Filter, jmap: string, drivenBy: string | null = null) params.push(...text.params); break; } - // header and the thread keyword filters: the server's job. + // The thread keyword filters: nothing in the app sends them. throw new MethodError('unsupportedFilter', `${key} isn't available offline`); } } @@ -118,7 +146,7 @@ function orderSql(ctx: MethodContext, sort: Comparator[]): Sql { // stored answer has to read it the way that server would. if (ctx.invertedKeywordSort) ascending = !ascending; parts.push(`EXISTS (SELECT 1 FROM email_keyword s WHERE s.email_rowid = e.rowid AND s.keyword = ?) ${ascending ? 'ASC' : 'DESC'}`); - params.push(String(level.keyword ?? '').toLowerCase()); + params.push(keywordKey(String(level.keyword ?? ''))); break; } default: @@ -142,6 +170,121 @@ function drivingMailbox(filter: Filter | undefined): string | null { return null; } +/** One step of a "pinned first" or "unread first" order. */ +interface KeywordLevel { keyword: string; first: boolean } + +/** A keyword this rare is read off its own index, then sorted. Commoner ones are checked along the folder. */ +const RARE_KEYWORD_ROWS = 5000; +const MAX_KEYWORD_LEVELS = 4; + +/** + * The keyword comparators of an order that goes on newest first, which is what + * the pinned and unread-first lists send. Null for any other order. + */ +function keywordLevels(ctx: MethodContext, sort: Comparator[]): KeywordLevel[] | null { + const levels: KeywordLevel[] = []; + let next = 0; + for (; sort[next]?.property === 'hasKeyword'; next += 1) { + const ascending = sort[next].isAscending !== false; + levels.push({ keyword: keywordKey(String(sort[next].keyword ?? '')), first: ctx.invertedKeywordSort ? ascending : !ascending }); + } + if (levels.length === 0 || levels.length > MAX_KEYWORD_LEVELS) return null; + return isNewestFirst(sort.slice(next)) ? levels : null; +} + +/** + * A page of an order by keywords and then date, without sorting the folder: the + * order splits into groups (pinned, then the rest), and each group is read in + * date order off the folder's index until the page is full. + */ +function keywordOrderedPage( + ctx: MethodContext, + jmap: string, + box: string | null, + where: Sql, + levels: readonly KeywordLevel[], + position: number, + limit: number, +): string[] { + const rowid = box ? 'd.email_rowid' : 'e.rowid'; + const rows = new Map(); + const rowsOf = (keyword: string): number => { + let n = rows.get(keyword); + if (n === undefined) { + n = Number(ctx.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM email_keyword INDEXED BY email_keyword_lookup WHERE keyword = ?', [keyword])?.n ?? 0); + rows.set(keyword, n); + } + return n; + }; + + const groupQuery = (group: ReadonlyArray<{ keyword: string; has: boolean }>) => { + const rare = group + .filter((level) => level.has && rowsOf(level.keyword) <= RARE_KEYWORD_ROWS) + .sort((a, b) => rowsOf(a.keyword) - rowsOf(b.keyword))[0]; + const checked = group.filter((level) => level !== rare); + const sql = [ + `${box ? 'd.jmap = ?' : 'e.jmap = ?'}${box && !rare ? ' AND d.mailbox_id = ?' : ''}`, + ...checked.map((level) => `${level.has ? '' : 'NOT '}EXISTS (SELECT 1 FROM email_keyword c WHERE c.email_rowid = ${rowid} AND c.keyword = ?)`), + `(${where.sql})`, + ].join(' AND '); + let head: string; + const params: SqlValue[] = []; + if (rare) { + head = box + ? 'email_keyword r INDEXED BY email_keyword_lookup CROSS JOIN email_mailbox d ON d.email_rowid = r.email_rowid AND d.mailbox_id = ?' + : 'email_keyword r INDEXED BY email_keyword_lookup'; + if (box) params.push(box); + } else { + head = box ? 'email_mailbox d INDEXED BY email_mailbox_list' : ''; + } + const emailJoin = !box + ? (rare ? ' CROSS JOIN email e ON e.rowid = r.email_rowid' : 'email e') + : ' CROSS JOIN email e ON e.rowid = d.email_rowid'; + const after: SqlValue[] = [jmap, ...(box && !rare ? [box] : []), ...checked.map((level) => level.keyword), ...where.params]; + const rareCondition = rare ? 'r.keyword = ? AND ' : ''; + const allParams = [...params, ...(rare ? [rare.keyword] : []), ...after]; + return { + // The message's own row is read only for what goes out, or if a condition needs it. + countFrom: box && !/\be\./.test(where.sql) ? head : `${head}${emailJoin}`, + from: `${head}${emailJoin}`, + where: `${rareCondition}${sql}`, + params: allParams, + order: box ? 'd.received_at DESC, d.email_rowid DESC' : 'e.received_at DESC, e.rowid DESC', + }; + }; + + const groups = levels.reduce>>( + (all, level) => all.flatMap((group) => [ + [...group, { keyword: level.keyword, has: level.first }], + [...group, { keyword: level.keyword, has: !level.first }], + ]), + [[]], + ); + + const ids: string[] = []; + let skip = position; + let need = limit; + for (const group of groups) { + if (need === 0) break; + const query = groupQuery(group); + if (skip > 0) { + const inGroup = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM ${query.countFrom} WHERE ${query.where}`, query.params)?.n ?? 0); + if (inGroup <= skip) { + skip -= inGroup; + continue; + } + } + const found = ctx.db.all<{ id: string }>( + `SELECT e.id FROM ${query.from} WHERE ${query.where} ORDER BY ${query.order} LIMIT ? OFFSET ?`, + [...query.params, need, skip], + ); + for (const row of found) ids.push(String(row.id)); + skip = 0; + if (need > 0) need -= found.length; + } + return ids; +} + const emailQuery: MethodHandler = (ctx, args) => { const jmap = jmapOf(ctx, args); const filter = limitTextConditions((args.filter ?? undefined) as Filter | undefined); @@ -149,7 +292,11 @@ const emailQuery: MethodHandler = (ctx, args) => { const box = drivingMailbox(filter); const where = filter ? filterSql(filter, jmap, box) : { sql: '1', params: [] }; const sort = (args.sort as Comparator[]) ?? []; - const order: Sql = !isNewestFirst(sort) + const collapse = Boolean(args.collapseThreads); + const levels = collapse ? null : keywordLevels(ctx, sort); + const order: Sql = levels + ? { sql: '', params: [] } + : !isNewestFirst(sort) ? orderSql(ctx, sort) : { sql: box ? 'd.received_at DESC, d.email_rowid DESC' : 'e.received_at DESC, e.rowid DESC', params: [] }; // Starting from the folder's own index keeps small folders fast in a large account. @@ -161,24 +308,30 @@ const emailQuery: MethodHandler = (ctx, args) => { : { sql: 'e.jmap = ?', params: [jmap] as SqlValue[] }; const position = Math.max(0, Number(args.position ?? 0)); const limit = typeof args.limit === 'number' && args.limit > 0 ? args.limit : -1; - const collapse = Boolean(args.collapseThreads); const group = collapse ? 'GROUP BY COALESCE(e.thread_id, e.id)' : ''; - const ids = ctx.db.all<{ id: string }>( - `SELECT e.id FROM ${from} WHERE ${base.sql} AND (${where.sql}) ${group} ORDER BY ${order.sql} LIMIT ? OFFSET ?`, - [...base.params, ...where.params, ...order.params, limit, position], - ).map((row) => String(row.id)); + const ids = levels + ? keywordOrderedPage(ctx, jmap, box, where, levels, position, limit) + : ctx.db.all<{ id: string }>( + `SELECT e.id FROM ${from} WHERE ${base.sql} AND (${where.sql}) ${group} ORDER BY ${order.sql} LIMIT ? OFFSET ?`, + [...base.params, ...where.params, ...order.params, limit, position], + ).map((row) => String(row.id)); + // Stalwart's queryState is the Email state. Absent until the first sync, as allowed. + const queryState = lastState(ctx, jmap, 'Email'); + // Nothing here doesn't mean nothing there: an empty folder of a first sync + // that hasn't finished mustn't paint over what's on screen. + if (ids.length === 0 && position === 0 && !queryState) throw new MethodError('serverUnavailable', 'Mail not kept yet'); const result: MethodResult = { accountId: jmap, canCalculateChanges: false, - // Local answers aren't a position in the server's log, so this is a token only. - queryState: 'local', + ...(queryState ? { queryState } : {}), position, ids, }; if (args.calculateTotal) { const count = collapse ? 'COUNT(DISTINCT COALESCE(e.thread_id, e.id))' : 'COUNT(*)'; - // A plain folder counts from its index alone, without reading a message. - const countFrom = box && !collapse && where.sql === '1' ? 'email_mailbox d INDEXED BY email_mailbox_list' : from; + // In a folder, what needs nothing of the message itself counts off the + // folder's index and the keyword rows, without reading a message. + const countFrom = box && !collapse && !/\be\./.test(where.sql) ? 'email_mailbox d INDEXED BY email_mailbox_list' : from; result.total = Number(ctx.db.get<{ n: number }>( `SELECT ${count} AS n FROM ${countFrom} WHERE ${base.sql} AND (${where.sql})`, [...base.params, ...where.params], @@ -187,12 +340,60 @@ const emailQuery: MethodHandler = (ctx, args) => { return result; }; -const BODY_PROPERTIES = new Set(['bodyValues', 'textBody', 'htmlBody', 'bodyStructure', 'headers', 'attachments', 'bcc', 'replyTo', 'messageId', 'inReplyTo', 'references']); +/** What only the stored body has, past what the list keeps. */ +const BODY_PROPERTIES = new Set(['bodyValues', 'textBody', 'htmlBody', 'bodyStructure', 'headers', 'attachments', 'bcc', 'replyTo', 'messageId', 'inReplyTo', 'references', 'sender']); -function pick(email: Record, properties: string[] | undefined): Record { - if (!properties) return email; - const out: Record = { id: email.id }; - for (const property of properties) if (property in email) out[property] = email[property]; +interface BodyPart { partId?: string | null; subParts?: BodyPart[]; [key: string]: unknown } + +function partWithProperties(part: BodyPart, wanted: readonly string[]): BodyPart { + const out: BodyPart = {}; + for (const name of wanted) if (name in part) out[name] = part[name]; + if (part.subParts) out.subParts = part.subParts.map((sub) => partWithProperties(sub, wanted)); + return out; +} + +/** The body values the call asked for. The whole kept text: never cut to maxBodyValueBytes. */ +function chosenBodyValues(body: Record, args: Record): Record { + const values = (body.bodyValues ?? {}) as Record; + if (args.fetchAllBodyValues) return values; + const parts = [ + ...(args.fetchTextBodyValues ? (body.textBody ?? []) as BodyPart[] : []), + ...(args.fetchHTMLBodyValues ? (body.htmlBody ?? []) as BodyPart[] : []), + ]; + const out: Record = {}; + for (const part of parts) { + const id = part.partId ?? ''; + if (id in values) out[id] = values[id]; + } + return out; +} + +function shapeEmail( + list: Record, + body: Record | null, + args: Record, +): Record { + const merged: Record = { ...(body ?? {}), ...list }; + if (body) merged.bodyValues = chosenBodyValues(body, args); + const bodyProperties = args.bodyProperties as string[] | undefined; + if (bodyProperties) { + for (const key of ['textBody', 'htmlBody', 'attachments']) { + if (Array.isArray(merged[key])) merged[key] = (merged[key] as BodyPart[]).map((part) => partWithProperties(part, bodyProperties)); + } + if (merged.bodyStructure) merged.bodyStructure = partWithProperties(merged.bodyStructure as BodyPart, bodyProperties); + } + const properties = args.properties as string[] | undefined; + if (!properties) return merged; + const out: Record = { id: list.id }; + for (const property of properties) { + if (property in merged) { + out[property] = merged[property]; + continue; + } + // Only a message kept with its text has the raw headers these come from. + const form = parseHeaderProperty(property); + if (form && body && Array.isArray(body.headers)) out[property] = headerValue(form, body.headers as RawHeader[]); + } return out; } @@ -201,7 +402,7 @@ const emailGet: MethodHandler = (ctx, args) => { if (!Array.isArray(args.ids)) throw new MethodError('requestTooLarge', 'ids: null needs the server'); const properties = args.properties as string[] | undefined; const wantsValues = Boolean(args.fetchTextBodyValues || args.fetchHTMLBodyValues || args.fetchAllBodyValues) || Boolean(properties?.includes('bodyValues')); - const wantsBody = wantsValues || !properties || properties.some((p) => BODY_PROPERTIES.has(p)); + const wantsBody = wantsValues || !properties || properties.some((p) => BODY_PROPERTIES.has(p) || p.startsWith('header:')); const list: Record[] = []; const notFound: string[] = []; for (const id of args.ids as string[]) { @@ -215,7 +416,7 @@ const emailGet: MethodHandler = (ctx, args) => { } const email = JSON.parse(row.json) as Record; if (!wantsBody) { - list.push(pick(email, properties)); + list.push(shapeEmail(email, null, args)); continue; } const body = ctx.db.get<{ json: string }>('SELECT json FROM email_body WHERE email_rowid = ?', [row.rowid]); @@ -224,56 +425,73 @@ const emailGet: MethodHandler = (ctx, args) => { notFound.push(id); continue; } - list.push(pick({ ...(body ? JSON.parse(body.json) as object : {}), ...email }, properties)); + list.push(shapeEmail(email, body ? JSON.parse(body.json) as Record : null, args)); } - const state = ctx.db.get<{ state: string | null }>("SELECT state FROM sync_state WHERE jmap = ? AND type = 'Email'", [jmap])?.state; + const state = lastState(ctx, jmap, 'Email'); return { accountId: jmap, ...(state ? { state } : {}), list, notFound }; }; const threadGet: MethodHandler = (ctx, args) => { const jmap = jmapOf(ctx, args); const ids = Array.isArray(args.ids) ? [...new Set(args.ids as string[])] : []; - const list: Array<{ id: string; emailIds: string[] }> = []; - const notFound: string[] = []; + const found: Array<{ id: string; emailIds: string[] }> = []; for (const id of ids) { const stored = ctx.db.get<{ email_ids: string }>('SELECT email_ids FROM thread WHERE jmap = ? AND id = ?', [jmap, id]); if (stored) { - list.push({ id, emailIds: JSON.parse(stored.email_ids) as string[] }); + found.push({ id, emailIds: JSON.parse(stored.email_ids) as string[] }); continue; } const rows = ctx.db.all<{ id: string }>('SELECT id FROM email WHERE jmap = ? AND thread_id = ? ORDER BY received_at ASC', [jmap, id]); - if (rows.length > 0) list.push({ id, emailIds: rows.map((row) => String(row.id)) }); - else notFound.push(id); + if (rows.length > 0) found.push({ id, emailIds: rows.map((row) => String(row.id)) }); } - return { accountId: jmap, list, notFound }; -}; - -const mailboxGet: MethodHandler = (ctx, args) => { - const jmap = jmapOf(ctx, args); - const boxes = listObjects>(ctx.db, jmap, 'Mailbox'); - if (boxes.length === 0) throw new MethodError('serverUnavailable', 'Folders not kept yet'); - const ids = Array.isArray(args.ids) ? new Set(args.ids as string[]) : null; - const chosen = ids ? boxes.filter((box) => ids.has(box.id as string)) : boxes; - const state = ctx.db.get<{ state: string | null }>("SELECT state FROM sync_state WHERE jmap = ? AND type = 'Mailbox'", [jmap])?.state; + const list = ctx.overlay(jmap, 'Thread', found).filter((thread) => thread.emailIds.length > 0); + const state = lastState(ctx, jmap, 'Thread'); return { accountId: jmap, ...(state ? { state } : {}), - list: chosen.map((box) => pick(box, args.properties as string[] | undefined)), - notFound: ids ? [...ids].filter((id) => !boxes.some((box) => box.id === id)) : [], + list, + notFound: ids.filter((id) => !list.some((thread) => thread.id === id)), }; }; +function pick(item: Record, properties: string[] | undefined): Record { + if (!properties) return item; + const out: Record = { id: item.id }; + for (const property of properties) if (property in item) out[property] = item[property]; + return out; +} + +/** The answer of a /get over objects the syncers keep. An error until they've run once. */ +function objectGet(ctx: MethodContext, args: Record, type: string, label: string): MethodResult { + const jmap = jmapOf(ctx, args); + const state = lastState(ctx, jmap, type); + const all = ctx.overlay(jmap, type, listObjects & { id: string }>(ctx.db, jmap, type)); + if (all.length === 0 && !state) throw new MethodError('serverUnavailable', `${label} not kept yet`); + const ids = Array.isArray(args.ids) ? args.ids as string[] : null; + const chosen = ids ? all.filter((item) => ids.includes(item.id)) : all; + return { + accountId: jmap, + ...(state ? { state } : {}), + list: chosen.map((item) => pick(item, args.properties as string[] | undefined)), + notFound: ids ? ids.filter((id) => !all.some((item) => item.id === id)) : [], + }; +} + const mailboxQuery: MethodHandler = (ctx, args) => { const jmap = jmapOf(ctx, args); const boxes = listObjects>(ctx.db, jmap, 'Mailbox'); - if (boxes.length === 0) throw new MethodError('serverUnavailable', 'Folders not kept yet'); - const filter = (args.filter ?? {}) as { role?: string }; + const state = lastState(ctx, jmap, 'Mailbox'); + if (boxes.length === 0 || !state) throw new MethodError('serverUnavailable', 'Folders not kept yet'); + const filter = (args.filter ?? {}) as Record; + for (const key of Object.keys(filter)) { + if (key !== 'role') throw new MethodError('unsupportedFilter', `${key} isn't available offline`); + } const chosen = filter.role ? boxes.filter((box) => box.role === filter.role) : boxes; const position = typeof args.position === 'number' ? args.position : 0; const limit = typeof args.limit === 'number' ? args.limit : chosen.length; return { accountId: jmap, - queryState: 'local', + queryState: state, canCalculateChanges: false, position, ids: chosen.slice(position, position + limit).map((box) => box.id), @@ -281,11 +499,44 @@ const mailboxQuery: MethodHandler = (ctx, args) => { }; }; +/** The client treats this as "do a full fetch": what it reads next comes from the store anyway. */ +const noChanges: MethodHandler = () => { + throw new MethodError('cannotCalculateChanges', 'Changes need the server'); +}; + +const keywordGet: MethodHandler = (ctx, args) => { + const jmap = jmapOf(ctx, args); + const rows = ctx.db.all<{ keyword: string; total: number; unread: number }>( + `SELECT k.keyword AS keyword, COUNT(*) AS total, + SUM(NOT EXISTS (SELECT 1 FROM email_keyword s WHERE s.email_rowid = k.email_rowid AND s.keyword = '$seen')) AS unread + FROM email_keyword k JOIN email e ON e.rowid = k.email_rowid + WHERE e.jmap = ? GROUP BY k.keyword`, + [jmap], + ); + const counts = new Map(rows.map((row) => [String(row.keyword), { id: String(row.keyword), total: Number(row.total), unread: Number(row.unread) }])); + const wanted = Array.isArray(args.ids) ? args.ids as string[] : null; + const list = wanted + ? wanted.map((id) => ({ ...(counts.get(keywordKey(id)) ?? { total: 0, unread: 0 }), id })) + : [...counts.values()]; + return { + accountId: jmap, + list, + notFound: [], + totalEmails: Number(ctx.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM email WHERE jmap = ?', [jmap])?.n ?? 0), + }; +}; + export const MAIL_METHODS: Readonly> = { + 'Core/echo': (_ctx, args) => args, 'Email/query': emailQuery, 'Email/get': emailGet, + 'Email/changes': noChanges, 'Thread/get': threadGet, - 'Mailbox/get': mailboxGet, + 'Mailbox/get': (ctx, args) => objectGet(ctx, args, 'Mailbox', 'Folders'), 'Mailbox/query': mailboxQuery, + 'Mailbox/changes': noChanges, + 'Identity/get': (ctx, args) => objectGet(ctx, args, 'Identity', 'Identities'), + 'Quota/get': (ctx, args) => objectGet(ctx, args, 'Quota', 'Quotas'), + 'Keyword/get': keywordGet, 'SearchSnippet/get': searchSnippetGet, }; diff --git a/lib/store/mail/records.ts b/lib/store/mail/records.ts index 72b2397f8..e9d575c44 100644 --- a/lib/store/mail/records.ts +++ b/lib/store/mail/records.ts @@ -2,6 +2,18 @@ import type { StorageAdapter } from '../storage'; +/** + * How keywords compare, measured on both Stalwart versions: the IMAP system + * flags fold to lower case, every other keyword is case-sensitive, so `Tag` and + * `tag` are two keywords and one message can carry both. + */ +const SYSTEM_KEYWORDS = new Set(['$seen', '$flagged', '$answered', '$draft', '$forwarded', '$mdnsent', '$junk', '$notjunk', '$phishing', '$recent', '$deleted']); + +export function keywordKey(keyword: string): string { + const lower = keyword.toLowerCase(); + return SYSTEM_KEYWORDS.has(lower) ? lower : keyword; +} + export const MAIL_USING = ['urn:ietf:params:jmap:core', 'urn:ietf:params:jmap:mail']; export const EMAIL_LIST_PROPERTIES = [ @@ -80,11 +92,19 @@ export function upsertEmail(db: StorageAdapter, jmap: string, email: EmailObject if (on) db.run('INSERT INTO email_mailbox (email_rowid, jmap, mailbox_id, received_at) VALUES (?, ?, ?, ?)', [rowid, jmap, box, receivedAt]); } for (const [keyword, on] of Object.entries((email.keywords ?? {}) as Record)) { - if (on) db.run('INSERT INTO email_keyword (email_rowid, keyword) VALUES (?, ?)', [rowid, keyword.toLowerCase()]); + if (on) db.run('INSERT OR IGNORE INTO email_keyword (email_rowid, keyword) VALUES (?, ?)', [rowid, keywordKey(keyword)]); } return rowid; } +export function putBlob(db: StorageAdapter, jmap: string, blobId: string, type: string | null, data: Uint8Array, at: number): void { + db.run( + `INSERT INTO blob (jmap, blob_id, type, size, at, data) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (jmap, blob_id) DO UPDATE SET type = excluded.type, size = excluded.size, at = excluded.at, data = excluded.data`, + [jmap, blobId, type, data.byteLength, at, data], + ); +} + export function deleteEmail(db: StorageAdapter, jmap: string, id: string): void { const row = db.get<{ rowid: number }>('SELECT rowid FROM email WHERE jmap = ? AND id = ?', [jmap, id]); if (!row) return; @@ -94,6 +114,7 @@ export function deleteEmail(db: StorageAdapter, jmap: string, id: string): void export function deleteEmailRow(db: StorageAdapter, rowid: number): void { deleteHeadIndex(db, rowid); deleteBodyIndex(db, rowid); + forgetParts(db, rowid); // Explicit rather than ON DELETE CASCADE: foreign keys stay off for bulk speed. db.run('DELETE FROM email_mailbox WHERE email_rowid = ?', [rowid]); db.run('DELETE FROM email_keyword WHERE email_rowid = ?', [rowid]); @@ -211,11 +232,37 @@ export function indexEmail(db: StorageAdapter, rowid: number, email: Record('SELECT jmap, blob_id FROM email_part WHERE email_rowid = ?', [rowid]); + db.run('DELETE FROM email_part WHERE email_rowid = ?', [rowid]); + for (const part of parts) { + db.run( + 'DELETE FROM blob WHERE jmap = ? AND blob_id = ? AND NOT EXISTS (SELECT 1 FROM email_part p WHERE p.jmap = ? AND p.blob_id = ?)', + [part.jmap, part.blob_id, part.jmap, part.blob_id], + ); + } +} + +function rememberParts(db: StorageAdapter, rowid: number, attachments: unknown): void { + const row = db.get<{ jmap: string; received_at: number }>('SELECT jmap, received_at FROM email WHERE rowid = ?', [rowid]); + if (!row || !Array.isArray(attachments)) return; + for (const part of attachments as Array<{ blobId?: unknown; size?: unknown; type?: unknown; name?: unknown }>) { + if (typeof part.blobId !== 'string') continue; + db.run( + `INSERT INTO email_part (jmap, blob_id, email_rowid, received_at, size, type, name) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (jmap, blob_id, email_rowid) DO NOTHING`, + [row.jmap, part.blobId, rowid, row.received_at, typeof part.size === 'number' ? part.size : 0, typeof part.type === 'string' ? part.type : null, typeof part.name === 'string' ? part.name : null], + ); + } +} + export function storeBody(db: StorageAdapter, rowid: number, listJson: string, full: Record): void { const list = JSON.parse(listJson) as Record; const body: Record = {}; for (const [key, value] of Object.entries(full)) if (!(key in list)) body[key] = value; const hadBody = Number(db.get<{ body_state: number }>('SELECT body_state FROM email WHERE rowid = ?', [rowid])?.body_state) === 1; + rememberParts(db, rowid, body.attachments); db.run('INSERT INTO email_body (email_rowid, json) VALUES (?, ?) ON CONFLICT (email_rowid) DO UPDATE SET json = excluded.json', [rowid, JSON.stringify(body)]); db.run('UPDATE email SET body_state = 1 WHERE rowid = ?', [rowid]); // A first body is one insert. Only a body stored again has an old row to replace. diff --git a/lib/store/mail/schema.ts b/lib/store/mail/schema.ts index 98f29576d..64def1243 100644 --- a/lib/store/mail/schema.ts +++ b/lib/store/mail/schema.ts @@ -62,4 +62,34 @@ export const MAIL_MIGRATIONS: readonly Migration[] = [ ); `, }, + { + id: 101, + name: 'mail attachments', + sql: ` + -- The attachments and inline images of kept messages. + CREATE TABLE email_part ( + jmap TEXT NOT NULL, + blob_id TEXT NOT NULL, + email_rowid INTEGER NOT NULL, + received_at INTEGER NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + type TEXT, + name TEXT, + -- 1 when the server wouldn't give it: not asked for again + refused INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (jmap, blob_id, email_rowid) + ) WITHOUT ROWID; + CREATE INDEX email_part_message ON email_part (email_rowid); + CREATE INDEX email_part_newest ON email_part (jmap, received_at DESC); + CREATE TABLE blob ( + jmap TEXT NOT NULL, + blob_id TEXT NOT NULL, + type TEXT, + size INTEGER NOT NULL, + at INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (jmap, blob_id) + ) WITHOUT ROWID; + `, + }, ]; diff --git a/lib/store/mail/sync.ts b/lib/store/mail/sync.ts index 30b8c7b7b..f3919d1e4 100644 --- a/lib/store/mail/sync.ts +++ b/lib/store/mail/sync.ts @@ -10,13 +10,14 @@ import type { SyncContext, Syncer } from '../area'; import { deleteObject, putObject } from '../engine/core-schema'; import { runChangesLoop } from '../engine/changes'; -import { responseFor, type MethodCall } from '../engine/transport'; +import { JmapMethodError, responseFor, TransportError, type MethodCall } from '../engine/transport'; import { deleteEmail, deleteEmailRow, EMAIL_BODY_PROPERTIES, EMAIL_LIST_PROPERTIES, MAIL_USING, + putBlob, storeBody, upsertEmail, type EmailObject, @@ -83,7 +84,12 @@ async function syncMailboxes(ctx: SyncContext, jmap: string): Promise { // ---- Email list ---- -interface ListCursor { position: number; gen: number; captured: string } +/** + * `anchor` is the last id a pass saw. A pass can be paused for days, and a + * position would shift with every deletion before it: a message would be + * skipped, and no /changes after the captured state would ever mention it. + */ +interface ListCursor { position: number; gen: number; captured: string; anchor?: string | null } /** The generation of the last full listing. One row is enough: every row of a pass has it or an older one. */ function currentGen(ctx: SyncContext, jmap: string): number { @@ -98,19 +104,50 @@ async function startFullList(ctx: SyncContext, jmap: string): Promise { + for (;;) { + if (ctx.stopped()) return false; + const unseen = ctx.db.all<{ rowid: number; id: string }>('SELECT rowid, id FROM email WHERE jmap = ? AND gen < ? LIMIT ?', [jmap, gen, PAGE]); + if (unseen.length === 0) return true; + const response = await ctx.transport.request([['Email/get', { + accountId: jmap, + ids: unseen.map((row) => String(row.id)), + properties: EMAIL_LIST_PROPERTIES, + }, 'g']], MAIL_USING); + const found = responseFor(response, 'g')[1].list as EmailObject[]; + ctx.db.transaction(() => { + const kept = new Set(); + for (const email of ctx.overlay(jmap, 'Email', found.filter((candidate) => inWindow(ctx, candidate)))) { + upsertEmail(ctx.db, jmap, email, gen); + kept.add(email.id); + } + for (const row of unseen) if (!kept.has(String(row.id))) deleteEmailRow(ctx.db, Number(row.rowid)); + }); + } +} + async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): Promise { const cursor = { ...start }; const filter = windowFilter(ctx); let total: number | null = null; for (;;) { if (ctx.stopped()) return false; + const pageArgs = (i: number): Record => (cursor.anchor + ? { anchor: cursor.anchor, anchorOffset: 1 + i * PAGE } + : { position: cursor.position + i * PAGE }); const calls: MethodCall[] = []; for (let i = 0; i < PAIRS_PER_REQUEST; i += 1) { calls.push(['Email/query', { accountId: jmap, ...(filter ? { filter } : {}), sort: [{ property: 'receivedAt', isAscending: false }], - position: cursor.position + i * PAGE, + ...pageArgs(i), limit: PAGE, calculateTotal: i === 0, }, `q${i}`]); @@ -124,30 +161,36 @@ async function runFullList(ctx: SyncContext, jmap: string, start: ListCursor): P let done = false; let got = 0; const lists: EmailObject[][] = []; - for (let i = 0; i < PAIRS_PER_REQUEST; i += 1) { - const [, query] = responseFor(response, `q${i}`); - if (i === 0 && typeof query.total === 'number') total = query.total; - lists.push(responseFor(response, `g${i}`)[1].list as EmailObject[]); - got += (query.ids as string[]).length; - if ((query.ids as string[]).length < PAGE) { - done = true; - break; + try { + for (let i = 0; i < PAIRS_PER_REQUEST; i += 1) { + const [, query] = responseFor(response, `q${i}`); + if (i === 0 && typeof query.total === 'number') total = query.total; + lists.push(responseFor(response, `g${i}`)[1].list as EmailObject[]); + const ids = query.ids as string[]; + got += ids.length; + if (ids.length > 0) cursor.anchor = ids[ids.length - 1]; + if (ids.length < PAGE) { + done = true; + break; + } } + } catch (error) { + if (!(error instanceof JmapMethodError) || error.type !== 'anchorNotFound' || !cursor.anchor) throw error; + // The anchor was deleted, or left the window. Back to positions, a page + // earlier, since deletions before the cursor have shifted them. + cursor.anchor = null; + cursor.position = Math.max(0, cursor.position - PAGE); + continue; } cursor.position += got; ctx.db.transaction(() => { - for (const list of lists) for (const email of list) upsertEmail(ctx.db, jmap, email, cursor.gen); - if (done) { - // Whatever this pass didn't see is gone from the server or the window. - const stale = ctx.db.all<{ rowid: number }>('SELECT rowid FROM email WHERE jmap = ? AND gen < ?', [jmap, cursor.gen]); - for (const row of stale) deleteEmailRow(ctx.db, Number(row.rowid)); - ctx.setSyncState(jmap, 'Email', { state: cursor.captured, phase: null, cursor: null }); - } else { - ctx.setSyncState(jmap, 'Email', { cursor: { ...cursor } }); - } + for (const list of lists) for (const email of ctx.overlay(jmap, 'Email', list)) upsertEmail(ctx.db, jmap, email, cursor.gen); + ctx.setSyncState(jmap, 'Email', { cursor: { ...cursor } }); }); ctx.emit({ type: 'progress', account: ctx.account, area: 'mail', phase: 'list', done: cursor.position, total }); if (done) { + if (!(await sweepUnseen(ctx, jmap, cursor.gen))) return false; + ctx.db.transaction(() => ctx.setSyncState(jmap, 'Email', { state: cursor.captured, phase: null, cursor: null })); ctx.emit({ type: 'changed', account: ctx.account, collections: ['Email'] }); return true; } @@ -178,7 +221,7 @@ async function syncEmailDelta(ctx: SyncContext, jmap: string, since: string, tar const newIds: string[] = []; ctx.db.transaction(() => { const gen = fetched.length > 0 ? currentGen(ctx, jmap) : 0; - for (const email of fetched) { + for (const email of ctx.overlay(jmap, 'Email', fetched)) { const known = ctx.db.get('SELECT 1 AS x FROM email WHERE jmap = ? AND id = ?', [jmap, email.id]); // An old message changing doesn't pull it into the window. if (!known && !inWindow(ctx, email)) continue; @@ -306,6 +349,110 @@ async function backfillBodies(ctx: SyncContext, jmap: string): Promise { } } +// ---- Attachments ---- + +/** + * The files of kept messages, newest first, within what the user chose: one + * at a time, each in its own transaction, so a reload loses at most one. + */ +async function fetchAttachments(ctx: SyncContext, jmap: string): Promise { + if (!ctx.scope.attachments) return; + let used = Number(ctx.db.get<{ n: number | null }>('SELECT SUM(size) AS n FROM blob')?.n ?? 0); + for (;;) { + if (ctx.stopped()) return; + const parts = ctx.db.all<{ blob_id: string; size: number; type: string | null; name: string | null; received_at: number }>( + `SELECT p.blob_id, p.size, p.type, p.name, p.received_at FROM email_part p + WHERE p.jmap = ? AND p.refused = 0 AND p.size <= ? + AND NOT EXISTS (SELECT 1 FROM blob b WHERE b.jmap = p.jmap AND b.blob_id = p.blob_id) + ORDER BY p.received_at DESC LIMIT 20`, + [jmap, ctx.scope.maxAttachmentBytes], + ); + if (parts.length === 0) return; + for (const part of parts) { + if (ctx.stopped()) return; + const blobId = String(part.blob_id); + if (used + Number(part.size) > ctx.scope.maxTotalBytes) return; + let data: Uint8Array; + try { + data = await ctx.transport.download(jmap, blobId, part.type ?? 'application/octet-stream', part.name ?? 'attachment'); + } catch (error) { + // Only what says this file won't come. Anything else, like a 500 or a 429, is tried at the next sync. + if (error instanceof TransportError && error.kind === 'http' && PERMANENT_DOWNLOAD_STATUSES.has(error.status ?? 0)) { + ctx.db.run('UPDATE email_part SET refused = 1 WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + continue; + } + if (error instanceof TransportError && error.kind === 'http') return; + throw error; + } + putBlob(ctx.db, jmap, blobId, part.type, data, Number(part.received_at)); + used += data.byteLength; + } + } +} + +const PERMANENT_DOWNLOAD_STATUSES = new Set([403, 404, 410]); + +// ---- Identities and quotas ---- + +const IDENTITY_USING = ['urn:ietf:params:jmap:core', 'urn:ietf:params:jmap:mail', 'urn:ietf:params:jmap:submission']; +const QUOTA_USING = ['urn:ietf:params:jmap:core', 'urn:ietf:params:jmap:quota']; + +async function fullIdentities(ctx: SyncContext, jmap: string): Promise { + const response = await ctx.transport.request([['Identity/get', { accountId: jmap, ids: null }, 'i']], IDENTITY_USING); + const [, result] = responseFor(response, 'i'); + ctx.db.transaction(() => { + ctx.db.run("DELETE FROM object WHERE jmap = ? AND type = 'Identity'", [jmap]); + for (const identity of result.list as Array<{ id: string }>) putObject(ctx.db, jmap, 'Identity', identity.id, identity); + ctx.setSyncState(jmap, 'Identity', { state: String(result.state), phase: null, cursor: null }); + }); +} + +export const identitySyncer: Syncer = { + name: 'identities', + capability: 'urn:ietf:params:jmap:submission', + async run(ctx, jmap) { + const { state, target } = ctx.syncState(jmap, 'Identity'); + if (!state) return fullIdentities(ctx, jmap); + const outcome = await runChangesLoop({ + transport: ctx.transport, + type: 'Identity', + jmap, + using: IDENTITY_USING, + since: state, + target, + stopped: ctx.stopped, + apply: async (page) => { + const ids = [...page.created, ...page.updated]; + const response = ids.length > 0 + ? await ctx.transport.request([['Identity/get', { accountId: jmap, ids }, 'i']], IDENTITY_USING) + : null; + const list = response ? responseFor(response, 'i')[1].list as Array<{ id: string }> : []; + ctx.db.transaction(() => { + for (const identity of list) putObject(ctx.db, jmap, 'Identity', identity.id, identity); + for (const id of page.destroyed) deleteObject(ctx.db, jmap, 'Identity', id); + ctx.setSyncState(jmap, 'Identity', { state: page.newState }); + }); + }, + }); + if (outcome.outcome === 'resync') await fullIdentities(ctx, jmap); + }, +}; + +/** Only the last value: Quota/changes always refuses on Stalwart 1.0. */ +export const quotaSyncer: Syncer = { + name: 'quota', + capability: 'urn:ietf:params:jmap:quota', + async run(ctx, jmap) { + const response = await ctx.transport.request([['Quota/get', { accountId: jmap, ids: null }, 'q']], QUOTA_USING); + const [, result] = responseFor(response, 'q'); + ctx.db.transaction(() => { + ctx.db.run("DELETE FROM object WHERE jmap = ? AND type = 'Quota'", [jmap]); + for (const quota of result.list as Array<{ id: string }>) putObject(ctx.db, jmap, 'Quota', quota.id, quota); + ctx.setSyncState(jmap, 'Quota', { state: typeof result.state === 'string' ? result.state : null, phase: null, cursor: null }); + }); + }, +}; + // ---- The whole pass ---- export const mailSyncer: Syncer = { @@ -334,5 +481,7 @@ export const mailSyncer: Syncer = { await syncThreads(ctx, jmap); if (ctx.stopped()) return; await backfillBodies(ctx, jmap); + if (ctx.stopped()) return; + await fetchAttachments(ctx, jmap); }, }; diff --git a/lib/store/outbox/calls.ts b/lib/store/outbox/calls.ts new file mode 100644 index 000000000..33b1f7228 --- /dev/null +++ b/lib/store/outbox/calls.ts @@ -0,0 +1,297 @@ +/** What the page asks the outbox. Each call is one SQLite transaction, so two tabs can't interleave. */ + +import type { AreaCall, StoreContext } from '../area'; +import { metaGet, metaSet } from '../engine/core-schema'; +import { deleteEmail, upsertEmail, type EmailObject } from '../mail/records'; +import { nextSeq, queueUpdate, type PatchRow } from './email-set'; +import type { PatchOutcome, PendingPatch } from './flush-patches'; +import { + heldBlobIds, + LOCAL_BLOB_PREFIX, + type ConflictSummary, + type OutboxError, + type OutboxStatus, + type OutboxSummary, + type PolicyFingerprint, + type SendItem, + type StoredConflict, + type StoredOutboxEntry, +} from './items'; +import type { Patch, RecordState } from './patches'; + +interface OutboxRow { + id: number; + seq: number; + kind: string; + created_at: number; + status: string; + sending_since: number | null; + first_attempt_at: number | null; + last_attempt_at: number | null; + payload: string; + summary: string; + message_id: string | null; + copy_id: string | null; + error: string | null; +} + +function toEntry(row: OutboxRow): StoredOutboxEntry { + const payload = JSON.parse(row.payload) as { item: SendItem; fingerprint?: PolicyFingerprint }; + return { + id: Number(row.id), + seq: Number(row.seq), + createdAt: Number(row.created_at), + status: row.status as OutboxStatus, + summary: JSON.parse(row.summary) as OutboxSummary, + item: payload.item, + ...(payload.fingerprint ? { fingerprint: payload.fingerprint } : {}), + ...(row.sending_since ? { sendingSince: Number(row.sending_since) } : {}), + ...(row.first_attempt_at ? { firstAttemptAt: Number(row.first_attempt_at) } : {}), + ...(row.last_attempt_at ? { lastAttemptAt: Number(row.last_attempt_at) } : {}), + ...(row.error ? { error: JSON.parse(row.error) as OutboxError } : {}), + ...(row.message_id ? { messageId: row.message_id } : {}), + ...(row.copy_id ? { copyId: row.copy_id } : {}), + }; +} + +interface QueueParams { + summary: OutboxSummary; + item: SendItem; + fingerprint?: PolicyFingerprint; + messageId?: string; + status?: OutboxStatus; + createdAt?: number; + error?: OutboxError; +} + +function insertEntry(ctx: StoreContext, params: QueueParams): number { + return ctx.db.run( + `INSERT INTO outbox (seq, kind, created_at, status, payload, summary, message_id, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + nextSeq(ctx.db), + params.item.kind, + params.createdAt ?? ctx.now(), + params.status ?? 'waiting', + JSON.stringify({ item: params.item, fingerprint: params.fingerprint }), + JSON.stringify(params.summary), + params.messageId ?? null, + params.error ? JSON.stringify(params.error) : null, + ], + ).lastInsertRowid; +} + +function queueSend(ctx: StoreContext, params: QueueParams): number { + return ctx.db.transaction(() => insertEntry(ctx, params)); +} + +function read(ctx: StoreContext): { patches: number; entries: StoredOutboxEntry[]; conflicts: StoredConflict[] } { + const patches = Number(ctx.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM outbox_patch')?.n ?? 0); + const entries = ctx.db.all('SELECT * FROM outbox ORDER BY seq ASC').map(toEntry); + const conflicts = ctx.db.all<{ id: number; jmap: string; record_id: string; summary: string; created_at: number }>( + 'SELECT id, jmap, record_id, summary, created_at FROM conflict ORDER BY id ASC', + ).map((row) => ({ + id: Number(row.id), + jmap: row.jmap, + recordId: row.record_id, + createdAt: Number(row.created_at), + summary: JSON.parse(row.summary) as ConflictSummary, + })); + return { patches, entries, conflicts }; +} + +/** Takes a waiting entry for sending: false when it isn't waiting any more (another tab took it, the user deleted it). */ +function claim(ctx: StoreContext, params: { id: number }): boolean { + const now = ctx.now(); + return ctx.db.run( + `UPDATE outbox SET status = 'sending', sending_since = ?, last_attempt_at = ? WHERE id = ? AND status = 'waiting'`, + [now, now, params.id], + ).changes === 1; +} + +/** Deleted meanwhile stays deleted: there's no row to update. */ +function update(ctx: StoreContext, params: { id: number; status: OutboxStatus; error?: OutboxError; copyId?: string | null }): void { + ctx.db.run( + `UPDATE outbox SET status = ?, error = ?, sending_since = NULL, + first_attempt_at = CASE WHEN ? THEN COALESCE(first_attempt_at, last_attempt_at) ELSE first_attempt_at END, + copy_id = CASE WHEN ? THEN ? ELSE copy_id END + WHERE id = ?`, + [ + params.status, + params.error ? JSON.stringify(params.error) : null, + // A try that may have reached the server is what a later lookup has to cover. + params.status === 'uncertain' ? 1 : 0, + params.copyId === undefined ? 0 : 1, + params.copyId ?? null, + params.id, + ], + ); +} + +/** A send a closed tab left behind may have gone out. */ +function settleStale(ctx: StoreContext, params: { before: number }): number { + return ctx.db.run( + `UPDATE outbox SET status = 'uncertain', sending_since = NULL, first_attempt_at = COALESCE(first_attempt_at, last_attempt_at) + WHERE status = 'sending' AND COALESCE(sending_since, 0) < ?`, + [params.before], + ).changes; +} + +/** Removes an entry and the files it held. */ +function remove(ctx: StoreContext, params: { id: number }): void { + ctx.db.transaction(() => { + const row = ctx.db.get('SELECT * FROM outbox WHERE id = ?', [params.id]); + if (!row) return; + for (const blobId of heldBlobIds(toEntry(row).item)) ctx.db.run('DELETE FROM upload WHERE local_id = ?', [blobId]); + ctx.db.run('DELETE FROM outbox WHERE id = ?', [params.id]); + }); +} + +// ---- Files attached offline ---- + +function holdFile(ctx: StoreContext, params: { jmap?: string; name: string; type: string; data: Uint8Array; localId?: string }): { blobId: string; size: number; type: string } { + const blobId = params.localId ?? `${LOCAL_BLOB_PREFIX}${crypto.randomUUID()}`; + ctx.db.run( + 'INSERT OR REPLACE INTO upload (local_id, jmap, type, name, size, data, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)', + [blobId, params.jmap ?? null, params.type, params.name, params.data.byteLength, params.data, ctx.now()], + ); + return { blobId, size: params.data.byteLength, type: params.type || 'application/octet-stream' }; +} + +function heldFile(ctx: StoreContext, params: { blobId: string }): { name: string; type: string; data: Uint8Array } | null { + const row = ctx.db.get<{ name: string | null; type: string | null; data: Uint8Array }>('SELECT name, type, data FROM upload WHERE local_id = ?', [params.blobId]); + return row ? { name: row.name ?? 'attachment', type: row.type ?? 'application/octet-stream', data: row.data } : null; +} + +const heldFileSources: Array<(ctx: StoreContext) => Iterable> = []; + +/** Another area's records that hold files by their local id, which dropOrphans must leave alone. */ +export function addHeldFileSource(source: (ctx: StoreContext) => Iterable): void { + heldFileSources.push(source); +} + +/** Files held for no entry (attached offline to a message that was then discarded). SEAM: the drafts agent keeps its drafts' files here too. */ +function dropOrphans(ctx: StoreContext, params: { olderThan: number }): number { + return ctx.db.transaction(() => { + const needed = new Set(ctx.db.all('SELECT * FROM outbox').flatMap((row) => heldBlobIds(toEntry(row).item))); + for (const source of heldFileSources) for (const id of source(ctx)) needed.add(id); + let dropped = 0; + for (const row of ctx.db.all<{ local_id: string }>('SELECT local_id FROM upload WHERE created_at < ?', [params.olderThan])) { + if (needed.has(String(row.local_id))) continue; + ctx.db.run('DELETE FROM upload WHERE local_id = ?', [row.local_id]); + dropped += 1; + } + return dropped; + }); +} + +// ---- Changes ---- + +function pendingPatches(ctx: StoreContext, params: { upToSeq?: number | null }): PendingPatch[] { + const rows = ctx.db.all( + params.upToSeq === undefined || params.upToSeq === null + ? 'SELECT * FROM outbox_patch ORDER BY seq ASC' + : 'SELECT * FROM outbox_patch WHERE seq < ? ORDER BY seq ASC', + params.upToSeq === undefined || params.upToSeq === null ? [] : [params.upToSeq], + ); + return rows.map((row) => ({ + jmap: row.jmap, + type: row.type, + recordId: row.record_id, + op: row.op, + patch: JSON.parse(row.patch) as Patch, + base: row.base ? JSON.parse(row.base) as RecordState : null, + subject: row.subject, + seq: Number(row.seq), + })); +} + +function subjectFor(row: PatchRow, server: EmailObject | null | undefined): string { + if (typeof server?.subject === 'string') return server.subject; + return row.subject ?? ''; +} + +/** + * What came of a flush. A change made while it ran stays and goes next time, + * without comparing against a state this flush itself just moved. After a + * conflict the local copy is the server's again: nothing else will correct it + * if sync already passed that point. + */ +function patchResults(ctx: StoreContext, params: { outcomes: PatchOutcome[] }): number { + let conflicts = 0; + ctx.db.transaction(() => { + for (const outcome of params.outcomes) { + const key = [outcome.jmap, outcome.type, outcome.recordId]; + const row = ctx.db.get('SELECT * FROM outbox_patch WHERE jmap = ? AND type = ? AND record_id = ?', key); + if (!row) continue; + if (outcome.outcome === 'done' && Number(row.seq) !== outcome.seq) { + ctx.db.run('UPDATE outbox_patch SET base = NULL WHERE jmap = ? AND type = ? AND record_id = ?', key); + continue; + } + ctx.db.run('DELETE FROM outbox_patch WHERE jmap = ? AND type = ? AND record_id = ?', key); + if (outcome.outcome === 'done') continue; + conflicts += 1; + const summary: ConflictSummary = { + subject: subjectFor(row, outcome.server), + op: row.op, + reason: outcome.reason, + ...(outcome.detail ? { detail: outcome.detail } : {}), + }; + ctx.db.run( + 'INSERT INTO conflict (jmap, type, record_id, summary, local, created_at) VALUES (?, ?, ?, ?, ?, ?)', + [outcome.jmap, outcome.type, outcome.recordId, JSON.stringify(summary), JSON.stringify({ op: row.op, patch: JSON.parse(row.patch) }), ctx.now()], + ); + if (outcome.server === null) { + deleteEmail(ctx.db, outcome.jmap, outcome.recordId); + } else if (outcome.server) { + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM email WHERE jmap = ?', [outcome.jmap])?.gen ?? 0); + upsertEmail(ctx.db, outcome.jmap, outcome.server, gen); + } + } + }); + if (conflicts > 0) ctx.emit({ type: 'changed', account: ctx.account, collections: ['Email', 'Mailbox', 'Thread'] }); + return conflicts; +} + +function dismissConflict(ctx: StoreContext, params: { id: number }): void { + ctx.db.run('DELETE FROM conflict WHERE id = ?', [params.id]); +} + +// ---- What the old offline database held ---- + +export interface LegacyImport { + entries: QueueParams[]; + patches: Array<{ jmap: string; update: Record }>; + files: Array<{ blobId: string; name: string; type: string; data: Uint8Array }>; +} + +const LEGACY_DONE = 'legacy_outbox_moved'; + +/** One transaction: all of it or none. A second run after a partial failure elsewhere adds nothing. */ +function importLegacy(ctx: StoreContext, params: LegacyImport): boolean { + return ctx.db.transaction(() => { + if (metaGet(ctx.db, LEGACY_DONE)) return false; + for (const file of params.files) holdFile(ctx, { name: file.name, type: file.type, data: file.data, localId: file.blobId }); + for (const entry of params.entries) insertEntry(ctx, entry); + for (const { jmap, update: changes } of params.patches) { + for (const [id, patch] of Object.entries(changes)) queueUpdate(ctx.db, jmap, id, patch, ctx.now()); + } + metaSet(ctx.db, LEGACY_DONE, '1'); + return true; + }); +} + +export const OUTBOX_CALLS: Readonly> = { + 'outbox.queueSend': queueSend, + 'outbox.read': read, + 'outbox.claim': claim, + 'outbox.update': update, + 'outbox.settleStale': settleStale, + 'outbox.remove': remove, + 'outbox.holdFile': holdFile, + 'outbox.heldFile': heldFile, + 'outbox.dropOrphans': dropOrphans, + 'outbox.pendingPatches': pendingPatches, + 'outbox.patchResults': patchResults, + 'outbox.dismissConflict': dismissConflict, + 'outbox.importLegacy': importLegacy, +}; diff --git a/lib/store/outbox/dedupe.ts b/lib/store/outbox/dedupe.ts new file mode 100644 index 000000000..ca50ca900 --- /dev/null +++ b/lib/store/outbox/dedupe.ts @@ -0,0 +1,149 @@ +/** + * Before a send whose response got lost goes out again: did it reach the + * server? JMAP has no idempotency key and a second create or submission is + * simply accepted, so this looks. The `header` and `text` filters find no + * Message-ID on either Stalwart, so it reads the recent Drafts and Sent and + * matches on the client (stalwart-changes-prf.md section 1, decision 4). + * + * | Lookup | Decision | + * | no match | send it again | + * | in Drafts, no submission | submit the copy that's there | + * | in Sent | done | + * | in Drafts, a submission waiting | done, it's scheduled | + * | in Drafts, a canceled submission | done, someone decided against it | + * + * It answers "undecided" when it can't tell, and the entry stays with the user. + */ + +import { responseFor, type JmapResponse, type MethodCall } from '../engine/transport'; +import { MAIL_USING } from '../mail/records'; + +/** + * `after` compares the server's receivedAt with this device's clock, so it + * looks a day further back than the attempt. Only an exact Message-ID matches, + * so looking further costs nothing but a page or two. + */ +export const CLOCK_SKEW_MARGIN_MS = 24 * 60 * 60 * 1000; +/** + * Stalwart drops submissions after 3 days (`expungeSubmissionsAfter`). Past + * this, "no submission" and "not found" no longer say the send never happened. + */ +export const LOOKUP_TRUSTED_MS = 2 * 24 * 60 * 60 * 1000; +const PAGE = 500; +const MAX_PAGES = 5; +const SUBMISSION_USING = [...MAIL_USING, 'urn:ietf:params:jmap:submission']; + +export type RetryDecision = + | { action: 'send' } + | { action: 'submit'; emailId: string } + | { action: 'done'; emailId: string; why: 'sent' | 'scheduled' | 'canceled' } + | { action: 'undecided'; why: 'no-message-id' | 'no-folders' | 'too-many' | 'too-old' | 'no-copy-kept' | 'unexpected' }; + +export interface LookupIo { + request(calls: MethodCall[], using: string[]): Promise; +} + +export interface LookupArgs { + /** The JMAP account the send creates its copy in. */ + jmap: string; + submissionJmap: string; + messageId: string | undefined; + /** When the first attempt was handed to the server. */ + attemptAt: number; + now: number; + /** + * False when a successful send leaves no copy (no copy in Sent, a submit-only + * raw send): then finding nothing doesn't mean it never arrived. + */ + copyKept: boolean; +} + +interface FoundEmail { + id: string; + messageId?: string[] | null; + mailboxIds?: Record; + keywords?: Record; +} + +function bare(id: string): string { + return id.trim().replace(/^<|>$/g, ''); +} + +async function findCopies(io: LookupIo, args: LookupArgs, folders: { drafts?: string; sent?: string }): Promise { + const boxes = [folders.drafts, folders.sent].filter((box): box is string => Boolean(box)); + const wanted = bare(args.messageId as string); + const filter = { + operator: 'AND', + conditions: [ + { operator: 'OR', conditions: boxes.map((inMailbox) => ({ inMailbox })) }, + { after: new Date(args.attemptAt - CLOCK_SKEW_MARGIN_MS).toISOString() }, + ], + }; + const found: FoundEmail[] = []; + for (let page = 0; page < MAX_PAGES; page += 1) { + const response = await io.request([ + ['Email/query', { + accountId: args.jmap, + filter, + sort: [{ property: 'receivedAt', isAscending: false }], + position: page * PAGE, + limit: PAGE, + }, 'q'], + ['Email/get', { + accountId: args.jmap, + '#ids': { resultOf: 'q', name: 'Email/query', path: '/ids' }, + properties: ['id', 'messageId', 'mailboxIds', 'keywords'], + }, 'g'], + ], MAIL_USING); + const ids = responseFor(response, 'q')[1].ids as string[]; + const list = responseFor(response, 'g')[1].list as FoundEmail[]; + found.push(...list.filter((email) => (email.messageId ?? []).some((id) => bare(id) === wanted))); + if (ids.length < PAGE) return found; + } + // More than this since the attempt, and nothing matched so far: it may be further on. + return found.length > 0 ? found : 'too-many'; +} + +async function submissionsOf(io: LookupIo, jmap: string, emailId: string): Promise> { + const response = await io.request([ + ['EmailSubmission/query', { accountId: jmap, filter: { emailIds: [emailId] } }, 'q'], + ['EmailSubmission/get', { + accountId: jmap, + '#ids': { resultOf: 'q', name: 'EmailSubmission/query', path: '/ids' }, + properties: ['id', 'undoStatus'], + }, 'g'], + ], SUBMISSION_USING); + return responseFor(response, 'g')[1].list as Array<{ undoStatus?: string }>; +} + +/** What to do with a send that may have reached the server. Throws when the lookup itself fails. */ +export async function decideRetry(io: LookupIo, args: LookupArgs): Promise { + if (!args.messageId) return { action: 'undecided', why: 'no-message-id' }; + const boxesResponse = await io.request([['Mailbox/get', { accountId: args.jmap, properties: ['id', 'role'] }, 'm']], MAIL_USING); + const boxes = responseFor(boxesResponse, 'm')[1].list as Array<{ id: string; role?: string | null }>; + const folders = { drafts: boxes.find((box) => box.role === 'drafts')?.id, sent: boxes.find((box) => box.role === 'sent')?.id }; + if (!folders.drafts && !folders.sent) return { action: 'undecided', why: 'no-folders' }; + + const copies = await findCopies(io, args, folders); + if (copies === 'too-many') return { action: 'undecided', why: 'too-many' }; + const trusted = args.now - args.attemptAt <= LOOKUP_TRUSTED_MS; + + const inSent = copies.find((email) => folders.sent && email.mailboxIds?.[folders.sent]); + if (inSent) return { action: 'done', emailId: inSent.id, why: 'sent' }; + + const inDrafts = copies.find((email) => folders.drafts && email.mailboxIds?.[folders.drafts]); + if (inDrafts) { + const submissions = await submissionsOf(io, args.submissionJmap, inDrafts.id); + if (submissions.length > 0) { + const live = submissions.some((submission) => submission.undoStatus !== 'canceled'); + return { action: 'done', emailId: inDrafts.id, why: live ? 'scheduled' : 'canceled' }; + } + if (!trusted) return { action: 'undecided', why: 'too-old' }; + if (!inDrafts.keywords?.$draft) return { action: 'undecided', why: 'unexpected' }; + return { action: 'submit', emailId: inDrafts.id }; + } + + if (!args.copyKept) return { action: 'undecided', why: 'no-copy-kept' }; + if (!trusted) return { action: 'undecided', why: 'too-old' }; + return { action: 'send' }; +} diff --git a/lib/store/outbox/email-set.ts b/lib/store/outbox/email-set.ts new file mode 100644 index 000000000..5451e1c21 --- /dev/null +++ b/lib/store/outbox/email-set.ts @@ -0,0 +1,213 @@ +/** + * Email/set answered from the store: changes wait in outbox_patch, merged per + * record, and show at once. Deleting for good and emptying a folder wait the + * same way. Answers carry no state: a made-up one would reach the server later. + */ + +import { MethodError, type MethodContext, type MethodHandler, type RecordOverlay } from '../area'; +import { metaGet, metaSet } from '../engine/core-schema'; +import { deleteEmail, upsertEmail, type EmailObject } from '../mail/records'; +import type { StorageAdapter } from '../storage'; +import { applyPatch, mergePatches, normalizePatch, patchProblem, stateOf, type Patch, type RecordState } from './patches'; + +export interface PatchRow { + jmap: string; + type: string; + record_id: string; + op: 'update' | 'destroy'; + patch: string; + base: string | null; + server: string | null; + subject: string | null; + seq: number; + queued_at: number; +} + +const EMAIL = 'Email'; + +/** The next place in the order changes and sends were made. */ +export function nextSeq(db: StorageAdapter): number { + const next = Number(metaGet(db, 'outbox_seq') ?? 0) + 1; + metaSet(db, 'outbox_seq', String(next)); + return next; +} + +function pendingRow(db: StorageAdapter, jmap: string, id: string): PatchRow | undefined { + return db.get('SELECT * FROM outbox_patch WHERE jmap = ? AND type = ? AND record_id = ?', [jmap, EMAIL, id]); +} + +function savePatch(db: StorageAdapter, row: Omit, now: number): void { + db.run( + `INSERT INTO outbox_patch (jmap, type, record_id, op, patch, base, server, subject, seq, queued_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (jmap, type, record_id) DO UPDATE SET op = excluded.op, patch = excluded.patch, seq = excluded.seq`, + [row.jmap, EMAIL, row.record_id, row.op, row.patch, row.base, row.server, row.subject, nextSeq(db), now], + ); +} + +function localEmail(db: StorageAdapter, jmap: string, id: string): { email: EmailObject; gen: number } | null { + const row = db.get<{ json: string; gen: number }>('SELECT json, gen FROM email WHERE jmap = ? AND id = ?', [jmap, id]); + return row ? { email: JSON.parse(row.json) as EmailObject, gen: Number(row.gen) } : null; +} + +function subjectOf(email: EmailObject | undefined): string | null { + return typeof email?.subject === 'string' ? email.subject : null; +} + +/** Queues one change and shows it. The error for `notUpdated`, or null. */ +export function queueUpdate(db: StorageAdapter, jmap: string, id: string, rawPatch: Patch, now: number): { type: string; description?: string } | null { + const problem = patchProblem(rawPatch); + if (problem) return problem; + const patch = normalizePatch(rawPatch); + const existing = pendingRow(db, jmap, id); + if (existing?.op === 'destroy') return { type: 'notFound' }; + const local = localEmail(db, jmap, id); + const after = applyPatch(local?.email ?? ({} as EmailObject), patch); + // JMAP has no message outside every folder. Deleting is a destroy. + if ((local || 'mailboxIds' in patch) && Object.keys(after.mailboxIds ?? {}).length === 0) { + return { type: 'invalidProperties', description: 'A message has to stay in a folder' }; + } + const state = local ? JSON.stringify(stateOf(local.email)) : null; + savePatch(db, { + jmap, + record_id: id, + op: 'update', + patch: JSON.stringify(existing ? mergePatches(JSON.parse(existing.patch) as Patch, patch) : patch), + base: state, + server: state, + subject: subjectOf(local?.email), + }, now); + if (local) upsertEmail(db, jmap, after as EmailObject, local.gen); + return null; +} + +/** Queues a delete and shows it. */ +export function queueDestroy(db: StorageAdapter, jmap: string, id: string, now: number): void { + const local = localEmail(db, jmap, id); + const state = local ? JSON.stringify(stateOf(local.email)) : null; + savePatch(db, { jmap, record_id: id, op: 'destroy', patch: '{}', base: state, server: state, subject: subjectOf(local?.email) }, now); + // savePatch keeps base and server of a row already there; only the op changes. + if (local) deleteEmail(db, jmap, id); +} + +/** + * SEAM for the drafts agent: Email/set create is theirs. Until it lands every + * create is refused per message, so createDraft throws and the composer shows + * its "draft not saved" state, as it does today. + */ +function createEmails(_ctx: MethodContext, _jmap: string, create: Record): Record { + return Object.fromEntries(Object.keys(create).map((id) => [id, { type: 'serverUnavailable', description: 'Saving a draft needs the server' }])); +} + +export const emailSet: MethodHandler = (ctx, args) => { + const jmap = (args.accountId as string | undefined) ?? ctx.defaultJmap; + if (args.destroy !== undefined && args.destroy !== null && !Array.isArray(args.destroy)) { + throw new MethodError('invalidArguments', 'destroy has to be a list'); + } + const result: Record = { accountId: jmap }; + ctx.db.transaction(() => { + if (args.create && Object.keys(args.create as object).length > 0) { + result.created = {}; + result.notCreated = createEmails(ctx, jmap, args.create as Record); + } + if (args.update) { + const updated: Record = {}; + const notUpdated: Record = {}; + for (const [id, patch] of Object.entries(args.update as Record)) { + const problem = queueUpdate(ctx.db, jmap, id, patch, ctx.now()); + if (problem) notUpdated[id] = problem; + else updated[id] = null; + } + result.updated = updated; + if (Object.keys(notUpdated).length > 0) result.notUpdated = notUpdated; + } + if (Array.isArray(args.destroy)) { + for (const id of args.destroy as string[]) queueDestroy(ctx.db, jmap, id, ctx.now()); + result.destroyed = args.destroy; + } + }); + ctx.emit({ type: 'changed', account: ctx.account, collections: ['Email', 'Mailbox', 'Thread'] }); + return result; +}; + +/** What waiting changes do to the folder counts: from the server's copy to what the change makes of it. */ +function countDeltas(db: StorageAdapter, jmap: string): Map { + const rows = db.all<{ op: string; patch: string; server: string | null }>('SELECT op, patch, server FROM outbox_patch WHERE jmap = ? AND type = ?', [jmap, EMAIL]); + const deltas = new Map(); + const bump = (box: string, total: number, unread: number) => { + const delta = deltas.get(box) ?? { total: 0, unread: 0 }; + delta.total += total; + delta.unread += unread; + deltas.set(box, delta); + }; + for (const row of rows) { + if (!row.server) continue; + const before = JSON.parse(row.server) as RecordState; + const wasUnread = before.keywords?.$seen ? 0 : 1; + const boxesBefore = Object.keys(before.mailboxIds ?? {}).filter((box) => before.mailboxIds?.[box]); + if (row.op === 'destroy') { + for (const box of boxesBefore) bump(box, -1, -wasUnread); + continue; + } + const after = applyPatch(before, JSON.parse(row.patch) as Patch); + const isUnread = after.keywords?.$seen ? 0 : 1; + const boxesAfter = Object.keys(after.mailboxIds ?? {}); + for (const box of boxesBefore) if (!after.mailboxIds?.[box]) bump(box, -1, -wasUnread); + for (const box of boxesAfter) { + if (!before.mailboxIds?.[box]) bump(box, 1, isUnread); + else if (wasUnread !== isUnread) bump(box, 0, isUnread - wasUnread); + } + } + return deltas; +} + +/** + * Sync must not lose a change that waits: what the server sends goes under it. + * A record waiting to be deleted stays away. The folder counts stay the + * server's, with what waits added when they're read, so a sync can't count a + * change twice or forget it. + */ +export const outboxOverlay: RecordOverlay = (ctx, jmap, type, records) => { + if (!ctx.db.get('SELECT 1 AS x FROM outbox_patch WHERE jmap = ? LIMIT 1', [jmap])) return records; + switch (type) { + case EMAIL: { + const out: Array<{ id: string }> = []; + for (const record of records) { + const row = pendingRow(ctx.db, jmap, record.id); + if (!row) { + out.push(record); + continue; + } + const email = record as EmailObject; + ctx.db.run('UPDATE outbox_patch SET server = ?, subject = COALESCE(subject, ?) WHERE jmap = ? AND type = ? AND record_id = ?', [ + JSON.stringify(stateOf(email)), subjectOf(email), jmap, EMAIL, record.id, + ]); + if (row.op === 'update') out.push(applyPatch(email, JSON.parse(row.patch) as Patch) as EmailObject); + } + return out; + } + case 'Thread': { + const gone = new Set(ctx.db.all<{ record_id: string }>( + "SELECT record_id FROM outbox_patch WHERE jmap = ? AND type = ? AND op = 'destroy'", + [jmap, EMAIL], + ).map((row) => String(row.record_id))); + return (records as Array<{ id: string; emailIds: string[] }>).map((thread) => ({ + ...thread, + emailIds: thread.emailIds.filter((id) => !gone.has(id)), + })); + } + case 'Mailbox': { + const deltas = countDeltas(ctx.db, jmap); + if (deltas.size === 0) return records; + return (records as Array<{ id: string; totalEmails?: unknown; unreadEmails?: unknown }>).map((box) => { + const delta = deltas.get(box.id); + if (!delta) return box; + const count = (value: unknown, by: number) => (typeof value === 'number' ? Math.max(0, value + by) : value); + // Messages only: how the conversations change can't be told from here. + return { ...box, totalEmails: count(box.totalEmails, delta.total), unreadEmails: count(box.unreadEmails, delta.unread) }; + }); + } + default: + return records; + } +}; diff --git a/lib/store/outbox/flush-patches.ts b/lib/store/outbox/flush-patches.ts new file mode 100644 index 000000000..ae0901828 --- /dev/null +++ b/lib/store/outbox/flush-patches.ts @@ -0,0 +1,175 @@ +/** + * Sends the changes that waited. Written against a small `request`, so the page + * can run it through the JMAP client (its auth, its retries) while the logic + * stays testable next to the store. + * + * Per batch: read what the server has now, keep back what it changed since the + * change was queued, send the rest, and report per record. A change the server + * took is dropped and the next /changes run brings the truth. A change it + * refused, or that met a record changed since, goes to the user as a conflict. + * Anything that looks like an outage stops the run: the changes wait. + */ + +import { JmapMethodError, responseFor, type JmapResponse, type MethodCall } from '../engine/transport'; +import { EMAIL_LIST_PROPERTIES, MAIL_USING, type EmailObject } from '../mail/records'; +import type { ConflictReason } from './items'; +import { destroyConflicts, patchConflicts, type Patch, type RecordState } from './patches'; + +export interface PendingPatch { + jmap: string; + type: string; + recordId: string; + op: 'update' | 'destroy'; + patch: Patch; + base: RecordState | null; + subject: string | null; + seq: number; +} + +export type PatchOutcome = + | { jmap: string; type: string; recordId: string; seq: number; outcome: 'done' } + | { + jmap: string; + type: string; + recordId: string; + seq: number; + outcome: 'conflict'; + reason: ConflictReason; + detail?: string; + /** The record as the server has it, or null when it's gone there. Undefined when unknown. */ + server?: EmailObject | null; + }; + +export interface PatchIo { + request(calls: MethodCall[], using: string[]): Promise; +} + +const BATCH = 200; +/** Method errors that mean "not now" rather than "no". */ +const RETRY_LATER = new Set(['serverUnavailable', 'serverFail', 'serverPartialFail', 'rateLimit', 'requestTooLarge']); + +/** + * The server refused the request itself, for good: a JMAP problem document with + * a status that says so. `limit` is the parallel-request ceiling, which passes, + * and an error page says something about a proxy, not about this request. + */ +export function isRequestRefusal(error: unknown): boolean { + if (!(error instanceof Error)) return false; + const problem = /^Request failed: (?:400|403|404|413|422)\b[\s\S]*?"type"\s*:\s*"urn:ietf:params:jmap:error:([A-Za-z]+)"/.exec(error.message); + return !!problem && problem[1] !== 'limit'; +} + +function isLater(error: unknown): boolean { + if (error instanceof JmapMethodError) return RETRY_LATER.has(error.type); + return !isRequestRefusal(error); +} + +function conflict(row: PendingPatch, reason: ConflictReason, server?: EmailObject | null, detail?: string): PatchOutcome { + return { jmap: row.jmap, type: row.type, recordId: row.recordId, seq: row.seq, outcome: 'conflict', reason, detail, server }; +} + +function done(row: PendingPatch): PatchOutcome { + return { jmap: row.jmap, type: row.type, recordId: row.recordId, seq: row.seq, outcome: 'done' }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function readRecords(io: PatchIo, jmap: string, rows: PendingPatch[]): Promise> { + const read = await io.request([['Email/get', { + accountId: jmap, + ids: rows.map((row) => row.recordId), + properties: EMAIL_LIST_PROPERTIES, + }, 'g']], MAIL_USING); + const [, got] = responseFor(read, 'g'); + return new Map((got.list as EmailObject[]).map((email) => [email.id, email])); +} + +/** + * The whole batch was refused. The server's copy of each record goes with the + * conflict so that this device shows what's true again. When even that read is + * refused the record stays as it is here until the next sync. + */ +async function refusedOutcomes(io: PatchIo, jmap: string, rows: PendingPatch[], refusal: unknown): Promise { + let current: Map | null = null; + try { + current = await readRecords(io, jmap, rows); + } catch (error) { + if (isLater(error)) return null; + } + return rows.map((row) => { + if (current && !current.has(row.recordId)) return row.op === 'destroy' ? done(row) : conflict(row, 'gone', null); + return conflict(row, 'refused', current?.get(row.recordId), errorText(refusal)); + }); +} + +async function flushBatch(io: PatchIo, jmap: string, rows: PendingPatch[]): Promise { + const current = await readRecords(io, jmap, rows); + + const outcomes: PatchOutcome[] = []; + const update: Record = {}; + const destroy: string[] = []; + const sent = new Map(); + for (const row of rows) { + const server = current.get(row.recordId); + if (!server) { + // Asked to delete what's already gone: that's the same result. + outcomes.push(row.op === 'destroy' ? done(row) : conflict(row, 'gone', null)); + continue; + } + const changed = row.op === 'destroy' ? destroyConflicts(row.base, server) : patchConflicts(row.patch, row.base, server); + if (changed) { + outcomes.push(conflict(row, 'changed', server)); + continue; + } + sent.set(row.recordId, row); + if (row.op === 'destroy') destroy.push(row.recordId); + else update[row.recordId] = row.patch; + } + if (sent.size === 0) return outcomes; + + const response = await io.request([['Email/set', { + accountId: jmap, + ...(Object.keys(update).length > 0 ? { update } : {}), + ...(destroy.length > 0 ? { destroy } : {}), + }, 's']], MAIL_USING); + const [, result] = responseFor(response, 's'); + const refused: Record = { ...(result.notUpdated ?? {}), ...(result.notDestroyed ?? {}) }; + for (const [id, row] of sent) { + const refusal = refused[id]; + if (!refusal) { + outcomes.push(done(row)); + } else if (refusal.type === 'notFound') { + outcomes.push(row.op === 'destroy' ? done(row) : conflict(row, 'gone', null)); + } else { + outcomes.push(conflict(row, 'refused', current.get(id), refusal.description ?? refusal.type)); + } + } + return outcomes; +} + +/** True when all went through to an answer, false when the run stopped on an outage. */ +export async function flushPatches( + io: PatchIo, + pending: readonly PendingPatch[], + report: (outcomes: PatchOutcome[]) => Promise, +): Promise { + const byAccount = new Map(); + for (const row of pending) byAccount.set(row.jmap, [...(byAccount.get(row.jmap) ?? []), row]); + for (const [jmap, rows] of byAccount) { + for (let i = 0; i < rows.length; i += BATCH) { + const batch = rows.slice(i, i + BATCH); + let outcomes: PatchOutcome[] | null; + try { + outcomes = await flushBatch(io, jmap, batch); + } catch (error) { + if (isLater(error)) return false; + outcomes = await refusedOutcomes(io, jmap, batch, error); + if (!outcomes) return false; + } + await report(outcomes); + } + } + return true; +} diff --git a/lib/store/outbox/index.ts b/lib/store/outbox/index.ts new file mode 100644 index 000000000..4d65885cb --- /dev/null +++ b/lib/store/outbox/index.ts @@ -0,0 +1,13 @@ +import type { StoreArea } from '../area'; +import { OUTBOX_CALLS } from './calls'; +import { emailSet, outboxOverlay } from './email-set'; +import { OUTBOX_MIGRATIONS } from './schema'; + +export const outboxArea: StoreArea = { + name: 'outbox', + migrations: OUTBOX_MIGRATIONS, + methods: { 'Email/set': emailSet }, + syncers: [], + calls: OUTBOX_CALLS, + overlay: outboxOverlay, +}; diff --git a/lib/store/outbox/items.ts b/lib/store/outbox/items.ts new file mode 100644 index 000000000..e802090ee --- /dev/null +++ b/lib/store/outbox/items.ts @@ -0,0 +1,106 @@ +/** + * What a finished send looks like when it waits. An outbox entry is a finished + * message: everything a send does up to the transport ran when the user clicked + * Send (the organisation's checks, the label, signatures, plugin send checks, + * S/MIME or PGP with the user present). What is kept is the arguments of the + * final send, or the finished raw message of a plugin's send. Sending it later + * can't skip any of that. + */ + +/** The arguments of useEmailStore.sendEmail after the client (positional, as the client takes them). */ +export interface EmailSendItem { + kind: 'email'; + args: unknown[]; +} + +export interface RawSendItem { + kind: 'raw'; + /** The finished message, kept as an upload until it is sent. */ + blobId: string; + identityId: string; + delayedUntil?: string; + envelopeRecipients?: string[]; + /** submitRaw: no copy in Sent. */ + submitOnly?: boolean; +} + +export type SendItem = EmailSendItem | RawSendItem; + +export interface OutboxSummary { + subject: string; + to: string[]; +} + +/** + * - waiting: goes out by itself when the server answers; + * - sending: on its way (a tab took it); + * - failed: the server refused it; the error says why; + * - uncertain: the connection broke after it may have reached the server. + * Looked up when the server is back; stays this way only when the lookup + * can't tell, and then it's the user's to send again or delete; + * - held: the organisation's rules got stricter while it waited. + */ +export type OutboxStatus = 'waiting' | 'sending' | 'failed' | 'uncertain' | 'held'; + +/** Why an entry failed: one of ours (a code the UI translates) or the server's own words. */ +export interface OutboxError { + code?: 'file_missing' | 'no_sent_mailbox' | 'no_drafts_mailbox'; + text?: string; +} + +/** What the organisation required when the message was finished. */ +export interface PolicyFingerprint { + labelRequired: boolean; + hasLabel: boolean; + guards: string[]; +} + +export interface StoredOutboxEntry { + id: number; + seq: number; + createdAt: number; + summary: OutboxSummary; + status: OutboxStatus; + sendingSince?: number; + /** When the first try that may have reached the server began. */ + firstAttemptAt?: number; + lastAttemptAt?: number; + error?: OutboxError; + item: SendItem; + fingerprint?: PolicyFingerprint; + messageId?: string; + copyId?: string; +} + +/** Stand-in blob ids of files kept on this device start with this. */ +export const LOCAL_BLOB_PREFIX = 'bulwark-local-'; + +export function isLocalBlobId(blobId: unknown): blobId is string { + return typeof blobId === 'string' && blobId.startsWith(LOCAL_BLOB_PREFIX); +} + +/** The stand-in blob ids an outbox entry needs uploaded first. */ +export function heldBlobIds(item: SendItem): string[] { + if (item.kind === 'raw') return isLocalBlobId(item.blobId) ? [item.blobId] : []; + const attachments = item.args[10]; + if (!Array.isArray(attachments)) return []; + return attachments.map((a) => (a as { blobId?: unknown }).blobId).filter(isLocalBlobId); +} + +export type ConflictReason = 'changed' | 'gone' | 'refused'; + +export interface ConflictSummary { + subject: string; + op: 'update' | 'destroy'; + reason: ConflictReason; + /** The server's word when it refused. */ + detail?: string; +} + +export interface StoredConflict { + id: number; + jmap: string; + recordId: string; + createdAt: number; + summary: ConflictSummary; +} diff --git a/lib/store/outbox/patches.ts b/lib/store/outbox/patches.ts new file mode 100644 index 000000000..e055ae7b9 --- /dev/null +++ b/lib/store/outbox/patches.ts @@ -0,0 +1,157 @@ +/** + * JMAP PatchObjects for the two properties that can change offline, keywords + * and mailboxIds, in both forms the client sends: `keywords/$seen` paths and + * the whole map. + */ + +import { keywordKey } from '../mail/records'; + +export type Patch = Record; + +export interface RecordState { + keywords?: Record; + mailboxIds?: Record; +} + +const MAPS = ['keywords', 'mailboxIds'] as const; +type MapName = (typeof MAPS)[number]; + +function unescapeToken(token: string): string { + return token.replace(/~1/g, '/').replace(/~0/g, '~'); +} + +/** The map and the key a path changes, or null when it isn't one of ours. */ +function parsePath(path: string): { map: MapName; key: string | null } | null { + const slash = path.indexOf('/'); + const name = slash < 0 ? path : path.slice(0, slash); + if (!(MAPS as readonly string[]).includes(name)) return null; + return { map: name as MapName, key: slash < 0 ? null : unescapeToken(path.slice(slash + 1)) }; +} + +/** Why this patch can't wait offline, or null when it can. */ +export function patchProblem(patch: Patch): { type: string; description: string } | null { + const paths = Object.keys(patch); + if (paths.length === 0) return { type: 'invalidPatch', description: 'An empty change' }; + for (const path of paths) { + const parsed = parsePath(path); + if (!parsed) return { type: 'forbidden', description: `Changing ${path} needs the server` }; + // A folder made in the same request: that part needs the server, so this does too. + if (parsed.map === 'mailboxIds') { + const keys = parsed.key !== null ? [parsed.key] : Object.keys((patch[path] ?? {}) as object); + if (keys.some((key) => key.startsWith('#'))) return { type: 'forbidden', description: 'A new folder needs the server' }; + } + } + return null; +} + +/** `false` means removal to the client's code; on the wire only null does. */ +export function normalizePatch(patch: Patch): Patch { + const out: Patch = {}; + for (const [path, value] of Object.entries(patch)) out[path] = path.includes('/') && !value ? null : value; + return out; +} + +export function applyPatch(record: T, patch: Patch): T & RecordState { + const state = record as RecordState; + const next: T & RecordState = { + ...record, + keywords: { ...(state.keywords ?? {}) }, + mailboxIds: { ...(state.mailboxIds ?? {}) }, + }; + for (const [path, value] of Object.entries(patch)) { + const parsed = parsePath(path); + if (!parsed) continue; + if (parsed.key === null) { + next[parsed.map] = { ...((value ?? {}) as Record) }; + continue; + } + const map = next[parsed.map] as Record; + if (value) map[parsed.key] = true; + else delete map[parsed.key]; + } + return next; +} + +/** + * A later change to the same record on top of an earlier one. A whole map + * replaces every earlier change to that map; a path into a map the earlier + * change replaced goes into that map, since the whole map is what gets sent. + */ +export function mergePatches(earlier: Patch, later: Patch): Patch { + const out: Patch = { ...earlier }; + for (const [path, value] of Object.entries(later)) { + const parsed = parsePath(path); + if (!parsed) { + out[path] = value; + continue; + } + if (parsed.key === null) { + for (const existing of Object.keys(out)) { + if (existing === parsed.map || existing.startsWith(`${parsed.map}/`)) delete out[existing]; + } + out[path] = value; + continue; + } + if (parsed.map in out) { + const map = { ...(out[parsed.map] as Record) }; + if (value) map[parsed.key] = true; + else delete map[parsed.key]; + out[parsed.map] = map; + continue; + } + out[path] = value; + } + return out; +} + +function sameMap(a: Record | undefined, b: Record | undefined): boolean { + const left = Object.keys(a ?? {}).filter((key) => a?.[key]).map(keywordKey).sort(); + const right = Object.keys(b ?? {}).filter((key) => b?.[key]).map(keywordKey).sort(); + return left.length === right.length && left.every((key, i) => key === right[i]); +} + +function has(map: Record | undefined, key: string): boolean { + const wanted = keywordKey(key); + return Object.keys(map ?? {}).some((name) => keywordKey(name) === wanted && map?.[name]); +} + +/** + * Whether the server changed what this patch touches since it was queued. A + * change elsewhere on the record isn't a conflict, and neither is the server + * already having what the patch wants. Folders count as a whole: a move here + * after a move on another device would leave the message in both places. + * Keywords stand alone. + */ +export function patchConflicts(patch: Patch, base: RecordState | null, serverRecord: object): boolean { + if (!base) return false; + const server = serverRecord as RecordState; + const paths = Object.keys(patch).map((path) => parsePath(path)); + if (paths.some((parsed) => parsed?.map === 'mailboxIds') + && !sameMap(base.mailboxIds, server.mailboxIds) + && !sameMap(applyPatch(base, patch).mailboxIds, server.mailboxIds)) return true; + for (const [path, value] of Object.entries(patch)) { + const parsed = parsePath(path); + if (!parsed || parsed.map === 'mailboxIds') continue; + if (parsed.key === null) { + const wanted = value as Record; + if (!sameMap(base[parsed.map], server[parsed.map]) && !sameMap(wanted, server[parsed.map])) return true; + continue; + } + const before = has(base[parsed.map], parsed.key); + const now = has(server[parsed.map], parsed.key); + if (before !== now && now !== Boolean(value)) return true; + } + return false; +} + +/** Deleting a record the server changed since: the user should see that. */ +export function destroyConflicts(base: RecordState | null, serverRecord: object): boolean { + if (!base) return false; + const server = serverRecord as RecordState; + return !sameMap(base.keywords, server.keywords) || !sameMap(base.mailboxIds, server.mailboxIds); +} + +export function stateOf(record: object): RecordState { + const state = record as RecordState; + return { keywords: state.keywords ?? {}, mailboxIds: state.mailboxIds ?? {} }; +} diff --git a/lib/store/outbox/schema.ts b/lib/store/outbox/schema.ts new file mode 100644 index 000000000..2f9ed83e5 --- /dev/null +++ b/lib/store/outbox/schema.ts @@ -0,0 +1,73 @@ +import type { Migration } from '../storage'; + +export const OUTBOX_MIGRATIONS: readonly Migration[] = [ + { + id: 201, + name: 'outbox_last_attempt', + sql: ` + -- When the latest try began. It becomes first_attempt_at only once a try + -- may have reached the server. + ALTER TABLE outbox ADD COLUMN last_attempt_at INTEGER; + `, + }, + { + id: 200, + name: 'outbox', + sql: ` + -- Finished messages waiting for the server. seq is shared with + -- outbox_patch, so changes and sends go out in the order they were made. + CREATE TABLE outbox ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + seq INTEGER NOT NULL, + kind TEXT NOT NULL, + created_at INTEGER NOT NULL, + status TEXT NOT NULL, + sending_since INTEGER, + -- When it was first handed to the server: where a retry's lookup starts. + first_attempt_at INTEGER, + payload TEXT NOT NULL, + summary TEXT NOT NULL, + message_id TEXT, + -- The copy already on the server that only needs submitting. + copy_id TEXT, + error TEXT + ); + -- One row per record, later changes merged in. base is the record's state + -- when first queued, to tell whether the server changed it since; server + -- is the newest state the server sent, for the folder counts. + CREATE TABLE outbox_patch ( + jmap TEXT NOT NULL, + type TEXT NOT NULL, + record_id TEXT NOT NULL, + op TEXT NOT NULL, + patch TEXT NOT NULL, + base TEXT, + server TEXT, + subject TEXT, + seq INTEGER NOT NULL, + queued_at INTEGER NOT NULL, + PRIMARY KEY (jmap, type, record_id) + ) WITHOUT ROWID; + CREATE INDEX outbox_patch_seq ON outbox_patch (seq); + CREATE TABLE conflict ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + jmap TEXT NOT NULL, + type TEXT NOT NULL, + record_id TEXT NOT NULL, + summary TEXT NOT NULL, + local TEXT NOT NULL, + created_at INTEGER NOT NULL + ); + -- Files attached offline, under a stand-in blob id. + CREATE TABLE upload ( + local_id TEXT PRIMARY KEY, + jmap TEXT, + type TEXT, + name TEXT, + size INTEGER NOT NULL, + data BLOB NOT NULL, + created_at INTEGER NOT NULL + ) WITHOUT ROWID; + `, + }, +]; diff --git a/lib/store/storage.ts b/lib/store/storage.ts index 5ca4c29af..d59a1b4c6 100644 --- a/lib/store/storage.ts +++ b/lib/store/storage.ts @@ -52,6 +52,16 @@ export function runTransaction( } } +/** + * The disk is full, the browser's quota ran out, the file can't be written. + * Trying again right away can't help, and each try costs network and power. + */ +export function isStorageFailure(error: unknown): boolean { + if (!(error instanceof Error)) return false; + if (error.name === 'QuotaExceededError') return true; + return /SQLITE_FULL|SQLITE_IOERR|SQLITE_CANTOPEN|disk is full|disk I\/O error|QuotaExceeded|No space left/i.test(error.message); +} + export interface Migration { /** Unique, ascending. Ranges per area are listed in lib/store/area.ts. */ id: number; diff --git a/lib/store/worker/host.ts b/lib/store/worker/host.ts index 636c231c9..3a2a7036f 100644 --- a/lib/store/worker/host.ts +++ b/lib/store/worker/host.ts @@ -9,6 +9,11 @@ import { FetchTransport, type StoreSession } from '../engine/transport'; import type { StorageAdapter } from '../storage'; import type { OpenAccountParams, StoreRpc, StoreStatus } from './protocol'; +/** Files in the pool that belong to no key. A name that isn't a store file's is left alone. */ +export function storeFilesToPrune(files: readonly string[], keep: readonly string[]): string[] { + return files.filter((file) => /^\/a[0-9a-f]+\.db$/.test(file) && !keep.includes(file)); +} + export interface HostStorage { open(file: string, keyHex: string): Promise; remove(file: string): boolean; @@ -30,6 +35,13 @@ export interface StoreHostOptions { needAuth(account: string): void; } +/** The owner hasn't opened this account yet. The caller may try again shortly. */ +function accountClosed(): Error { + const error = new Error('The local store is not open for this account'); + error.name = 'StoreAccountClosedError'; + return error; +} + export class StoreHost { private readonly accounts = new Map(); private online = true; @@ -37,17 +49,18 @@ export class StoreHost { constructor(private readonly options: StoreHostOptions) {} - async openAccount(params: OpenAccountParams): Promise { + async openAccount(params: OpenAccountParams): Promise<{ created: boolean }> { const existing = this.accounts.get(params.account); if (existing) { if (params.session) this.setSession(params.account, params.session); this.setAuth(params.account, params.auth); existing.engine.setScope(params.scope); - existing.engine.setInvertedKeywordSort(params.invertedKeywordSort); + if (params.invertedKeywordSort !== null) existing.engine.setInvertedKeywordSort(params.invertedKeywordSort); void existing.engine.requestSync(); - return; + return { created: false }; } const db = await this.options.storage.open(params.file, params.keyHex); + const created = !db.get("SELECT 1 AS yes FROM sqlite_master WHERE name = 'schema_migration'"); const session = params.session ?? AccountEngine.storedSession(db); if (!session) { db.close(); @@ -69,7 +82,7 @@ export class StoreHost { transport, areas: this.options.areas, scope: params.scope, - invertedKeywordSort: params.invertedKeywordSort, + invertedKeywordSort: params.invertedKeywordSort ?? undefined, emit: (event) => { if (event.type === 'synced') this.synced.set(event.account, event.at); this.options.emit(event); @@ -78,6 +91,7 @@ export class StoreHost { account.engine.setOnline(this.online); this.accounts.set(params.account, account); void account.engine.requestSync(); + return { created }; } setAuth(account: string, auth: string | null): void { @@ -101,6 +115,10 @@ export class StoreHost { void open.engine.requestSync(); } + setInvertedKeywordSort(account: string, inverted: boolean): void { + this.accounts.get(account)?.engine.setInvertedKeywordSort(inverted); + } + async closeAccount(account: string): Promise { const open = this.accounts.get(account); if (!open) return; @@ -126,7 +144,7 @@ export class StoreHost { case 'answer': { const { account, calls } = params as Parameters[0]; const open = this.accounts.get(account); - if (!open) throw new Error('The local store is not open for this account'); + if (!open) throw accountClosed(); return open.engine.answer(calls); } case 'sync': { @@ -152,9 +170,13 @@ export class StoreHost { case 'area': { const { account, name, params: callParams } = params as Parameters[0]; const open = this.accounts.get(account); - if (!open) throw new Error('The local store is not open for this account'); + if (!open) throw accountClosed(); return open.engine.areaCall(name, callParams); } + case 'removeAccount': { + const { account, file } = params as Parameters[0]; + return this.removeAccount(account, file); + } default: throw new Error(`Unknown store call ${String(method)}`); } diff --git a/lib/store/worker/protocol.ts b/lib/store/worker/protocol.ts index ed5a08f96..5fefc7e0f 100644 --- a/lib/store/worker/protocol.ts +++ b/lib/store/worker/protocol.ts @@ -5,6 +5,8 @@ import type { JmapResponse, MethodCall, StoreSession } from '../engine/transport export const RPC_CHANNEL = 'bulwark-store-rpc'; export const OWNER_LOCK = 'bulwark-store-owner'; +/** The OPFS folder of the database pool. The page can delete it without loading wasm. */ +export const POOL_DIRECTORY = '.bulwark-store'; export interface OpenAccountParams { account: string; @@ -14,7 +16,8 @@ export interface OpenAccountParams { keyHex: string; session: StoreSession | null; scope: StoreScope; - invertedKeywordSort: boolean; + /** Null when this tab's client hasn't probed it: the stored answer stays. */ + invertedKeywordSort: boolean | null; /** The page's current Authorization header. */ auth: string | null; } @@ -28,6 +31,8 @@ export interface StoreRpc { status(params: { account: string }): StoreStatus | null; /** A non-JMAP call into an area of the store. */ area(params: { account: string; name: string; params: unknown }): unknown; + /** Closes an account and deletes its file. The name holds no account data. */ + removeAccount(params: { account: string; file: string }): boolean; } export interface StoreStatus { @@ -52,21 +57,27 @@ export type RpcReply = { to: string; result?: unknown; error?: string; + errorName?: string; }; export type ChannelMessage = | RpcRequest | RpcReply | { kind: 'event'; event: StoreEvent } - | { kind: 'owner-changed'; owner: string | null }; + | { kind: 'owner-changed'; owner: string | null } + // A tab started after the owner announced itself asks. + | { kind: 'who-owns' } + // Everyone signed out: every tab stops its worker, so the pool can go. + | { kind: 'shutdown' }; /** From this tab's page to its own worker, never on the channel. */ export type PageToWorker = | { kind: 'init'; clientId: string; moduleUrl: string } | { kind: 'open'; id: string; params: OpenAccountParams } | { kind: 'close-account'; id: string; account: string } - | { kind: 'remove-account'; id: string; account: string; file: string } + | { kind: 'prune-files'; id: string; keep: string[] } | { kind: 'set-auth'; account: string; auth: string | null } + | { kind: 'set-inverted'; account: string; inverted: boolean } | { kind: 'set-session'; account: string; session: StoreSession } | { kind: 'set-scope'; account: string; scope: StoreScope } | { kind: 'release'; id: string }; diff --git a/lib/store/worker/worker.ts b/lib/store/worker/worker.ts index 003340c82..a41ee1405 100644 --- a/lib/store/worker/worker.ts +++ b/lib/store/worker/worker.ts @@ -9,7 +9,7 @@ import { WasmStorage } from '../adapters/wasm'; import { STORE_AREAS } from '../areas'; import { RPC_CHANNEL, OWNER_LOCK, type ChannelMessage, type PageToWorker, type WorkerToPage } from './protocol'; -import { StoreHost } from './host'; +import { StoreHost, storeFilesToPrune } from './host'; const scope = self as unknown as { postMessage(message: WorkerToPage): void; @@ -56,6 +56,10 @@ async function becomeOwner(): Promise { channel = new BroadcastChannel(RPC_CHANNEL); channel.onmessage = async (event: MessageEvent) => { const message = event.data; + if (message.kind === 'who-owns') { + channel?.postMessage({ kind: 'owner-changed', owner: clientId } satisfies ChannelMessage); + return; + } if (message.kind !== 'rpc') return; try { const result = await requireHost().call(message.method, message.params); @@ -66,6 +70,7 @@ async function becomeOwner(): Promise { id: message.id, to: message.from, error: error instanceof Error ? error.message : String(error), + errorName: error instanceof Error ? error.name : undefined, } satisfies ChannelMessage); } }; @@ -105,7 +110,14 @@ scope.addEventListener('message', async (event) => { break; case 'open': try { - await requireHost().openAccount(message.params); + reply(message.id, await requireHost().openAccount(message.params)); + } catch (error) { + replyError(message.id, error); + } + break; + case 'prune-files': + try { + for (const file of storeFilesToPrune(storage?.files() ?? [], message.keep)) storage?.remove(file); reply(message.id); } catch (error) { replyError(message.id, error); @@ -119,13 +131,6 @@ scope.addEventListener('message', async (event) => { replyError(message.id, error); } break; - case 'remove-account': - try { - reply(message.id, await requireHost().removeAccount(message.account, message.file)); - } catch (error) { - replyError(message.id, error); - } - break; case 'set-auth': host?.setAuth(message.account, message.auth); break; @@ -135,6 +140,9 @@ scope.addEventListener('message', async (event) => { case 'set-scope': host?.setScope(message.account, message.scope); break; + case 'set-inverted': + host?.setInvertedKeywordSort(message.account, message.inverted); + break; case 'release': try { await release(); diff --git a/locales/ar/common.json b/locales/ar/common.json index 91db0f8ea..4ddf9d120 100644 --- a/locales/ar/common.json +++ b/locales/ar/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "تعذّر العثور على مجلد المسودات." } }, + "offlineConflicts": { + "title": "تغييرات لم تُطبَّق", + "description": "أجريت هذه التغييرات دون اتصال. لم تُطبَّق وتبقى الرسالة كما هي على الخادم.", + "changed": "تم التغيير على جهاز آخر، لذا لم يُطبَّق تغييرك", + "changed_delete": "تم التغيير على جهاز آخر، لذا لم تُحذف", + "gone": "لم تعد على الخادم، لذا لم يُطبَّق تغييرك", + "refused": "رفض الخادم هذا التغيير: {reason}", + "dismiss": "إغلاق", + "dismiss_label": "إغلاق الملاحظة حول {subject}", + "banner": "تغييرات لم تُطبَّق: {count}", + "toast": "تغييرات أُجريت دون اتصال ولم تُطبَّق: {count}. يوضح صندوق الصادر أيها.", + "storage_failed": "لم تعد هناك مساحة على هذا الجهاز للبريد دون اتصال. أفرغ مساحة أو احتفظ بأقل من الإعدادات.", + "signout_changes_title": "هل تريد تسجيل الخروج مع تغييرات غير محفوظة؟", + "signout_changes": "تغييرات لم تصل إلى الخادم بعد: {count}. يؤدي تسجيل الخروج إلى حذفها.", + "store_unavailable": "لا يمكن فتح البريد دون اتصال على هذا الجهاز الآن. يبقى البريد غير المرسل هنا ويواصل التطبيق المحاولة.", + "store_reset": "تعذرت قراءة البريد دون اتصال على هذا الجهاز، لذا يجري تنزيله من جديد. ما لم يُرسل قد فُقد.", + "legacy_stranded": "لا يمكن لهذا المتصفح فتح البريد غير المرسل المحفوظ من إصدار سابق. قد يعيده تحديث المتصفح.", + "store_private": "لا يمكن للنوافذ الخاصة الاحتفاظ بالبريد للاستخدام دون اتصال.", + "server_not_answering": "خادم البريد لا يستجيب. حاول مرة أخرى بعد قليل." + }, "deep_link": { "message_not_found": "لم تعد هذه الرسالة متاحة.", "thread_not_found": "لم تعد هذه المحادثة متاحة.", diff --git a/locales/ca/common.json b/locales/ca/common.json index d2e062a69..d878d3899 100644 --- a/locales/ca/common.json +++ b/locales/ca/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "No s'ha trobat la carpeta Esborranys." } }, + "offlineConflicts": { + "title": "Canvis no aplicats", + "description": "Heu fet aquests canvis sense connexió. No s'han aplicat i el missatge queda tal com és al servidor.", + "changed": "Canviat en un altre dispositiu, el vostre canvi no s'ha aplicat", + "changed_delete": "Canviat en un altre dispositiu, no s'ha suprimit", + "gone": "Ja no és al servidor, el vostre canvi no s'ha aplicat", + "refused": "El servidor ha rebutjat aquest canvi: {reason}", + "dismiss": "Descarta", + "dismiss_label": "Descarta la nota sobre {subject}", + "banner": "Canvis no aplicats: {count}", + "toast": "Canvis fets sense connexió que no s'han aplicat: {count}. La safata de sortida indica quins.", + "storage_failed": "Aquest dispositiu no té més espai per al correu sense connexió. Allibereu espai o conserveu-ne menys a la configuració.", + "signout_changes_title": "Voleu tancar la sessió amb canvis sense desar?", + "signout_changes": "Canvis que encara no són al servidor: {count}. En tancar la sessió s'eliminen.", + "store_unavailable": "El correu sense connexió d'aquest dispositiu no es pot obrir ara. El correu no enviat es queda aquí i l'aplicació ho continua provant.", + "store_reset": "No s'ha pogut llegir el correu sense connexió d'aquest dispositiu, així que es torna a baixar. El que no s'havia enviat s'ha perdut.", + "legacy_stranded": "Aquest navegador no pot obrir el correu no enviat desat per una versió anterior. Actualitzar el navegador el pot recuperar.", + "store_private": "Les finestres privades no poden desar correu per fer-lo servir sense connexió.", + "server_not_answering": "El servidor de correu no respon. Torneu-ho a provar d'aquí a un moment." + }, "deep_link": { "message_not_found": "Aquest missatge ja no està disponible.", "thread_not_found": "Aquesta conversa ja no està disponible.", diff --git a/locales/cs/common.json b/locales/cs/common.json index 6b1e41754..a07558c21 100644 --- a/locales/cs/common.json +++ b/locales/cs/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Složka Koncepty nebyla nalezena." } }, + "offlineConflicts": { + "title": "Nepoužité změny", + "description": "Tyto změny jste provedli offline. Nebyly použity a zpráva zůstává tak, jak je na serveru.", + "changed": "Změněno na jiném zařízení, vaše změna nebyla použita", + "changed_delete": "Změněno na jiném zařízení, zpráva nebyla smazána", + "gone": "Už není na serveru, vaše změna nebyla použita", + "refused": "Server tuto změnu odmítl: {reason}", + "dismiss": "Zavřít", + "dismiss_label": "Zavřít poznámku o {subject}", + "banner": "Nepoužité změny: {count}", + "toast": "Nepoužité změny provedené offline: {count}. Které to jsou, ukazuje pošta k odeslání.", + "storage_failed": "Na tomto zařízení už není místo pro poštu offline. Uvolněte místo nebo v nastavení ponechte méně.", + "signout_changes_title": "Odhlásit se s neuloženými změnami?", + "signout_changes": "Změny, které ještě nejsou na serveru: {count}. Odhlášení je smaže.", + "store_unavailable": "Poštu offline na tomto zařízení teď nelze otevřít. Neodeslaná pošta tu zůstává a aplikace to dál zkouší.", + "store_reset": "Poštu offline na tomto zařízení se nepodařilo přečíst, proto se stahuje znovu. Co nebylo odesláno, je pryč.", + "legacy_stranded": "Tento prohlížeč neotevře neodeslanou poštu uloženou starší verzí. Aktualizace prohlížeče ji může vrátit.", + "store_private": "Soukromá okna nemohou uchovávat poštu pro použití offline.", + "server_not_answering": "Poštovní server neodpovídá. Zkuste to za chvíli znovu." + }, "deep_link": { "message_not_found": "Tato zpráva už není dostupná.", "thread_not_found": "Tato konverzace už není dostupná.", diff --git a/locales/da/common.json b/locales/da/common.json index 61d6a1412..368d343bf 100644 --- a/locales/da/common.json +++ b/locales/da/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Mappen Kladder blev ikke fundet." } }, + "offlineConflicts": { + "title": "Ændringer, der ikke blev gennemført", + "description": "Du lavede disse ændringer offline. De blev ikke gennemført, og beskeden forbliver, som den er på serveren.", + "changed": "Ændret på en anden enhed, din ændring blev ikke gennemført", + "changed_delete": "Ændret på en anden enhed, den blev ikke slettet", + "gone": "Findes ikke længere på serveren, din ændring blev ikke gennemført", + "refused": "Serveren afviste denne ændring: {reason}", + "dismiss": "Afvis", + "dismiss_label": "Afvis noten om {subject}", + "banner": "Ændringer, der ikke blev gennemført: {count}", + "toast": "Ændringer lavet offline, der ikke blev gennemført: {count}. Udbakken viser hvilke.", + "storage_failed": "Denne enhed har ikke mere plads til offlinemail. Frigør plads, eller behold mindre i indstillingerne.", + "signout_changes_title": "Log ud med ændringer, der ikke er gemt?", + "signout_changes": "Ændringer, der endnu ikke er på serveren: {count}. Når du logger ud, slettes de.", + "store_unavailable": "Offlinemail på denne enhed kan ikke åbnes lige nu. Ikke-sendt mail bliver her, og appen bliver ved med at prøve.", + "store_reset": "Offlinemail på denne enhed kunne ikke læses og hentes derfor igen. Det, der ikke var sendt, er væk.", + "legacy_stranded": "Denne browser kan ikke åbne ikke-sendt mail fra en tidligere version. En opdatering af browseren kan hente den tilbage.", + "store_private": "Private vinduer kan ikke gemme mail til brug offline.", + "server_not_answering": "Mailserveren svarer ikke. Prøv igen om lidt." + }, "deep_link": { "message_not_found": "Denne besked er ikke længere tilgængelig.", "thread_not_found": "Denne samtale er ikke længere tilgængelig.", diff --git a/locales/de/common.json b/locales/de/common.json index eb1ec9a29..a815148d2 100644 --- a/locales/de/common.json +++ b/locales/de/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Ihr Ordner „Entwürfe“ wurde nicht gefunden." } }, + "offlineConflicts": { + "title": "Nicht übernommene Änderungen", + "description": "Sie haben diese Änderungen offline gemacht. Sie wurden nicht übernommen, die Nachricht bleibt wie auf dem Server.", + "changed": "Auf einem anderen Gerät geändert, Ihre Änderung wurde nicht übernommen", + "changed_delete": "Auf einem anderen Gerät geändert, die Nachricht wurde nicht gelöscht", + "gone": "Nicht mehr auf dem Server, Ihre Änderung wurde nicht übernommen", + "refused": "Der Server hat diese Änderung abgelehnt: {reason}", + "dismiss": "Ausblenden", + "dismiss_label": "Hinweis zu {subject} ausblenden", + "banner": "Nicht übernommene Änderungen: {count}", + "toast": "Offline gemachte Änderungen wurden nicht übernommen: {count}. Welche, steht im Postausgang.", + "storage_failed": "Auf diesem Gerät ist kein Platz mehr für Offline-E-Mails. Geben Sie Speicher frei oder behalten Sie in den Einstellungen weniger.", + "signout_changes_title": "Mit nicht gespeicherten Änderungen abmelden?", + "signout_changes": "Änderungen, die noch nicht auf dem Server sind: {count}. Beim Abmelden werden sie gelöscht.", + "store_unavailable": "Die Offline-E-Mails auf diesem Gerät lassen sich gerade nicht öffnen. Nicht gesendete E-Mails bleiben hier, und die App versucht es weiter.", + "store_reset": "Die Offline-E-Mails auf diesem Gerät konnten nicht gelesen werden und werden neu heruntergeladen. Was hier noch nicht gesendet war, ist verloren.", + "legacy_stranded": "Dieser Browser kann nicht gesendete E-Mails einer früheren Version nicht öffnen. Ein Browser-Update kann sie zurückholen.", + "store_private": "Private Fenster können keine E-Mails für die Offline-Nutzung behalten.", + "server_not_answering": "Der Mailserver antwortet nicht. Versuchen Sie es gleich noch einmal." + }, "deep_link": { "message_not_found": "Diese Nachricht ist nicht mehr verfügbar.", "thread_not_found": "Diese Unterhaltung ist nicht mehr verfügbar.", diff --git a/locales/en/common.json b/locales/en/common.json index 0af0e143f..177ba38b4 100644 --- a/locales/en/common.json +++ b/locales/en/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Your Drafts folder couldn't be found." } }, + "offlineConflicts": { + "title": "Changes that didn't go through", + "description": "You made these changes offline. They weren't applied, and the message stays as it is on the server.", + "changed": "Changed on another device, so your change wasn't applied", + "changed_delete": "Changed on another device, so it wasn't deleted", + "gone": "No longer on the server, so your change wasn't applied", + "refused": "The server refused this change: {reason}", + "dismiss": "Dismiss", + "dismiss_label": "Dismiss the note about {subject}", + "banner": "Changes that didn't go through: {count}", + "toast": "Changes you made offline weren't applied: {count}. The outbox shows which.", + "storage_failed": "This device has no room left for offline mail. Free up space, or keep less in Settings.", + "signout_changes_title": "Sign out with unsaved changes?", + "signout_changes": "Changes not yet on the server: {count}. Signing out deletes them.", + "store_unavailable": "Offline mail on this device can't be opened right now. Unsent mail stays here, and the app keeps trying.", + "store_reset": "Offline mail on this device couldn't be read, so it's being downloaded again. Anything unsent that was kept here is gone.", + "legacy_stranded": "This browser can't open unsent mail kept by an earlier version. Updating the browser may bring it back.", + "store_private": "Private windows can't keep mail for offline use.", + "server_not_answering": "The mail server isn't answering. Try again in a moment." + }, "deep_link": { "message_not_found": "This message is no longer available.", "thread_not_found": "This conversation is no longer available.", diff --git a/locales/es/common.json b/locales/es/common.json index 83aeead77..3061c636f 100644 --- a/locales/es/common.json +++ b/locales/es/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "No se encontró tu carpeta Borradores." } }, + "offlineConflicts": { + "title": "Cambios no aplicados", + "description": "Hiciste estos cambios sin conexión. No se aplicaron y el mensaje queda como está en el servidor.", + "changed": "Cambiado en otro dispositivo, tu cambio no se aplicó", + "changed_delete": "Cambiado en otro dispositivo, no se eliminó", + "gone": "Ya no está en el servidor, tu cambio no se aplicó", + "refused": "El servidor rechazó este cambio: {reason}", + "dismiss": "Descartar", + "dismiss_label": "Descartar la nota sobre {subject}", + "banner": "Cambios no aplicados: {count}", + "toast": "Cambios hechos sin conexión que no se aplicaron: {count}. La bandeja de salida indica cuáles.", + "storage_failed": "Este dispositivo no tiene espacio para el correo sin conexión. Libera espacio o guarda menos en Ajustes.", + "signout_changes_title": "¿Cerrar sesión con cambios sin guardar?", + "signout_changes": "Cambios que aún no están en el servidor: {count}. Al cerrar sesión se eliminan.", + "store_unavailable": "El correo sin conexión de este dispositivo no se puede abrir ahora. El correo sin enviar sigue aquí y la aplicación sigue intentándolo.", + "store_reset": "No se pudo leer el correo sin conexión de este dispositivo, así que se está descargando de nuevo. Lo que estaba sin enviar se ha perdido.", + "legacy_stranded": "Este navegador no puede abrir el correo sin enviar guardado por una versión anterior. Actualizar el navegador puede recuperarlo.", + "store_private": "Las ventanas privadas no pueden guardar correo para usarlo sin conexión.", + "server_not_answering": "El servidor de correo no responde. Inténtalo de nuevo en un momento." + }, "deep_link": { "message_not_found": "Este mensaje ya no está disponible.", "thread_not_found": "Esta conversación ya no está disponible.", diff --git a/locales/et/common.json b/locales/et/common.json index 788fb5171..f26b001c5 100644 --- a/locales/et/common.json +++ b/locales/et/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Kausta Mustandid ei leitud." } }, + "offlineConflicts": { + "title": "Rakendamata muudatused", + "description": "Tegite need muudatused võrguühenduseta. Neid ei rakendatud ja kiri jääb selliseks, nagu see on serveris.", + "changed": "Muudetud teises seadmes, teie muudatust ei rakendatud", + "changed_delete": "Muudetud teises seadmes, seda ei kustutatud", + "gone": "Pole enam serveris, teie muudatust ei rakendatud", + "refused": "Server keeldus sellest muudatusest: {reason}", + "dismiss": "Sulge", + "dismiss_label": "Sulge märkus teemal {subject}", + "banner": "Rakendamata muudatused: {count}", + "toast": "Võrguühenduseta tehtud rakendamata muudatused: {count}. Väljundkaust näitab, millised.", + "storage_failed": "Selles seadmes pole enam ruumi võrguühenduseta kirjade jaoks. Vabastage ruumi või hoidke seadetes vähem.", + "signout_changes_title": "Kas logida välja salvestamata muudatustega?", + "signout_changes": "Muudatused, mida serveris veel pole: {count}. Väljalogimine kustutab need.", + "store_unavailable": "Selle seadme võrguühenduseta kirju ei saa praegu avada. Saatmata kirjad jäävad siia ja rakendus proovib edasi.", + "store_reset": "Selle seadme võrguühenduseta kirju ei õnnestunud lugeda, seega laaditakse need uuesti alla. Mis oli saatmata, on kadunud.", + "legacy_stranded": "See brauser ei saa avada varasema versiooni hoitud saatmata kirju. Brauseri uuendamine võib need tagasi tuua.", + "store_private": "Privaatsed aknad ei saa kirju võrguühenduseta kasutamiseks hoida.", + "server_not_answering": "Meiliserver ei vasta. Proovige kohe uuesti." + }, "deep_link": { "message_not_found": "See sõnum ei ole enam saadaval.", "thread_not_found": "See vestlus ei ole enam saadaval.", diff --git a/locales/fa/common.json b/locales/fa/common.json index 9eadf3199..8e39bcc95 100644 --- a/locales/fa/common.json +++ b/locales/fa/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "پوشهٔ پیش‌نویس‌ها پیدا نشد." } }, + "offlineConflicts": { + "title": "تغییراتی که اعمال نشدند", + "description": "این تغییرات را در حالت آفلاین انجام دادید. اعمال نشدند و پیام همان‌طور که روی سرور است می‌ماند.", + "changed": "در دستگاه دیگری تغییر کرده، پس تغییر شما اعمال نشد", + "changed_delete": "در دستگاه دیگری تغییر کرده، پس حذف نشد", + "gone": "دیگر روی سرور نیست، پس تغییر شما اعمال نشد", + "refused": "سرور این تغییر را رد کرد: {reason}", + "dismiss": "بستن", + "dismiss_label": "بستن یادداشت درباره {subject}", + "banner": "تغییراتی که اعمال نشدند: {count}", + "toast": "تغییرات آفلاین که اعمال نشدند: {count}. صندوق خروجی نشان می‌دهد کدام‌ها.", + "storage_failed": "در این دستگاه جایی برای ایمیل آفلاین نمانده است. فضا را آزاد کنید یا در تنظیمات مقدار کمتری نگه دارید.", + "signout_changes_title": "با تغییرات ذخیره‌نشده از حساب خارج می‌شوید؟", + "signout_changes": "تغییراتی که هنوز روی سرور نیستند: {count}. خروج از حساب آن‌ها را حذف می‌کند.", + "store_unavailable": "ایمیل آفلاین این دستگاه اکنون باز نمی‌شود. ایمیل ارسال‌نشده اینجا می‌ماند و برنامه همچنان تلاش می‌کند.", + "store_reset": "ایمیل آفلاین این دستگاه خوانده نشد و دوباره بارگیری می‌شود. آنچه ارسال نشده بود از بین رفته است.", + "legacy_stranded": "این مرورگر نمی‌تواند ایمیل ارسال‌نشدهٔ نگه‌داشته‌شده از نسخهٔ قبلی را باز کند. به‌روزرسانی مرورگر ممکن است آن را برگرداند.", + "store_private": "پنجره‌های خصوصی نمی‌توانند ایمیل را برای استفادهٔ آفلاین نگه دارند.", + "server_not_answering": "سرور ایمیل پاسخ نمی‌دهد. کمی بعد دوباره امتحان کنید." + }, "deep_link": { "message_not_found": "این پیام دیگر در دسترس نیست.", "thread_not_found": "این گفتگو دیگر در دسترس نیست.", diff --git a/locales/fr/common.json b/locales/fr/common.json index a44af9dbe..a7def2938 100644 --- a/locales/fr/common.json +++ b/locales/fr/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Votre dossier Brouillons est introuvable." } }, + "offlineConflicts": { + "title": "Modifications non appliquées", + "description": "Vous avez fait ces modifications hors ligne. Elles n'ont pas été appliquées et le message reste tel qu'il est sur le serveur.", + "changed": "Modifié sur un autre appareil, votre modification n'a pas été appliquée", + "changed_delete": "Modifié sur un autre appareil, le message n'a pas été supprimé", + "gone": "N'est plus sur le serveur, votre modification n'a pas été appliquée", + "refused": "Le serveur a refusé cette modification : {reason}", + "dismiss": "Ignorer", + "dismiss_label": "Ignorer la note sur {subject}", + "banner": "Modifications non appliquées : {count}", + "toast": "Des modifications faites hors ligne n'ont pas été appliquées : {count}. La boîte d'envoi indique lesquelles.", + "storage_failed": "Cet appareil n'a plus de place pour le courrier hors ligne. Libérez de l'espace ou conservez moins dans les paramètres.", + "signout_changes_title": "Se déconnecter avec des modifications non enregistrées ?", + "signout_changes": "Modifications pas encore sur le serveur : {count}. La déconnexion les supprime.", + "store_unavailable": "Le courrier hors ligne de cet appareil ne peut pas être ouvert pour le moment. Les messages non envoyés restent ici et l'application continue d'essayer.", + "store_reset": "Le courrier hors ligne de cet appareil n'a pas pu être lu : il est de nouveau téléchargé. Ce qui n'était pas envoyé est perdu.", + "legacy_stranded": "Ce navigateur ne peut pas ouvrir les messages non envoyés conservés par une version précédente. Le mettre à jour peut les récupérer.", + "store_private": "Les fenêtres privées ne peuvent pas conserver le courrier pour une utilisation hors ligne.", + "server_not_answering": "Le serveur de messagerie ne répond pas. Réessayez dans un instant." + }, "deep_link": { "message_not_found": "Ce message n'est plus disponible.", "thread_not_found": "Cette conversation n'est plus disponible.", diff --git a/locales/he/common.json b/locales/he/common.json index 17580b9a8..ce9459e20 100644 --- a/locales/he/common.json +++ b/locales/he/common.json @@ -5744,6 +5744,26 @@ "no_drafts_mailbox": "התיקייה 'טיוטות' לא נמצאה." } }, + "offlineConflicts": { + "title": "שינויים שלא הוחלו", + "description": "ביצעת את השינויים האלה במצב לא מקוון. הם לא הוחלו וההודעה נשארת כפי שהיא בשרת.", + "changed": "שונה במכשיר אחר, ולכן השינוי שלך לא הוחל", + "changed_delete": "שונה במכשיר אחר, ולכן לא נמחק", + "gone": "כבר לא בשרת, ולכן השינוי שלך לא הוחל", + "refused": "השרת דחה את השינוי הזה: {reason}", + "dismiss": "סגירה", + "dismiss_label": "סגירת ההערה על {subject}", + "banner": "שינויים שלא הוחלו: {count}", + "toast": "שינויים במצב לא מקוון שלא הוחלו: {count}. דואר יוצא מראה אילו.", + "storage_failed": "אין יותר מקום במכשיר הזה לדואר לא מקוון. יש לפנות מקום או לשמור פחות בהגדרות.", + "signout_changes_title": "לצאת מהחשבון עם שינויים שלא נשמרו?", + "signout_changes": "שינויים שעדיין לא בשרת: {count}. היציאה מהחשבון מוחקת אותם.", + "store_unavailable": "אי אפשר לפתוח כרגע את הדואר הלא מקוון במכשיר הזה. דואר שלא נשלח נשאר כאן, והאפליקציה ממשיכה לנסות.", + "store_reset": "לא ניתן היה לקרוא את הדואר הלא מקוון במכשיר הזה, ולכן הוא יורד מחדש. מה שלא נשלח אבד.", + "legacy_stranded": "הדפדפן הזה לא יכול לפתוח דואר שלא נשלח שנשמר בגרסה קודמת. עדכון הדפדפן עשוי להחזיר אותו.", + "store_private": "חלונות פרטיים לא יכולים לשמור דואר לשימוש לא מקוון.", + "server_not_answering": "שרת הדואר לא עונה. אפשר לנסות שוב בעוד רגע." + }, "deep_link": { "message_not_found": "הודעה זו אינה זמינה עוד.", "thread_not_found": "שיחה זו אינה זמינה עוד.", diff --git a/locales/hu/common.json b/locales/hu/common.json index 474772591..f82879100 100644 --- a/locales/hu/common.json +++ b/locales/hu/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "A Piszkozatok mappa nem található." } }, + "offlineConflicts": { + "title": "Nem alkalmazott módosítások", + "description": "Ezeket a módosításokat offline végezte. Nem lettek alkalmazva, és az üzenet úgy marad, ahogy a szerveren van.", + "changed": "Másik eszközön módosult, az Ön módosítása nem lett alkalmazva", + "changed_delete": "Másik eszközön módosult, nem lett törölve", + "gone": "Már nincs a szerveren, az Ön módosítása nem lett alkalmazva", + "refused": "A szerver elutasította ezt a módosítást: {reason}", + "dismiss": "Bezárás", + "dismiss_label": "A(z) {subject} megjegyzésének bezárása", + "banner": "Nem alkalmazott módosítások: {count}", + "toast": "Offline végzett, nem alkalmazott módosítások: {count}. A postázandók mappa mutatja, melyek.", + "storage_failed": "Ezen az eszközön nincs több hely az offline leveleknek. Szabadítson fel helyet, vagy tartson meg kevesebbet a beállításokban.", + "signout_changes_title": "Kijelentkezik a nem mentett módosításokkal?", + "signout_changes": "Még nem a szerveren lévő módosítások: {count}. A kijelentkezés törli őket.", + "store_unavailable": "Az eszközön lévő offline levelek most nem nyithatók meg. Az el nem küldött levelek itt maradnak, az alkalmazás pedig tovább próbálkozik.", + "store_reset": "Az eszközön lévő offline leveleket nem sikerült beolvasni, ezért újra letöltődnek. Ami nem lett elküldve, elveszett.", + "legacy_stranded": "Ez a böngésző nem tudja megnyitni egy korábbi verzió által megőrzött, el nem küldött leveleket. A böngésző frissítése visszahozhatja őket.", + "store_private": "A privát ablakok nem tudnak leveleket megőrizni offline használatra.", + "server_not_answering": "A levelezőszerver nem válaszol. Próbálja újra egy pillanat múlva." + }, "deep_link": { "message_not_found": "Ez az üzenet már nem érhető el.", "thread_not_found": "Ez a beszélgetés már nem érhető el.", diff --git a/locales/it/common.json b/locales/it/common.json index 8cc39c364..2b4c6afbe 100644 --- a/locales/it/common.json +++ b/locales/it/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Impossibile trovare la cartella Bozze." } }, + "offlineConflicts": { + "title": "Modifiche non applicate", + "description": "Hai fatto queste modifiche offline. Non sono state applicate e il messaggio resta com'è sul server.", + "changed": "Modificato su un altro dispositivo, la tua modifica non è stata applicata", + "changed_delete": "Modificato su un altro dispositivo, non è stato eliminato", + "gone": "Non è più sul server, la tua modifica non è stata applicata", + "refused": "Il server ha rifiutato questa modifica: {reason}", + "dismiss": "Ignora", + "dismiss_label": "Ignora la nota su {subject}", + "banner": "Modifiche non applicate: {count}", + "toast": "Modifiche fatte offline non applicate: {count}. La posta in uscita indica quali.", + "storage_failed": "Questo dispositivo non ha più spazio per la posta offline. Libera spazio o conservane meno nelle impostazioni.", + "signout_changes_title": "Uscire con modifiche non salvate?", + "signout_changes": "Modifiche non ancora sul server: {count}. Uscendo vengono eliminate.", + "store_unavailable": "La posta offline di questo dispositivo non si può aprire ora. La posta non inviata resta qui e l'app continua a provare.", + "store_reset": "Non è stato possibile leggere la posta offline di questo dispositivo, quindi viene scaricata di nuovo. Ciò che non era stato inviato è perso.", + "legacy_stranded": "Questo browser non può aprire la posta non inviata conservata da una versione precedente. Aggiornare il browser può recuperarla.", + "store_private": "Le finestre private non possono conservare la posta per l'uso offline.", + "server_not_answering": "Il server di posta non risponde. Riprova tra un momento." + }, "deep_link": { "message_not_found": "Questo messaggio non è più disponibile.", "thread_not_found": "Questa conversazione non è più disponibile.", diff --git a/locales/ja/common.json b/locales/ja/common.json index e051ceb66..d0e59ab1e 100644 --- a/locales/ja/common.json +++ b/locales/ja/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "下書きフォルダーが見つかりません。" } }, + "offlineConflicts": { + "title": "適用されなかった変更", + "description": "これらの変更はオフライン中に行われました。適用されず、メッセージはサーバー上の状態のままです。", + "changed": "別のデバイスで変更されたため、変更は適用されませんでした", + "changed_delete": "別のデバイスで変更されたため、削除されませんでした", + "gone": "サーバーにもうないため、変更は適用されませんでした", + "refused": "サーバーがこの変更を拒否しました: {reason}", + "dismiss": "閉じる", + "dismiss_label": "{subject} のメモを閉じる", + "banner": "適用されなかった変更: {count}", + "toast": "オフライン中の変更で適用されなかったもの: {count} 件。送信トレイで確認できます。", + "storage_failed": "このデバイスにはオフラインメール用の空き容量がありません。空き容量を増やすか、設定で保存する量を減らしてください。", + "signout_changes_title": "未保存の変更があるままサインアウトしますか?", + "signout_changes": "サーバーにまだ届いていない変更: {count} 件。サインアウトすると削除されます。", + "store_unavailable": "このデバイスのオフラインメールを今は開けません。未送信のメールはここに残り、アプリは再試行を続けます。", + "store_reset": "このデバイスのオフラインメールを読み取れなかったため、再ダウンロードしています。未送信だったものは失われました。", + "legacy_stranded": "このブラウザーでは、以前のバージョンが保存した未送信メールを開けません。ブラウザーを更新すると戻る場合があります。", + "store_private": "プライベートウィンドウではオフライン用にメールを保存できません。", + "server_not_answering": "メールサーバーが応答していません。少ししてからもう一度お試しください。" + }, "deep_link": { "message_not_found": "このメッセージは利用できなくなりました。", "thread_not_found": "この会話は利用できなくなりました。", diff --git a/locales/ko/common.json b/locales/ko/common.json index e7f7be907..b9e4bdd72 100644 --- a/locales/ko/common.json +++ b/locales/ko/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "임시보관함 폴더를 찾을 수 없습니다." } }, + "offlineConflicts": { + "title": "적용되지 않은 변경 사항", + "description": "오프라인 상태에서 변경한 내용입니다. 적용되지 않았으며 메시지는 서버에 있는 그대로 유지됩니다.", + "changed": "다른 기기에서 변경되어 변경 사항이 적용되지 않았습니다", + "changed_delete": "다른 기기에서 변경되어 삭제되지 않았습니다", + "gone": "서버에 더 이상 없어 변경 사항이 적용되지 않았습니다", + "refused": "서버가 이 변경을 거부했습니다: {reason}", + "dismiss": "닫기", + "dismiss_label": "{subject} 관련 메모 닫기", + "banner": "적용되지 않은 변경 사항: {count}", + "toast": "오프라인에서 변경했지만 적용되지 않은 항목: {count}개. 보낼 편지함에서 확인할 수 있습니다.", + "storage_failed": "이 기기에 오프라인 메일을 저장할 공간이 없습니다. 공간을 확보하거나 설정에서 보관량을 줄이세요.", + "signout_changes_title": "저장하지 않은 변경 사항이 있는 채로 로그아웃할까요?", + "signout_changes": "아직 서버에 없는 변경 사항: {count}개. 로그아웃하면 삭제됩니다.", + "store_unavailable": "지금은 이 기기의 오프라인 메일을 열 수 없습니다. 보내지 않은 메일은 여기에 남아 있고 앱이 계속 시도합니다.", + "store_reset": "이 기기의 오프라인 메일을 읽을 수 없어 다시 내려받는 중입니다. 보내지 않은 내용은 사라졌습니다.", + "legacy_stranded": "이 브라우저에서는 이전 버전이 보관한 보내지 않은 메일을 열 수 없습니다. 브라우저를 업데이트하면 되찾을 수 있습니다.", + "store_private": "개인 정보 보호 창에서는 오프라인용 메일을 보관할 수 없습니다.", + "server_not_answering": "메일 서버가 응답하지 않습니다. 잠시 후 다시 시도하세요." + }, "deep_link": { "message_not_found": "이 메시지를 더 이상 사용할 수 없습니다.", "thread_not_found": "이 대화를 더 이상 사용할 수 없습니다.", diff --git a/locales/lv/common.json b/locales/lv/common.json index c07fe9c31..ed0da4a56 100644 --- a/locales/lv/common.json +++ b/locales/lv/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Mape Melnraksti netika atrasta." } }, + "offlineConflicts": { + "title": "Nepiemērotās izmaiņas", + "description": "Šīs izmaiņas veicāt bezsaistē. Tās netika piemērotas, un ziņojums paliek tāds, kāds tas ir serverī.", + "changed": "Mainīts citā ierīcē, jūsu izmaiņas netika piemērotas", + "changed_delete": "Mainīts citā ierīcē, tas netika izdzēsts", + "gone": "Vairs nav serverī, jūsu izmaiņas netika piemērotas", + "refused": "Serveris noraidīja šīs izmaiņas: {reason}", + "dismiss": "Aizvērt", + "dismiss_label": "Aizvērt piezīmi par {subject}", + "banner": "Nepiemērotās izmaiņas: {count}", + "toast": "Bezsaistē veiktās nepiemērotās izmaiņas: {count}. Izsūtne parāda, kuras.", + "storage_failed": "Šajā ierīcē vairs nav vietas bezsaistes pastam. Atbrīvojiet vietu vai iestatījumos glabājiet mazāk.", + "signout_changes_title": "Vai izrakstīties ar nesaglabātām izmaiņām?", + "signout_changes": "Izmaiņas, kas vēl nav serverī: {count}. Izrakstoties tās tiks izdzēstas.", + "store_unavailable": "Bezsaistes pastu šajā ierīcē pašlaik nevar atvērt. Nenosūtītais pasts paliek šeit, un lietotne turpina mēģināt.", + "store_reset": "Bezsaistes pastu šajā ierīcē neizdevās nolasīt, tāpēc tas tiek lejupielādēts vēlreiz. Tas, kas nebija nosūtīts, ir zudis.", + "legacy_stranded": "Šī pārlūkprogramma nevar atvērt iepriekšējās versijas saglabāto nenosūtīto pastu. Pārlūkprogrammas atjaunināšana var to atgūt.", + "store_private": "Privātie logi nevar glabāt pastu lietošanai bezsaistē.", + "server_not_answering": "Pasta serveris neatbild. Mēģiniet vēlreiz pēc brīža." + }, "deep_link": { "message_not_found": "Šī ziņa vairs nav pieejama.", "thread_not_found": "Šī sarakste vairs nav pieejama.", diff --git a/locales/mn/common.json b/locales/mn/common.json index 4633a5d6d..79d8fdfc1 100644 --- a/locales/mn/common.json +++ b/locales/mn/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Ноорог хавтас олдсонгүй." } }, + "offlineConflicts": { + "title": "Хэрэгжээгүй өөрчлөлтүүд", + "description": "Та эдгээр өөрчлөлтийг офлайн хийсэн. Тэдгээр нь хэрэгжээгүй бөгөөд захидал серверт байгаа хэвээрээ үлдэнэ.", + "changed": "Өөр төхөөрөмж дээр өөрчлөгдсөн тул таны өөрчлөлт хэрэгжээгүй", + "changed_delete": "Өөр төхөөрөмж дээр өөрчлөгдсөн тул устгагдаагүй", + "gone": "Серверт байхгүй болсон тул таны өөрчлөлт хэрэгжээгүй", + "refused": "Сервер энэ өөрчлөлтөөс татгалзлаа: {reason}", + "dismiss": "Хаах", + "dismiss_label": "{subject}-ийн тэмдэглэлийг хаах", + "banner": "Хэрэгжээгүй өөрчлөлтүүд: {count}", + "toast": "Офлайн хийсэн, хэрэгжээгүй өөрчлөлтүүд: {count}. Аль нь болохыг Илгээх хайрцгаас харна уу.", + "storage_failed": "Энэ төхөөрөмжид офлайн захидалд зай үлдсэнгүй. Зай чөлөөлөх эсвэл тохиргооноос цөөнийг хадгална уу.", + "signout_changes_title": "Хадгалаагүй өөрчлөлттэй гарах уу?", + "signout_changes": "Сервер дээр хараахан ороогүй өөрчлөлт: {count}. Гарвал устгагдана.", + "store_unavailable": "Энэ төхөөрөмжийн офлайн захидлыг одоо нээж чадахгүй байна. Илгээгээгүй захидал энд үлдэх бөгөөд апп үргэлжлүүлэн оролдоно.", + "store_reset": "Энэ төхөөрөмжийн офлайн захидлыг уншиж чадсангүй тул дахин татаж байна. Илгээгээгүй зүйл алга болсон.", + "legacy_stranded": "Энэ хөтөч өмнөх хувилбарын хадгалсан илгээгээгүй захидлыг нээж чадахгүй. Хөтчөө шинэчилбэл буцаж магадгүй.", + "store_private": "Хувийн цонх офлайн ашиглах захидал хадгалж чадахгүй.", + "server_not_answering": "Шуудангийн сервер хариу өгөхгүй байна. Хэсэг хугацааны дараа дахин оролдоно уу." + }, "deep_link": { "message_not_found": "Энэ зурвас дахин ашиглах боломжгүй болсон.", "thread_not_found": "Энэ харилцан яриа дахин ашиглах боломжгүй болсон.", diff --git a/locales/nb/common.json b/locales/nb/common.json index d2e1a518e..c500ca8b8 100644 --- a/locales/nb/common.json +++ b/locales/nb/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Fant ikke mappen Utkast." } }, + "offlineConflicts": { + "title": "Endringer som ikke ble utført", + "description": "Du gjorde disse endringene uten nett. De ble ikke utført, og meldingen forblir som den er på serveren.", + "changed": "Endret på en annen enhet, endringen din ble ikke utført", + "changed_delete": "Endret på en annen enhet, den ble ikke slettet", + "gone": "Finnes ikke lenger på serveren, endringen din ble ikke utført", + "refused": "Serveren avviste denne endringen: {reason}", + "dismiss": "Lukk", + "dismiss_label": "Lukk merknaden om {subject}", + "banner": "Endringer som ikke ble utført: {count}", + "toast": "Endringer gjort uten nett som ikke ble utført: {count}. Utboksen viser hvilke.", + "storage_failed": "Denne enheten har ikke mer plass til e-post uten nett. Frigjør plass, eller behold mindre i innstillingene.", + "signout_changes_title": "Logge ut med endringer som ikke er lagret?", + "signout_changes": "Endringer som ennå ikke er på serveren: {count}. Når du logger ut, slettes de.", + "store_unavailable": "E-post uten nett på denne enheten kan ikke åpnes nå. Usendt e-post blir her, og appen fortsetter å prøve.", + "store_reset": "E-post uten nett på denne enheten kunne ikke leses og lastes derfor ned på nytt. Det som ikke var sendt, er borte.", + "legacy_stranded": "Denne nettleseren kan ikke åpne usendt e-post fra en tidligere versjon. En oppdatering av nettleseren kan hente den tilbake.", + "store_private": "Private vinduer kan ikke beholde e-post for bruk uten nett.", + "server_not_answering": "E-postserveren svarer ikke. Prøv igjen om litt." + }, "deep_link": { "message_not_found": "Denne meldingen er ikke lenger tilgjengelig.", "thread_not_found": "Denne samtalen er ikke lenger tilgjengelig.", diff --git a/locales/nl/common.json b/locales/nl/common.json index b704a66fc..9fa70a38d 100644 --- a/locales/nl/common.json +++ b/locales/nl/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Je map Concepten is niet gevonden." } }, + "offlineConflicts": { + "title": "Niet doorgevoerde wijzigingen", + "description": "Je hebt deze wijzigingen offline gemaakt. Ze zijn niet doorgevoerd en het bericht blijft zoals het op de server staat.", + "changed": "Gewijzigd op een ander apparaat, je wijziging is niet doorgevoerd", + "changed_delete": "Gewijzigd op een ander apparaat, niet verwijderd", + "gone": "Staat niet meer op de server, je wijziging is niet doorgevoerd", + "refused": "De server heeft deze wijziging geweigerd: {reason}", + "dismiss": "Sluiten", + "dismiss_label": "Melding over {subject} sluiten", + "banner": "Niet doorgevoerde wijzigingen: {count}", + "toast": "Offline gemaakte wijzigingen niet doorgevoerd: {count}. Het Postvak UIT laat zien welke.", + "storage_failed": "Dit apparaat heeft geen ruimte meer voor offline mail. Maak ruimte vrij of bewaar minder via Instellingen.", + "signout_changes_title": "Afmelden met niet-opgeslagen wijzigingen?", + "signout_changes": "Wijzigingen die nog niet op de server staan: {count}. Afmelden verwijdert ze.", + "store_unavailable": "De offline mail op dit apparaat kan nu niet worden geopend. Niet-verzonden mail blijft hier en de app blijft het proberen.", + "store_reset": "De offline mail op dit apparaat kon niet worden gelezen en wordt opnieuw gedownload. Wat hier nog niet verzonden was, is weg.", + "legacy_stranded": "Deze browser kan niet-verzonden mail van een eerdere versie niet openen. De browser bijwerken kan die terughalen.", + "store_private": "Privévensters kunnen geen mail bewaren voor offline gebruik.", + "server_not_answering": "De mailserver antwoordt niet. Probeer het zo meteen opnieuw." + }, "deep_link": { "message_not_found": "Dit bericht is niet meer beschikbaar.", "thread_not_found": "Dit gesprek is niet meer beschikbaar.", diff --git a/locales/pl/common.json b/locales/pl/common.json index 8ccc30784..4608b20df 100644 --- a/locales/pl/common.json +++ b/locales/pl/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Nie znaleziono folderu Wersje robocze." } }, + "offlineConflicts": { + "title": "Niezastosowane zmiany", + "description": "Te zmiany wprowadzono w trybie offline. Nie zostały zastosowane, a wiadomość pozostaje taka jak na serwerze.", + "changed": "Zmieniono na innym urządzeniu, Twoja zmiana nie została zastosowana", + "changed_delete": "Zmieniono na innym urządzeniu, wiadomość nie została usunięta", + "gone": "Nie ma już na serwerze, Twoja zmiana nie została zastosowana", + "refused": "Serwer odrzucił tę zmianę: {reason}", + "dismiss": "Odrzuć", + "dismiss_label": "Odrzuć uwagę o {subject}", + "banner": "Niezastosowane zmiany: {count}", + "toast": "Niezastosowane zmiany wprowadzone offline: {count}. Skrzynka nadawcza pokazuje które.", + "storage_failed": "Na tym urządzeniu nie ma już miejsca na pocztę offline. Zwolnij miejsce lub zachowuj mniej w ustawieniach.", + "signout_changes_title": "Wylogować z niezapisanymi zmianami?", + "signout_changes": "Zmiany, których jeszcze nie ma na serwerze: {count}. Wylogowanie je usunie.", + "store_unavailable": "Nie można teraz otworzyć poczty offline na tym urządzeniu. Niewysłane wiadomości zostają tutaj, a aplikacja próbuje dalej.", + "store_reset": "Nie udało się odczytać poczty offline na tym urządzeniu, więc jest pobierana ponownie. To, co nie zostało wysłane, przepadło.", + "legacy_stranded": "Ta przeglądarka nie może otworzyć niewysłanych wiadomości z wcześniejszej wersji. Aktualizacja przeglądarki może je przywrócić.", + "store_private": "Okna prywatne nie mogą przechowywać poczty do użytku offline.", + "server_not_answering": "Serwer poczty nie odpowiada. Spróbuj ponownie za chwilę." + }, "deep_link": { "message_not_found": "Ta wiadomość nie jest już dostępna.", "thread_not_found": "Ta rozmowa nie jest już dostępna.", diff --git a/locales/pt/common.json b/locales/pt/common.json index 188b256f8..cca6bb8e2 100644 --- a/locales/pt/common.json +++ b/locales/pt/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Não foi possível encontrar a pasta Rascunhos." } }, + "offlineConflicts": { + "title": "Alterações não aplicadas", + "description": "Fez estas alterações offline. Não foram aplicadas e a mensagem fica como está no servidor.", + "changed": "Alterada noutro dispositivo, a sua alteração não foi aplicada", + "changed_delete": "Alterada noutro dispositivo, não foi eliminada", + "gone": "Já não está no servidor, a sua alteração não foi aplicada", + "refused": "O servidor recusou esta alteração: {reason}", + "dismiss": "Dispensar", + "dismiss_label": "Dispensar a nota sobre {subject}", + "banner": "Alterações não aplicadas: {count}", + "toast": "Alterações feitas offline não aplicadas: {count}. A caixa de saída mostra quais.", + "storage_failed": "Este dispositivo não tem espaço para o correio offline. Liberte espaço ou guarde menos nas definições.", + "signout_changes_title": "Terminar sessão com alterações não guardadas?", + "signout_changes": "Alterações ainda não enviadas para o servidor: {count}. Terminar sessão elimina-as.", + "store_unavailable": "O correio offline deste dispositivo não pode ser aberto agora. O correio por enviar fica aqui e a aplicação continua a tentar.", + "store_reset": "Não foi possível ler o correio offline deste dispositivo, por isso está a ser transferido de novo. O que estava por enviar perdeu-se.", + "legacy_stranded": "Este navegador não consegue abrir o correio por enviar guardado por uma versão anterior. Atualizar o navegador pode recuperá-lo.", + "store_private": "As janelas privadas não podem guardar correio para uso offline.", + "server_not_answering": "O servidor de correio não está a responder. Tente novamente daqui a pouco." + }, "deep_link": { "message_not_found": "Esta mensagem não está mais disponível.", "thread_not_found": "Esta conversa não está mais disponível.", diff --git a/locales/ro/common.json b/locales/ro/common.json index ae31839c2..1f04eda3a 100644 --- a/locales/ro/common.json +++ b/locales/ro/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Dosarul Ciorne nu a fost găsit." } }, + "offlineConflicts": { + "title": "Modificări neaplicate", + "description": "Ați făcut aceste modificări offline. Nu au fost aplicate, iar mesajul rămâne așa cum este pe server.", + "changed": "Modificat pe alt dispozitiv, modificarea dvs. nu a fost aplicată", + "changed_delete": "Modificat pe alt dispozitiv, nu a fost șters", + "gone": "Nu mai este pe server, modificarea dvs. nu a fost aplicată", + "refused": "Serverul a refuzat această modificare: {reason}", + "dismiss": "Închide", + "dismiss_label": "Închide nota despre {subject}", + "banner": "Modificări neaplicate: {count}", + "toast": "Modificări făcute offline care nu au fost aplicate: {count}. Căsuța de trimitere arată care.", + "storage_failed": "Acest dispozitiv nu mai are loc pentru e-mailul offline. Eliberați spațiu sau păstrați mai puțin din setări.", + "signout_changes_title": "Vă deconectați cu modificări nesalvate?", + "signout_changes": "Modificări care nu sunt încă pe server: {count}. Deconectarea le șterge.", + "store_unavailable": "E-mailul offline de pe acest dispozitiv nu poate fi deschis acum. E-mailul netrimis rămâne aici, iar aplicația continuă să încerce.", + "store_reset": "E-mailul offline de pe acest dispozitiv nu a putut fi citit, așa că se descarcă din nou. Ce nu fusese trimis s-a pierdut.", + "legacy_stranded": "Acest browser nu poate deschide e-mailul netrimis păstrat de o versiune anterioară. Actualizarea browserului îl poate recupera.", + "store_private": "Ferestrele private nu pot păstra e-mailuri pentru utilizare offline.", + "server_not_answering": "Serverul de e-mail nu răspunde. Încercați din nou imediat." + }, "deep_link": { "message_not_found": "Acest mesaj nu mai este disponibil.", "thread_not_found": "Această conversație nu mai este disponibilă.", diff --git a/locales/ru/common.json b/locales/ru/common.json index a9ee1c65c..4a11da8b0 100644 --- a/locales/ru/common.json +++ b/locales/ru/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Папка «Черновики» не найдена." } }, + "offlineConflicts": { + "title": "Неприменённые изменения", + "description": "Вы внесли эти изменения офлайн. Они не применены, письмо остаётся таким, как на сервере.", + "changed": "Изменено на другом устройстве, ваше изменение не применено", + "changed_delete": "Изменено на другом устройстве, письмо не удалено", + "gone": "Больше нет на сервере, ваше изменение не применено", + "refused": "Сервер отклонил это изменение: {reason}", + "dismiss": "Закрыть", + "dismiss_label": "Закрыть заметку о {subject}", + "banner": "Неприменённые изменения: {count}", + "toast": "Изменения, внесённые офлайн и не применённые: {count}. Какие именно, видно в исходящих.", + "storage_failed": "На этом устройстве не осталось места для офлайн-почты. Освободите место или храните меньше в настройках.", + "signout_changes_title": "Выйти с несохранёнными изменениями?", + "signout_changes": "Изменения, которых ещё нет на сервере: {count}. При выходе они удалятся.", + "store_unavailable": "Офлайн-почту на этом устройстве сейчас не удаётся открыть. Неотправленные письма остаются здесь, а приложение продолжает попытки.", + "store_reset": "Офлайн-почту на этом устройстве не удалось прочитать, поэтому она загружается заново. Всё, что не было отправлено, потеряно.", + "legacy_stranded": "Этот браузер не может открыть неотправленные письма, сохранённые предыдущей версией. Обновление браузера может их вернуть.", + "store_private": "Приватные окна не могут хранить почту для работы офлайн.", + "server_not_answering": "Почтовый сервер не отвечает. Повторите попытку через минуту." + }, "deep_link": { "message_not_found": "Это письмо больше недоступно.", "thread_not_found": "Эта переписка больше недоступна.", diff --git a/locales/sk/common.json b/locales/sk/common.json index a018ba55a..004c61727 100644 --- a/locales/sk/common.json +++ b/locales/sk/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Priečinok Koncepty sa nenašiel." } }, + "offlineConflicts": { + "title": "Nepoužité zmeny", + "description": "Tieto zmeny ste urobili offline. Neboli použité a správa zostáva tak, ako je na serveri.", + "changed": "Zmenené na inom zariadení, vaša zmena nebola použitá", + "changed_delete": "Zmenené na inom zariadení, správa nebola odstránená", + "gone": "Už nie je na serveri, vaša zmena nebola použitá", + "refused": "Server túto zmenu odmietol: {reason}", + "dismiss": "Zavrieť", + "dismiss_label": "Zavrieť poznámku o {subject}", + "banner": "Nepoužité zmeny: {count}", + "toast": "Nepoužité zmeny urobené offline: {count}. Ktoré to sú, ukazuje pošta na odoslanie.", + "storage_failed": "V tomto zariadení už nie je miesto na poštu offline. Uvoľnite miesto alebo v nastaveniach ponechajte menej.", + "signout_changes_title": "Odhlásiť sa s neuloženými zmenami?", + "signout_changes": "Zmeny, ktoré ešte nie sú na serveri: {count}. Odhlásenie ich odstráni.", + "store_unavailable": "Poštu offline v tomto zariadení teraz nemožno otvoriť. Neodoslaná pošta tu zostáva a aplikácia to skúša ďalej.", + "store_reset": "Poštu offline v tomto zariadení sa nepodarilo prečítať, preto sa sťahuje znova. Čo nebolo odoslané, je preč.", + "legacy_stranded": "Tento prehliadač neotvorí neodoslanú poštu uloženú staršou verziou. Aktualizácia prehliadača ju môže vrátiť.", + "store_private": "Súkromné okná nemôžu uchovávať poštu na používanie offline.", + "server_not_answering": "Poštový server neodpovedá. Skúste to o chvíľu znova." + }, "deep_link": { "message_not_found": "Táto správa už nie je dostupná.", "thread_not_found": "Táto konverzácia už nie je dostupná.", diff --git a/locales/th/common.json b/locales/th/common.json index 065a7afbc..4e87f1d6d 100644 --- a/locales/th/common.json +++ b/locales/th/common.json @@ -2088,6 +2088,26 @@ "no_drafts_mailbox": "ไม่พบโฟลเดอร์แบบร่าง" } }, + "offlineConflicts": { + "title": "การเปลี่ยนแปลงที่ไม่ได้นำไปใช้", + "description": "คุณทำการเปลี่ยนแปลงเหล่านี้ขณะออฟไลน์ การเปลี่ยนแปลงไม่ได้ถูกนำไปใช้ และข้อความยังคงเหมือนบนเซิร์ฟเวอร์", + "changed": "มีการเปลี่ยนแปลงบนอุปกรณ์อื่น การเปลี่ยนแปลงของคุณจึงไม่ถูกนำไปใช้", + "changed_delete": "มีการเปลี่ยนแปลงบนอุปกรณ์อื่น จึงไม่ถูกลบ", + "gone": "ไม่มีบนเซิร์ฟเวอร์แล้ว การเปลี่ยนแปลงของคุณจึงไม่ถูกนำไปใช้", + "refused": "เซิร์ฟเวอร์ปฏิเสธการเปลี่ยนแปลงนี้: {reason}", + "dismiss": "ปิด", + "dismiss_label": "ปิดหมายเหตุเกี่ยวกับ {subject}", + "banner": "การเปลี่ยนแปลงที่ไม่ได้นำไปใช้: {count}", + "toast": "การเปลี่ยนแปลงขณะออฟไลน์ที่ไม่ได้นำไปใช้: {count} ดูรายการได้ในกล่องขาออก", + "storage_failed": "อุปกรณ์นี้ไม่มีที่ว่างสำหรับอีเมลออฟไลน์แล้ว โปรดเพิ่มพื้นที่ว่างหรือเก็บให้น้อยลงในการตั้งค่า", + "signout_changes_title": "ออกจากระบบทั้งที่มีการเปลี่ยนแปลงที่ยังไม่บันทึกหรือไม่", + "signout_changes": "การเปลี่ยนแปลงที่ยังไม่อยู่บนเซิร์ฟเวอร์: {count} การออกจากระบบจะลบรายการเหล่านี้", + "store_unavailable": "ตอนนี้เปิดอีเมลออฟไลน์บนอุปกรณ์นี้ไม่ได้ อีเมลที่ยังไม่ส่งยังอยู่ที่นี่ และแอปจะลองต่อไป", + "store_reset": "อ่านอีเมลออฟไลน์บนอุปกรณ์นี้ไม่ได้ จึงกำลังดาวน์โหลดใหม่ สิ่งที่ยังไม่ได้ส่งหายไปแล้ว", + "legacy_stranded": "เบราว์เซอร์นี้เปิดอีเมลที่ยังไม่ส่งซึ่งเวอร์ชันก่อนหน้าเก็บไว้ไม่ได้ การอัปเดตเบราว์เซอร์อาจนำกลับมาได้", + "store_private": "หน้าต่างส่วนตัวเก็บอีเมลไว้ใช้แบบออฟไลน์ไม่ได้", + "server_not_answering": "เซิร์ฟเวอร์อีเมลไม่ตอบสนอง โปรดลองอีกครั้งในอีกสักครู่" + }, "plugin_pages": { "loading": "กำลังโหลดหน้าเว็บ...", "not_available_title": "หน้านี้ไม่พร้อมใช้งาน", diff --git a/locales/tr/common.json b/locales/tr/common.json index 81acedbf7..2fe4bf8f1 100644 --- a/locales/tr/common.json +++ b/locales/tr/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Taslaklar klasörü bulunamadı." } }, + "offlineConflicts": { + "title": "Uygulanmayan değişiklikler", + "description": "Bu değişiklikleri çevrimdışıyken yaptınız. Uygulanmadı ve ileti sunucudaki haliyle kalıyor.", + "changed": "Başka bir cihazda değiştirildi, değişikliğiniz uygulanmadı", + "changed_delete": "Başka bir cihazda değiştirildi, silinmedi", + "gone": "Artık sunucuda yok, değişikliğiniz uygulanmadı", + "refused": "Sunucu bu değişikliği reddetti: {reason}", + "dismiss": "Kapat", + "dismiss_label": "{subject} ile ilgili notu kapat", + "banner": "Uygulanmayan değişiklikler: {count}", + "toast": "Çevrimdışı yapılan ve uygulanmayan değişiklikler: {count}. Giden kutusu hangileri olduğunu gösterir.", + "storage_failed": "Bu cihazda çevrimdışı e-posta için yer kalmadı. Yer açın veya Ayarlar'dan daha azını saklayın.", + "signout_changes_title": "Kaydedilmemiş değişikliklerle oturum kapatılsın mı?", + "signout_changes": "Henüz sunucuda olmayan değişiklikler: {count}. Oturumu kapatmak bunları siler.", + "store_unavailable": "Bu cihazdaki çevrimdışı e-postalar şu anda açılamıyor. Gönderilmemiş e-postalar burada kalır ve uygulama denemeye devam eder.", + "store_reset": "Bu cihazdaki çevrimdışı e-postalar okunamadı, bu yüzden yeniden indiriliyor. Gönderilmemiş olanlar kayboldu.", + "legacy_stranded": "Bu tarayıcı, önceki bir sürümün sakladığı gönderilmemiş e-postaları açamıyor. Tarayıcıyı güncellemek bunları geri getirebilir.", + "store_private": "Gizli pencereler e-postaları çevrimdışı kullanım için saklayamaz.", + "server_not_answering": "E-posta sunucusu yanıt vermiyor. Birazdan yeniden deneyin." + }, "deep_link": { "message_not_found": "Bu ileti artık kullanılamıyor.", "thread_not_found": "Bu sohbet artık kullanılamıyor.", diff --git a/locales/uk/common.json b/locales/uk/common.json index 42dbd1aba..88c17f3ef 100644 --- a/locales/uk/common.json +++ b/locales/uk/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "Теку «Чернетки» не знайдено." } }, + "offlineConflicts": { + "title": "Незастосовані зміни", + "description": "Ви внесли ці зміни офлайн. Їх не застосовано, і лист залишається таким, як на сервері.", + "changed": "Змінено на іншому пристрої, вашу зміну не застосовано", + "changed_delete": "Змінено на іншому пристрої, лист не видалено", + "gone": "Більше немає на сервері, вашу зміну не застосовано", + "refused": "Сервер відхилив цю зміну: {reason}", + "dismiss": "Закрити", + "dismiss_label": "Закрити примітку про {subject}", + "banner": "Незастосовані зміни: {count}", + "toast": "Зміни, внесені офлайн і не застосовані: {count}. Які саме, видно у вихідних.", + "storage_failed": "На цьому пристрої не залишилося місця для офлайн-пошти. Звільніть місце або зберігайте менше в налаштуваннях.", + "signout_changes_title": "Вийти з незбереженими змінами?", + "signout_changes": "Зміни, яких ще немає на сервері: {count}. Під час виходу вони видаляться.", + "store_unavailable": "Офлайн-пошту на цьому пристрої зараз не вдається відкрити. Ненадіслані листи залишаються тут, а застосунок продовжує спроби.", + "store_reset": "Офлайн-пошту на цьому пристрої не вдалося прочитати, тому вона завантажується знову. Усе, що не було надіслано, втрачено.", + "legacy_stranded": "Цей браузер не може відкрити ненадіслані листи, збережені попередньою версією. Оновлення браузера може їх повернути.", + "store_private": "Приватні вікна не можуть зберігати пошту для роботи офлайн.", + "server_not_answering": "Поштовий сервер не відповідає. Спробуйте ще раз за хвилину." + }, "deep_link": { "message_not_found": "Цей лист більше недоступний.", "thread_not_found": "Ця розмова більше недоступна.", diff --git a/locales/vi/common.json b/locales/vi/common.json index 9cbcd8bd3..6fd6e2c04 100644 --- a/locales/vi/common.json +++ b/locales/vi/common.json @@ -5755,6 +5755,26 @@ "no_drafts_mailbox": "Không tìm thấy thư mục Thư nháp." } }, + "offlineConflicts": { + "title": "Các thay đổi chưa được áp dụng", + "description": "Bạn đã thực hiện các thay đổi này khi ngoại tuyến. Chúng chưa được áp dụng và thư vẫn giữ nguyên như trên máy chủ.", + "changed": "Đã thay đổi trên thiết bị khác nên thay đổi của bạn chưa được áp dụng", + "changed_delete": "Đã thay đổi trên thiết bị khác nên chưa bị xóa", + "gone": "Không còn trên máy chủ nên thay đổi của bạn chưa được áp dụng", + "refused": "Máy chủ đã từ chối thay đổi này: {reason}", + "dismiss": "Đóng", + "dismiss_label": "Đóng ghi chú về {subject}", + "banner": "Các thay đổi chưa được áp dụng: {count}", + "toast": "Thay đổi khi ngoại tuyến chưa được áp dụng: {count}. Hộp thư đi cho biết là những thay đổi nào.", + "storage_failed": "Thiết bị này không còn chỗ cho thư ngoại tuyến. Hãy giải phóng dung lượng hoặc giữ ít hơn trong Cài đặt.", + "signout_changes_title": "Đăng xuất khi còn thay đổi chưa lưu?", + "signout_changes": "Thay đổi chưa có trên máy chủ: {count}. Đăng xuất sẽ xóa chúng.", + "store_unavailable": "Hiện không mở được thư ngoại tuyến trên thiết bị này. Thư chưa gửi vẫn ở đây và ứng dụng tiếp tục thử.", + "store_reset": "Không đọc được thư ngoại tuyến trên thiết bị này nên đang tải lại. Những gì chưa gửi đã mất.", + "legacy_stranded": "Trình duyệt này không mở được thư chưa gửi do phiên bản trước lưu. Cập nhật trình duyệt có thể lấy lại chúng.", + "store_private": "Cửa sổ riêng tư không thể giữ thư để dùng ngoại tuyến.", + "server_not_answering": "Máy chủ thư không phản hồi. Hãy thử lại sau giây lát." + }, "deep_link": { "message_not_found": "Thư này không còn nữa.", "thread_not_found": "Cuộc trò chuyện này không còn tồn tại nữa.", diff --git a/locales/zh-TW/common.json b/locales/zh-TW/common.json index 5ba44316f..ead0cce96 100644 --- a/locales/zh-TW/common.json +++ b/locales/zh-TW/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "找不到「草稿」資料夾。" } }, + "offlineConflicts": { + "title": "未套用的變更", + "description": "這些變更是您離線時做的。它們沒有被套用,郵件維持伺服器上的狀態。", + "changed": "已在其他裝置上變更,您的變更未被套用", + "changed_delete": "已在其他裝置上變更,未被刪除", + "gone": "伺服器上已沒有此郵件,您的變更未被套用", + "refused": "伺服器拒絕了此變更:{reason}", + "dismiss": "關閉", + "dismiss_label": "關閉關於 {subject} 的提示", + "banner": "未套用的變更:{count}", + "toast": "離線時的變更未被套用:{count} 項。可在寄件匣中查看。", + "storage_failed": "此裝置已沒有空間存放離線郵件。請釋出空間,或在設定中減少保留的內容。", + "signout_changes_title": "要在有未儲存變更時登出嗎?", + "signout_changes": "尚未到達伺服器的變更:{count} 項。登出會刪除它們。", + "store_unavailable": "目前無法開啟此裝置上的離線郵件。未傳送的郵件仍保留在這裡,應用程式會繼續嘗試。", + "store_reset": "無法讀取此裝置上的離線郵件,正在重新下載。未傳送的內容已遺失。", + "legacy_stranded": "此瀏覽器無法開啟舊版本保存的未傳送郵件。更新瀏覽器可能會找回它們。", + "store_private": "私密視窗無法保留供離線使用的郵件。", + "server_not_answering": "郵件伺服器沒有回應。請稍後再試。" + }, "deep_link": { "message_not_found": "已無法使用此郵件。", "thread_not_found": "已無法使用此對話串。", diff --git a/locales/zh/common.json b/locales/zh/common.json index f53c82da4..47e79990b 100644 --- a/locales/zh/common.json +++ b/locales/zh/common.json @@ -5743,6 +5743,26 @@ "no_drafts_mailbox": "找不到“草稿”文件夹。" } }, + "offlineConflicts": { + "title": "未应用的更改", + "description": "这些更改是您离线时做的。它们没有被应用,邮件保持服务器上的状态。", + "changed": "已在其他设备上更改,您的更改未被应用", + "changed_delete": "已在其他设备上更改,未被删除", + "gone": "服务器上已没有此邮件,您的更改未被应用", + "refused": "服务器拒绝了此更改:{reason}", + "dismiss": "关闭", + "dismiss_label": "关闭关于 {subject} 的提示", + "banner": "未应用的更改:{count}", + "toast": "离线时的更改未被应用:{count} 项。可在发件箱中查看。", + "storage_failed": "此设备已没有空间存放离线邮件。请释放空间,或在设置中减少保留的内容。", + "signout_changes_title": "要在有未保存更改时退出登录吗?", + "signout_changes": "尚未到达服务器的更改:{count} 项。退出登录会删除它们。", + "store_unavailable": "现在无法打开此设备上的离线邮件。未发送的邮件仍保留在这里,应用会继续尝试。", + "store_reset": "无法读取此设备上的离线邮件,正在重新下载。未发送的内容已丢失。", + "legacy_stranded": "此浏览器无法打开早期版本保存的未发送邮件。更新浏览器可能会找回它们。", + "store_private": "隐私窗口无法保留供离线使用的邮件。", + "server_not_answering": "邮件服务器没有响应。请稍后再试。" + }, "deep_link": { "message_not_found": "此邮件已不可用。", "thread_not_found": "此会话已不可用。", diff --git a/public/sw.js b/public/sw.js index f46f5fee7..22e4174ba 100644 --- a/public/sw.js +++ b/public/sw.js @@ -126,7 +126,10 @@ function appPagePath(url) { function staticPath(url) { if (url.origin !== self.location.origin) return false; - return url.pathname.startsWith(BASE_PATH + "/_next/static/"); + if (url.pathname.startsWith(BASE_PATH + "/_next/static/")) return true; + // The local store's engine: named by build (?v=) and by version, so it can be + // kept like a hashed file, and an offline start needs it. + return url.pathname === BASE_PATH + "/store-worker.js" || url.pathname.startsWith(BASE_PATH + "/sqlite3mc/"); } /** Where a page of the app is kept: its address without the query. */ diff --git a/scripts/lite/lib.mjs b/scripts/lite/lib.mjs index 3760b208a..9396192d3 100644 --- a/scripts/lite/lib.mjs +++ b/scripts/lite/lib.mjs @@ -49,6 +49,13 @@ export const LITE_REMOVED_PATHS = [ "public/sqlite3mc", ]; +/** The local store's engine in an export: Lite has no offline mode, and must not ship or load it. */ +export function unexpectedEngineFiles(files) { + return files + .map((file) => file.replace(/\\/g, "/")) + .filter((file) => /(^|\/)store-worker\.js$/.test(file) || /(^|\/)sqlite3mc\//.test(file) || /(^|\/)sqlite3\.(wasm|mjs)$/.test(file)); +} + /** Test trees are pruned too: they import the routes removed above. */ export const LITE_TEST_DIR_NAME = "__tests__"; export const LITE_TEST_FILE_PATTERN = /\.test\.[cm]?[jt]sx?$/; diff --git a/scripts/lite/verify.mjs b/scripts/lite/verify.mjs index 3e2800168..4a19c6e30 100644 --- a/scripts/lite/verify.mjs +++ b/scripts/lite/verify.mjs @@ -10,7 +10,7 @@ import { existsSync, readFileSync, statSync } from "node:fs"; import { join } from "node:path"; import { LITE_PENDING_PATH_KEY, STALWART_EXCLUDED_FILES, STALWART_MAX_BUNDLE_BYTES, STALWART_SAFE_BUNDLE_BYTES, STALWART_ZIP_NAME, - collectApiStrings, collectZipEntries, discoverBuiltLocales, flightTextRowsWithNeedles, inlineScriptHashes, isMainModule, listZipEntries, parseLiteTarget, resolveRepoRoot, stalwartEntryProblems, stalwartZipEntryProblems, unexpectedApiStrings, + collectApiStrings, collectZipEntries, discoverBuiltLocales, flightTextRowsWithNeedles, inlineScriptHashes, isMainModule, listZipEntries, parseLiteTarget, resolveRepoRoot, stalwartEntryProblems, stalwartZipEntryProblems, unexpectedApiStrings, unexpectedEngineFiles, } from "./lib.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); @@ -83,6 +83,10 @@ export function verifyExport({ root = repoRoot, target, safeBundleBytes = STALWA const unexpected = unexpectedApiStrings(apiStrings); for (const s of unexpected) problems.push(`client chunk references undocumented server endpoint ${s} (add an IS_LITE gate or document it in scripts/lite/lib.mjs)`); + if (existsSync(outDir)) { + for (const file of unexpectedEngineFiles(collectZipEntries(outDir).map(({ name }) => name))) problems.push(`out/${file} is the local store's engine, which Lite must not ship`); + } + let zipBytes = 0; if (resolvedTarget === "stalwart") { const entry = join(outDir, "index.html"); diff --git a/stores/__tests__/auth-store-keep-local.test.ts b/stores/__tests__/auth-store-keep-local.test.ts new file mode 100644 index 000000000..99ba77f90 --- /dev/null +++ b/stores/__tests__/auth-store-keep-local.test.ts @@ -0,0 +1,225 @@ +import 'fake-indexeddb/auto'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +// What an automatic sign-out does to the mail on this device. A real refusal of +// the login wipes it, like a sign-out. An outage, or a refusal that says nothing +// about the login, keeps it for the next sign-in. + +const purge = vi.hoisted(() => ({ files: 0 })); +vi.mock('@/lib/store/client/purge', () => ({ purgeStoreFiles: async () => { purge.files += 1; }, deleteStorePool: async () => {} })); +vi.mock('@/lib/compose-flush-registry', () => ({ flushComposers: vi.fn(async () => ({ total: 0, settled: 0, failed: 0, timedOut: false })) })); + +let useAuthStore: typeof import('../auth-store').useAuthStore; +let useAccountStore: typeof import('../account-store').useAccountStore; +let state: typeof import('@/lib/offline/state'); +let offline: typeof import('@/lib/offline/offline-mode'); +let queue: typeof import('@/lib/offline/queue'); +let lost: typeof import('@/lib/offline/lost'); +let store: import('@/lib/store/__tests__/helpers/in-process-store').InProcessStore; + +const ALICE = 'alice@mail.example.test'; + +function entry(id: string, cookieSlot: number) { + return { + id, label: id, serverUrl: 'https://mail.example.test', username: id.split('@mail')[0], authMode: 'oauth', rememberMe: true, + displayName: id, email: id, lastLoginAt: Date.now(), isConnected: true, hasError: false, isDefault: cookieSlot === 0, cookieSlot, avatarColor: '#000', + }; +} + +function answer(status: number, body: unknown = {}): Response { + return new Response(typeof body === 'string' ? body : JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); +} + +function tokenAnswers(next: () => Response): void { + vi.stubGlobal('fetch', vi.fn(async (input: unknown) => (String(input).includes('/api/auth/token') ? next() : answer(200)))); +} + +/** `times`: as often as the retry timer would run it. */ +async function refresh(times = 1): Promise { + for (let i = 0; i < times; i += 1) await useAuthStore.getState().refreshAccessToken({ accountId: ALICE }); + await vi.waitFor(() => expect(useAuthStore.getState().isAuthenticated).toBe(false)); +} + +beforeEach(async () => { + vi.resetModules(); + purge.files = 0; + localStorage.clear(); + sessionStorage.clear(); + ({ useAuthStore } = await import('../auth-store')); + ({ useAccountStore } = await import('../account-store')); + state = await import('@/lib/offline/state'); + offline = await import('@/lib/offline/offline-mode'); + queue = await import('@/lib/offline/queue'); + lost = await import('@/lib/offline/lost'); + const browserNavigation = await import('@/lib/browser-navigation'); + vi.spyOn(browserNavigation, 'replaceWindowLocation').mockImplementation(() => {}); + const { InProcessStore } = await import('@/lib/store/__tests__/helpers/in-process-store'); + const { setStoreBackend } = await import('@/lib/store/client/api'); + store = new InProcessStore(); + setStoreBackend(store); + useAccountStore.setState({ accounts: [entry(ALICE, 0)], activeAccountId: ALICE, defaultAccountId: ALICE } as never); + useAuthStore.setState({ isAuthenticated: true, activeAccountId: ALICE, authMode: 'oauth', serverUrl: 'https://mail.example.test', username: 'alice' }); + state.writeEnabled([ALICE]); + await queue.queueSend(ALICE, { subject: 'Unsent', to: [] }, { kind: 'email', args: [] }); +}, 60_000); + +function expectKept(): void { + expect(purge.files).toBe(0); + expect(state.useOfflineStore.getState().enabled).toEqual([ALICE]); + expect(localStorage.getItem(state.OFFLINE_ACCOUNTS_KEY)).toContain(ALICE); + expect(state.isEvicted(ALICE)).toBe(true); + expect(store.removed).toEqual([]); + expect(store.stopped).toBe(0); + // Nothing was lost, so nothing is named at the sign-in page. + expect(lost.consumeLost()).toEqual([]); +} + +describe('an automatic sign-out of the active account', () => { + it('keeps the data through a run of 502s from a proxy', async () => { + tokenAnswers(() => answer(502, 'Bad Gateway')); + await refresh(5); + expectKept(); + }); + + it('keeps the data through a run of the route\'s own 500s', async () => { + tokenAnswers(() => answer(500, { error: 'Internal server error' })); + await refresh(5); + expectKept(); + }); + + it('keeps the data on a refusal that says nothing about the login', async () => { + tokenAnswers(() => answer(401, { error: 'Refresh failed' })); + await refresh(); + expectKept(); + }); + + it('keeps the data on a refusal it can\'t read', async () => { + tokenAnswers(() => answer(401, 'nope')); + await refresh(); + expectKept(); + }); + + it('wipes on a real invalid_grant, and names the lost mail', async () => { + tokenAnswers(() => answer(401, { error: 'invalid_grant' })); + offline.startOfflineMode(); + await refresh(); + await vi.waitFor(() => expect(purge.files).toBeGreaterThan(0)); + expect(localStorage.getItem(state.OFFLINE_ACCOUNTS_KEY)).toBeNull(); + expect(lost.consumeLost()).toEqual(['Unsent']); + }); + + it('wipes at the organisation\'s session limit', async () => { + const { SESSION_TOO_OLD } = await import('@/lib/session-policy/constants'); + tokenAnswers(() => answer(401, { error: SESSION_TOO_OLD, maxHours: 8 })); + await refresh(); + await vi.waitFor(() => expect(purge.files).toBeGreaterThan(0)); + }); +}); + +describe('a background account', () => { + const BOB = 'bob@mail.example.test'; + const token = async () => useAuthStore.getState().refreshAccessToken({ accountId: BOB }); + + beforeEach(() => { + useAccountStore.setState({ accounts: [entry(ALICE, 0), entry(BOB, 1)] } as never); + state.writeEnabled([ALICE, BOB]); + offline.startOfflineMode(); + }); + + it('keeps its data on an unknown refusal', async () => { + tokenAnswers(() => answer(401, { error: 'Refresh failed' })); + await token(); + await vi.waitFor(() => expect(useAccountStore.getState().getAccountById(BOB)).toBeUndefined()); + expect(state.isEvicted(BOB)).toBe(true); + expect(store.removed).toEqual([]); + }); + + it('wipes on a real invalid_grant', async () => { + tokenAnswers(() => answer(401, { error: 'invalid_grant' })); + await token(); + await vi.waitFor(() => expect(store.removed).toEqual([BOB])); + expect(state.isEvicted(BOB)).toBe(false); + }); +}); + +describe('data waiting for a sign-in', () => { + const BOB = 'bob@mail.example.test'; + + beforeEach(async () => { + state.writeEnabled([ALICE, BOB]); + await queue.queueSend(BOB, { subject: 'Bob\'s', to: [] }, { kind: 'email', args: [] }); + useAccountStore.setState({ accounts: [entry(ALICE, 0), entry(BOB, 1)] } as never); + useAccountStore.getState().removeAccount(BOB, 'evicted'); + }); + + it('is hidden from anyone else, in every list', async () => { + await offline.refreshQueueState(); + expect(state.useOfflineStore.getState().outbox.map((row) => row.subject)).toEqual(['Unsent']); + expect((await queue.readQueue()).outbox.map((row) => row.account)).toEqual([ALICE]); + await offline.refreshUsage(); + expect(state.visibleAccounts()).toEqual([ALICE]); + }); + + it('shows again when the account signs in', async () => { + useAccountStore.getState().addAccount({ username: 'bob', serverUrl: 'https://mail.example.test', authMode: 'oauth', rememberMe: true, label: 'bob', displayName: 'bob', email: 'bob', lastLoginAt: 1, isConnected: true, hasError: false, isDefault: false } as never); + expect(state.isEvicted(BOB)).toBe(false); + await offline.refreshQueueState(); + expect(state.useOfflineStore.getState().outbox.map((row) => row.subject).sort()).toEqual(['Bob\'s', 'Unsent']); + }); + + it('goes after 30 days', async () => { + localStorage.setItem(state.OFFLINE_EVICTED_KEY, JSON.stringify({ [BOB]: Date.now() - state.EVICTED_KEEP_MS - 1000 })); + offline.startOfflineMode(); + await vi.waitFor(() => expect(store.removed).toEqual([BOB])); + expect(state.isEvicted(BOB)).toBe(false); + expect(state.useOfflineStore.getState().enabled).toEqual([ALICE]); + }); + + it('stays before that', async () => { + offline.startOfflineMode(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(store.removed).toEqual([]); + expect(state.useOfflineStore.getState().enabled).toEqual([ALICE, BOB]); + }); + + it('survives someone else\'s full sign-out, whose own data goes', async () => { + offline.startOfflineMode(); + await useAuthStore.getState().logout({ endProviderSession: false }); + await vi.waitFor(() => expect(store.removed).toEqual([ALICE])); + expect(purge.files).toBe(0); + expect(state.useOfflineStore.getState().enabled).toEqual([BOB]); + expect(state.isEvicted(BOB)).toBe(true); + }); +}); + +describe('a sign-out that has to drop another account', () => { + const BOB = 'bob@mail.example.test'; + + it('keeps that account\'s data for its next sign-in, since nobody asked it', async () => { + useAccountStore.setState({ accounts: [entry(ALICE, 0), { ...entry(BOB, 1), hasError: true }] } as never); + state.writeEnabled([ALICE, BOB]); + await queue.queueSend(BOB, { subject: 'Bob\'s', to: [] }, { kind: 'email', args: [] }); + offline.startOfflineMode(); + tokenAnswers(() => answer(200)); + await useAuthStore.getState().logout({ endProviderSession: false }); + await vi.waitFor(() => expect(useAccountStore.getState().accounts).toEqual([])); + await vi.waitFor(() => expect(store.removed).toEqual([ALICE])); + expect(purge.files).toBe(0); + expect(state.isEvicted(BOB)).toBe(true); + expect(state.useOfflineStore.getState().enabled).toEqual([BOB]); + expect(lost.consumeLost()).not.toContain('Bob\'s'); + }); +}); + +describe('a login without "remember me"', () => { + it('ends with its data at a reload, naming lost mail, even before the host runs', async () => { + useAccountStore.setState({ accounts: [{ ...entry(ALICE, 0), rememberMe: false }] } as never); + tokenAnswers(() => answer(200)); + await useAuthStore.getState().checkAuth(); + expect(useAccountStore.getState().getAccountById(ALICE)).toBeUndefined(); + expect(state.isEvicted(ALICE)).toBe(false); + offline.startOfflineMode(); + await vi.waitFor(() => expect(purge.files).toBeGreaterThan(0)); + expect(lost.consumeLost()).toEqual(['Unsent']); + }); +}); diff --git a/stores/__tests__/auth-store-local-copy.test.ts b/stores/__tests__/auth-store-local-copy.test.ts new file mode 100644 index 000000000..ee514fb01 --- /dev/null +++ b/stores/__tests__/auth-store-local-copy.test.ts @@ -0,0 +1,212 @@ +import { beforeAll, beforeEach, expect, it, vi } from 'vitest'; +import { createConfig } from '@/lib/__tests__/fixtures/config'; + +// With the local store on, a start paints from the local copy at once, then +// connects and swaps. The hooks are there before the first request. +const offline = vi.hoisted(() => ({ + order: [] as string[], + session: { apiUrl: 'x' } as unknown, + gone: false, + ready: true, + // A local copy with no identities kept answers an empty list. + localIdentities: Promise.resolve([]) as Promise, + releaseLocal: () => {}, +})); + +vi.mock('@/lib/offline/offline-mode', () => ({ + registerOfflineHooks: () => { offline.order.push('hooks'); }, + offlineSessionFor: async (_account: string, options?: { withFolders?: boolean }) => (options?.withFolders && !offline.ready ? null : offline.session), + serverUnreachable: async () => offline.gone, + markStartedOffline: () => { offline.order.push('startedOffline'); }, +})); + +let connectGate: Promise = Promise.resolve(); + +vi.mock('@/lib/jmap/client', () => { + class JMAPClient { + offlineOnly = false; + constructor(public serverUrl: string, public username: string, public password: string) {} + static withBearer() { throw new Error('not used'); } + static fromOfflineSession(serverUrl: string, username: string) { + offline.order.push('local client'); + const client = new JMAPClient(serverUrl, username, ''); + client.offlineOnly = true; + return client; + } + isOfflineOnly() { return this.offlineOnly; } + async connect() { offline.order.push('connect'); await connectGate; } + getSessionUsername() { return this.username; } + getUsername() { return this.username; } + async getIdentities() { + return this.offlineOnly ? offline.localIdentities : [{ id: 'a', email: this.username, name: this.username, mayDelete: false }]; + } + getAuthHeader() { return 'Basic eA=='; } + onConnectionChange() {} + onRateLimit() {} + getRateLimitRemainingMs() { return 0; } + supportsContacts() { return false; } + supportsPrincipals() { return false; } + supportsVacationResponse() { return false; } + supportsCalendars() { return false; } + supportsSieve() { return false; } + disconnect() {} + getAccountId() { return 'acct'; } + } + class RateLimitError extends Error {} + class RequestTimeoutError extends Error {} + return { JMAPClient, RateLimitError, RequestTimeoutError }; +}); +vi.mock('@/lib/stalwart/principal', () => ({ fetchPrincipalDisplayName: async () => null })); + +import { useAuthStore } from '@/stores/auth-store'; +import { useAccountStore } from '@/stores/account-store'; +import { useOfflineStore } from '@/lib/offline/state'; + +// The auth store keeps its clients for good, so each test gets an account of its own. +let run = 0; +let ACCOUNT = makeAccount(); + +function makeAccount() { + run += 1; + const username = `bob${run}@x.test`; + return { + id: `${username}@mail.x.test`, label: 'Bob', serverUrl: 'https://mail.x.test', username, authMode: 'basic' as const, + rememberMe: true, cookieSlot: 0, displayName: 'Bob', email: username, isConnected: false, hasError: false, + lastLoginAt: 1, avatarColor: '#000000', isDefault: true, + }; +} + +let sessionAnswer: () => Response; + +beforeAll(() => { + vi.stubGlobal('fetch', vi.fn(async (input: unknown) => { + const url = String(input); + if (url.includes('/api/auth/session')) return sessionAnswer(); + if (url.includes('/api/config')) return new Response(JSON.stringify(createConfig()), { status: 200 }); + return new Response('{}', { status: 200 }); + })); +}); + +const goodSession = () => new Response(JSON.stringify({ serverUrl: ACCOUNT.serverUrl, username: ACCOUNT.username, password: 'x' }), { status: 200 }); + +beforeEach(async () => { + ACCOUNT = makeAccount(); + offline.order.length = 0; + offline.session = { apiUrl: 'x' }; + offline.gone = false; + offline.ready = true; + offline.localIdentities = Promise.resolve([]); + connectGate = Promise.resolve(); + sessionAnswer = goodSession; + useAccountStore.setState({ accounts: [ACCOUNT] as never, activeAccountId: ACCOUNT.id, defaultAccountId: ACCOUNT.id }); + useOfflineStore.setState({ enabled: [ACCOUNT.id], startedOffline: false }); + useAuthStore.setState({ client: null, isAuthenticated: false, activeAccountId: ACCOUNT.id, identities: [], primaryIdentity: null }); +}); + +type ClientLike = { isOfflineOnly(): boolean }; +const current = () => useAuthStore.getState().client as unknown as ClientLike | null; + +it('registers the hooks first, shows the local copy before connecting, then swaps', async () => { + let open: () => void = () => {}; + connectGate = new Promise((resolve) => { open = resolve; }); + await useAuthStore.getState().checkAuth(); + + // The real connect hasn't finished, and the app is up on the local copy. + expect(useAuthStore.getState().isAuthenticated).toBe(true); + expect(useAuthStore.getState().isLoading).toBe(false); + expect(current()?.isOfflineOnly()).toBe(true); + expect(offline.order[0]).toBe('hooks'); + expect(offline.order).toContain('local client'); + // No "you're offline" while the server is simply being asked. + expect(offline.order).not.toContain('startedOffline'); + + open(); + await vi.waitFor(() => expect(current()?.isOfflineOnly()).toBe(false)); + expect(useAuthStore.getState().isAuthenticated).toBe(true); + expect(useAuthStore.getState().getClientForAccount(ACCOUNT.id)).toBe(useAuthStore.getState().client); +}); + +it('stays on the local copy when the server can\'t be reached', async () => { + sessionAnswer = () => new Response('{}', { status: 503 }); + offline.gone = true; + await useAuthStore.getState().checkAuth(); + await vi.waitFor(() => expect(offline.order).toContain('startedOffline')); + expect(useAuthStore.getState().isAuthenticated).toBe(true); + expect(current()?.isOfflineOnly()).toBe(true); + expect(useAccountStore.getState().getAccountById(ACCOUNT.id)).toBeTruthy(); +}); + +it('stays on the local copy, and says so, when the server answers a start with a 5xx or 429', async () => { + for (const status of [502, 429]) { + ACCOUNT = makeAccount(); + offline.order.length = 0; + useAccountStore.setState({ accounts: [ACCOUNT] as never, activeAccountId: ACCOUNT.id, defaultAccountId: ACCOUNT.id }); + useOfflineStore.setState({ enabled: [ACCOUNT.id] }); + useAuthStore.setState({ client: null, isAuthenticated: false, activeAccountId: ACCOUNT.id }); + sessionAnswer = () => new Response('{}', { status }); + // The probe passes: the server itself is there. + offline.gone = false; + await useAuthStore.getState().checkAuth(); + await vi.waitFor(() => expect(offline.order).toContain('startedOffline')); + expect(current()?.isOfflineOnly()).toBe(true); + expect(useAccountStore.getState().getAccountById(ACCOUNT.id)).toBeTruthy(); + } +}); + +it('goes the usual way when the store has no copy', async () => { + offline.session = null; + await useAuthStore.getState().checkAuth(); + expect(offline.order).not.toContain('local client'); + expect(current()?.isOfflineOnly()).toBe(false); +}); + +it('doesn\'t touch the store for an account without it', async () => { + useOfflineStore.setState({ enabled: [] }); + await useAuthStore.getState().checkAuth(); + expect(offline.order).toEqual(['connect']); + expect(current()?.isOfflineOnly()).toBe(false); +}); + +it('goes the usual way while the store has no folders yet', async () => { + offline.ready = false; + await useAuthStore.getState().checkAuth(); + expect(offline.order).not.toContain('local client'); + expect(current()?.isOfflineOnly()).toBe(false); +}); + +it('still starts on an empty store when the server can\'t be reached', async () => { + offline.ready = false; + offline.gone = true; + sessionAnswer = () => new Response('{}', { status: 503 }); + await useAuthStore.getState().checkAuth(); + await vi.waitFor(() => expect(current()?.isOfflineOnly()).toBe(true)); +}); + +it('fills the From list from the server after the swap, when the local copy had none', async () => { + let open: () => void = () => {}; + connectGate = new Promise((resolve) => { open = resolve; }); + await useAuthStore.getState().checkAuth(); + await vi.waitFor(() => expect(current()?.isOfflineOnly()).toBe(true)); + expect(useAuthStore.getState().identities).toEqual([]); + open(); + await vi.waitFor(() => expect(useAuthStore.getState().identities.map((identity) => identity.email)).toEqual([ACCOUNT.username])); + expect(useAuthStore.getState().primaryIdentity?.email).toBe(ACCOUNT.username); +}); + +it('doesn\'t let a late local answer overwrite the server\'s From list', async () => { + offline.localIdentities = new Promise((resolve) => { offline.releaseLocal = () => resolve([{ id: 'old', email: 'stale@x.test', name: 'Stale', mayDelete: false }]); }); + await useAuthStore.getState().checkAuth(); + await vi.waitFor(() => expect(useAuthStore.getState().identities.map((identity) => identity.email)).toEqual([ACCOUNT.username])); + offline.releaseLocal(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(useAuthStore.getState().identities.map((identity) => identity.email)).toEqual([ACCOUNT.username]); +}); + +it('leaves the login path to a refused account, and keeps its data', async () => { + sessionAnswer = () => new Response('{}', { status: 401 }); + await useAuthStore.getState().checkAuth(); + await vi.waitFor(() => expect(useAccountStore.getState().getAccountById(ACCOUNT.id)).toBeUndefined()); + await vi.waitFor(() => expect(useAuthStore.getState().isAuthenticated).toBe(false)); + expect(useAuthStore.getState().client).toBeNull(); + expect(useOfflineStore.getState().enabled).toEqual([ACCOUNT.id]); +}); diff --git a/stores/__tests__/email-snapshot-account.test.ts b/stores/__tests__/email-snapshot-account.test.ts index a7d2c4e4f..c2ee1933d 100644 --- a/stores/__tests__/email-snapshot-account.test.ts +++ b/stores/__tests__/email-snapshot-account.test.ts @@ -38,6 +38,72 @@ describe('email boot snapshot', { timeout: 30_000 }, () => { }); }); +// With the local store on, mail doesn't go to localStorage in plain text too. +describe('email boot snapshot for an account in the local store', { timeout: 30_000 }, () => { + const STATE = { mailboxes: MAILBOXES, selectedMailbox: 'inbox', emails: [{ id: 'e1', mailboxIds: { inbox: true } }] }; + + async function fresh(account: string) { + vi.useFakeTimers(); + persistSession(account); + vi.resetModules(); + const { useEmailStore } = await import('@/stores/email-store'); + const offline = await import('@/lib/offline/state'); + return { useEmailStore, offline }; + } + + afterEach(() => vi.useRealTimers()); + + it('isn\'t written', async () => { + const { useEmailStore, offline } = await fresh('alice@mail.example'); + offline.useOfflineStore.setState({ enabled: ['alice@mail.example'] }); + useEmailStore.setState(STATE as never); + await vi.advanceTimersByTimeAsync(1500); + expect(localStorage.getItem('email-snapshot')).toBeNull(); + }); + + it('is written for an account outside it', async () => { + const { useEmailStore, offline } = await fresh('alice@mail.example'); + offline.useOfflineStore.setState({ enabled: ['bob@mail.example'] }); + useEmailStore.setState(STATE as never); + await vi.advanceTimersByTimeAsync(1500); + expect(JSON.parse(localStorage.getItem('email-snapshot') ?? '{}').account).toBe('alice@mail.example'); + }); + + it('goes only when that account\'s store can take over', async () => { + const { offline } = await fresh('alice@mail.example'); + localStorage.setItem('email-snapshot', JSON.stringify({ v: 3, account: 'alice@mail.example', mailboxes: MAILBOXES })); + offline.forgetMailSnapshot('bob@mail.example'); + expect(localStorage.getItem('email-snapshot')).not.toBeNull(); + offline.forgetMailSnapshot('alice@mail.example'); + expect(localStorage.getItem('email-snapshot')).toBeNull(); + }); +}); + +// A client's late answer doesn't land on the client that replaced it. +describe('a list answer from a client that was replaced', { timeout: 30_000 }, () => { + it('is dropped', async () => { + persistSession('alice@mail.example'); + vi.resetModules(); + const { useEmailStore } = await import('@/stores/email-store'); + const { useAuthStore } = await import('@/stores/auth-store'); + const rows = [{ id: 'stale', mailboxIds: { inbox: true } }]; + let answer: (value: unknown) => void = () => {}; + const local = { + getAccountId: () => 'acc', + getEmails: vi.fn(() => new Promise((resolve) => { answer = resolve; })), + getThreadEmailCounts: vi.fn(async () => ({})), + }; + useAuthStore.setState({ client: local as never, activeAccountId: 'alice@mail.example' }); + useEmailStore.setState({ mailboxes: MAILBOXES as never, selectedMailbox: 'inbox', emails: [] }); + const fetching = useEmailStore.getState().fetchEmails(local as never, 'inbox'); + await vi.waitFor(() => expect(local.getEmails).toHaveBeenCalled()); + useAuthStore.setState({ client: { getAccountId: () => 'acc' } as never }); + answer({ emails: rows, hasMore: false, total: 1, state: 's1' }); + await fetching; + expect(useEmailStore.getState().emails).toEqual([]); + }); +}); + // The snapshot keeps the last plain folder that was open; a session that // ended in a unified view left Trash there, and every visit to "/" opened it. describe('email boot snapshot folder', { timeout: 30_000 }, () => { diff --git a/stores/account-store.ts b/stores/account-store.ts index ebaf2b6dc..836dada92 100644 --- a/stores/account-store.ts +++ b/stores/account-store.ts @@ -5,6 +5,17 @@ import { IS_DESKTOP_BUILD } from '@/lib/desktop/flag'; import { forgetDesktopServer } from '@/lib/desktop/bridge'; import { generateAccountId, generateAvatarColor, getMaxAccounts } from '@/lib/account-utils'; import { authHooks } from '@/lib/plugin-hooks'; +import { clearEvicted, markEvicted } from '@/lib/offline/state'; + +export type AccountRemovalReason = 'explicit' | 'evicted'; + +const removalListeners = new Set<(accountId: string, reason: AccountRemovalReason) => void>(); + +/** Told before an account leaves the registry, and why. */ +export function onAccountRemoved(listener: (accountId: string, reason: AccountRemovalReason) => void): () => void { + removalListeners.add(listener); + return () => removalListeners.delete(listener); +} function sameServerOrigin(a: string, b: string): boolean { try { @@ -85,7 +96,11 @@ interface AccountState { defaultAccountId: string | null; addAccount: (entry: Omit) => string; - removeAccount: (accountId: string) => void; + /** + * `reason`: 'explicit' when someone signed the account out or removed it, its + * offline data goes with it; 'evicted' when the app dropped it by itself. + */ + removeAccount: (accountId: string, reason?: AccountRemovalReason) => void; setActiveAccount: (accountId: string) => void; setDefaultAccount: (accountId: string) => void; getDefaultAccount: () => AccountEntry | null; @@ -108,6 +123,7 @@ export const useAccountStore = create()( const state = get(); const id = generateAccountId(entry.username, entry.serverUrl); + clearEvicted(id); if (state.accounts.some((a) => a.id === id)) { // Already exists - update mutable fields and return existing id set((s) => ({ @@ -158,9 +174,11 @@ export const useAccountStore = create()( return id; }, - removeAccount: (accountId) => { + removeAccount: (accountId, reason = 'explicit') => { if (!get().accounts.some((a) => a.id === accountId)) return; const leaving = get().accounts.find((a) => a.id === accountId); + if (reason === 'evicted') markEvicted(accountId); + for (const listener of [...removalListeners]) listener(accountId, reason); set((s) => { const remaining = s.accounts.filter((a) => a.id !== accountId); const wasDefault = s.defaultAccountId === accountId; diff --git a/stores/auth-store.ts b/stores/auth-store.ts index 9f6579916..9fc0c7ca3 100644 --- a/stores/auth-store.ts +++ b/stores/auth-store.ts @@ -9,7 +9,7 @@ import { useVacationStore } from './vacation-store'; import { subscriptionOwner, useCalendarStore } from './calendar-store'; import { useFilterStore } from './filter-store'; import { useSettingsStore } from './settings-store'; -import { useAccountStore, type AccountEntry } from './account-store'; +import { useAccountStore, type AccountEntry, type AccountRemovalReason } from './account-store'; import { fetchPrincipalDisplayName } from '@/lib/stalwart/principal'; import { fetchConfig, getCachedConfig } from '@/hooks/use-config'; import { debug } from '@/lib/debug'; @@ -22,7 +22,7 @@ import { DESKTOP_CLIENT_ID_KEY } from '@/lib/desktop/browser-sign-in'; import { isEmbedded, notifyParent } from '@/lib/iframe-bridge'; import { snapshotAccount, restoreAccount, clearAllStores, evictAccount, evictAll, forgetAccountRows } from '@/lib/account-state-manager'; import { clearAllPluginStorage, clearPluginStorageForAccount } from '@/lib/plugin-sandbox/storage-scope'; -import { broadcastSignOut, onSignedOutElsewhere, passSignOutGates, purgeSignedOutData, runBeforeSignOut } from '@/lib/sign-out-cleanup'; +import { broadcastSignOut, onSignedOutElsewhere, passSignOutGates, purgeSignedOutData, runBeforeSignOut, type SignOutCleanupOptions } from '@/lib/sign-out-cleanup'; import { forgetPrincipals } from '@/lib/delegation/known-principals'; import { useSearchHistoryStore } from './search-history-store'; import { useCalendarNotificationStore } from './calendar-notification-store'; @@ -36,6 +36,8 @@ import { IS_LITE, IS_LITE_STALWART } from '@/lib/lite'; import { withTokenLock } from '@/lib/auth/token-lock'; import { requestWindowSession, type SharedAccountSession } from '@/lib/popout/channel'; import { closePopoutWindow, isPopoutWindow, mainWindowStorage } from '@/lib/popout/window-role'; +import { noteCredentialsRejected } from '@/lib/offline/credentials-rejected'; +import { readEvicted, useOfflineStore } from '@/lib/offline/state'; import { toAsciiEmail } from '@/lib/idn'; import { flushComposers } from '@/lib/compose-flush-registry'; import { sortIdentities } from '@/lib/identity-order'; @@ -122,7 +124,7 @@ interface AuthState { * (the organisation's session limits, which sign out without asking). */ logoutAll: (options?: LogoutOptions) => Promise; - removeAccount: (accountId: string) => void; + removeAccount: (accountId: string, reason?: AccountRemovalReason) => void; switchAccount: (accountId: string) => Promise; checkAuth: () => Promise; /** @@ -164,6 +166,12 @@ export interface LogoutOptions { * provider, and signing in again should stay a single click. */ endProviderSession?: boolean; + /** + * The app, not the user, ends this login, and nothing says the login itself + * is over (an outage, an unknown refusal): local mail and the outbox stay, + * hidden, for the next sign-in or 30 days. + */ + keepLocalData?: boolean; } const ERROR_PATTERNS: Array<{ key: string; matches: string[] }> = [ @@ -1241,6 +1249,20 @@ function recordPermanentRefreshFailure(status: number, accountId?: string): bool return true; } +/** + * Whether a refused renewal ends the login for real, which wipes the account's + * local data: the server said invalid_grant, or the organisation's session limit + * was reached. Anything else, including a body we can't read, keeps the data. + */ +async function loginReallyOver(res: Response): Promise { + try { + const { error } = await res.clone().json() as { error?: unknown }; + return error === 'invalid_grant' || error === SESSION_TOO_OLD; + } catch { + return false; + } +} + function notifySignInAgain(): void { void import('@/stores/toast-store').then(({ toast }) => { toast.error('Your session could not be renewed', 'Sign in again to continue.'); @@ -1417,7 +1439,27 @@ async function startOfflineClient(accountId: string, serverUrl: string, username } } -function performFullLogout(set: (state: Partial) => void): void { +/** + * The account's local copy, to show right away while the real connect runs. A + * client that answers from the device only; null without the local store for + * that account, and the start goes the usual way. + */ +async function startLocalCopyClient(accountId: string, serverUrl: string, username: string): Promise { + if (IS_LITE || skipLocalCopy || !useOfflineStore.getState().enabled.includes(accountId)) return null; + try { + const offline = await import('@/lib/offline/offline-mode'); + // A store that has no folders yet has nothing to show: the snapshot stays up. + const session = await offline.offlineSessionFor(accountId, { withFolders: true }); + return session ? JMAPClient.fromOfflineSession(serverUrl, username, session) : null; + } catch { + return null; + } +} + +// While device-only clients get replaced, restoring goes straight to the server. +let skipLocalCopy = false; + +function performFullLogout(set: (state: Partial) => void, options: SignOutCleanupOptions = {}): void { useSettingsStore.getState().disableSync(); // With settings sync on, the local settings and templates are a copy of the // signed-out account's server file. Left in place, the next account to sign @@ -1456,7 +1498,9 @@ function performFullLogout(set: (state: Partial) => void): void { // doesn't re-write stale values. try { localStorage.removeItem('auth-storage'); } catch { /* noop */ } try { localStorage.removeItem('account-registry'); } catch { /* noop */ } - purgeSignedOutData(); + // A dropped account's data outlives anyone else's sign-out. The account that + // signed out takes its own along when it leaves the registry. + purgeSignedOutData({ keepLocalStore: options.keepLocalStore || Object.keys(readEvicted()).length > 0 }); // Other tabs still hold the accounts' mail in memory. broadcastSignOut(); } @@ -1477,9 +1521,9 @@ function notifySignedIn(state: { activeAccountId: string | null; username: strin void authHooks.onLogin.emit({ accountId: state.activeAccountId, username: state.username ?? '' }); } -function beforeLogoutHooks(accountId: string): Promise { +function beforeLogoutHooks(accountId: string, options: SignOutCleanupOptions = {}): Promise { return Promise.race([ - Promise.all([authHooks.onBeforeLogout.emit({ accountId }), runBeforeSignOut(accountId)]).then(() => {}), + Promise.all([authHooks.onBeforeLogout.emit({ accountId }), runBeforeSignOut(accountId, options)]).then(() => {}), new Promise((resolve) => setTimeout(resolve, LOGOUT_HOOK_BUDGET_MS)), ]); } @@ -2288,15 +2332,17 @@ export const useAuthStore = create()( // maintenance windows and offline spells. if (res.status === 401) { resetRefreshBackoff(accountId ?? undefined); + const over = await loginReallyOver(res); + if (over && accountId) noteCredentialsRejected(accountId); // Only this login's refresh token was refused. if (accountId && !isActive) { - get().removeAccount(accountId); + get().removeAccount(accountId, over ? 'explicit' : 'evicted'); return null; } await saveDraftsBeforeExpiry(slot); notifyParent('sso:session-expired'); markSessionExpired(); - get().logout({ endProviderSession: false }); + get().logout({ endProviderSession: false, keepLocalData: !over }); return null; } // A Bulwark-side failure (500/502) that keeps repeating will not @@ -2308,10 +2354,11 @@ export const useAuthStore = create()( notifyParent('sso:session-expired'); markSessionExpired(); // removeAccount() on the active account would sign out as if the user asked to. - if (accountId && accountId !== get().activeAccountId) get().removeAccount(accountId); + if (accountId && accountId !== get().activeAccountId) get().removeAccount(accountId, 'evicted'); else { await saveDraftsBeforeExpiry(slot); - get().logout({ endProviderSession: false }); + // An outage on our side says nothing against the mail on this device. + get().logout({ endProviderSession: false, keepLocalData: true }); } return null; } @@ -2381,13 +2428,14 @@ export const useAuthStore = create()( const slot = account?.cookieSlot ?? 0; // `options` can be a click event when the action is wired straight to onClick. const userInitiated = options?.endProviderSession !== false; + const keepLocalData = options?.keepLocalData === true; const endProviderSession = userInitiated && !wasDemoMode && shouldEndProviderSession(account); // Signing out on purpose may be stopped by the user (unsent mail in // the offline outbox would be lost); an automatic one may not. if (userInitiated && !(await passSignOutGates(accountId ?? 'all'))) return; - await beforeLogoutHooks(accountId ?? 'all'); + await beforeLogoutHooks(accountId ?? 'all', { keepLocalStore: keepLocalData }); // From here on nothing may renew this slot's session; the cleanup // below reopens it once the credentials are gone. @@ -2405,7 +2453,7 @@ export const useAuthStore = create()( if (accountId) { clients.delete(accountId); evictAccount(accountId); - accountStore.removeAccount(accountId); + accountStore.removeAccount(accountId, keepLocalData ? 'evicted' : 'explicit'); forgetPrincipals(accountId); if (!wasDemoMode) clearPluginStorageForAccount(accountId); } @@ -2499,7 +2547,9 @@ export const useAuthStore = create()( clients.get(remaining.id)?.disconnect(); clients.delete(remaining.id); evictAccount(remaining.id); - accountStore.removeAccount(remaining.id); + // Dropped by this sign-out, not signed out by anyone: their unsent + // mail never passed a question, so it waits for their next sign-in. + accountStore.removeAccount(remaining.id, 'evicted'); } } @@ -2513,7 +2563,7 @@ export const useAuthStore = create()( ...droppedAccounts.map((dropped) => clearSlotCredentials(dropped.cookieSlot ?? 0, dropped.authMode === 'oauth')), ]); - performFullLogout(set); + performFullLogout(set, { keepLocalStore: keepLocalData }); notifyParent('sso:logout'); @@ -2524,7 +2574,7 @@ export const useAuthStore = create()( // drop it from the registry, and clear its per-slot cookies. If asked to // remove the active account, defer to logout() which handles switching // away or redirecting. - removeAccount: (accountId: string) => { + removeAccount: (accountId: string, reason = 'explicit') => { if (accountId === get().activeAccountId) { void get().logout(); return; } const accountStore = useAccountStore.getState(); const account = accountStore.getAccountById(accountId); @@ -2544,7 +2594,7 @@ export const useAuthStore = create()( clients.delete(accountId); evictAccount(accountId); forgetAccountRows(accountId, get().client); - accountStore.removeAccount(accountId); + accountStore.removeAccount(accountId, reason); forgetPrincipals(accountId); clearPluginStorageForAccount(accountId); }, @@ -2704,7 +2754,7 @@ export const useAuthStore = create()( // Cannot restore - remove the stale account and redirect to login evictAccount(accountId); - accountStore.removeAccount(accountId); + accountStore.removeAccount(accountId, 'evicted'); void clearSlotCredentials(targetAccount.cookieSlot, targetAccount.authMode === 'oauth'); // Restore the previous account if still available @@ -2882,6 +2932,12 @@ export const useAuthStore = create()( // with a stale client reference while we're restoring accounts. set({ isLoading: true, client: null }); + // The offline hooks have to be there before the first request, or a + // start without the server has nothing to paint from. + if (!IS_LITE && useOfflineStore.getState().enabled.length > 0) { + await import('@/lib/offline/offline-mode').then((offline) => offline.registerOfflineHooks()).catch(() => {}); + } + // Determine which account to activate first const defaultAccount = accountStore.getDefaultAccount(); const activeId = get().activeAccountId; @@ -2910,7 +2966,8 @@ export const useAuthStore = create()( if (!account.rememberMe && !IS_LITE) { forgottenAccounts += 1; evictAccount(account.id); - accountStore.removeAccount(account.id); + // The user's choice on what may be a shared computer: its local data goes too. + accountStore.removeAccount(account.id, 'explicit'); if (account.authMode === 'oauth') void clearSlotCredentials(account.cookieSlot, true); return; } @@ -2935,7 +2992,7 @@ export const useAuthStore = create()( // Repeated 500/502 (see recordPermanentRefreshFailure) falls // through here and evicts the account like a rejection. (#972) if (res.status >= 500) notifySignInAgain(); - throw new Error(`Token refresh failed: ${res.status}`); + throw Object.assign(new Error(`Token refresh failed: ${res.status}`), { loginOver: await loginReallyOver(res) }); } } else { const res = await fetchSlotSession(account.cookieSlot); @@ -2953,7 +3010,7 @@ export const useAuthStore = create()( impersonated: session.impersonated === true, }); void syncAccountDisplayName(account.id, client); - } else if (res.status >= 500) { + } else if (res.status >= 500 || res.status === 429) { throw new TransientAuthError('Session restore failed', res.status); } else { throw new Error(`Session cookie missing: ${res.status}`); @@ -2992,9 +3049,10 @@ export const useAuthStore = create()( } // Remove unrestorable accounts so the user is prompted to log in // again rather than seeing a stale error entry forever. Its - // refresh token goes with it: nothing would use or revoke it. + // refresh token goes with it: nothing would use or revoke it. Its + // local data waits, unless the server really ended the login. evictAccount(account.id); - accountStore.removeAccount(account.id); + accountStore.removeAccount(account.id, (err as { loginOver?: boolean }).loginOver ? 'explicit' : 'evicted'); void clearSlotCredentials(account.cookieSlot, account.authMode === 'oauth'); } }; @@ -3005,7 +3063,56 @@ export const useAuthStore = create()( // bump re-runs per-client effects (push binding) over the full set. const targetEntry = accounts.find((account) => account.id === targetId); const otherAccounts = accounts.filter((account) => account.id !== targetId); - if (targetEntry) await restoreAccount(targetEntry); + // With the local store on, the local copy shows at once and the real + // connect takes over when it's done (swapInRealClient below). + const localCopy = targetEntry && !targetEntry.awaitingSignIn && targetEntry.rememberMe + ? await startLocalCopyClient(targetEntry.id, targetEntry.serverUrl, targetEntry.username) + : null; + if (localCopy && targetEntry) { + clients.set(targetEntry.id, localCopy); + accountStore.updateAccount(targetEntry.id, { isConnected: true, hasError: false }); + } else if (targetEntry) { + await restoreAccount(targetEntry); + } + + /** + * The real connect behind a local copy on screen. No reload and no + * loading state: the client in the store changes and the pages + * refetch from the server. A refused account goes through the usual + * restore path, an unreachable server keeps the local copy. + */ + const swapInRealClient = async (entry: (typeof accounts)[number], local: JMAPClient) => { + clients.delete(entry.id); + await restoreAccount(entry); + const real = clients.get(entry.id); + if (!real) { + if (useAccountStore.getState().getAccountById(entry.id)) { + clients.set(entry.id, local); + // A 5xx or a rate limit, with the server up otherwise. The app stays + // on the local copy, and says so, and leaves it when the server answers. + void import('@/lib/offline/offline-mode').then((offline) => offline.markStartedOffline()).catch(() => {}); + return; + } + if (get().activeAccountId !== entry.id) return; + // Refused, and evicted by the restore. Whoever is left, or the login page. + set({ client: null, isAuthenticated: false, activeAccountId: null }); + if (useAccountStore.getState().accounts.length > 0) await get().checkAuth(); + else markSessionExpired(); + return; + } + if (get().activeAccountId !== entry.id) return; + initializeFeatureStores(real); + set({ client: real, ...getClientRateLimitState(real), connectionLost: false, error: null }); + // The local copy may have had none yet, and the server's are newer. + void real.getIdentities() + .then((raw) => { + if (get().client !== real) return; + const { identities, primaryIdentity } = loadIdentities(raw, entry.username); + set({ identities, primaryIdentity }); + applyPreferredIdentity(entry.id); + }) + .catch((err) => debug.error('Failed to load identities after the local copy:', err)); + }; // A few at a time rather than one after the other: with many logins // the serial walk left the last of them - and with it a complete @@ -3049,6 +3156,8 @@ export const useAuthStore = create()( // spending a serial round trip before the UI unblocks. const identitiesLoaded = targetClient.getIdentities() .then((raw) => { + // The real client took over meanwhile, and asks for itself. + if (localCopy && get().client !== null && get().client !== targetClient) return; const { identities, primaryIdentity } = loadIdentities(raw, targetAccount.username); set({ identities, primaryIdentity }); }) @@ -3067,6 +3176,7 @@ export const useAuthStore = create()( error: null, activeAccountId: targetId, }); + if (localCopy && targetEntry) void swapInRealClient(targetEntry, localCopy); fetchConfig().then(config => { if (!config.settingsSyncEnabled) return; @@ -3484,7 +3594,12 @@ export const useAuthStore = create()( const offline = [...clients.entries()].filter(([, c]) => c.isOfflineOnly()); if (offline.length === 0) return true; for (const [id] of offline) clients.delete(id); - await get().checkAuth(); + skipLocalCopy = true; + try { + await get().checkAuth(); + } finally { + skipLocalCopy = false; + } return offline.every(([id]) => { const client = clients.get(id); return !!client && !client.isOfflineOnly(); diff --git a/stores/email-store.ts b/stores/email-store.ts index 0f1b60161..8bff6f55d 100644 --- a/stores/email-store.ts +++ b/stores/email-store.ts @@ -38,6 +38,7 @@ import { SendCancelledCleanupError } from "@/lib/send-failure-message"; import type { StorageQuota } from "@/lib/quota"; import { isArchiveActionEnabled } from "@/lib/archive-gate"; import { getOutboxHooks } from "@/lib/offline/outbox-hooks"; +import { useOfflineStore } from "@/lib/offline/state"; import { READ_FILTER, UNREAD_FILTER, UNREAD_SECTION_SIZE, andFilters, isUnreadFirstView, mergeUnreadSection, openSectionRows, sectionFromUnreadRows, unreadSectionRowCount, withoutReached, type UnreadSection, @@ -1626,10 +1627,13 @@ function threadCountsPatch( function captureEmailListView(): () => boolean { const view = useEmailStore.getState(); const activeAccountId = useAuthStore.getState().activeAccountId; + // A local copy's late answer must not overwrite what the real client loaded. + const authClient = useAuthStore.getState().client; const epoch = currentStoreEpoch(); return () => { const current = useEmailStore.getState(); return currentStoreEpoch() === epoch + && useAuthStore.getState().client === authClient && current.selectedMailbox === view.selectedMailbox && current.selectedKeyword === view.selectedKeyword && current.viewingAccountId === view.viewingAccountId @@ -8728,6 +8732,8 @@ if (typeof window !== 'undefined') { if (s.searchQuery || s.selectedKeyword || s.isScheduledView || s.viewingAccountId || s.isUnifiedView) return; const account = persistedAuthState()?.activeAccountId; if (typeof account !== 'string' || !account) return; + // Its mail lives encrypted in the store, and not in plain localStorage too. + if (useOfflineStore.getState().enabled.includes(account)) return; try { window.localStorage.setItem(EMAIL_SNAPSHOT_KEY, JSON.stringify({ v: EMAIL_SNAPSHOT_VERSION, diff --git a/stores/file-store.ts b/stores/file-store.ts index f9ec1a6b5..190c37264 100644 --- a/stores/file-store.ts +++ b/stores/file-store.ts @@ -1,3 +1,4 @@ +import { getOfflineHooks, type OfflineClientRef } from '@/lib/jmap/offline-hooks'; import { create } from 'zustand'; import type { FileNodeListings, IJMAPClient } from '@/lib/jmap/client-interface'; import type { FileNode, FileNodeRights } from '@/lib/jmap/types'; @@ -1688,7 +1689,8 @@ export const useFileStore = create((set, get) => ({ }, addRecentFile: (name: string, id: string) => { - const { recentFiles } = get(); + const { recentFiles, client } = get(); + if (client && id) getOfflineHooks()?.fileOpened?.(client as OfflineClientRef, client.getFilesAccountId(), id); const entry = { name, id, timestamp: Date.now() }; const filtered = recentFiles.filter(r => r.id !== id); const next = [entry, ...filtered].slice(0, 20);