From 5c5d982f9d84b633cd79c96b9b24bfc5b71817df Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:49:19 +0100 Subject: [PATCH 1/7] feat: save drafts and keep attachments while offline A draft saved offline now stays on the device under a stand-in id and goes to the server by itself when it's back. Until now every save was refused and the composer showed "not saved". - `lib/store/drafts`: one pending create per draft, however often it's saved. A draft from the server edited offline is replaced by create, then destroy, so a failure leaves a copy. Each create has a fixed Message-ID, looked up in Drafts after a lost response. A refused draft stays on the device with its error and can be opened. Changed on another device: both are kept and the user is told. - `JMAPClient`: stand-in ids in Email/get and Email/set resolve like calendar's, also when answering from the device. A file attached offline stays readable, and is uploaded before a save that names it. - Held files and kept mail files are stored in 1 MB chunks. Orphans are dropped at the end of a flush, counting what drafts need. - `lib/store/mail/attachments.ts`: attachments, inline images and raw messages per scope, newest first, after everything else in a pass, with eviction that spares pinned mail and never sees held files. - `mail.rawBlob` and `mail.rawStatus` for an offline export. Changed test assertions: `outbox.heldFile` and `mail.getBlob` return `blob` instead of `data`, so a file isn't held whole in memory. The attachment tests ignore raw downloads, and mail-sync runs with attachments off, because raw messages add a commit per message. --- components/email/email-composer.tsx | 7 +- components/mail/mail-app.tsx | 14 + .../offline/__tests__/outbox-drafts.test.tsx | 124 ++++ components/offline/offline-banner.tsx | 17 +- components/offline/offline-host.tsx | 14 +- components/offline/outbox-list.tsx | 58 +- lib/jmap/client-interface.ts | 1 + lib/jmap/client.ts | 78 ++- lib/jmap/offline-hooks.ts | 13 +- lib/jmap/stand-in-ids.ts | 18 +- lib/offline/__tests__/drafts-flush.test.ts | 194 ++++++ lib/offline/__tests__/migrate-legacy.test.ts | 4 +- lib/offline/flush.ts | 50 +- lib/offline/offline-mode.ts | 100 ++- lib/offline/queue.ts | 38 +- lib/offline/state.ts | 8 +- lib/store/__tests__/attachments.test.ts | 382 +++++++++++ lib/store/__tests__/drafts-client.test.ts | 303 +++++++++ lib/store/__tests__/drafts.test.ts | 599 ++++++++++++++++++ lib/store/__tests__/helpers/fake-drafts.ts | 120 ++++ .../__tests__/helpers/in-process-store.ts | 17 +- lib/store/__tests__/mail-answers.test.ts | 8 +- lib/store/__tests__/mail-sync.test.ts | 3 +- lib/store/__tests__/outbox.test.ts | 4 +- lib/store/area.ts | 10 + lib/store/drafts/calls.ts | 199 ++++++ lib/store/drafts/flush.ts | 208 ++++++ lib/store/drafts/local-email.ts | 135 ++++ lib/store/drafts/pending.ts | 264 ++++++++ lib/store/drafts/schema.ts | 58 ++ lib/store/engine/account-engine.ts | 20 +- lib/store/engine/responder.ts | 4 +- lib/store/engine/transport.ts | 5 + lib/store/mail/attachments.ts | 295 +++++++++ lib/store/mail/calls.ts | 17 +- lib/store/mail/index.ts | 4 +- lib/store/mail/records.ts | 71 ++- lib/store/mail/schema.ts | 18 + lib/store/mail/sync.ts | 47 +- lib/store/outbox/calls.ts | 93 ++- lib/store/outbox/email-set.ts | 39 +- lib/store/outbox/index.ts | 14 +- lib/store/outbox/items.ts | 3 +- locales/ar/common.json | 19 + locales/ca/common.json | 19 + locales/cs/common.json | 19 + locales/da/common.json | 19 + locales/de/common.json | 19 + locales/en/common.json | 19 + locales/es/common.json | 19 + locales/et/common.json | 19 + locales/fa/common.json | 19 + locales/fr/common.json | 19 + locales/he/common.json | 19 + locales/hu/common.json | 19 + locales/it/common.json | 19 + locales/ja/common.json | 19 + locales/ko/common.json | 19 + locales/lv/common.json | 19 + locales/mn/common.json | 19 + locales/nb/common.json | 19 + locales/nl/common.json | 19 + locales/pl/common.json | 19 + locales/pt/common.json | 19 + locales/ro/common.json | 19 + locales/ru/common.json | 19 + locales/sk/common.json | 19 + locales/th/common.json | 19 + locales/tr/common.json | 19 + locales/uk/common.json | 19 + locales/vi/common.json | 19 + locales/zh-TW/common.json | 19 + locales/zh/common.json | 19 + 73 files changed, 4044 insertions(+), 204 deletions(-) create mode 100644 components/offline/__tests__/outbox-drafts.test.tsx create mode 100644 lib/offline/__tests__/drafts-flush.test.ts create mode 100644 lib/store/__tests__/attachments.test.ts create mode 100644 lib/store/__tests__/drafts-client.test.ts create mode 100644 lib/store/__tests__/drafts.test.ts create mode 100644 lib/store/__tests__/helpers/fake-drafts.ts create mode 100644 lib/store/drafts/calls.ts create mode 100644 lib/store/drafts/flush.ts create mode 100644 lib/store/drafts/local-email.ts create mode 100644 lib/store/drafts/pending.ts create mode 100644 lib/store/drafts/schema.ts create mode 100644 lib/store/mail/attachments.ts diff --git a/components/email/email-composer.tsx b/components/email/email-composer.tsx index 13418b806..c12ae34ad 100644 --- a/components/email/email-composer.tsx +++ b/components/email/email-composer.tsx @@ -89,6 +89,7 @@ import { FilePickerDialog } from "@/components/files/file-picker-dialog"; import { DirectoryPickerDialog } from "@/components/contacts/directory-picker-dialog"; import type { DirectoryEntry } from "@/stores/contact-store"; import type { IJMAPClient } from "@/lib/jmap/client-interface"; +import { isStandInId } from "@/lib/jmap/stand-in-ids"; import type { FileNode } from "@/lib/jmap/types"; import { useContactStore, getContactDisplayName, getContactPrimaryEmail } from "@/stores/contact-store"; import { renderTemplateWithPlugins, notifyTemplateApplied } from "@/stores/template-store"; @@ -521,6 +522,7 @@ export function EmailComposer({ const tViewer = useTranslations('email_viewer'); const tClassification = useTranslations('classification'); const tProofreading = useTranslations('proofreading'); + const tOfflineDrafts = useTranslations('offlineDrafts'); const tOffline = useTranslations('offline'); const tSendGuards = useTranslations('send_guards'); const timeFormat = useSettingsStore((state) => state.timeFormat); @@ -1010,6 +1012,8 @@ export function EmailComposer({ const [draftLocalAccountId, setDraftLocalAccountId] = useState(initialData?.draftLocalAccountId); const draftLocalAccountIdRef = useRef(initialData?.draftLocalAccountId); const [saveStatus, setSaveStatus] = useState<'idle' | 'saving' | 'saved' | 'error'>('idle'); + // The last save stayed on this device: the server wasn't there. + const [savedOnDevice, setSavedOnDevice] = useState(false); const saveTimeoutRef = useRef(null); const saveCloseAttemptRef = useRef<{ cancelled: boolean; saving: boolean } | null>(null); const [isSavingAndClosing, setIsSavingAndClosing] = useState(false); @@ -2907,6 +2911,7 @@ export function EmailComposer({ } } + setSavedOnDevice(isStandInId(savedDraftId)); setSaveStatus('saved'); saveFailedRef.current = false; lastDraftIdRef.current = savedDraftId; @@ -4453,7 +4458,7 @@ export function EmailComposer({ {saveStatus === 'saved' && (
- {t('draft_saved')} + {savedOnDevice ? tOfflineDrafts('saved_on_device') : t('draft_saved')}
)} {saveStatus === 'error' && ( diff --git a/components/mail/mail-app.tsx b/components/mail/mail-app.tsx index 70e8f5d98..d83de30b1 100644 --- a/components/mail/mail-app.tsx +++ b/components/mail/mail-app.tsx @@ -55,6 +55,7 @@ import { useAuthStore, redirectToLogin, saveRedirectAfterLogin, canEnterLoginInP import { useRouter } from '@/i18n/navigation'; import { useSettingsStore } from "@/stores/settings-store"; import { useEffectiveMailLayout } from "@/hooks/use-effective-mail-layout"; +import { useOfflineStore } from "@/lib/offline/state"; import { useContactStore } from "@/stores/contact-store"; import { useIdentityStore } from "@/stores/identity-store"; import { useUIStore, SIDEBAR_MIN, SIDEBAR_MAX, EMAIL_LIST_MIN, EMAIL_LIST_MAX, EMAIL_LIST_HEIGHT_MIN, EMAIL_LIST_HEIGHT_MAX } from "@/stores/ui-store"; @@ -2668,6 +2669,19 @@ export function MailApp({ linkSegments: routeSegments }: MailAppProps = {}) { console.error('Cancelled the scheduled send, but opening the draft failed:', error); }; + // A draft that's only on this device, opened from the outbox: the server + // doesn't list it, so nothing else can open it while online. + const draftToOpen = useOfflineStore((s) => s.draftToOpen); + useEffect(() => { + if (!draftToOpen || !client) return; + useOfflineStore.setState({ draftToOpen: null }); + const draftClient = useAuthStore.getState().getClientForAccount(draftToOpen.account) ?? client; + void draftClient.getEmail(draftToOpen.id, draftToOpen.jmap) + .then((draft) => (draft ? handleEditDraft(draft, { accountId: draftToOpen.jmap, localAccountId: draftToOpen.account }) : undefined)) + .catch((error) => console.error('Opening a draft kept on this device failed:', error)); + // eslint-disable-next-line react-hooks/exhaustive-deps -- runs when the outbox asks, with the handler of that render + }, [draftToOpen, client]); + /** * "Cancel and edit" from the list or the viewer, neither of which waits for * it: the draft to open (null for S/MIME, which is composed again), or diff --git a/components/offline/__tests__/outbox-drafts.test.tsx b/components/offline/__tests__/outbox-drafts.test.tsx new file mode 100644 index 000000000..6bb708538 --- /dev/null +++ b/components/offline/__tests__/outbox-drafts.test.tsx @@ -0,0 +1,124 @@ +import 'fake-indexeddb/auto'; +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setStoreBackend, type StoreBackend } from '@/lib/store/client/api'; +import { useOfflineStore } from '@/lib/offline/state'; +import type { LocalDraftRow } from '@/lib/offline/queue'; +import { useAuthStore } from '@/stores/auth-store'; +import { OutboxList } from '../outbox-list'; +import { OfflineBanner } from '../offline-banner'; + +const push = vi.fn(); +let pathname = '/settings/backup/offline'; + +vi.mock('@/i18n/navigation', () => ({ + useRouter: () => ({ push }), + usePathname: () => pathname, +})); + +const draft = (overrides: Partial = {}): LocalDraftRow => ({ + account: 'acc', + localId: 'local-1', + jmap: 'j1', + subject: 'Only here', + status: 'waiting', + hasFiles: false, + createdAt: Date.UTC(2026, 10, 3), + ...overrides, +}); + +let area: ReturnType; + +beforeEach(() => { + push.mockClear(); + pathname = '/settings/backup/offline'; + area = vi.fn(async (_account: string, name: string) => { + if (name === 'outbox.read') return { patches: 0, entries: [], conflicts: [] }; + if (name === 'drafts.list') return []; + return undefined; + }); + setStoreBackend({ supported: () => true, area } as unknown as StoreBackend); + useAuthStore.setState({ activeAccountId: 'acc' }); + useOfflineStore.setState({ enabled: ['acc'], outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [], draftToOpen: null, browserOffline: false, downServers: [], startedOffline: false }); +}); + +afterEach(() => { + setStoreBackend(null); + useOfflineStore.setState({ enabled: [], conflicts: [], localDrafts: [], draftToOpen: null }); +}); + +describe('drafts that are only on this device', () => { + it('shows nothing when there are none', () => { + render(); + expect(screen.queryByTestId('offline-local-drafts')).toBeNull(); + }); + + it('shows a refused one with the server\'s reason, and a way to open and retry it', () => { + useOfflineStore.setState({ localDrafts: [draft({ status: 'refused', error: { type: 'overQuota', text: 'No room left' } })] }); + render(); + const row = screen.getByTestId('offline-local-draft-row'); + expect(row.getAttribute('data-status')).toBe('refused'); + expect(row.textContent).toContain('Only here'); + expect(row.textContent).toContain('status_refused'); + expect(screen.getByRole('button', { name: 'open_label' })).toBeTruthy(); + expect(screen.getByRole('button', { name: 'retry_label' })).toBeTruthy(); + }); + + it('offers no retry for one that waits by itself', () => { + useOfflineStore.setState({ localDrafts: [draft()] }); + render(); + expect(screen.getByTestId('offline-local-draft-row').textContent).toContain('status_waiting'); + expect(screen.queryByRole('button', { name: 'retry_label' })).toBeNull(); + }); + + it('opens it on the mail page, going there from elsewhere', () => { + useOfflineStore.setState({ localDrafts: [draft()] }); + const onOpenDraft = vi.fn(); + render(); + fireEvent.click(screen.getByRole('button', { name: 'open_label' })); + expect(useOfflineStore.getState().draftToOpen).toEqual({ account: 'acc', jmap: 'j1', id: 'local-1' }); + expect(onOpenDraft).toHaveBeenCalled(); + expect(push).toHaveBeenCalledWith('/mail'); + }); + + it('stays on the mail page when it\'s already there', () => { + pathname = '/mail'; + useOfflineStore.setState({ localDrafts: [draft()] }); + render(); + fireEvent.click(screen.getByRole('button', { name: 'open_label' })); + expect(push).not.toHaveBeenCalled(); + }); + + it('asks the store to try again, and reads the queue', async () => { + useOfflineStore.setState({ localDrafts: [draft({ status: 'refused' })] }); + render(); + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'retry_label' })); + }); + expect(area).toHaveBeenCalledWith('acc', 'drafts.retry', { localId: 'local-1' }); + expect(area).toHaveBeenCalledWith('acc', 'drafts.list'); + expect(useOfflineStore.getState().localDrafts).toEqual([]); + }); + + it('says both versions are kept for a draft changed elsewhere', () => { + useOfflineStore.setState({ + conflicts: [ + { id: 1, account: 'acc', recordId: 'r', createdAt: 0, summary: { subject: 'A', op: 'draft', reason: 'changed' } }, + { id: 2, account: 'acc', recordId: 's', createdAt: 0, summary: { subject: 'B', op: 'draft', reason: 'gone' } }, + ], + }); + render(); + const rows = screen.getAllByTestId('offline-conflict-row'); + expect(rows[0].textContent).toContain('conflict_changed'); + expect(rows[1].textContent).toContain('conflict_gone'); + }); + + it('puts a refused one in the banner while online, and not one that only waits', () => { + useOfflineStore.setState({ localDrafts: [draft()] }); + const { rerender } = render(); + expect(screen.queryByTestId('offline-banner-drafts')).toBeNull(); + act(() => useOfflineStore.setState({ localDrafts: [draft({ status: 'refused' })] })); + rerender(); + expect(screen.getByTestId('offline-banner-drafts').textContent).toContain('banner_refused'); + }); +}); diff --git a/components/offline/offline-banner.tsx b/components/offline/offline-banner.tsx index dc179e362..e87b70e3a 100644 --- a/components/offline/offline-banner.tsx +++ b/components/offline/offline-banner.tsx @@ -25,10 +25,12 @@ export function OfflineBanner() { const serverDown = useOfflineStore((s) => !!server && s.downServers.includes(serverKey(server))); const waiting = useOfflineStore((s) => s.outbox.length); const conflicts = useOfflineStore((s) => s.conflicts.length); + const refusedDrafts = useOfflineStore((s) => s.localDrafts.filter((draft) => draft.status === 'refused').length); const tConflicts = useTranslations('offlineConflicts'); + const tDrafts = useTranslations('offlineDrafts'); const [open, setOpen] = useState(false); const offline = browserOffline || serverDown; - if (IS_LITE || !active || (!offline && waiting === 0 && conflicts === 0 && !open)) return null; + if (IS_LITE || !active || (!offline && waiting === 0 && conflicts === 0 && refusedDrafts === 0 && !open)) return null; return ( <>
)} + {refusedDrafts > 0 && ( + + )}
setOpen(false)} title={t('outbox.title')} testId="offline-outbox-dialog"> - + setOpen(false)} /> ); diff --git a/components/offline/offline-host.tsx b/components/offline/offline-host.tsx index 13e675950..8d1bf8b3e 100644 --- a/components/offline/offline-host.tsx +++ b/components/offline/offline-host.tsx @@ -23,6 +23,9 @@ export function OfflineHost() { const tConflicts = useTranslations('offlineConflicts'); const tConflictsRef = useRef(tConflicts); tConflictsRef.current = tConflicts; + const tDrafts = useTranslations('offlineDrafts'); + const tDraftsRef = useRef(tDrafts); + tDraftsRef.current = tDrafts; const { dialogProps, confirm } = useConfirmDialog(); const confirmRef = useRef(confirm); confirmRef.current = confirm; @@ -51,6 +54,14 @@ export function OfflineHost() { case 'conflict': toast.warning(tConflictsRef.current('toast', { count: notice.count })); break; + case 'draft-refused': + toast.error(tDraftsRef.current('toast_refused', { subject: notice.subject || tr('outbox.no_subject') }), { + message: notice.error.code ? tDraftsRef.current(notice.error.code) : notice.error.text, + }); + break; + case 'draft-conflict': + toast.warning(tDraftsRef.current('toast_conflict', { subject: notice.subject || tr('outbox.no_subject') })); + break; case 'storage-failed': toast.warning(tConflictsRef.current('storage_failed'), { duration: 0 }); break; @@ -75,7 +86,8 @@ export function OfflineHost() { const stop = startOfflineMode(); // Messages that would be lost: ask before a sign-out the user started. const removeGate = addSignOutGate(async (accountId) => { - const unsent = useOfflineStore.getState().outbox.filter((row) => accountId === 'all' || row.account === accountId).length; + const { outbox, localDrafts } = useOfflineStore.getState(); + const unsent = [...outbox, ...localDrafts].filter((row) => accountId === 'all' || row.account === accountId).length; if (unsent === 0) return true; const tr = tRef.current; const ok = await confirmRef.current({ diff --git a/components/offline/outbox-list.tsx b/components/offline/outbox-list.tsx index 7e8c3a864..b68de536a 100644 --- a/components/offline/outbox-list.tsx +++ b/components/offline/outbox-list.tsx @@ -2,11 +2,12 @@ import { useState } from 'react'; import { useFormatter, useTranslations } from 'next-intl'; +import { usePathname, useRouter } from '@/i18n/navigation'; import { Button } from '@/components/ui/button'; import { ConfirmDialog } from '@/components/ui/confirm-dialog'; import { deleteOutboxEntry, retryOutboxEntry } from '@/lib/offline/flush'; import { refreshQueueState, scheduleFlushAll } from '@/lib/offline/offline-mode'; -import { dismissConflict, type ConflictEntry } from '@/lib/offline/queue'; +import { dismissConflict, retryLocalDraft, type ConflictEntry, type LocalDraftRow } from '@/lib/offline/queue'; import { isOfflineNow, useOfflineStore, type OutboxRow } from '@/lib/offline/state'; import { useAccountStore } from '@/stores/account-store'; @@ -16,12 +17,18 @@ import { useAccountStore } from '@/stores/account-store'; * be sent again or deleted; one whose connection broke on the way is never * sent again on its own (it may have gone out); one held because the * organisation's rules changed can only be deleted (and written again). - * Changes that didn't go through ("changed on another device") are listed too. + * Changes that didn't go through ("changed on another device") are listed too, + * and drafts that are only on this device: on their way up, or refused by the + * server, in which case they can still be opened and tried again. */ -export function OutboxList({ heading = true }: { heading?: boolean }) { +export function OutboxList({ heading = true, onOpenDraft }: { heading?: boolean; onOpenDraft?: () => void }) { const t = useTranslations('offline.outbox'); const tErrors = useTranslations('offline.errors'); const tConflicts = useTranslations('offlineConflicts'); + const tDrafts = useTranslations('offlineDrafts'); + const router = useRouter(); + const pathname = usePathname(); + const localDrafts = useOfflineStore((s) => s.localDrafts); const format = useFormatter(); const rows = useOfflineStore((s) => s.outbox); const changes = useOfflineStore((s) => s.pendingChanges); @@ -46,7 +53,29 @@ export function OutboxList({ heading = true }: { heading?: boolean }) { if (!isOfflineNow()) scheduleFlushAll(0); }; + const draftStatus = (draft: LocalDraftRow) => { + switch (draft.status) { + case 'refused': return tDrafts('status_refused', { error: draft.error?.code ? tDrafts(draft.error.code) : (draft.error?.text ?? '') }); + case 'sending': return tDrafts('status_sending'); + case 'uncertain': return tDrafts('status_uncertain'); + default: return tDrafts('status_waiting'); + } + }; + + const openDraft = (draft: LocalDraftRow) => { + useOfflineStore.setState({ draftToOpen: { account: draft.account, jmap: draft.jmap, id: draft.localId } }); + onOpenDraft?.(); + if (!pathname.startsWith('/mail')) router.push('/mail'); + }; + + const retryDraft = async (draft: LocalDraftRow) => { + await retryLocalDraft(draft.account, draft.localId); + await refreshQueueState(); + if (!isOfflineNow()) scheduleFlushAll(0); + }; + const conflictText = (conflict: ConflictEntry) => { + if (conflict.summary.op === 'draft') return tDrafts(conflict.summary.reason === 'gone' ? 'conflict_gone' : 'conflict_changed'); if (conflict.summary.reason === 'refused') return tConflicts('refused', { reason: conflict.summary.detail ?? '' }); if (conflict.summary.reason === 'gone') return tConflicts('gone'); return tConflicts(conflict.summary.op === 'destroy' ? 'changed_delete' : 'changed'); @@ -89,6 +118,29 @@ export function OutboxList({ heading = true }: { heading?: boolean }) { ))} )} + {localDrafts.length > 0 && ( +
+

{tDrafts('description')}

+
    + {localDrafts.map((draft) => ( +
  • +
    +

    {draft.subject || t('no_subject')}

    +

    + {draftStatus(draft)} · {format.dateTime(draft.createdAt, { dateStyle: 'short', timeStyle: 'short' })} +

    +
    +
    + + {draft.status === 'refused' && ( + + )} +
    +
    • + ))} +
+
+ )} {changes > 0 && (

{t('changes', { count: changes })}

)} diff --git a/lib/jmap/client-interface.ts b/lib/jmap/client-interface.ts index 0c6e9c5fb..487b060e7 100644 --- a/lib/jmap/client-interface.ts +++ b/lib/jmap/client-interface.ts @@ -553,6 +553,7 @@ export interface IJMAPClient { accountId?: string; onProgress?: (loaded: number, total: number) => void; signal?: AbortSignal; + noHold?: boolean; }, ): Promise<{ blobId: string; size: number; type: string }>; getBlobDownloadUrl(blobId: string, name?: string, type?: string, accountId?: string): string; diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index 1282c7c15..1d4f2df59 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -4,7 +4,8 @@ import { IncompleteDeleteError } from "./delete-error"; import { downloadAllowed, saveBlobAs, type DownloadKind } from "@/lib/download-guard"; import { ArchiveEmailsError } from "./archive-error"; import { generateUUID } from '@/lib/utils'; -import { standInsIn, withRealIds, withStandInsBack } from './stand-in-ids'; +import { isStandInId, standInsIn, withRealIds, withStandInsBack } from './stand-in-ids'; +import { isLocalBlobId } from '@/lib/store/outbox/items'; import type { Attachment, Email, Mailbox, MailboxRights, StateChange, AccountStates, CollectionChanges, ShareNotification, BusyPeriod, CalendarParticipantIdentity, CalendarEventNotification, Thread, Identity, EmailAddress, ContactCard, AddressBook, AddressBookRights, VacationResponse, Calendar, CalendarComponentType, CalendarRights, CalendarEvent, CalendarEventFilter, CalendarTask, CreateCalendarOptions, FileNode, FileNodeFilter, FileNodeRights, Principal, PrincipalAvailability, PrincipalListing, PushSubscription, EmailPushConfig, DeliveryStatus, EmailSubmission, RejectedRecipient, ScheduledEmail, SendEmailResult, SharedAccount } from "./types"; import type { SieveScript, SieveCapabilities } from "./sieve-types"; import type { BlobBatch, CalendarEventUpdateOptions, CalendarQueryOptions, FileNodeListings, IJMAPClient, KeywordDiscoveryResult, KeywordInfo, KeywordMigration, SearchEmailsResult, SearchRequestOptions, UploadedEmailImport, UploadedEmailImportResult } from "./client-interface"; @@ -2190,8 +2191,10 @@ export class JMAPClient implements IJMAPClient { * `networkOnly`: only the server's own word counts. A send or a change the * outbox hands over must never be answered by the device: the device would * queue it again, or answer for the server. + * `deviceOnly`: the other way round. What belongs to a draft saved on the + * device stays with it, even if the server came back in between. */ - private async request(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean; skipOfflineRecord?: boolean; networkOnly?: boolean }): Promise { + private async request(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean; skipOfflineRecord?: boolean; networkOnly?: boolean; deviceOnly?: boolean }): Promise { if (!this.apiUrl) { throw new Error('Not connected. Call connect() first.'); } @@ -2201,32 +2204,30 @@ export class JMAPClient implements IJMAPClient { // browser is offline or the network fails. Without offline mode no hooks // are set and this is the plain network request. const offline = getOfflineHooks(); - if (this.offlineOnly || offline?.skipNetwork(this)) { + // A stand-in id from an event or a draft created offline, still held by an + // open dialog or a list: the server knows it by its real id, or not yet at + // all. The store files what reached the server under the real id too. + const standIns = offline?.resolveStandIns ? standInsIn(methodCalls) : []; + let realIds: Record | null = null; + let onDeviceOnly = false; + if (standIns.length > 0) { + const resolved = await offline!.resolveStandIns!(this, standIns); + const known = Object.entries(resolved).filter((entry): entry is [string, string] => !!entry[1]); + if (known.length > 0) { + realIds = Object.fromEntries(known); + methodCalls = withRealIds(methodCalls, realIds); + } + onDeviceOnly = standIns.some((id) => !resolved[id]); + } + const asAsked = (response: JMAPResponse) => (realIds ? withStandInsBack(response, realIds) : response); + if (this.offlineOnly || opts?.deviceOnly || onDeviceOnly || offline?.skipNetwork(this)) { const local = offline && !opts?.networkOnly ? await offline.answer(this, methodCalls) : null; if (local) { this.lastAnswerFromDevice = true; - return local as JMAPResponse; + return asAsked(local as JMAPResponse); } throw new OfflineUnavailableError(); } - // A stand-in id from an event created offline, still held by an open dialog - // or a list: the server knows it by its real id, or not yet at all. - const standIns = offline?.resolveStandIns ? standInsIn(methodCalls) : []; - let realIds: Record | null = null; - if (standIns.length > 0) { - const resolved = await offline!.resolveStandIns!(this, standIns); - if (standIns.some((id) => !resolved[id])) { - const local = opts?.networkOnly ? null : await offline!.answer(this, methodCalls); - if (local) { - this.lastAnswerFromDevice = true; - return local as JMAPResponse; - } - throw new OfflineUnavailableError(); - } - realIds = resolved as Record; - methodCalls = withRealIds(methodCalls, realIds); - } - const asAsked = (response: JMAPResponse) => (realIds ? withStandInsBack(response, realIds) : response); let answer: JMAPResponse; try { answer = await this.networkRequest(methodCalls, using, opts); @@ -2234,7 +2235,7 @@ export class JMAPClient implements IJMAPClient { // Only when the server is really gone: a refusal without CORS // headers looks the same to the browser and must stay an error. if (offline && isNetworkFailure(error) && await offline.serverGone(this, error)) { - const local = opts?.networkOnly ? null : await offline.answer(this, methodCalls); + const local = opts?.networkOnly ? null : await offline.answer(this, methodCalls, { networkTried: true }); if (local) { this.lastAnswerFromDevice = true; return asAsked(local as JMAPResponse); @@ -5081,9 +5082,21 @@ export class JMAPClient implements IJMAPClient { throw new Error('No drafts mailbox found'); } + // Files attached offline go up now if the server answers. If it doesn't, + // the draft stays on the device with them, as when offline. + const hooks = getOfflineHooks(); + if (attachments?.some(att => isLocalBlobId(att.blobId)) && hooks?.resolveHeldAttachments && !this.offlineOnly && !hooks.skipNetwork(this)) { + try { + attachments = await hooks.resolveHeldAttachments(this, attachments, draftAccountId); + } catch (error) { + if (!(error instanceof Error) || error.name !== 'SendNotStartedError') throw error; + } + } + const emailId = `draft-${Date.now()}`; interface EmailDraft { + messageId: string[]; from: { name?: string; email: string }[]; to: { name?: string; email: string }[]; cc?: { name?: string; email: string }[]; @@ -5107,6 +5120,9 @@ export class JMAPClient implements IJMAPClient { const draftInReplyTo = inReplyTo?.map(stripMessageIdBrackets).filter(Boolean); const draftReferences = references?.map(stripMessageIdBrackets).filter(Boolean); const emailData: EmailDraft = { + // Fixed here: when the connection breaks on the way, this is how the + // device later finds out whether the server took the draft after all. + messageId: [generateMessageId(parseRecipientString(fromEmail || this.username).email)], from: [{ ...(sanitizedFromName ? { name: sanitizedFromName } : {}), email: fromEmail || this.username }], // "Name " must be split into the two JMAP fields: storing the whole // mailbox as the address makes the server treat the display name as part @@ -5171,9 +5187,12 @@ export class JMAPClient implements IJMAPClient { if (draftId) { try { + // A version saved on the device replaces the old one when it + // reaches the server, not before: the old one stays there until then. + const onDevice = isStandInId(newDraftId); const destroyResponse = await this.request([ - ["Email/set", { accountId: options?.previousDraftAccountId || this.accountId, destroy: [draftId] }, "0"], - ]); + ["Email/set", { accountId: options?.previousDraftAccountId || this.accountId, destroy: [draftId], ...(onDevice ? { replacedBy: newDraftId } : {}) }, "0"], + ], undefined, onDevice ? { deviceOnly: true } : undefined); const destroyResult = destroyResponse.methodResponses?.[0]?.[1]; if (destroyResult?.notDestroyed) { console.warn('Failed to destroy old draft:', destroyResult.notDestroyed); @@ -6281,6 +6300,8 @@ export class JMAPClient implements IJMAPClient { accountId?: string; onProgress?: (loaded: number, total: number) => void; signal?: AbortSignal; + /** A file already held on this device that's on its way up: an outage is an error, not a second copy. */ + noHold?: boolean; }, ): Promise<{ blobId: string; size: number; type: string }> { const opts = @@ -6303,7 +6324,7 @@ export class JMAPClient implements IJMAPClient { // is uploaded when the message it belongs to is sent from the outbox. const offline = getOfflineHooks(); if (this.offlineOnly || offline?.skipNetwork(this)) { - const held = offline ? await offline.holdUpload(this, file, targetAccountId) : null; + const held = offline && !opts.noHold ? await offline.holdUpload(this, file, targetAccountId) : null; if (held) return held; throw new OfflineUnavailableError(); } @@ -6338,7 +6359,7 @@ export class JMAPClient implements IJMAPClient { // The server is gone though the browser says it's online (#523): the // file waits on this device as when offline. Asking whether it is // gone tells an outage from a refusal (a too-large file, a quota). - if (offline && !opts.signal?.aborted && await offline.serverGone(this)) { + if (offline && !opts.noHold && !opts.signal?.aborted && await offline.serverGone(this)) { const held = await offline.holdUpload(this, file, targetAccountId); if (held) return held; } @@ -6519,7 +6540,8 @@ export class JMAPClient implements IJMAPClient { private async offlineAwareDownload(url: string, blobId: string, accountId: string | undefined): Promise { const offline = getOfflineHooks(); const account = accountId || this.accountId; - if (this.offlineOnly || offline?.skipNetwork(this)) { + // A file attached offline only exists on this device, whatever the network does. + if (this.offlineOnly || isLocalBlobId(blobId) || offline?.skipNetwork(this)) { const local = offline ? await offline.blob(this, blobId, account) : null; if (local) return local; throw new OfflineUnavailableError(); diff --git a/lib/jmap/offline-hooks.ts b/lib/jmap/offline-hooks.ts index b908eafb7..9f4f83597 100644 --- a/lib/jmap/offline-hooks.ts +++ b/lib/jmap/offline-hooks.ts @@ -27,8 +27,12 @@ export interface OfflineClientRef { export interface OfflineHooks { /** Answer without trying the network: the browser says it is offline (or the client was started offline). */ skipNetwork(client: OfflineClientRef): boolean; - /** An answer from the device for a request the network couldn't take, or null. */ - answer(client: OfflineClientRef, methodCalls: OfflineMethodCall[]): Promise; + /** + * An answer from the device for a request the network couldn't take, or null. + * `networkTried`: the request went out and its connection broke, so the server + * may have taken it even though no answer came. + */ + answer(client: OfflineClientRef, methodCalls: OfflineMethodCall[], options?: { networkTried?: boolean }): Promise; /** What the server answered, for the cache. */ record(client: OfflineClientRef, methodCalls: OfflineMethodCall[], response: OfflineResponse): void; /** A request reached the server (true) or failed on the network (false). */ @@ -59,6 +63,11 @@ export interface OfflineHooks { * ids as they are. */ resolveStandIns?(client: OfflineClientRef, ids: string[]): Promise>; + /** + * Files attached offline, uploaded now that the server answers, for a save + * that names them. Throws SendNotStartedError when the upload hits an outage. + */ + resolveHeldAttachments?(client: OfflineClientRef, attachments: T[], accountId?: string): Promise; /** The user opened this file: the local store keeps recently opened files. */ fileOpened?(client: OfflineClientRef, accountId: string, id: string): void; } diff --git a/lib/jmap/stand-in-ids.ts b/lib/jmap/stand-in-ids.ts index 00da263e6..302664c72 100644 --- a/lib/jmap/stand-in-ids.ts +++ b/lib/jmap/stand-in-ids.ts @@ -1,7 +1,8 @@ /** - * Stand-in ids: what the local store hands out for an event created offline. + * Stand-in ids: what the local store hands out for an event or a draft created + * offline. * - * A page can hold one long after the event reached the server: an open dialog, + * A page can hold one long after the record reached the server: an open dialog, * a list not refetched yet, a plugin. So the client swaps them for the real ids * on the way to the network, and swaps them back in the answer, because callers * look up the outcome under the id they asked about. @@ -19,14 +20,17 @@ export function newStandInId(): string { return `${STAND_IN_PREFIX}${crypto.randomUUID()}`; } +const SET_METHODS = new Set(['CalendarEvent/set', 'Email/set']); +const GET_METHODS = new Set(['CalendarEvent/get', 'Email/get']); + /** The stand-in ids a request names. */ export function standInsIn(calls: readonly OfflineMethodCall[]): string[] { const found = new Set(); for (const [name, args] of calls) { - if (name === 'CalendarEvent/set') { + if (SET_METHODS.has(name)) { for (const id of Object.keys((args.update ?? {}) as object)) if (isStandInId(id)) found.add(id); for (const id of (args.destroy ?? []) as string[]) if (isStandInId(id)) found.add(id); - } else if (name === 'CalendarEvent/get') { + } else if (GET_METHODS.has(name)) { for (const id of (args.ids ?? []) as string[]) if (isStandInId(id)) found.add(id); } } @@ -42,13 +46,13 @@ export function withRealIds(calls: readonly T[], re const rename = (id: string) => real[id] ?? id; return calls.map((call) => { const [name, args, callId] = call; - if (name === 'CalendarEvent/set') { + if (SET_METHODS.has(name)) { const next = { ...args }; if (args.update) next.update = renamedKeys(args.update as Record, rename); if (args.destroy) next.destroy = (args.destroy as string[]).map(rename); return [name, next, callId] as unknown as T; } - if (name === 'CalendarEvent/get' && args.ids) { + if (GET_METHODS.has(name) && args.ids) { return [name, { ...args, ids: (args.ids as string[]).map(rename) }, callId] as unknown as T; } return call; @@ -62,7 +66,7 @@ export function withStandInsBack(response: T, real: R return { ...response, methodResponses: response.methodResponses.map(([name, result, callId]) => { - if (name !== 'CalendarEvent/set' && name !== 'CalendarEvent/get') return [name, result, callId]; + if (!SET_METHODS.has(name) && !GET_METHODS.has(name)) return [name, result, callId]; const next = { ...result }; for (const key of ['updated', 'notUpdated', 'notDestroyed']) { if (next[key]) next[key] = renamedKeys(next[key], rename); diff --git a/lib/offline/__tests__/drafts-flush.test.ts b/lib/offline/__tests__/drafts-flush.test.ts new file mode 100644 index 000000000..683a8378c --- /dev/null +++ b/lib/offline/__tests__/drafts-flush.test.ts @@ -0,0 +1,194 @@ +import 'fake-indexeddb/auto'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { useAuthStore } from '@/stores/auth-store'; +import { useAccountStore } from '@/stores/account-store'; +import { usePolicyStore } from '@/stores/policy-store'; +import { DEFAULT_ORGANISATION_POLICY } from '@/lib/admin/organisation-policy'; +import { setStoreBackend } from '@/lib/store/client/api'; +import { FakeJmapDrafts } from '@/lib/store/__tests__/helpers/fake-drafts'; +import { JMAP_ACCOUNT } from '@/lib/store/__tests__/helpers/fake-jmap'; +import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store'; +import { refreshQueueState, startOfflineMode, wipeEverything } from '../offline-mode'; +import { flushQueue } from '../flush'; +import { heldFile, holdFile, queueSend, readLocalDrafts, retryLocalDraft } from '../queue'; +import { forgetLegacyMove } from '../migrate-legacy'; +import { useOfflineStore, writeEnabled } from '../state'; +import type { JMAPClient } from '@/lib/jmap/client'; + +let server: FakeJmapDrafts; +let store: InProcessStore; + +function fakeClient(overrides: Record = {}) { + return { + getServerUrl: () => 'https://mail.example.org', + getUsername: () => 'alice@example.org', + getAccountId: () => JMAP_ACCOUNT, + isOfflineOnly: () => false, + getSessionSnapshot: () => null, + getAuthHeader: () => 'Basic x', + keywordSortInverted: () => null, + sendEmail: vi.fn(async () => ({ scheduled: false, emailId: 'sent-1', emailSubmissionId: 'sub-1' })), + uploadBlob: vi.fn(async (file: File) => server.upload(JMAP_ACCOUNT, new Uint8Array(await file.arrayBuffer()))), + requestNetworkOnly: vi.fn((calls: never) => server.request(calls)), + resolveSendAccountId: vi.fn(async () => JMAP_ACCOUNT), + submissionAccountFor: () => JMAP_ACCOUNT, + pruneEmailSubmissions: vi.fn(async () => {}), + ...overrides, + }; +} + +let stop: () => void = () => {}; + +async function saveOffline(subject: string, attachments?: Array>): Promise { + const answer = await store.answer('acc', [['Email/set', { + accountId: JMAP_ACCOUNT, + create: { + c: { + messageId: [`${subject}@example.org`], + subject, + keywords: { $seen: true, $draft: true }, + mailboxIds: { drafts: true }, + bodyValues: { 1: { value: 'text' } }, + textBody: [{ partId: '1' }], + ...(attachments ? { attachments } : {}), + }, + }, + }, 's']]); + return answer?.methodResponses[0][1].created.c.id as string; +} + +beforeEach(async () => { + server = new FakeJmapDrafts({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'drafts', role: 'drafts' }, { id: 'sent', role: 'sent' }] }); + server.now = Date.now(); + store = new InProcessStore(server); + setStoreBackend(store); + forgetLegacyMove(); + localStorage.clear(); + useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [], browserOffline: false, downServers: [], startedOffline: false }); + usePolicyStore.setState({ loaded: true, subject: null, policy: { ...usePolicyStore.getState().policy, organisation: DEFAULT_ORGANISATION_POLICY } }); + useAccountStore.setState({ accounts: [{ id: 'acc', label: 'alice', serverUrl: 'https://mail.example.org', username: 'alice@example.org' } as never] }); + const client = fakeClient(); + useAuthStore.setState({ client: client as never, activeAccountId: 'acc' }); + writeEnabled(['acc']); + stop = startOfflineMode(); + await store.sync('acc'); +}); + +afterEach(async () => { + stop(); + await wipeEverything(); + setStoreBackend(null); +}); + +describe('flushQueue and drafts', () => { + it('uploads a draft\'s files, creates it, tells the page, and leaves nothing behind', async () => { + const client = fakeClient(); + const held = await holdFile('acc', new File(['hello'], 'note.txt', { type: 'text/plain' }), 'note.txt'); + await saveOffline('With a file', [{ blobId: held.blobId, type: 'text/plain', name: 'note.txt' }]); + const onDraftSent = vi.fn(); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true, onDraftSent }); + expect(server.drafts().map((email) => email.subject)).toEqual(['With a file']); + expect(client.uploadBlob).toHaveBeenCalledWith(expect.any(File), { accountId: JMAP_ACCOUNT, noHold: true }); + expect(onDraftSent).toHaveBeenCalledTimes(1); + expect(await readLocalDrafts('acc')).toEqual([]); + }); + + it('holds a large file in pieces and reads it back whole', async () => { + const size = 2 * 1024 * 1024 + 3; + const held = await holdFile('acc', new File([new Uint8Array(size).fill(4)], 'big.bin'), 'big.bin'); + expect(held.size).toBe(size); + const area = vi.spyOn(store, 'area'); + const file = await heldFile('acc', held.blobId); + expect(file?.name).toBe('big.bin'); + expect(file?.size).toBe(size); + expect(new Uint8Array(await (file as File).arrayBuffer()).every((value) => value === 4)).toBe(true); + area.mockRestore(); + }); + + it('tells the user about a refused draft, keeps it, and lists it with its error', async () => { + const client = fakeClient(); + server.refuseCreate = { type: 'overQuota', description: 'No room' }; + const id = await saveOffline('Too much'); + const notify = vi.fn(); + await flushQueue('acc', client as unknown as JMAPClient, { notify, allowed: () => true }); + expect(notify).toHaveBeenCalledWith({ kind: 'draft-refused', subject: 'Too much', error: { type: 'overQuota', text: 'No room' } }); + await refreshQueueState(); + expect(useOfflineStore.getState().localDrafts).toMatchObject([{ account: 'acc', localId: id, status: 'refused' }]); + + server.refuseCreate = null; + await retryLocalDraft('acc', id); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(server.drafts()).toHaveLength(1); + }); + + it('stops on an outage and carries on with what waited after it', async () => { + const client = fakeClient(); + await saveOffline('One'); + server.offline = true; + const sendEmail = client.sendEmail; + const entry = await queueSend('acc', { subject: 'Queued', to: [] }, { kind: 'email', args: [['b@example.org'], 'Queued', 'x'] }); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(sendEmail).not.toHaveBeenCalled(); + server.offline = false; + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(server.drafts()).toHaveLength(1); + expect(sendEmail).toHaveBeenCalledTimes(1); + expect(entry).toBeGreaterThan(0); + }); + + it('sends a draft kept here through the send path with its files, and the draft doesn\'t go up as well', async () => { + const held = await holdFile('acc', new File(['hello'], 'note.txt', { type: 'text/plain' }), 'note.txt'); + const id = await saveOffline('To send', [{ blobId: held.blobId, type: 'text/plain', name: 'note.txt' }]); + const client = fakeClient({ + // As the client does: the draft goes once the message is out. + sendEmail: vi.fn(async () => { + await store.answer('acc', [['Email/set', { accountId: JMAP_ACCOUNT, destroy: [id] }, 'd']]); + return { scheduled: false, emailId: 'sent-1', emailSubmissionId: 'sub-1' }; + }), + }); + await queueSend('acc', { subject: 'To send', to: [] }, { + kind: 'email', + args: [['b@example.org'], 'To send', 'x', undefined, undefined, 'i', 'alice@example.org', id, undefined, undefined, [{ blobId: held.blobId, name: 'note.txt', type: 'text/plain', size: 5 }]], + }); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(client.sendEmail).toHaveBeenCalledTimes(1); + expect(((client.sendEmail as ReturnType).mock.calls[0] as unknown[])[10]).toMatchObject([{ blobId: server.uploads[0].blobId }]); + expect(server.drafts()).toEqual([]); + expect(await readLocalDrafts('acc')).toEqual([]); + }); + + it('drops the draft of a send the lookup found went out', async () => { + const id = await saveOffline('Went out'); + const client = fakeClient(); + server.addEmail({ id: 'in-sent', subject: 'Went out', mailboxIds: { sent: true }, messageId: ['fixed@example.org'], receivedAt: new Date().toISOString() }); + const entryId = await queueSend('acc', { subject: 'Went out', to: [] }, { + kind: 'email', + args: [['b@example.org'], 'Went out', 'x', undefined, undefined, 'i', 'alice@example.org', id], + }, undefined, 'fixed@example.org'); + await store.area('acc', 'outbox.claim', { id: entryId }); + await store.area('acc', 'outbox.update', { id: entryId, status: 'uncertain' }); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(await readLocalDrafts('acc')).toEqual([]); + expect(server.drafts()).toEqual([]); + }); + + it('cleans held files nothing needs, only long after', async () => { + const client = fakeClient(); + const orphan = await holdFile('acc', new File(['a'], 'a.txt'), 'a.txt'); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + expect(await heldFile('acc', orphan.blobId)).not.toBeNull(); + const now = Date.now(); + const clock = vi.spyOn(Date, 'now').mockReturnValue(now + 8 * 24 * 60 * 60 * 1000); + await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true }); + clock.mockRestore(); + expect(await heldFile('acc', orphan.blobId)).toBeNull(); + }); +}); + +describe('a sign-out and local drafts', () => { + it('counts them as unsent, by name', async () => { + await saveOffline('Only here'); + await refreshQueueState(); + expect(useOfflineStore.getState().localDrafts.map((row) => row.subject)).toEqual(['Only here']); + }); +}); diff --git a/lib/offline/__tests__/migrate-legacy.test.ts b/lib/offline/__tests__/migrate-legacy.test.ts index 37e3fa492..3fe68c22e 100644 --- a/lib/offline/__tests__/migrate-legacy.test.ts +++ b/lib/offline/__tests__/migrate-legacy.test.ts @@ -105,9 +105,9 @@ describe('moving the old outbox into the store', () => { expect(outbox[0].messageId).toMatch(/@example\.org$/); expect(outbox[1].messageId).toBeUndefined(); expect(outbox[3].messageId).toBe('raw-1@example.org'); - const file = await store.area<{ name: string; data: Uint8Array }>('acc', 'outbox.heldFile', { blobId: 'bulwark-local-raw' }); + const file = await store.area<{ name: string; blob: Blob }>('acc', 'outbox.heldFile', { blobId: 'bulwark-local-raw' }); expect(file.name).toBe('message.eml'); - expect(new TextDecoder().decode(file.data)).toBe(RAW); + expect(await file.blob.text()).toBe(RAW); expect(await oldExists()).toBe(false); expect(store.has('stranger')).toBe(false); }); diff --git a/lib/offline/flush.ts b/lib/offline/flush.ts index 021c39348..6f8f7efdd 100644 --- a/lib/offline/flush.ts +++ b/lib/offline/flush.ts @@ -28,7 +28,9 @@ import type { JMAPClient } from '@/lib/jmap/client'; import { isNetworkFailure } from '@/lib/jmap/offline-hooks'; import type { SendEmailResult } from '@/lib/jmap/types'; +import { isStandInId } from '@/lib/jmap/stand-in-ids'; import { getStore } from '@/lib/store/client/api'; +import { flushDrafts, type DraftFlushIo } from '@/lib/store/drafts/flush'; import type { MethodCall } from '@/lib/store/engine/transport'; import { decideRetry, type LookupIo, type RetryDecision } from '@/lib/store/outbox/dedupe'; import { flushPatches, type PatchIo, type PatchOutcome, type PendingPatch } from '@/lib/store/outbox/flush-patches'; @@ -54,11 +56,19 @@ export interface FlushDeps { onScheduled?: (entry: OutboxEntry, result: SendEmailResult) => void; /** A send's connection broke though the server may be fine: look it up again soon. */ retryLater?: () => void; + /** A draft saved on this device reached the server: lists and open pages read again. */ + onDraftSent?: () => void; } /** A send is taken for at most this long before a closed tab's send counts as uncertain. */ export const SENDING_STALE_MS = 60_000; +/** + * How long a held file stays once nothing names it. An open composer may still + * show it after its draft went up or was replaced. + */ +const HELD_FILE_GRACE_MS = 7 * 24 * 60 * 60 * 1000; + const flushing = new Set(); /** A hold shorter than this is dropped: the undo window passed while the message waited. */ @@ -90,7 +100,7 @@ async function uploadHeld(account: string, client: JMAPClient, blobId: string): const file = await heldFile(account, blobId); if (!file) throw new OutboxPrepError('file_missing'); try { - return (await client.uploadBlob(file)).blobId; + return (await client.uploadBlob(file, { noHold: true })).blobId; } catch (error) { if (isNetworkFailure(error)) throw notStartedError('upload failed'); throw error; @@ -158,7 +168,31 @@ export async function settleStaleSends(account?: string, now = Date.now()): Prom /** The server refused the request itself: more tries won't change that. Outages and auth wait instead. */ function isRequestRefusal(error: unknown): boolean { - return error instanceof Error && /^Request failed: (400|403|404|413|422)\b/.test(error.message); + return error instanceof Error && /^(Request failed|Failed to upload file): (400|403|404|413|422)\b/.test(error.message); +} + +/** The drafts saved on this device. False when the run stopped on an outage. */ +function sendDrafts(account: string, client: JMAPClient, deps: FlushDeps): Promise { + const store = getStore(); + const io: DraftFlushIo = { + store: { + read: () => store.area(account, 'drafts.read', { before: Date.now() - SENDING_STALE_MS }), + claim: (localId) => store.area(account, 'drafts.claim', { localId }), + failed: (localId, status, error, neverSent) => store.area(account, 'drafts.failed', { localId, status, error, neverSent }), + created: (result) => store.area(account, 'drafts.created', result), + ghostsLooked: (jmap, messageIds, found) => store.area(account, 'drafts.ghostsLooked', { jmap, messageIds, found }), + }, + request: (calls, using) => client.requestNetworkOnly(calls, using), + upload: async (jmap, blobId) => { + const file = await heldFile(account, blobId); + return file ? (await client.uploadBlob(file, { accountId: jmap, noHold: true })).blobId : null; + }, + isRefusal: isRequestRefusal, + onRefused: (subject, error) => deps.notify({ kind: 'draft-refused', subject, error }), + onConflict: (subject) => deps.notify({ kind: 'draft-conflict', subject }), + onSent: () => deps.onDraftSent?.(), + }; + return flushDrafts(io); } /** The changes queued before `upToSeq` (all, with null). False when the run stopped on an outage. */ @@ -194,8 +228,14 @@ async function lookUp(client: JMAPClient, entry: OutboxEntry): Promise { if (entry.item.kind !== 'email' || typeof entry.item.args[7] !== 'string') return; const options = entry.item.args[15] as { draftAccountId?: string } | undefined; + const jmap = options?.draftAccountId || client.getAccountId(); try { - await client.requestNetworkOnly([['Email/set', { accountId: options?.draftAccountId || client.getAccountId(), destroy: [entry.item.args[7]] }, '0']]); + if (isStandInId(entry.item.args[7])) { + // Never reached the server: it only has to stop going there. + await getStore().area(entry.account, 'drafts.discard', { jmap, id: entry.item.args[7] }); + return; + } + await client.requestNetworkOnly([['Email/set', { accountId: jmap, destroy: [entry.item.args[7]] }, '0']]); } catch { // An orphan in Drafts, nothing worse. } @@ -241,6 +281,9 @@ async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps) flushing.add(account); try { await settleStaleSends(account); + // A draft goes before the changes that may be waiting: what it replaces + // only starts to go once it's there. + if (!(await sendDrafts(account, client, deps))) return; if (!(await settleUncertain(account, client, deps))) return; // In the order they were made: a reply's $answered, queued after the // reply, goes after it. @@ -290,6 +333,7 @@ async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps) } if (stillOffline() || !deps.allowed()) return; await sendPatches(account, client, deps, null); + await getStore().area(account, 'outbox.dropOrphans', { olderThan: Date.now() - HELD_FILE_GRACE_MS }); } finally { flushing.delete(account); } diff --git a/lib/offline/offline-mode.ts b/lib/offline/offline-mode.ts index 1993f154d..5e1ab9522 100644 --- a/lib/offline/offline-mode.ts +++ b/lib/offline/offline-mode.ts @@ -28,6 +28,7 @@ import { getSendGuardSource, guardState, guardsSending } from '@/lib/plugin-send import { emitPushStateChange } from '@/lib/push-events'; import { requestOutboxSync } from '@/lib/pwa-notify/outbox-sync'; import { STORAGE_ERROR_AREA } from '@/lib/store/area'; +import type { DraftError } from '@/lib/store/drafts/pending'; import { configureStore, getStore, type StorePageHooks } from '@/lib/store/client/api'; import { purgeStoreFiles } from '@/lib/store/client/purge'; import type { StoreSession } from '@/lib/store/engine/transport'; @@ -37,7 +38,7 @@ import { useEmailStore } from '@/stores/email-store'; import { usePolicyStore, whenPolicySettled } from '@/stores/policy-store'; import { onBeforeSignOut, onFullSignOut } from '@/lib/sign-out-cleanup'; import { generateMessageId } from '@/lib/jmap/client'; -import { setOfflineHooks, type OfflineClientRef, type OfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks'; +import { isNetworkFailure, SendNotStartedError, setOfflineHooks, type OfflineClientRef, type OfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks'; import type { SendEmailResult } from '@/lib/jmap/types'; import { setOutboxHooks, type OutboxHooks } from './outbox-hooks'; import { @@ -45,6 +46,7 @@ import { holdFile, isLocalBlobId, queueSend, + readLocalDrafts, readQueue, type OutboxEntry, type OutboxError, @@ -65,6 +67,8 @@ export type OfflineNotice = | { kind: 'held'; subject: string } | { kind: 'lost'; subjects: string[] } | { kind: 'conflict'; count: number } + | { kind: 'draft-refused'; subject: string; error: DraftError } + | { kind: 'draft-conflict'; subject: string } | { kind: 'storage-failed' } | { kind: 'refused-delete' } | { kind: 'refused-other' } @@ -228,12 +232,16 @@ const hooks: OfflineHooks = { if (typeof navigator === 'undefined' || navigator.onLine !== false) return false; return offlineActive(accountForClient(client)); }, - async answer(client, methodCalls) { + async answer(client, methodCalls, options) { const account = accountForClient(client); if (!offlineActive(account) || !account) return null; let answered: OfflineResponse | null = null; try { - answered = await getStore().answer(account, methodCalls); + // A draft whose connection broke may be on the server already. + const calls = options?.networkTried + ? methodCalls.map((call): OfflineMethodCall => (call[0] === 'Email/set' && call[1].create ? [call[0], { ...call[1], networkTried: true }, call[2]] : call)) + : methodCalls; + answered = await getStore().answer(account, calls); } catch { // The store couldn't start or lost its owner: as if nothing were kept. } @@ -247,7 +255,12 @@ const hooks: OfflineHooks = { async resolveStandIns(client, ids) { const account = accountForClient(client); if (!offlineActive(account) || !account) return {}; - return getStore().area>(account, 'calendar.resolveStandIns', { ids }).catch(() => ({})); + const ask = (call: string) => getStore().area>(account, call, { ids }).catch(() => ({} as Record)); + const [drafts, events] = await Promise.all([ask('drafts.resolveStandIns'), ask('calendar.resolveStandIns')]); + return Object.fromEntries(ids.map((id) => [id, events[id] ?? drafts[id] ?? null])); + }, + resolveHeldAttachments(client, attachments, accountId) { + return resolveHeld(client, attachments, accountId); }, fileOpened(client, accountId, id) { const account = accountForClient(client); @@ -308,7 +321,8 @@ const hooks: OfflineHooks = { // An attachment, or a file from the Files area. const kept = await getStore().area(account, 'mail.getBlob', { jmap, blobId }).catch(() => null) ?? await getStore().area(account, 'files.blob', { jmap, blobId }).catch(() => null); - return kept ? new Blob([kept.data as BlobPart], { type: kept.type || 'application/octet-stream' }) : null; + if (!kept) return null; + return new Blob(kept.blob ? [kept.blob] : [kept.data as BlobPart], { type: kept.type || 'application/octet-stream' }); }, recordBlob(client, blobId, jmap, blob) { const account = accountForClient(client); @@ -326,7 +340,29 @@ const hooks: OfflineHooks = { }, }; -interface KeptBlob { type: string | null; data: Uint8Array } +interface KeptBlob { type: string | null; blob?: Blob; data?: Uint8Array } + +/** Files attached offline, uploaded now that the server answers. An outage here means nothing went anywhere. */ +async function resolveHeld(client: OfflineClientRef, attachments: T[], accountId?: string): Promise { + const account = accountForClient(client); + if (!account || !attachments.some((a) => isLocalBlobId(a.blobId))) return attachments; + const real = onlineClientFor(account); + if (!real) return attachments; + const out: T[] = []; + for (const attachment of attachments) { + if (!isLocalBlobId(attachment.blobId)) { out.push(attachment); continue; } + const file = await heldFile(account, attachment.blobId); + if (!file) throw Object.assign(new Error('A file attached offline is no longer on this device'), { name: 'OfflineFileMissingError' }); + try { + const { blobId } = await real.uploadBlob(file, { accountId, noHold: true }); + out.push({ ...attachment, blobId }); + } catch (error) { + if (isNetworkFailure(error)) throw new SendNotStartedError(error); + throw error; + } + } + return out; +} /** Attachments and images opened while online are kept up to this size. */ const MAX_RECORDED_BLOB_BYTES = 5 * 1024 * 1024; @@ -421,20 +457,8 @@ const outbox: OutboxHooks = { notify({ kind: 'queued', subject: summary.subject }); return queuedResult(id); }, - async resolveHeldAttachments(client, attachments) { - const account = accountForClient(client); - if (!account || !attachments.some((a) => isLocalBlobId(a.blobId))) return attachments; - const real = onlineClientFor(account); - if (!real) return attachments; - const out = []; - for (const attachment of attachments) { - if (!isLocalBlobId(attachment.blobId)) { out.push(attachment); continue; } - const file = await heldFile(account, attachment.blobId); - if (!file) throw Object.assign(new Error('A file attached offline is no longer on this device'), { name: 'OfflineFileMissingError' }); - const { blobId } = await real.uploadBlob(file); - out.push({ ...attachment, blobId }); - } - return out; + resolveHeldAttachments(client, attachments) { + return resolveHeld(client, attachments); }, }; @@ -471,13 +495,13 @@ export async function refreshQueueState(): Promise { // Nothing is kept for an account that hasn't it on; and opening the // store would create it (on the sign-in page, after a sign-out). if (useOfflineStore.getState().enabled.length === 0) { - useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [] }); + useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [] }); return; } try { // Sends of a tab closed on the way: uncertain, so they can be looked up. await settleStaleSends(); - const { patches, outbox: entries, conflicts } = await readQueue(); + const [{ patches, outbox: entries, conflicts }, localDrafts] = await Promise.all([readQueue(), readLocalDrafts()]); const rows: OutboxRow[] = entries.map((entry) => ({ id: entry.id, account: entry.account, @@ -487,7 +511,7 @@ export async function refreshQueueState(): Promise { status: entry.status, error: entry.error, })); - useOfflineStore.setState({ outbox: rows, pendingChanges: patches, conflicts }); + useOfflineStore.setState({ outbox: rows, pendingChanges: patches, conflicts, localDrafts }); } catch { // The store is being deleted, or can't start. } @@ -580,6 +604,13 @@ const AREA_FLUSHES = [ { call: 'contacts.flush', types: ['AddressBook', 'ContactCard'] }, ]; +/** The page reads these types again. */ +function emitTypesChanged(client: NonNullable>, types: string[]): void { + const role = (useAuthStore.getState().client as unknown) === client ? 'active' : 'background'; + const changed = { [client.getAccountId()]: Object.fromEntries(types.map((type) => [type, 'flushed'])) }; + emitPushStateChange({ '@type': 'StateChange', changed } as never, client, role); +} + async function flushAreas(account: string, client: NonNullable>): Promise { for (const { call, types } of AREA_FLUSHES) { // Contacts may not be there in this build: nothing to flush then. @@ -587,9 +618,7 @@ async function flushAreas(account: string, client: NonNullable [type, 'flushed'])) }; - emitPushStateChange({ '@type': 'StateChange', changed } as never, client, role); + emitTypesChanged(client, types); } } @@ -620,6 +649,7 @@ export async function flushAll(): Promise { stricterNow, onScheduled: offerUndo, retryLater: () => scheduleFlushAll(LOOKUP_AGAIN_MS), + onDraftSent: () => emitTypesChanged(client, ['Email', 'Mailbox', 'Thread']), }).catch(() => {}); await flushAreas(account, client); } @@ -665,13 +695,14 @@ export function confirmLoss(scope: string): void { } async function unsentSubjects(scope: string): Promise { - let rows = useOfflineStore.getState().outbox; + const known = useOfflineStore.getState(); + let rows: Array<{ account: string; subject: string }> = [...known.outbox, ...known.localDrafts]; // Reading an account's store would create it if it never had one. - if (scope !== 'all' && !useOfflineStore.getState().enabled.includes(scope)) return rows.filter((row) => row.account === scope).map((row) => row.subject); + if (scope !== 'all' && !known.enabled.includes(scope)) return rows.filter((row) => row.account === scope).map((row) => row.subject); try { - rows = (await readQueue(scope === 'all' ? undefined : scope)).outbox.map((entry) => ({ - id: entry.id, account: entry.account, createdAt: entry.createdAt, subject: entry.summary.subject, to: entry.summary.to, status: entry.status, - })); + const wanted = scope === 'all' ? undefined : scope; + const [{ outbox: entries }, drafts] = await Promise.all([readQueue(wanted), readLocalDrafts(wanted)]); + rows = [...entries.map((entry) => ({ account: entry.account, subject: entry.summary.subject })), ...drafts]; } catch { // No store: what the state last saw. } @@ -699,7 +730,8 @@ export async function wipeAccount(account: string): Promise { const timer = syncTimers.get(account); if (timer) clearTimeout(timer); syncTimers.delete(account); - if (useOfflineStore.getState().enabled.includes(account) || useOfflineStore.getState().outbox.some((row) => row.account === account)) { + const before = useOfflineStore.getState(); + if (before.enabled.includes(account) || before.outbox.some((row) => row.account === account) || before.localDrafts.some((row) => row.account === account)) { await nameLost(account, useAuthStore.getState().isAuthenticated); } named.delete(account); @@ -738,7 +770,7 @@ export async function wipeEverything(): Promise { clearHandoffs(); confirmed.clear(); named.clear(); - useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [], usageBytes: 0, syncing: [], lastSyncedAt: {} }); + useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [], usageBytes: 0, syncing: [], lastSyncedAt: {} }); } /** Whether the policy of an account is known well enough to decide on it. */ @@ -934,7 +966,7 @@ onFullSignOut(() => { getStore().stop(); confirmed.clear(); named.clear(); - useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, conflicts: [], usageBytes: 0, syncing: [], lastSyncedAt: {}, startedOffline: false }); + useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [], usageBytes: 0, syncing: [], lastSyncedAt: {}, startedOffline: false }); }); // Every sign-out the user didn't agree to lose mail at (the session limits, diff --git a/lib/offline/queue.ts b/lib/offline/queue.ts index 5686f1fc5..2bb4bef74 100644 --- a/lib/offline/queue.ts +++ b/lib/offline/queue.ts @@ -8,6 +8,7 @@ import { requestOutboxSync } from '@/lib/pwa-notify/outbox-sync'; import { getStore } from '@/lib/store/client/api'; +import type { LocalDraftSummary } from '@/lib/store/drafts/calls'; import type { ConflictSummary, OutboxError, OutboxStatus, OutboxSummary, PolicyFingerprint, SendItem, StoredConflict, StoredOutboxEntry } from '@/lib/store/outbox/items'; import { useOfflineStore } from './state'; @@ -72,20 +73,49 @@ export function removeOutboxEntry(entry: Pick): P return getStore().area(entry.account, 'outbox.remove', { id: entry.id }); } +export interface LocalDraftRow extends LocalDraftSummary { + account: string; +} + +/** Drafts that are only on this device, oldest first. */ +export async function readLocalDrafts(account?: string): Promise { + const accounts = account ? [account] : useOfflineStore.getState().enabled; + const read = await Promise.all(accounts.map(async (id) => ((await getStore().area(id, 'drafts.list')) ?? []).map((row) => ({ ...row, account: id })))); + return read.flat(); +} + +/** Asks the server again about a draft it refused. */ +export function retryLocalDraft(account: string, localId: string): Promise { + return getStore().area(account, 'drafts.retry', { localId }); +} + export function dismissConflict(account: string, id: number): Promise { return getStore().area(account, 'outbox.dismissConflict', { id }); } // ---- Files attached while offline ---- +/** A file on its way to the store is cut into pieces of this size, so it's never in memory as a whole. */ +const HOLD_CHUNK_BYTES = 1024 * 1024; + /** Keeps a file under a stand-in blob id until its message is sent. */ export async function holdFile(account: string, file: Blob, name: string, jmap?: string): Promise<{ blobId: string; size: number; type: string }> { - const data = new Uint8Array(await file.arrayBuffer()); - return getStore().area(account, 'outbox.holdFile', { jmap, name, type: file.type, data }); + const store = getStore(); + const { blobId } = await store.area<{ blobId: string }>(account, 'outbox.holdStart', { jmap, name, type: file.type, size: file.size }); + try { + for (let at = 0, seq = 0; at < file.size; at += HOLD_CHUNK_BYTES, seq += 1) { + const data = new Uint8Array(await file.slice(at, at + HOLD_CHUNK_BYTES).arrayBuffer()); + await store.area(account, 'outbox.holdChunk', { blobId, seq, data }); + } + return await store.area(account, 'outbox.holdEnd', { blobId }); + } catch (error) { + await store.area(account, 'outbox.holdAbort', { blobId }).catch(() => {}); + throw error; + } } export async function heldFile(account: string, blobId: string): Promise { - const held = await getStore().area<{ name: string; type: string; data: Uint8Array } | null>(account, 'outbox.heldFile', { blobId }); + const held = await getStore().area<{ name: string; type: string; blob: Blob } | null>(account, 'outbox.heldFile', { blobId }); if (!held) return null; - return new File([held.data as BlobPart], held.name, { type: held.type }); + return new File([held.blob], held.name, { type: held.type }); } diff --git a/lib/offline/state.ts b/lib/offline/state.ts index 1ba184282..e5bacb7f9 100644 --- a/lib/offline/state.ts +++ b/lib/offline/state.ts @@ -7,7 +7,7 @@ */ import { create } from 'zustand'; -import type { ConflictEntry, OutboxEntry, OutboxError } from './queue'; +import type { ConflictEntry, LocalDraftRow, OutboxEntry, OutboxError } from './queue'; /** localStorage: the local account ids that keep mail on this device. */ export const OFFLINE_ACCOUNTS_KEY = 'bulwark-offline-accounts'; @@ -35,6 +35,10 @@ interface OfflineState { pendingChanges: number; /** Changes the server refused, or that met a message changed on another device. */ conflicts: ConflictEntry[]; + /** Drafts that are only on this device: on their way up, or refused by the server. */ + localDrafts: LocalDraftRow[]; + /** A local draft the user asked to open, for the mail page to pick up. */ + draftToOpen: { account: string; jmap: string; id: string } | null; usageBytes: number; /** Accounts whose mail is being brought onto the device right now. */ syncing: string[]; @@ -59,6 +63,8 @@ export const useOfflineStore = create()(() => ({ outbox: [], pendingChanges: 0, conflicts: [], + localDrafts: [], + draftToOpen: null, usageBytes: 0, syncing: [], lastSyncedAt: {}, diff --git a/lib/store/__tests__/attachments.test.ts b/lib/store/__tests__/attachments.test.ts new file mode 100644 index 000000000..0d3df51d5 --- /dev/null +++ b/lib/store/__tests__/attachments.test.ts @@ -0,0 +1,382 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { DEFAULT_SCOPE, type StoreScope } from '../area'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError } from '../engine/transport'; +import { mailArea } from '../mail'; +import { outboxArea } from '../outbox'; +import { FakeJmapDrafts } from './helpers/fake-drafts'; +import { FAKE_SESSION, JMAP_ACCOUNT, type FakeEmail } from './helpers/fake-jmap'; +import { openMemoryStore } from './helpers/node-adapter'; + +const SCOPE: StoreScope = { ...DEFAULT_SCOPE, days: null }; +const RAW = 'message/rfc822'; + +/** Hands out files in pieces, so a test can tell a stream from a whole download. */ +class Files extends FakeJmapDrafts { + readonly downloads: string[] = []; + readonly rawDownloads: string[] = []; + readonly files = new Map(); + gone = new Set(); + breakAfter: { blobId: string; bytes: number } | null = null; + pieces = 0; + onDownload: ((blobId: string) => void) | null = null; + + private record(blobId: string, type: string): Uint8Array { + if (type === RAW) this.rawDownloads.push(blobId); + else this.downloads.push(blobId); + this.onDownload?.(blobId); + if (this.gone.has(blobId)) throw new TransportError('http', 'Gone', 404); + return this.files.get(blobId) ?? new Uint8Array(type === RAW ? [9, 9] : [1, 2, 3]); + } + + override async download(_jmap?: string, blobId?: string, type?: string): Promise { + return this.record(String(blobId), String(type)); + } + + async downloadStream(_jmap: string, blobId: string, type: string, _name: string, sink: (part: Uint8Array) => void): Promise { + const data = this.record(blobId, type); + const step = 256 * 1024; + for (let at = 0; at < data.length; at += step) { + if (this.breakAfter?.blobId === blobId && at >= this.breakAfter.bytes) throw new TransportError('network', 'The download broke off'); + this.pieces += 1; + sink(data.subarray(at, at + step)); + } + } +} + +function engineFor(server: Files, scope: StoreScope = SCOPE, areas = [mailArea]) { + const db = openMemoryStore(); + const engine = new AccountEngine({ + account: 'a@fake.test', + db, + session: FAKE_SESSION, + transport: server, + areas, + scope, + now: () => Date.UTC(2026, 10, 3), + sleep: async () => {}, + }); + return { engine, db }; +} + +const kept = (engine: AccountEngine, blobId: string) => engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId }) as Promise<{ type: string; size: number; blob: Blob } | null>; + +function withFile(server: Files, id: string, blobId: string, size = 3, day = 1, extra: Partial = {}): FakeEmail { + server.files.set(blobId, new Uint8Array(size).fill(7)); + return server.addEmail({ + id, + size: 100_000_000, + receivedAt: new Date(Date.UTC(2026, 9, day)).toISOString(), + attachments: [{ partId: '2', blobId, name: `${blobId}.pdf`, type: 'application/pdf', size }], + ...extra, + }); +} + +describe('what\'s fetched', () => { + it('goes newest mail first, after the rest of the pass', async () => { + const server = new Files(); + withFile(server, 'old', 'b-old', 3, 1); + withFile(server, 'new', 'b-new', 3, 20); + withFile(server, 'mid', 'b-mid', 3, 10); + const { engine } = engineFor(server); + await engine.requestSync(); + expect(server.downloads).toEqual(['b-new', 'b-mid', 'b-old']); + }); + + it('runs after every other syncer, and a newer pass goes first', async () => { + const server = new Files(); + withFile(server, 'one', 'b-1', 3, 2); + withFile(server, 'two', 'b-2', 3, 1); + const order: string[] = []; + const later = { name: 'later', migrations: [], methods: {}, syncers: [{ name: 'later', capability: 'urn:ietf:params:jmap:mail', run: async () => { order.push('later'); } }] }; + const { engine } = engineFor(server, SCOPE, [mailArea, later]); + let wanted = false; + server.onDownload = () => { + order.push('file'); + // Asked for while a file comes in: the next file waits for that pass. + if (!wanted) { + wanted = true; + void engine.requestSync(); + } + }; + await engine.requestSync(); + expect(order[0]).toBe('later'); + // later, file 1, then the wanted pass: later again before file 2. + expect(order.slice(0, 3)).toEqual(['later', 'file', 'later']); + expect(server.downloads).toEqual(['b-1', 'b-2']); + }); + + it('includes inline images that only the body lists, as Stalwart 1.0 answers', async () => { + const server = new Files(); + server.addEmail({ + id: 'm', + attachments: [], + htmlBody: [{ partId: '1', type: 'text/html' }, { partId: '2', blobId: 'b-inline', type: 'image/png', name: 'i.png', size: 3, cid: 'i', disposition: 'inline' }] as never, + bodyStructure: { + type: 'multipart/related', + subParts: [{ partId: '1', blobId: 'b-text', type: 'text/html', size: 10 }, { partId: '3', blobId: 'b-tree', type: 'image/gif', size: 3, cid: 'j', disposition: 'inline' }], + } as never, + }); + const { engine } = engineFor(server); + await engine.requestSync(); + expect(server.downloads.sort()).toEqual(['b-inline', 'b-tree']); + expect(await kept(engine, 'b-inline')).toMatchObject({ type: 'image/png' }); + }); + + it('comes through the stream in chunks that are each kept, and is readable whole', async () => { + const server = new Files(); + const size = 2 * 1024 * 1024 + 5; + withFile(server, 'm', 'b-big', size); + server.files.get('b-big')?.fill(3); + const { engine, db } = engineFor(server); + await engine.requestSync(); + expect(server.pieces).toBeGreaterThan(3); + expect(Number(db.get<{ n: number }>("SELECT COUNT(*) AS n FROM blob_chunk WHERE blob_id = 'b-big'")?.n)).toBe(3); + const file = await kept(engine, 'b-big'); + expect(file?.size).toBe(size); + const bytes = new Uint8Array(await (file as { blob: Blob }).blob.arrayBuffer()); + expect(bytes.length).toBe(size); + expect(bytes.every((value) => value === 3)).toBe(true); + }); + + it('leaves nothing readable when a download breaks, and tries again next time', async () => { + const server = new Files(); + withFile(server, 'm', 'b-big', 2 * 1024 * 1024); + server.breakAfter = { blobId: 'b-big', bytes: 1024 * 1024 + 1 }; + const { engine, db } = engineFor(server); + await engine.requestSync(); + expect(await kept(engine, 'b-big')).toBeNull(); + expect(db.get("SELECT 1 AS x FROM blob_chunk WHERE blob_id = 'b-big'")).toBeUndefined(); + server.breakAfter = null; + await engine.requestSync(); + expect((await kept(engine, 'b-big'))?.size).toBe(2 * 1024 * 1024); + }); + + it('drops a file larger than it said, and doesn\'t fetch it again', async () => { + const server = new Files(); + withFile(server, 'm', 'b-liar', 3); + server.files.set('b-liar', new Uint8Array(5000)); + const { engine } = engineFor(server, { ...SCOPE, maxAttachmentBytes: 1000 }); + await engine.requestSync(); + await engine.requestSync(); + expect(await kept(engine, 'b-liar')).toBeNull(); + expect(server.downloads).toEqual(['b-liar']); + }); +}); + +describe('the cap', () => { + it('pushes out older mail\'s files for a newer message\'s', async () => { + const server = new Files(); + withFile(server, 'old', 'b-old', 40, 1); + const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }); + await engine.requestSync(); + expect(await kept(engine, 'b-old')).not.toBeNull(); + withFile(server, 'new', 'b-new', 40, 20); + await engine.requestSync(); + expect(await kept(engine, 'b-new')).not.toBeNull(); + expect(await kept(engine, 'b-old')).toBeNull(); + }); + + it('doesn\'t fetch an older file back that a newer one pushed out', async () => { + const server = new Files(); + withFile(server, 'old', 'b-old', 40, 1); + withFile(server, 'new', 'b-new', 40, 20); + const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }); + await engine.requestSync(); + await engine.requestSync(); + expect(server.downloads).toEqual(['b-new']); + }); + + it('doesn\'t let a file of older mail the user opened push out a newer one', async () => { + const server = new Files(); + withFile(server, 'new', 'b-new', 40, 20); + server.addEmail({ id: 'old', size: 100_000_000, receivedAt: new Date(Date.UTC(2026, 9, 1)).toISOString(), attachments: [{ partId: '2', blobId: 'b-old', size: 40, type: 'image/png' }] }); + server.gone.add('b-old'); + const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }); + await engine.requestSync(); + expect(await engine.areaCall('mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-old', type: null, data: new Uint8Array(40) })).toBe(false); + expect(await kept(engine, 'b-new')).not.toBeNull(); + }); + + it('keeps a smaller older file that still fits past one that doesn\'t', async () => { + const server = new Files(); + withFile(server, 'new', 'b-new', 40, 20); + withFile(server, 'mid', 'b-mid', 40, 10); + withFile(server, 'old', 'b-old', 10, 1); + const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }); + await engine.requestSync(); + expect(server.downloads).toEqual(['b-new', 'b-old']); + }); + + it('stops looking after a long run of files that don\'t fit, instead of reading the kept list for each', async () => { + const server = new Files(); + withFile(server, 'fills', 'b-fills', 50, 28); + for (let n = 0; n < 60; n += 1) withFile(server, `no-${n}`, `b-no-${n}`, 40, 27); + withFile(server, 'small-old', 'b-small-old', 5, 1); + const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }); + await engine.requestSync(); + expect(server.downloads).toEqual(['b-fills']); + }); + + it('never pushes out the files of a pinned message', async () => { + const server = new Files(); + withFile(server, 'pinned', 'b-pinned', 40, 1, { keywords: { $pinned: true } }); + const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }); + await engine.requestSync(); + withFile(server, 'new', 'b-new', 40, 20); + await engine.requestSync(); + expect(await kept(engine, 'b-pinned')).not.toBeNull(); + expect(await kept(engine, 'b-new')).toBeNull(); + }); + + it('never touches a file held for a draft', async () => { + const server = new Files(); + withFile(server, 'old', 'b-old', 40, 1); + const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }, [mailArea, outboxArea]); + const held = await engine.areaCall('outbox.holdFile', { name: 'mine', type: '', data: new Uint8Array(100) }) as { blobId: string }; + await engine.requestSync(); + withFile(server, 'new', 'b-new', 40, 20); + await engine.requestSync(); + expect(await kept(engine, 'b-old')).toBeNull(); + expect(await engine.areaCall('outbox.heldFile', { blobId: held.blobId })).not.toBeNull(); + }); + + it('applies a lowered total at once, oldest first', async () => { + const server = new Files(); + withFile(server, 'old', 'b-old', 40, 1); + withFile(server, 'new', 'b-new', 40, 20); + const { engine } = engineFor(server); + await engine.requestSync(); + engine.setScope({ ...SCOPE, maxTotalBytes: 50 }); + expect(await kept(engine, 'b-old')).toBeNull(); + expect(await kept(engine, 'b-new')).not.toBeNull(); + }); + + it('applies a lowered size limit at once', async () => { + const server = new Files(); + withFile(server, 'big', 'b-big', 40, 1); + withFile(server, 'small', 'b-small', 4, 2); + const { engine } = engineFor(server); + await engine.requestSync(); + engine.setScope({ ...SCOPE, maxAttachmentBytes: 10 }); + expect(await kept(engine, 'b-big')).toBeNull(); + expect(await kept(engine, 'b-small')).not.toBeNull(); + }); +}); + +describe('a policy forbidding attachments', () => { + it('fetches nothing, drops what\'s kept at once, and gives nothing out', async () => { + const server = new Files(); + withFile(server, 'm', 'b-1'); + const { engine, db } = engineFor(server); + await engine.requestSync(); + expect(await kept(engine, 'b-1')).not.toBeNull(); + + engine.setScope({ ...SCOPE, attachments: false }); + expect(db.get('SELECT 1 AS x FROM blob')).toBeUndefined(); + expect(db.get('SELECT 1 AS x FROM blob_chunk')).toBeUndefined(); + expect(await kept(engine, 'b-1')).toBeNull(); + + server.downloads.length = 0; + server.rawDownloads.length = 0; + server.addEmail({ id: 'more', attachments: [{ partId: '2', blobId: 'b-2', size: 3, type: 'application/pdf' }] }); + await engine.requestSync(); + expect(server.downloads).toEqual([]); + expect(server.rawDownloads).toEqual([]); + }); + + it('refuses a file the user opened online too', async () => { + const server = new Files(); + server.addEmail({ id: 'm', attachments: [{ partId: '2', blobId: 'b-1', size: 3, type: 'image/png' }] }); + const { engine } = engineFor(server, { ...SCOPE, attachments: false }); + await engine.requestSync(); + expect(await engine.areaCall('mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-1', type: null, data: new Uint8Array([1]) })).toBe(false); + }); + + it('applies when the store opens with it, without waiting for a sync', async () => { + const server = new Files(); + withFile(server, 'm', 'b-1'); + const { engine, db } = engineFor(server); + await engine.requestSync(); + expect(db.get('SELECT 1 AS x FROM blob')).toBeTruthy(); + new AccountEngine({ account: 'a@fake.test', db, session: FAKE_SESSION, transport: server, areas: [mailArea], scope: { ...SCOPE, attachments: false }, sleep: async () => {} }); + expect(db.get('SELECT 1 AS x FROM blob')).toBeUndefined(); + }); + + it('keeps a file the user attached to a draft: it\'s theirs, not a copy of mail', async () => { + const server = new Files(); + const { engine } = engineFor(server, { ...SCOPE, attachments: false }, [mailArea, outboxArea]); + const held = await engine.areaCall('outbox.holdFile', { name: 'mine', type: '', data: new Uint8Array(3) }) as { blobId: string }; + await engine.requestSync(); + engine.setScope({ ...SCOPE, attachments: false }); + expect(await engine.areaCall('outbox.heldFile', { blobId: held.blobId })).not.toBeNull(); + }); +}); + +describe('raw messages for an export', () => { + it('are kept with the attachments, and handed out by Email.blobId', async () => { + const server = new Files(); + server.addEmail({ id: 'm', blobId: 'raw-m', size: 2 }); + server.files.set('raw-m', new TextEncoder().encode('From: a\r\n\r\nx')); + const { engine } = engineFor(server); + await engine.requestSync(); + expect(server.rawDownloads).toEqual(['raw-m']); + const raw = await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-m' }) as { ok: true; type: string; blob: Blob }; + expect(raw.ok).toBe(true); + expect(raw.type).toBe(RAW); + expect(await raw.blob.text()).toBe('From: a\r\n\r\nx'); + expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT })).toEqual({ total: 1, kept: 1 }); + }); + + it('come after the files of the same message', async () => { + const server = new Files(); + const order: string[] = []; + server.onDownload = (blobId) => order.push(blobId); + withFile(server, 'm', 'b-1', 3, 1, { blobId: 'raw-m', size: 3 }); + const { engine } = engineFor(server); + await engine.requestSync(); + expect(order).toEqual(['b-1', 'raw-m']); + }); + + it('are refused plainly with attachments off, and counted as not there', async () => { + const server = new Files(); + server.addEmail({ id: 'm', blobId: 'raw-m', mailboxIds: { inbox: true } }); + const { engine } = engineFor(server, { ...SCOPE, attachments: false }); + await engine.requestSync(); + expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-m' })).toEqual({ ok: false, reason: 'attachments_off' }); + expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT })).toEqual({ total: 1, kept: 0 }); + }); + + it('are refused when they aren\'t kept, whatever else is', async () => { + const server = new Files(); + server.addEmail({ id: 'm', blobId: 'raw-m', size: 5_000, attachments: [{ partId: '2', blobId: 'b-1', size: 3, type: 'image/png' }] }); + const { engine } = engineFor(server, { ...SCOPE, maxAttachmentBytes: 1000 }); + await engine.requestSync(); + expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-m' })).toEqual({ ok: false, reason: 'not_kept' }); + // An attachment isn't a whole message. + expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-1' })).toEqual({ ok: false, reason: 'not_kept' }); + expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT })).toEqual({ total: 1, kept: 0 }); + }); + + it('count per folder for an export of one', async () => { + const server = new Files({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'other', name: 'Other' }] }); + server.addEmail({ id: 'a', blobId: 'raw-a', size: 2, mailboxIds: { inbox: true } }); + server.addEmail({ id: 'b', blobId: 'raw-b', size: 2, mailboxIds: { other: true } }); + server.gone.add('raw-b'); + const { engine } = engineFor(server); + await engine.requestSync(); + expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT, mailboxId: 'inbox' })).toEqual({ total: 1, kept: 1 }); + expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT, mailboxId: 'other' })).toEqual({ total: 1, kept: 0 }); + }); + + it('don\'t exist for a draft on the device', async () => { + const server = new Files({ mailboxes: [{ id: 'drafts', role: 'drafts' }] }); + const { engine, db } = engineFor(server, SCOPE, [mailArea, outboxArea]); + await engine.requestSync(); + const answer = await engine.answer([['Email/set', { create: { c: { keywords: { $draft: true }, mailboxIds: { drafts: true }, bodyValues: { 1: { value: 'x' } }, textBody: [{ partId: '1' }] } } }, 's']]); + const id = answer?.methodResponses[0][1].created.c.id as string; + expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: id })).toEqual({ ok: false, reason: 'not_kept' }); + expect(db.get('SELECT 1 AS x FROM email_part')).toBeUndefined(); + }); +}); diff --git a/lib/store/__tests__/drafts-client.test.ts b/lib/store/__tests__/drafts-client.test.ts new file mode 100644 index 000000000..e34c08779 --- /dev/null +++ b/lib/store/__tests__/drafts-client.test.ts @@ -0,0 +1,303 @@ +/** + * The real JMAPClient's createDraft, getEmail and blob calls against the store + * while offline and a fake server online, through a stubbed fetch: what the + * composer does around a save. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { JMAPClient } from '@/lib/jmap/client'; +import { setOfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks'; +import { DEFAULT_SCOPE } from '../area'; +import type { DraftGhost, PendingDraft } from '../drafts/calls'; +import { flushDrafts } from '../drafts/flush'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError, type MethodCall } from '../engine/transport'; +import { mailArea } from '../mail'; +import { outboxArea } from '../outbox'; +import { flushPatches, type PatchOutcome, type PendingPatch } from '../outbox/flush-patches'; +import { FakeJmapDrafts } from './helpers/fake-drafts'; +import { JMAP_ACCOUNT } from './helpers/fake-jmap'; +import { openMemoryStore } from './helpers/node-adapter'; + +const NOW = Date.UTC(2026, 10, 3); +const MAIL = 'urn:ietf:params:jmap:mail'; +const SESSION = { + capabilities: { 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500, maxObjectsInSet: 500 }, [MAIL]: {} }, + accounts: { [JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [MAIL]: {} } } }, + primaryAccounts: { [MAIL]: JMAP_ACCOUNT }, + username: 'a@example.org', + apiUrl: 'https://fake.test/api', + downloadUrl: 'https://fake.test/d/{accountId}/{blobId}/{name}?type={type}', + uploadUrl: 'https://fake.test/u/{accountId}', + eventSourceUrl: '', + state: 's', +}; + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { status: 200, headers: { 'Content-Type': 'application/json' } }); +} + +let server: FakeJmapDrafts; +let engine: AccountEngine; +let online: boolean; +/** The server comes back right after the next create was answered on the device. */ +let backAfterCreate: boolean; +let spy: ReturnType; +let client: JMAPClient; +/** What reached the network. */ +let wire: MethodCall[]; + +beforeEach(async () => { + server = new FakeJmapDrafts({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'drafts', role: 'drafts' }] }); + server.now = NOW; + engine = new AccountEngine({ + account: 'a@fake.test', + db: openMemoryStore(), + session: SESSION as never, + transport: server, + areas: [mailArea, outboxArea], + scope: { ...DEFAULT_SCOPE, days: null, attachments: false }, + now: () => NOW, + sleep: async () => {}, + }); + online = true; + backAfterCreate = false; + wire = []; + spy = vi.spyOn(globalThis, 'fetch'); + spy.mockImplementation(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (!online) throw new TypeError('Failed to fetch'); + if (url.includes('/u/')) { + const uploaded = await server.upload(JMAP_ACCOUNT, new Uint8Array(await (init?.body as Blob).arrayBuffer())); + return json(uploaded); + } + if (url.includes('/d/')) return new Response(new Uint8Array([9]), { status: 200 }); + if (!init?.body) return json(SESSION); + const calls = (JSON.parse(String(init.body)) as { methodCalls: MethodCall[] }).methodCalls; + wire.push(...calls); + return json(await server.request(calls)); + }); + setOfflineHooks({ + skipNetwork: () => !online, + answer: async (_client, calls: OfflineMethodCall[], options) => { + const marked = options?.networkTried + ? calls.map((call): OfflineMethodCall => (call[0] === 'Email/set' && call[1].create ? [call[0], { ...call[1], networkTried: true }, call[2]] : call)) + : calls; + const answer = await engine.answer(marked); + if (backAfterCreate && calls.some(([name, args]) => name === 'Email/set' && args.create)) online = true; + return answer as OfflineResponse | null; + }, + record: () => {}, + noteNetwork: () => {}, + serverGone: async () => true, + blob: async (_client, blobId) => { + const held = await engine.areaCall('outbox.heldFile', { blobId }) as { blob: Blob } | null; + return held?.blob ?? null; + }, + recordBlob: () => {}, + holdUpload: async (_client, file) => engine.areaCall('outbox.holdFile', { name: file.name, type: file.type, data: new Uint8Array(await file.arrayBuffer()) }) as Promise<{ blobId: string; size: number; type: string }>, + resolveStandIns: (_client, ids) => engine.areaCall('drafts.resolveStandIns', { ids }) as Promise>, + resolveHeldAttachments: async (_client, attachments) => { + const out = []; + for (const attachment of attachments) { + const held = await engine.areaCall('outbox.heldFile', { blobId: attachment.blobId }) as { blob: Blob } | null; + out.push(held ? { ...attachment, blobId: (await server.upload(JMAP_ACCOUNT, new Uint8Array(await held.blob.arrayBuffer()))).blobId } : attachment); + } + return out; + }, + }); + await engine.requestSync(); + client = new JMAPClient('https://fake.test', 'a@example.org', 'pw'); + await client.connect(); + wire.length = 0; +}); + +afterEach(() => { + spy.mockRestore(); + setOfflineHooks(null); +}); + +const saveDraft = (text: string, previous?: string, attachments?: Array<{ blobId: string; name: string; type: string; size: number }>) => + client.createDraft(['b@example.org'], 'Subject', text, undefined, undefined, undefined, 'a@example.org', previous, attachments); + +async function flushAll(): Promise { + await flushDrafts({ + store: { + read: () => engine.areaCall('drafts.read', { before: NOW - 60_000 }) as Promise<{ rows: PendingDraft[]; ghosts: DraftGhost[] }>, + claim: (localId) => engine.areaCall('drafts.claim', { localId }) as Promise, + failed: async (localId, status, error, neverSent) => { await engine.areaCall('drafts.failed', { localId, status, error, neverSent }); }, + created: (result) => engine.areaCall('drafts.created', result) as Promise<{ conflicts: string[] }>, + ghostsLooked: async (jmap, messageIds, found) => { await engine.areaCall('drafts.ghostsLooked', { jmap, messageIds, found }); }, + }, + request: (calls) => server.request(calls), + upload: async (jmap, blobId) => { + const held = await engine.areaCall('outbox.heldFile', { blobId }) as { blob: Blob } | null; + return held ? (await server.upload(jmap, new Uint8Array(await held.blob.arrayBuffer()))).blobId : null; + }, + isRefusal: (error) => error instanceof TransportError && error.kind === 'http', + onRefused: () => {}, + onConflict: () => {}, + onSent: () => {}, + }); + const pending = await engine.areaCall('outbox.pendingPatches', {}) as PendingPatch[]; + await flushPatches({ request: (calls) => server.request(calls), isRefusal: () => false }, pending, async (outcomes: PatchOutcome[]) => { + await engine.areaCall('outbox.patchResults', { outcomes }); + }); +} + +describe('createDraft offline', () => { + it('saves on the device, and the composer can read the draft straight back', async () => { + online = false; + const id = await saveDraft('Written offline'); + expect(id).toMatch(/^local-/); + expect(server.drafts()).toEqual([]); + const email = await client.getEmail(id); + expect(email?.id).toBe(id); + expect(email?.bodyValues?.['1']?.value).toBe('Written offline'); + expect(email?.to?.[0]?.email).toBe('b@example.org'); + }); + + it('merges saves, so a hundred leave one draft on the server with the latest text', async () => { + online = false; + let id = await saveDraft('Save 0'); + for (let n = 1; n < 100; n += 1) id = await saveDraft(`Save ${n}`, id); + online = true; + await flushAll(); + expect(server.drafts()).toHaveLength(1); + expect((server.drafts()[0].bodyValues as Record)['1'].value).toBe('Save 99'); + }); + + it('leaves the old version alone on the server, even when it comes back between the two steps', async () => { + const real = await saveDraft('Online version'); + online = false; + backAfterCreate = true; + wire.length = 0; + const id = await saveDraft('Offline version', real); + expect(id).toMatch(/^local-/); + expect(online).toBe(true); + expect(server.emails.has(real)).toBe(true); + expect(wire.filter(([name, args]) => name === 'Email/set' && args.destroy)).toEqual([]); + await flushAll(); + expect(server.drafts().map((email) => (email.bodyValues as Record)['1'].value)).toEqual(['Offline version']); + }); + + it('gives each create a Message-ID of its own, online too, for finding it after a lost answer', async () => { + await saveDraft('One'); + const create = wire.find(([name]) => name === 'Email/set')?.[1].create as Record; + expect(Object.values(create)[0].messageId[0]).toMatch(/@example\.org$/); + }); + + it('marks a create whose connection broke, so the flush looks before it creates', async () => { + const original = spy.getMockImplementation(); + spy.mockImplementation(async (input: RequestInfo | URL, init?: RequestInit) => { + const body = init?.body ? String(init.body) : ''; + if (body.includes('"Email/set"')) throw new TypeError('Failed to fetch'); + return original!(input, init); + }); + const id = await saveDraft('On the way'); + spy.mockImplementation(original!); + expect(id).toMatch(/^local-/); + expect(await engine.areaCall('drafts.list', {})).toMatchObject([{ status: 'uncertain' }]); + }); +}); + +describe('after a flush the page still holds the stand-in', () => { + it('reads the real message by it, and the answer says what the caller asked', async () => { + online = false; + const id = await saveDraft('Text'); + online = true; + await flushAll(); + const real = server.drafts()[0].id; + wire.length = 0; + const email = await client.getEmail(id); + expect(email?.id).toBe(real); + expect((wire.find(([name]) => name === 'Email/get')?.[1].ids as string[])).toEqual([real]); + }); + + it('replaces the real message on the next save, online', async () => { + online = false; + const id = await saveDraft('First'); + online = true; + await flushAll(); + const next = await saveDraft('Second', id); + expect(server.drafts().map((email) => email.id)).toEqual([next]); + }); + + it('also finds it offline again', async () => { + online = false; + const id = await saveDraft('First'); + online = true; + await flushAll(); + await engine.requestSync(); + online = false; + const email = await client.getEmail(id); + expect(email?.bodyValues?.['1']?.value).toBe('First'); + }); + + it('replaces the real message on the next save, offline, after the new one is up', async () => { + online = false; + const id = await saveDraft('First'); + online = true; + await flushAll(); + const real = server.drafts()[0].id; + online = false; + await saveDraft('Second', id); + expect(server.emails.has(real)).toBe(true); + online = true; + await flushAll(); + expect(server.emails.has(real)).toBe(false); + expect((server.drafts()[0].bodyValues as Record)['1'].value).toBe('Second'); + }); +}); + +describe('before the flush, with the server back', () => { + it('answers a read of the stand-in from the device, not the server', async () => { + online = false; + const id = await saveDraft('Text'); + online = true; + wire.length = 0; + expect((await client.getEmail(id))?.bodyValues?.['1']?.value).toBe('Text'); + expect(wire.filter(([name]) => name === 'Email/get')).toEqual([]); + }); + + it('replaces the stand-in with a save that goes straight to the server', async () => { + online = false; + const id = await saveDraft('Offline'); + online = true; + const real = await saveDraft('Online', id); + expect(real).not.toMatch(/^local-/); + await flushAll(); + expect(server.drafts().map((email) => email.id)).toEqual([real]); + expect(await engine.areaCall('drafts.list', {})).toEqual([]); + }); + + it('uploads files attached offline before the save goes to the server', async () => { + online = false; + const { blobId } = await client.uploadBlob(new File(['hello'], 'note.txt', { type: 'text/plain' })); + expect(blobId).toMatch(/^bulwark-local-/); + const id = await saveDraft('With a file', undefined, [{ blobId, name: 'note.txt', type: 'text/plain', size: 5 }]); + online = true; + await saveDraft('With a file, later', id, [{ blobId, name: 'note.txt', type: 'text/plain', size: 5 }]); + const [draft] = server.drafts(); + expect((draft.attachments as Array<{ blobId: string }>)[0].blobId).toBe(server.uploads[0].blobId); + }); +}); + +describe('a file attached offline', () => { + it('stays readable and downloadable, offline and once the server is back', async () => { + online = false; + const { blobId } = await client.uploadBlob(new File(['hello'], 'note.txt', { type: 'text/plain' })); + expect(await (await client.fetchBlob(blobId, 'note.txt', 'text/plain')).text()).toBe('hello'); + online = true; + expect(await (await client.fetchBlob(blobId, 'note.txt', 'text/plain')).text()).toBe('hello'); + expect(spy.mock.calls.filter(([url]: [unknown]) => String(url).includes('/d/'))).toEqual([]); + }); + + it('isn\'t held a second time when its own upload hits an outage', async () => { + const holdUpload = vi.fn(); + const hooks = { skipNetwork: () => false, answer: async () => null, record: () => {}, noteNetwork: () => {}, serverGone: async () => true, blob: async () => null, recordBlob: () => {}, holdUpload }; + setOfflineHooks(hooks as never); + online = false; + await expect(client.uploadBlob(new File(['x'], 'x.txt'), { noHold: true })).rejects.toThrow(); + expect(holdUpload).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/store/__tests__/drafts.test.ts b/lib/store/__tests__/drafts.test.ts new file mode 100644 index 000000000..31fa738a8 --- /dev/null +++ b/lib/store/__tests__/drafts.test.ts @@ -0,0 +1,599 @@ +// @vitest-environment node +import { beforeEach, describe, expect, it } from 'vitest'; +import { DEFAULT_SCOPE } from '../area'; +import type { DraftError } from '../drafts/pending'; +import type { LocalDraftSummary, PendingDraft, DraftGhost } from '../drafts/calls'; +import { flushDrafts, type DraftFlushIo } from '../drafts/flush'; +import { AccountEngine } from '../engine/account-engine'; +import { TransportError, type MethodCall } from '../engine/transport'; +import { mailArea } from '../mail'; +import { outboxArea } from '../outbox'; +import { flushPatches, type PatchOutcome, type PendingPatch } from '../outbox/flush-patches'; +import type { StoredConflict } from '../outbox/items'; +import { FakeJmapDrafts } from './helpers/fake-drafts'; +import { FAKE_SESSION, JMAP_ACCOUNT } from './helpers/fake-jmap'; +import { openMemoryStore } from './helpers/node-adapter'; + +const NOW = Date.UTC(2026, 10, 3); +const BOXES = [ + { id: 'inbox', name: 'Inbox', role: 'inbox', totalEmails: 0, unreadEmails: 0 }, + { id: 'drafts', name: 'Drafts', role: 'drafts', totalEmails: 0, unreadEmails: 0 }, + { id: 'trash', name: 'Trash', role: 'trash', totalEmails: 0, unreadEmails: 0 }, +]; + +function setup() { + const server = new FakeJmapDrafts({ mailboxes: BOXES.map((box) => ({ ...box })) }); + server.now = NOW; + const db = openMemoryStore(); + const engine = new AccountEngine({ + account: 'a@fake.test', + db, + session: FAKE_SESSION, + transport: server, + areas: [mailArea, outboxArea], + scope: { ...DEFAULT_SCOPE, days: null, attachments: false }, + now: () => NOW, + sleep: async () => {}, + }); + return { server, db, engine }; +} + +let t: ReturnType; + +beforeEach(async () => { + t = setup(); + await t.engine.requestSync(); +}); + +interface SaveOptions { + subject?: string; + text?: string; + previous?: string; + attachments?: Array>; + networkTried?: boolean; + messageId?: string; +} + +let nextMessage = 0; + +/** What createDraft sends: a new version, then the old one goes. */ +async function save(options: SaveOptions = {}): Promise { + nextMessage += 1; + const answer = await t.engine.answer([['Email/set', { + accountId: JMAP_ACCOUNT, + ...(options.networkTried ? { networkTried: true } : {}), + create: { + c1: { + messageId: [options.messageId ?? `draft-${nextMessage}@example.org`], + from: [{ email: 'a@example.org' }], + to: [{ email: 'b@example.org' }], + subject: options.subject ?? 'A draft', + keywords: { $seen: true, $draft: true }, + mailboxIds: { drafts: true }, + bodyValues: { 1: { value: options.text ?? 'Text' } }, + textBody: [{ partId: '1' }], + ...(options.attachments ? { attachments: options.attachments } : {}), + }, + }, + }, 's']]); + const result = answer?.methodResponses[0][1]; + const id = result.created?.c1?.id as string; + if (!id) throw new Error(`not created: ${JSON.stringify(result.notCreated)}`); + if (options.previous) { + await t.engine.answer([['Email/set', { accountId: JMAP_ACCOUNT, destroy: [options.previous], replacedBy: id }, 'd']]); + } + return id; +} + +async function get(id: string, properties?: string[]) { + const answer = await t.engine.answer([['Email/get', { accountId: JMAP_ACCOUNT, ids: [id], fetchAllBodyValues: true, ...(properties ? { properties } : {}) }, 'g']]); + return answer?.methodResponses[0][1] as { list: Array>; notFound: string[] }; +} + +async function draftBox(): Promise { + const answer = await t.engine.answer([['Email/query', { accountId: JMAP_ACCOUNT, filter: { inMailbox: 'drafts' } }, 'q']]); + return answer?.methodResponses[0][1].ids; +} + +const list = () => t.engine.areaCall('drafts.list', {}) as Promise; +const conflicts = async () => (await t.engine.areaCall('outbox.read', {}) as { conflicts: StoredConflict[] }).conflicts; + +interface Run { + refused: Array<[string, DraftError]>; + conflicted: string[]; + sent: number; +} + +/** The flush as the page runs it, with the file upload on the fake server. */ +function ioFor(run: Run): DraftFlushIo { + const { engine, server } = t; + return { + store: { + read: () => engine.areaCall('drafts.read', { before: NOW - 60_000 }) as Promise<{ rows: PendingDraft[]; ghosts: DraftGhost[] }>, + claim: (localId) => engine.areaCall('drafts.claim', { localId }) as Promise, + failed: async (localId, status, error, neverSent) => { await engine.areaCall('drafts.failed', { localId, status, error, neverSent }); }, + created: (result) => engine.areaCall('drafts.created', result) as Promise<{ conflicts: string[] }>, + ghostsLooked: async (jmap, messageIds, found) => { await engine.areaCall('drafts.ghostsLooked', { jmap, messageIds, found }); }, + }, + request: (calls: MethodCall[]) => server.request(calls), + upload: async (jmap, blobId) => { + const held = await engine.areaCall('outbox.heldFile', { blobId }) as { blob: Blob } | null; + if (!held) return null; + return (await server.upload(jmap, new Uint8Array(await held.blob.arrayBuffer()))).blobId; + }, + isRefusal: (error) => error instanceof TransportError && error.kind === 'http', + onRefused: (subject, error) => run.refused.push([subject, error]), + onConflict: (subject) => run.conflicted.push(subject), + onSent: () => { run.sent += 1; }, + }; +} + +async function flush(): Promise { + const run: Run = { refused: [], conflicted: [], sent: 0 }; + const finished = await flushDrafts(ioFor(run)); + return { ...run, finished }; +} + +/** And what the flush left in the usual queue: the old versions go here. */ +async function flushQueue(): Promise { + const pending = await t.engine.areaCall('outbox.pendingPatches', {}) as PendingPatch[]; + return flushPatches({ request: (calls) => t.server.request(calls), isRefusal: () => false }, pending, async (outcomes: PatchOutcome[]) => { + await t.engine.areaCall('outbox.patchResults', { outcomes }); + }); +} + +function serverDraft(subject: string, id = 'R') { + t.server.addEmail({ id, subject, mailboxIds: { drafts: true }, keywords: { $seen: true, $draft: true } }); +} + +async function hold(text: string, name = 'note.txt'): Promise { + const held = await t.engine.areaCall('outbox.holdFile', { name, type: 'text/plain', data: new TextEncoder().encode(text) }) as { blobId: string }; + return held.blobId; +} + +describe('a draft saved offline', () => { + it('gets a stand-in id and answers as a server would, without going to the server', async () => { + const requests = t.server.requests; + const id = await save({ subject: 'Offline', text: 'Written on a train' }); + expect(id).toMatch(/^local-/); + expect(t.server.requests).toBe(requests); + expect(await draftBox()).toEqual([id]); + + const [email] = (await get(id)).list; + expect(email.subject).toBe('Offline'); + expect(email.keywords).toEqual({ $seen: true, $draft: true }); + expect(email.preview).toBe('Written on a train'); + expect((email.bodyValues as Record)['1'].value).toBe('Written on a train'); + expect(email.textBody).toEqual([{ partId: '1', type: 'text/plain', size: 18 }]); + }); + + it('adds itself to the folder count', async () => { + await save(); + const answer = await t.engine.answer([['Mailbox/get', {}, 'm']]); + const drafts = (answer?.methodResponses[0][1].list as Array<{ id: string; totalEmails: number }>).find((box) => box.id === 'drafts'); + expect(drafts?.totalEmails).toBe(1); + }); + + it('can be found by its words', async () => { + const id = await save({ text: 'quarterly figures' }); + const answer = await t.engine.answer([['Email/query', { filter: { text: 'quarterly' } }, 'q']]); + expect(answer?.methodResponses[0][1].ids).toEqual([id]); + }); + + it('reads headers back the way the composer does', async () => { + const answer = await t.engine.answer([['Email/set', { + create: { c: { keywords: { $draft: true }, mailboxIds: { drafts: true }, subject: 's', 'header:X-Priority:asText': '1 (Highest)', bodyValues: { 1: { value: 'x' } }, textBody: [{ partId: '1' }] } }, + }, 's']]); + const id = answer?.methodResponses[0][1].created.c.id as string; + const [email] = (await get(id, ['header:X-Priority:asText'])).list; + expect(email['header:X-Priority:asText']).toBe('1 (Highest)'); + }); + + it('survives a full resync, which asks the server about what it didn\'t list', async () => { + const id = await save(); + t.server.addEmail({ id: 'new-mail', subject: 'Trigger' }); + t.server.truncateLog('Email'); + t.server.calls.length = 0; + await t.engine.requestSync(); + // It did list everything again, and asked about what it didn't see. + expect(t.server.calls.some(([name]) => name === 'Email/query')).toBe(true); + expect((await get(id)).list).toHaveLength(1); + }); + + it('takes a flag and a move, which go up with it', async () => { + const id = await save(); + await t.engine.answer([['Email/set', { update: { [id]: { 'keywords/$flagged': true } } }, 'u']]); + expect((await get(id, ['keywords'])).list[0].keywords).toEqual({ $seen: true, $draft: true, $flagged: true }); + await flush(); + expect(t.server.drafts()[0].keywords).toEqual({ $seen: true, $draft: true, $flagged: true }); + }); + + it('is refused as a draft when the same request sends it, or when it isn\'t in Drafts', async () => { + const sendLike = await t.engine.answer([ + ['Email/set', { create: { c: { keywords: { $draft: true }, mailboxIds: { drafts: true } } } }, '0'], + ['EmailSubmission/set', { create: { s: { emailId: '#c' } } }, '1'], + ]); + expect(sendLike?.methodResponses[0][1].notCreated.c.type).toBe('serverUnavailable'); + const elsewhere = await t.engine.answer([['Email/set', { create: { c: { keywords: { $draft: true }, mailboxIds: { inbox: true } } } }, '0']]); + expect(elsewhere?.methodResponses[0][1].notCreated.c.type).toBe('serverUnavailable'); + const noKeyword = await t.engine.answer([['Email/set', { create: { c: { keywords: {}, mailboxIds: { drafts: true } } } }, '0']]); + expect(noKeyword?.methodResponses[0][1].notCreated.c.type).toBe('serverUnavailable'); + expect(await list()).toEqual([]); + }); +}); + +describe('saving again', () => { + it('leaves one pending draft after a hundred saves, and one on the server with the latest text', async () => { + let id = await save({ text: 'Save 0' }); + for (let n = 1; n < 100; n += 1) id = await save({ text: `Save ${n}`, previous: id }); + expect(await list()).toHaveLength(1); + expect(await draftBox()).toEqual([id]); + + const { finished, sent } = await flush(); + expect(finished).toBe(true); + expect(sent).toBe(1); + expect(t.server.drafts()).toHaveLength(1); + expect((t.server.drafts()[0].bodyValues as Record)['1'].value).toBe('Save 99'); + expect(await list()).toEqual([]); + }); + + it('removes the older stand-in from everything that reads', async () => { + const first = await save(); + const second = await save({ previous: first }); + expect((await get(first)).notFound).toEqual([first]); + expect((await get(second)).list).toHaveLength(1); + }); +}); + +describe('a stand-in in a destroy', () => { + it('discards the draft, and nothing reaches the server', async () => { + const id = await save(); + const answer = await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]); + expect(answer?.methodResponses[0][1].destroyed).toEqual([id]); + expect((await get(id)).notFound).toEqual([id]); + await flush(); + expect(t.server.drafts()).toEqual([]); + expect(await t.engine.areaCall('outbox.pendingPatches', {})).toEqual([]); + }); + + it('after the flush goes to the real message, by the map', async () => { + const id = await save(); + await flush(); + const real = t.server.drafts()[0].id; + expect(await t.engine.areaCall('drafts.resolveStandIns', { ids: [id, 'local-unknown'] })).toEqual({ [id]: real, 'local-unknown': null }); + await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]); + await flushQueue(); + expect(t.server.emails.has(real)).toBe(false); + }); +}); + +describe('a draft from the server edited offline', () => { + beforeEach(async () => { + serverDraft('On the server'); + await t.engine.requestSync(); + }); + + it('hides the old version here and leaves it on the server until the new one has arrived', async () => { + const id = await save({ subject: 'Edited', previous: 'R' }); + expect(await draftBox()).toEqual([id]); + expect(t.server.emails.has('R')).toBe(true); + expect(await t.engine.areaCall('outbox.pendingPatches', {})).toEqual([]); + // A sync doesn't bring it back. + t.server.updateEmail('R', { keywords: { $seen: true, $draft: true } }); + t.server.coll.Email.push('updated', 'R'); + await t.engine.requestSync(); + expect(await draftBox()).toEqual([id]); + }); + + it('creates the new version first and destroys the old one after', async () => { + await save({ subject: 'Edited', previous: 'R' }); + await flush(); + expect(t.server.emails.has('R')).toBe(true); + await flushQueue(); + expect(t.server.log.filter((line) => /^(create|destroy)/.test(line))).toEqual(['create srv1', 'destroy R']); + expect(t.server.drafts().map((email) => email.subject)).toEqual(['Edited']); + }); + + it('keeps both when the destroy fails: nothing is lost, and it goes again', async () => { + await save({ subject: 'Edited', previous: 'R' }); + await flush(); + t.server.offline = true; + expect(await flushQueue()).toBe(false); + t.server.offline = false; + expect(t.server.drafts()).toHaveLength(2); + await flushQueue(); + expect(t.server.drafts().map((email) => email.subject)).toEqual(['Edited']); + }); + + it('keeps the old version when the create fails', async () => { + t.server.refuseCreate = { type: 'overQuota' }; + await save({ subject: 'Edited', previous: 'R' }); + await flush(); + await flushQueue(); + expect(t.server.emails.has('R')).toBe(true); + }); + + it('carries what an earlier save replaced through later saves', async () => { + const first = await save({ text: 'One', previous: 'R' }); + await save({ text: 'Two', previous: first }); + await flush(); + await flushQueue(); + expect(t.server.drafts()).toHaveLength(1); + expect((t.server.drafts()[0].bodyValues as Record)['1'].value).toBe('Two'); + }); + + it('removes the old version too when the new one is discarded before it went up', async () => { + const id = await save({ previous: 'R' }); + await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]); + await flushQueue(); + expect(t.server.drafts()).toEqual([]); + }); + + it('keeps both and says so when another device changed it meanwhile', async () => { + await save({ subject: 'Mine', previous: 'R' }); + t.server.updateEmail('R', { keywords: { $seen: true, $draft: true, $flagged: true } }); + const { conflicted } = await flush(); + await flushQueue(); + expect(conflicted).toEqual(['On the server']); + expect(t.server.drafts().map((email) => email.subject).sort()).toEqual(['Mine', 'On the server']); + expect((await conflicts())[0].summary).toMatchObject({ op: 'draft', reason: 'changed', subject: 'On the server' }); + // The other version is back in view: nothing else would bring it. + expect((await draftBox()).sort()).toEqual(['R', 'srv1'].sort()); + }); + + it('keeps ours and says so when another device replaced or removed it', async () => { + await save({ subject: 'Mine', previous: 'R' }); + t.server.destroyEmail('R'); + t.server.addEmail({ id: 'R2', subject: 'Theirs', mailboxIds: { drafts: true }, keywords: { $seen: true, $draft: true } }); + const { conflicted } = await flush(); + await flushQueue(); + expect(conflicted).toEqual(['On the server']); + expect(t.server.drafts().map((email) => email.subject).sort()).toEqual(['Mine', 'Theirs']); + expect((await conflicts())[0].summary).toMatchObject({ op: 'draft', reason: 'gone' }); + }); +}); + +describe('a lost response', () => { + it('doesn\'t make a second draft: the copy is found by its Message-ID', async () => { + const id = await save({ messageId: 'fixed@example.org' }); + t.server.loseAnswerOf = 'Email/set'; + const first = await flush(); + expect(first.finished).toBe(false); + expect(t.server.drafts()).toHaveLength(1); + expect((await list())[0].status).toBe('uncertain'); + // It stays readable under its stand-in meanwhile. + expect((await get(id)).list).toHaveLength(1); + + const second = await flush(); + expect(second.finished).toBe(true); + expect(t.server.drafts()).toHaveLength(1); + expect(await list()).toEqual([]); + expect(await t.engine.areaCall('drafts.resolveStandIns', { ids: [id] })).toEqual({ [id]: t.server.drafts()[0].id }); + }); + + it('creates it when the lookup finds that it never arrived', async () => { + await save(); + t.server.offline = true; + await flush(); + t.server.offline = false; + await flush(); + expect(t.server.drafts()).toHaveLength(1); + }); + + it('removes the copy of a version replaced while the answer was missing', async () => { + const first = await save({ text: 'Old', messageId: 'old@example.org' }); + t.server.loseAnswerOf = 'Email/set'; + await flush(); + await save({ text: 'New', previous: first }); + await flush(); + await flushQueue(); + expect(t.server.drafts()).toHaveLength(1); + expect((t.server.drafts()[0].bodyValues as Record)['1'].value).toBe('New'); + expect(await t.engine.areaCall('drafts.read', { before: 0 })).toMatchObject({ ghosts: [] }); + }); + + it('looks a create up that came to the store after its own request broke', async () => { + t.server.addEmail({ id: 'landed', subject: 'A draft', mailboxIds: { drafts: true }, keywords: { $draft: true }, messageId: ['tried@example.org'], receivedAt: new Date(NOW).toISOString() }); + const id = await save({ messageId: 'tried@example.org', networkTried: true }); + expect((await list())[0].status).toBe('uncertain'); + await flush(); + expect(t.server.drafts().map((email) => email.id)).toEqual(['landed']); + expect(await t.engine.areaCall('drafts.resolveStandIns', { ids: [id] })).toEqual({ [id]: 'landed' }); + }); + + it('counts a tab closed on the way as uncertain', async () => { + const id = await save({ messageId: 'closed@example.org' }); + expect(await t.engine.areaCall('drafts.claim', { localId: id })).toBe(true); + await t.engine.areaCall('drafts.read', { before: NOW + 1 }); + expect((await list())[0].status).toBe('uncertain'); + }); + + it('removes a copy that arrives after the draft was replaced on the way', async () => { + const id = await save({ text: 'Old' }); + await t.engine.areaCall('drafts.claim', { localId: id }); + const next = await save({ text: 'New', previous: id }); + await t.engine.areaCall('drafts.created', { localId: id, seq: 1, jmap: JMAP_ACCOUNT, messageId: 'x', serverId: 'late', email: null, server: {} }); + const pending = await t.engine.areaCall('outbox.pendingPatches', {}) as PendingPatch[]; + expect(pending.map((row) => [row.recordId, row.op])).toEqual([['late', 'destroy']]); + expect((await get(next)).list).toHaveLength(1); + }); +}); + +describe('a refused draft', () => { + it('stays on the device with its error, and the composer can still open it', async () => { + t.server.refuseCreate = { type: 'overQuota', description: 'No room left' }; + const id = await save({ subject: 'Too much', text: 'Keep me' }); + const { refused, finished } = await flush(); + expect(finished).toBe(true); + expect(refused).toEqual([['Too much', { type: 'overQuota', text: 'No room left' }]]); + expect(await list()).toMatchObject([{ localId: id, status: 'refused', error: { type: 'overQuota', text: 'No room left' } }]); + expect((await get(id)).list[0].bodyValues).toMatchObject({ 1: { value: 'Keep me' } }); + expect(await draftBox()).toEqual([id]); + }); + + it('isn\'t asked about again by itself, and goes up when the user tries again', async () => { + t.server.refuseCreate = { type: 'overQuota' }; + const id = await save(); + await flush(); + t.server.refuseCreate = null; + await flush(); + expect(t.server.drafts()).toEqual([]); + await t.engine.areaCall('drafts.retry', { localId: id }); + await flush(); + expect(t.server.drafts()).toHaveLength(1); + expect(await list()).toEqual([]); + }); + + it('goes up again once it\'s edited', async () => { + t.server.refuseCreate = { type: 'overQuota' }; + const id = await save({ text: 'One' }); + await flush(); + t.server.refuseCreate = null; + await save({ text: 'Two', previous: id }); + await flush(); + expect((t.server.drafts()[0].bodyValues as Record)['1'].value).toBe('Two'); + }); + + it('says so when the request itself is refused for good', async () => { + const id = await save(); + const original = t.server.request.bind(t.server); + t.server.request = async () => { throw new TransportError('http', 'Too large', 413); }; + const { refused } = await flush(); + t.server.request = original; + expect(refused).toHaveLength(1); + expect((await list())[0]).toMatchObject({ localId: id, status: 'refused' }); + }); +}); + +describe('files attached offline', () => { + it('are shown on the draft with their size, stay readable, and go up on the flush', async () => { + const blobId = await hold('hello'); + const id = await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt', disposition: 'attachment' }] }); + const [email] = (await get(id)).list; + expect(email.hasAttachment).toBe(true); + expect(email.attachments).toMatchObject([{ blobId, name: 'note.txt', size: 5, type: 'text/plain', disposition: 'attachment' }]); + const file = await t.engine.areaCall('outbox.heldFile', { blobId }) as { name: string; blob: Blob }; + expect(await file.blob.text()).toBe('hello'); + + await flush(); + const [draft] = t.server.drafts(); + expect(t.server.uploads).toHaveLength(1); + expect((draft.attachments as Array<{ blobId: string; size: number }>)[0]).toMatchObject({ blobId: t.server.uploads[0].blobId, size: 5 }); + }); + + it('go up before the create that names them', async () => { + const blobId = await hold('hello'); + await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt' }] }); + await flush(); + expect(t.server.log.slice(0, 2)).toEqual(['upload up1', 'create srv1']); + }); + + it('come in pieces, and a broken one leaves nothing readable', async () => { + const started = await t.engine.areaCall('outbox.holdStart', { name: 'big.bin', type: '', size: 6 }) as { blobId: string }; + await t.engine.areaCall('outbox.holdChunk', { blobId: started.blobId, seq: 0, data: new Uint8Array([1, 2, 3]) }); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: started.blobId })).toBeNull(); + await t.engine.areaCall('outbox.holdChunk', { blobId: started.blobId, seq: 1, data: new Uint8Array([4, 5, 6]) }); + await t.engine.areaCall('outbox.holdEnd', { blobId: started.blobId }); + const file = await t.engine.areaCall('outbox.heldFile', { blobId: started.blobId }) as { blob: Blob }; + expect([...new Uint8Array(await file.blob.arrayBuffer())]).toEqual([1, 2, 3, 4, 5, 6]); + await t.engine.areaCall('outbox.holdAbort', { blobId: started.blobId }); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: started.blobId })).toBeNull(); + expect(t.db.get('SELECT 1 AS x FROM upload_chunk')).toBeUndefined(); + }); + + it('wait when the upload hits an outage, without counting as maybe sent', async () => { + const blobId = await hold('hello'); + await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt' }] }); + t.server.failUpload = true; + const { finished } = await flush(); + expect(finished).toBe(false); + expect(t.server.drafts()).toEqual([]); + expect((await list())[0].status).toBe('waiting'); + }); + + it('refuse the draft when a file is gone from the device', async () => { + const blobId = await hold('hello'); + const id = await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt' }] }); + await t.engine.areaCall('outbox.holdAbort', { blobId }); + const { refused } = await flush(); + expect(refused).toEqual([['A draft', { code: 'file_missing' }]]); + expect((await list())[0]).toMatchObject({ localId: id, status: 'refused', error: { code: 'file_missing' } }); + }); + + it('count as needed for as long as a draft or a send names them', async () => { + const forDraft = await hold('a'); + const forNothing = await hold('b'); + const id = await save({ attachments: [{ blobId: forDraft, type: 'text/plain', name: 'a' }] }); + expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW + 1 })).toBe(1); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: forNothing })).toBeNull(); + expect(await t.engine.areaCall('outbox.heldFile', { blobId: forDraft })).not.toBeNull(); + await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]); + expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW + 1 })).toBe(1); + }); + + it('survive removing the send that shared them with a draft', async () => { + const blobId = await hold('a'); + await save({ attachments: [{ blobId, type: 'text/plain', name: 'a' }] }); + const entry = await t.engine.areaCall('outbox.queueSend', { + summary: { subject: 's', to: [] }, + item: { kind: 'email', args: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, [{ blobId }]] }, + }) as number; + await t.engine.areaCall('outbox.remove', { id: entry }); + expect(await t.engine.areaCall('outbox.heldFile', { blobId })).not.toBeNull(); + }); + + it('are found inside a body structure too, as a draft with inline images writes them', async () => { + const blobId = await hold('png'); + const answer = await t.engine.answer([['Email/set', { + create: { + c: { + keywords: { $draft: true }, + mailboxIds: { drafts: true }, + bodyValues: { text: { value: 'text' }, html: { value: '' } }, + bodyStructure: { + type: 'multipart/alternative', + subParts: [ + { partId: 'text', type: 'text/plain' }, + { type: 'multipart/related', subParts: [{ partId: 'html', type: 'text/html' }, { blobId, type: 'image/png', name: 'i.png', disposition: 'inline', cid: 'i' }] }, + ], + }, + }, + }, + }, 's']]); + const id = answer?.methodResponses[0][1].created.c.id as string; + const [email] = (await get(id)).list; + expect(email.hasAttachment).toBe(false); + expect(email.attachments).toMatchObject([{ blobId, cid: 'i', disposition: 'inline', size: 3 }]); + expect(email.htmlBody).toMatchObject([{ partId: 'html', type: 'text/html' }]); + await flush(); + const leaf = (t.server.calls.find(([name, args]) => name === 'Email/set' && args.create)?.[1].create as Record }> } }>).draft.bodyStructure.subParts[1].subParts[1]; + expect(leaf.blobId).toBe(t.server.uploads[0].blobId); + }); +}); + +describe('sending a draft that\'s on the device', () => { + it('leaves the draft alone while the send waits, and the send takes care of it', async () => { + const id = await save({ previous: undefined }); + await t.engine.areaCall('outbox.queueSend', { + summary: { subject: 's', to: [] }, + item: { kind: 'email', args: [['b@example.org'], 's', 'x', undefined, undefined, undefined, undefined, id] }, + }); + await flush(); + expect(t.server.drafts()).toEqual([]); + expect(await list()).toHaveLength(1); + + await t.engine.areaCall('drafts.discard', { jmap: JMAP_ACCOUNT, id }); + expect(await list()).toEqual([]); + }); + + it('goes up as a draft if the send fails, so nothing depends on the outbox', async () => { + const id = await save(); + const entry = await t.engine.areaCall('outbox.queueSend', { + summary: { subject: 's', to: [] }, + item: { kind: 'email', args: [['b@example.org'], 's', 'x', undefined, undefined, undefined, undefined, id] }, + }) as number; + await t.engine.areaCall('outbox.update', { id: entry, status: 'failed', error: { text: 'refused' } }); + await flush(); + expect(t.server.drafts()).toHaveLength(1); + }); +}); diff --git a/lib/store/__tests__/helpers/fake-drafts.ts b/lib/store/__tests__/helpers/fake-drafts.ts new file mode 100644 index 000000000..b8364a56e --- /dev/null +++ b/lib/store/__tests__/helpers/fake-drafts.ts @@ -0,0 +1,120 @@ +/** + * FakeJmapMail with what drafts talk to: Email/set create, uploads and blob + * downloads. Follows Stalwart: a part names a blob it knows or the create is + * refused with `blobNotFound`, two creates with one Message-ID are both accepted, + * and each create makes a new message. + */ + +import { TransportError, type JmapResponse, type MethodCall } from '../../engine/transport'; +import { type FakeEmail } from './fake-jmap'; +import { FakeJmapMail } from './fake-jmap-mail'; + +interface Part { + partId?: string; + blobId?: string; + type?: string; + name?: string; + disposition?: string; + cid?: string; + subParts?: Part[]; +} + +export class FakeJmapDrafts extends FakeJmapMail { + /** What was uploaded or attached, by blob id. */ + readonly blobs = new Map(); + readonly uploads: Array<{ blobId: string; size: number }> = []; + /** The order things happened in, for tests about order. */ + readonly log: string[] = []; + /** Every create is refused with this. */ + refuseCreate: { type: string; description?: string } | null = null; + /** The server does the work of the next request naming this method, and the answer never arrives. */ + loseAnswerOf: string | null = null; + /** The next upload fails like a dropped connection. */ + failUpload = false; + now = Date.UTC(2026, 10, 3); + private nextCreated = 1; + private nextBlob = 1; + + drafts(): FakeEmail[] { + return [...this.emails.values()].filter((email) => (email.mailboxIds as Record).drafts); + } + + private knows(blobId: string): boolean { + if (this.blobs.has(blobId)) return true; + return [...this.emails.values()].some((email) => (email.attachments as Part[] | undefined)?.some((part) => part.blobId === blobId)); + } + + private partsOf(create: Record): Part[] { + const leaves = (part: Part): Part[] => (part.subParts ? part.subParts.flatMap(leaves) : [part]); + return [...((create.attachments ?? []) as Part[]), ...(create.bodyStructure ? leaves(create.bodyStructure as Part) : [])].filter((part) => part.blobId); + } + + private createOne(create: Record): { id: string; blobId: string; threadId: string; size: number } | { type: string; description?: string } { + if (this.refuseCreate) return this.refuseCreate; + const parts = this.partsOf(create); + if (parts.some((part) => !this.knows(part.blobId as string))) return { type: 'blobNotFound' }; + const id = `srv${this.nextCreated++}`; + const values = (create.bodyValues ?? {}) as Record; + const size = Object.values(values).reduce((sum, value) => sum + value.value.length, 0); + this.addEmail({ + id, + threadId: `t-${id}`, + mailboxIds: create.mailboxIds as Record, + keywords: create.keywords as Record, + subject: String(create.subject ?? ''), + from: create.from as never, + to: create.to as never, + messageId: create.messageId as never, + receivedAt: new Date(this.now).toISOString(), + size, + blobId: `raw-${id}`, + preview: '', + hasAttachment: parts.length > 0, + textBody: (create.textBody ?? []) as never, + htmlBody: (create.htmlBody ?? create.textBody ?? []) as never, + bodyValues: Object.fromEntries(Object.entries(values).map(([key, value]) => [key, { value: value.value, isTruncated: false, isEncodingProblem: false }])) as never, + attachments: parts.map((part) => ({ ...part, size: this.blobs.get(part.blobId as string)?.length ?? 0 })) as never, + }); + this.log.push(`create ${id}`); + return { id, blobId: `raw-${id}`, threadId: `t-${id}`, size }; + } + + protected override handle(name: string, args: Record): Record { + if (name !== 'Email/set') return super.handle(name, args); + const created: Record = {}; + const notCreated: Record = {}; + for (const [creationId, create] of Object.entries((args.create ?? {}) as Record>)) { + const answer = this.createOne(create); + if ('id' in answer) created[creationId] = answer; + else notCreated[creationId] = answer; + } + for (const id of (args.destroy ?? []) as string[]) this.log.push(`destroy ${id}`); + return { + ...super.handle(name, args), + ...(args.create ? { created } : {}), + ...(Object.keys(notCreated).length > 0 ? { notCreated } : {}), + }; + } + + override async request(calls: MethodCall[]): Promise { + const answer = await super.request(calls); + if (this.loseAnswerOf && calls.some(([name]) => name === this.loseAnswerOf)) { + this.loseAnswerOf = null; + throw new TransportError('network', 'The connection broke'); + } + return answer; + } + + override async upload(_jmap?: string, data?: Uint8Array): Promise<{ blobId: string; type: string; size: number }> { + if (this.offline || this.failUpload) { + this.failUpload = false; + throw new TransportError('network', 'Failed to fetch'); + } + const blobId = `up${this.nextBlob++}`; + const bytes = data ?? new Uint8Array(); + this.blobs.set(blobId, bytes); + this.uploads.push({ blobId, size: bytes.length }); + this.log.push(`upload ${blobId}`); + return { blobId, type: 'application/octet-stream', size: bytes.length }; + } +} diff --git a/lib/store/__tests__/helpers/in-process-store.ts b/lib/store/__tests__/helpers/in-process-store.ts index 81699f3a3..337bcb734 100644 --- a/lib/store/__tests__/helpers/in-process-store.ts +++ b/lib/store/__tests__/helpers/in-process-store.ts @@ -19,6 +19,19 @@ const NO_SERVER: JmapTransport = { upload: async () => ({ blobId: 'x', type: '', size: 0 }), }; +/** + * What crosses to the worker and back is a copy, except a Blob: that goes by + * reference there, and jsdom's Blob can't be cloned here. + */ +function acrossWorker(value: T): T { + if (value instanceof Blob) return value; + if (Array.isArray(value)) return value.map(acrossWorker) as T; + if (value && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, acrossWorker(item)])) as T; + } + return structuredClone(value); +} + export class InProcessStore implements StoreBackend { readonly engines = new Map(); readonly dbs = new Map(); @@ -71,8 +84,8 @@ export class InProcessStore implements StoreBackend { async area(account: string, name: string, params?: unknown): Promise { if (this.failArea?.(name)) throw new Error(`${name} failed`); - const result = await this.engine(account).areaCall(name, structuredClone(params)); - return (result === undefined ? undefined : structuredClone(result)) as T; + const result = await this.engine(account).areaCall(name, acrossWorker(params)); + return (result === undefined ? undefined : acrossWorker(result)) as T; } async sync(account: string): Promise { diff --git a/lib/store/__tests__/mail-answers.test.ts b/lib/store/__tests__/mail-answers.test.ts index d27aaca1a..924d690b3 100644 --- a/lib/store/__tests__/mail-answers.test.ts +++ b/lib/store/__tests__/mail-answers.test.ts @@ -361,7 +361,9 @@ describe('attachments', () => { class Files extends FakeJmapMail { downloads: string[] = []; missing = new Set(); - override async download(_jmap?: string, blobId?: string): Promise { + override async download(_jmap?: string, blobId?: string, type?: string): Promise { + // Raw messages are kept too since exports read them; these tests are about attachments. + if (type === 'message/rfc822') return new Uint8Array([9]); this.downloads.push(String(blobId)); if (this.offline) throw new TransportError('network', 'Failed to fetch'); if (this.missing.has(String(blobId))) throw new TransportError('http', 'Gone', 404); @@ -381,9 +383,9 @@ describe('attachments', () => { const { engine, db } = engineFor(server, { ...EVERYTHING, maxAttachmentBytes: 1000 }); await engine.requestSync(); expect(server.downloads).toEqual(['b-small']); - const stored = await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-small' }) as { type: string; data: Uint8Array }; + const stored = await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-small' }) as { type: string; blob: Blob }; expect(stored.type).toBe('application/pdf'); - expect([...stored.data]).toEqual([1, 2, 3]); + expect([...new Uint8Array(await stored.blob.arrayBuffer())]).toEqual([1, 2, 3]); server.destroyEmail('small'); await engine.requestSync(); diff --git a/lib/store/__tests__/mail-sync.test.ts b/lib/store/__tests__/mail-sync.test.ts index 13d438c6c..066b6767e 100644 --- a/lib/store/__tests__/mail-sync.test.ts +++ b/lib/store/__tests__/mail-sync.test.ts @@ -13,7 +13,8 @@ import { mailArea } from '../mail'; import { FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './helpers/fake-jmap'; import { openFileStore, openMemoryStore } from './helpers/node-adapter'; -const EVERYTHING = { ...DEFAULT_SCOPE, days: null }; +// No files: raw messages would add a commit per message, and these tests are about mail. +const EVERYTHING = { ...DEFAULT_SCOPE, days: null, attachments: false }; const INBOX = { inMailbox: 'inbox' }; function engineFor(server: FakeJmap, db = openMemoryStore(), emit?: (e: StoreEvent) => void) { diff --git a/lib/store/__tests__/outbox.test.ts b/lib/store/__tests__/outbox.test.ts index 610974797..2af42fffc 100644 --- a/lib/store/__tests__/outbox.test.ts +++ b/lib/store/__tests__/outbox.test.ts @@ -420,9 +420,9 @@ describe('sends', () => { const held = await t.engine.areaCall('outbox.holdFile', { name: 'note.txt', type: 'text/plain', data }) as { blobId: string; size: number }; expect(held).toMatchObject({ size: 5, type: 'text/plain' }); expect(held.blobId).toMatch(/^bulwark-local-/); - const file = await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId }) as { name: string; data: Uint8Array }; + const file = await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId }) as { name: string; blob: Blob }; expect(file.name).toBe('note.txt'); - expect(new TextDecoder().decode(file.data)).toBe('hello'); + expect(await file.blob.text()).toBe('hello'); const id = await t.engine.areaCall('outbox.queueSend', { summary, item: { kind: 'raw', blobId: held.blobId, identityId: 'i' } }) as number; await t.engine.areaCall('outbox.remove', { id }); expect(await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId })).toBeNull(); diff --git a/lib/store/area.ts b/lib/store/area.ts index c435a4bce..fba893280 100644 --- a/lib/store/area.ts +++ b/lib/store/area.ts @@ -71,6 +71,10 @@ export interface SyncContext extends StoreContext { transport: JmapTransport; /** True once the engine is closing or lost the network: stop at the next batch. */ stopped(): boolean; + /** A newer pass was asked for. Work that can wait stops, and the next pass carries on with it. */ + syncWanted(): boolean; + /** Waits, to stay polite to the server. */ + pause(ms: number): Promise; syncState(jmap: string, type: string): SyncStateRow; /** Call inside the transaction that writes the data, so the two can't disagree. */ setSyncState(jmap: string, type: string, row: Partial): void; @@ -81,6 +85,8 @@ export interface Syncer { name: string; /** The JMAP capability an account must advertise for this to run for it. */ capability: string; + /** Runs after every other syncer of the pass, for work that can wait. */ + low?: boolean; /** Runs one pass for one JMAP account. Must be safe to stop between batches and start again. */ run(ctx: SyncContext, jmap: string): Promise; } @@ -102,6 +108,8 @@ export interface MethodContext extends StoreContext { invertedKeywordSort: boolean; /** Earlier responses in this request, for creation-id references. */ createdIds: Map; + /** The whole request this call came in. */ + calls: readonly MethodCall[]; } export type MethodHandler = (ctx: MethodContext, args: Record, call: MethodCall) => MethodResult | Promise; @@ -121,4 +129,6 @@ export interface StoreArea { syncers: readonly Syncer[]; calls?: Readonly>; overlay?: RecordOverlay; + /** At start and when the scope changes: a tighter policy applies at once, not at the next sync. */ + scopeChanged?(ctx: StoreContext): void; } diff --git a/lib/store/drafts/calls.ts b/lib/store/drafts/calls.ts new file mode 100644 index 000000000..3378c36ed --- /dev/null +++ b/lib/store/drafts/calls.ts @@ -0,0 +1,199 @@ +/** What the page's flush asks the store. Each call is one transaction, so two tabs can't interleave. */ + +import type { AreaCall, StoreContext } from '../area'; +import { upsertEmail, type EmailObject } from '../mail/records'; +import { savePatch, EMAIL_TYPE } from '../outbox/email-set'; +import type { ConflictSummary } from '../outbox/items'; +import { applyPatch, destroyConflicts, type RecordState } from '../outbox/patches'; +import { blobPartsOf, type DraftCreate } from './local-email'; +import { allPending, destroyForDraft, pendingForFlush, replaceLocalRow, replacesOf, resolveStandIns, type DraftError, type DraftRow, type DraftStatus } from './pending'; + +export interface PendingDraft { + localId: string; + jmap: string; + seq: number; + subject: string; + create: DraftCreate; + messageId: string; + replaces: ReturnType; + status: DraftStatus; + firstAttemptAt: number | null; +} + +export interface DraftGhost { + jmap: string; + messageId: string; + at: number; +} + +/** What the outbox view shows of a draft that's only on this device. */ +export interface LocalDraftSummary { + localId: string; + jmap: string; + subject: string; + status: DraftStatus; + error?: DraftError; + hasFiles: boolean; + createdAt: number; +} + +/** A stand-in the page may still hold is resolved this long after it reached the server. */ +const MAP_KEPT_MS = 30 * 24 * 60 * 60 * 1000; + +function toPending(row: DraftRow): PendingDraft { + return { + localId: row.local_id, + jmap: row.jmap, + seq: Number(row.seq), + subject: row.subject ?? '', + create: JSON.parse(row.create_json) as DraftCreate, + messageId: row.message_id, + replaces: replacesOf(row), + status: row.status, + firstAttemptAt: row.first_attempt_at === null ? null : Number(row.first_attempt_at), + }; +} + +function read(ctx: StoreContext, params: { before: number }): { rows: PendingDraft[]; ghosts: DraftGhost[] } { + return ctx.db.transaction(() => { + // What a closed tab left on the way may have arrived. + ctx.db.run("UPDATE draft_pending SET status = 'uncertain', sending_since = NULL WHERE status = 'sending' AND COALESCE(sending_since, 0) < ?", [params.before]); + ctx.db.run('DELETE FROM draft_id_map WHERE at < ?', [ctx.now() - MAP_KEPT_MS]); + return { + rows: pendingForFlush(ctx.db).map(toPending), + ghosts: ctx.db.all<{ jmap: string; message_id: string; at: number }>('SELECT jmap, message_id, at FROM draft_ghost').map((row) => ({ jmap: row.jmap, messageId: row.message_id, at: Number(row.at) })), + }; + }); +} + +function list(ctx: StoreContext): LocalDraftSummary[] { + return allPending(ctx.db).map((row) => ({ + localId: row.local_id, + jmap: row.jmap, + subject: row.subject ?? '', + status: row.status, + hasFiles: blobPartsOf(JSON.parse(row.create_json) as DraftCreate).length > 0, + createdAt: Number(row.created_at), + ...(row.error ? { error: JSON.parse(row.error) as DraftError } : {}), + })); +} + +/** False when it isn't there any more, or it's no longer waiting: replaced, discarded, another tab took it. */ +function claim(ctx: StoreContext, params: { localId: string }): boolean { + const now = ctx.now(); + return ctx.db.run( + `UPDATE draft_pending SET status = 'sending', sending_since = ?, first_attempt_at = COALESCE(first_attempt_at, ?) + WHERE local_id = ? AND status IN ('waiting', 'uncertain')`, + [now, now, params.localId], + ).changes === 1; +} + +/** Nothing arrived (waiting), it may have (uncertain), or the server said no (refused). */ +function failed(ctx: StoreContext, params: { localId: string; status: 'waiting' | 'uncertain' | 'refused'; error?: DraftError; neverSent?: boolean }): void { + ctx.db.run( + `UPDATE draft_pending SET status = ?, error = ?, sending_since = NULL, + first_attempt_at = CASE WHEN ? THEN NULL ELSE first_attempt_at END + WHERE local_id = ?`, + [params.status, params.error ? JSON.stringify(params.error) : null, params.neverSent ? 1 : 0, params.localId], + ); +} + +/** Try a refused draft again, once there may be room for it. */ +function retry(ctx: StoreContext, params: { localId: string }): void { + ctx.db.run("UPDATE draft_pending SET status = 'waiting', error = NULL WHERE local_id = ? AND status = 'refused'", [params.localId]); +} + +interface CreatedParams { + localId: string; + /** The row's place when it was taken: a later one means it was flagged or moved on the way. */ + seq: number; + jmap: string; + messageId: string; + serverId: string; + /** The new message as the server has it. Null when reading it back failed. */ + email: EmailObject | null; + /** What the server has of the versions this replaces, with null for one it no longer has. */ + server: Record; +} + +function addConflict(ctx: StoreContext, jmap: string, recordId: string, summary: ConflictSummary): void { + ctx.db.run( + 'INSERT INTO conflict (jmap, type, record_id, summary, local, created_at) VALUES (?, ?, ?, ?, ?, ?)', + [jmap, EMAIL_TYPE, recordId, JSON.stringify(summary), JSON.stringify({ op: 'draft' }), ctx.now()], + ); +} + +/** + * The new version is on the server. Only now do the versions it replaces start + * to go, by the usual queue: a failure there leaves an old copy, never no copy. + * One changed elsewhere stays, with the user told: both are kept. + */ +function created(ctx: StoreContext, params: CreatedParams): { conflicts: string[] } { + const conflicts: string[] = []; + ctx.db.transaction(() => { + const now = ctx.now(); + ctx.db.run('DELETE FROM draft_ghost WHERE jmap = ? AND message_id = ?', [params.jmap, params.messageId]); + const row = ctx.db.get('SELECT * FROM draft_pending WHERE local_id = ?', [params.localId]); + if (!row) { + // Replaced or discarded while it was on its way: this copy has no place. + savePatch(ctx.db, { jmap: params.jmap, record_id: params.serverId, op: 'destroy', patch: '{}', base: null, server: null, subject: null }, now); + return; + } + ctx.db.run('INSERT OR REPLACE INTO draft_id_map (local_id, jmap, server_id, at) VALUES (?, ?, ?, ?)', [row.local_id, row.jmap, params.serverId, now]); + replaceLocalRow(ctx.db, row, params.serverId, params.email); + if (Number(row.seq) !== params.seq) { + const after = JSON.parse(row.create_json) as RecordState; + const patch = JSON.stringify({ keywords: after.keywords, mailboxIds: after.mailboxIds }); + savePatch(ctx.db, { jmap: row.jmap, record_id: params.serverId, op: 'update', patch, base: null, server: null, subject: row.subject }, now); + const kept = ctx.db.get<{ json: string; gen: number }>('SELECT json, gen FROM email WHERE jmap = ? AND id = ?', [row.jmap, params.serverId]); + if (kept) upsertEmail(ctx.db, row.jmap, applyPatch(JSON.parse(kept.json) as EmailObject, JSON.parse(patch)), Number(kept.gen)); + } + ctx.db.run('DELETE FROM draft_pending WHERE local_id = ?', [row.local_id]); + for (const target of replacesOf(row)) { + const server = params.server[target.id]; + const state = target.base ? JSON.stringify(target.base) : null; + // Not read on this run (added on the way): the queue checks it again anyway. + if (server === undefined || (server && !destroyConflicts(target.base, server))) { + savePatch(ctx.db, { jmap: target.jmap, record_id: target.id, op: 'destroy', patch: '{}', base: state, server: state, subject: target.subject }, now); + continue; + } + if (server) { + const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM email WHERE jmap = ?', [target.jmap])?.gen ?? 0); + upsertEmail(ctx.db, target.jmap, server, gen); + } + const subject = (typeof server?.subject === 'string' ? server.subject : target.subject) ?? row.subject ?? ''; + addConflict(ctx, row.jmap, params.serverId, { subject, op: 'draft', reason: server ? 'changed' : 'gone' }); + conflicts.push(subject); + } + }); + ctx.emit({ type: 'changed', account: ctx.account, collections: ['Email', 'Mailbox', 'Thread'] }); + return { conflicts }; +} + +/** The ghosts looked up: copies found go through the usual queue. */ +function ghostsLooked(ctx: StoreContext, params: { jmap: string; messageIds: string[]; found: string[] }): void { + ctx.db.transaction(() => { + for (const messageId of params.messageIds) ctx.db.run('DELETE FROM draft_ghost WHERE jmap = ? AND message_id = ?', [params.jmap, messageId]); + for (const id of params.found) savePatch(ctx.db, { jmap: params.jmap, record_id: id, op: 'destroy', patch: '{}', base: null, server: null, subject: null }, ctx.now()); + }); +} + +/** A send took care of this draft without the page holding a client's word for it. */ +function discard(ctx: StoreContext, params: { jmap: string; id: string }): void { + ctx.db.transaction(() => { + destroyForDraft(ctx, params.jmap, params.id, undefined); + }); + ctx.emit({ type: 'changed', account: ctx.account, collections: ['Email', 'Mailbox', 'Thread'] }); +} + +export const DRAFT_CALLS: Readonly> = { + 'drafts.read': read, + 'drafts.list': list, + 'drafts.claim': claim, + 'drafts.failed': failed, + 'drafts.retry': retry, + 'drafts.created': created, + 'drafts.ghostsLooked': ghostsLooked, + 'drafts.discard': discard, + 'drafts.resolveStandIns': (ctx, params: { ids: string[] }) => resolveStandIns(ctx.db, params.ids), +}; diff --git a/lib/store/drafts/flush.ts b/lib/store/drafts/flush.ts new file mode 100644 index 000000000..9c743e41a --- /dev/null +++ b/lib/store/drafts/flush.ts @@ -0,0 +1,208 @@ +/** + * Sends the drafts saved on this device, written against a small `io` so the + * page can run it through the JMAP client while the logic is tested next to the + * store (decision 15). + * + * For each draft, in order: look up whether it arrived already when an answer got + * lost, read the versions it replaces, upload its files, create it, and report. + * The store only then queues the old versions for removal, so a failure anywhere + * leaves a copy on the server or on the device, never neither. Anything that + * looks like an outage ends the run and the draft waits. A refusal stays with + * the draft, on the device, for the user to see. + * + * Message-ID lookups match on the client: `header` and `text` filters find no + * Message-ID on either Stalwart (decision 4). + */ + +import { JmapMethodError, responseFor, type JmapResponse, type MethodCall } from '../engine/transport'; +import { EMAIL_BODY_PROPERTIES, EMAIL_LIST_PROPERTIES, MAIL_USING, type EmailObject } from '../mail/records'; +import { CLOCK_SKEW_MARGIN_MS } from '../outbox/dedupe'; +import type { DraftGhost, PendingDraft } from './calls'; +import { heldBlobsOf, withUploadedBlobs } from './local-email'; +import type { DraftError } from './pending'; + +export interface DraftFlushStore { + read(): Promise<{ rows: PendingDraft[]; ghosts: DraftGhost[] }>; + claim(localId: string): Promise; + failed(localId: string, status: 'waiting' | 'uncertain' | 'refused', error?: DraftError, neverSent?: boolean): Promise; + created(result: { + localId: string; + seq: number; + jmap: string; + messageId: string; + serverId: string; + email: EmailObject | null; + server: Record; + }): Promise<{ conflicts: string[] }>; + ghostsLooked(jmap: string, messageIds: string[], found: string[]): Promise; +} + +export interface DraftFlushIo { + store: DraftFlushStore; + request(calls: MethodCall[], using: string[]): Promise; + /** Puts a held file on the server. Null when it's no longer on this device. */ + upload(jmap: string, blobId: string): Promise; + /** The server refused for good: no use retrying. Anything else counts as an outage. */ + isRefusal(error: unknown): boolean; + onRefused(subject: string, error: DraftError): void; + onConflict(subject: string): void; + /** One more draft is on the server. */ + onSent(): void; +} + +const PAGE = 500; +const MAX_PAGES = 5; +/** Set errors that mean "not now" rather than "no". */ +const RETRY_LATER = new Set(['serverUnavailable', 'serverFail', 'serverPartialFail', 'rateLimit']); + +function bare(id: string): string { + return id.trim().replace(/^<|>$/g, ''); +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isOutage(error: unknown, io: DraftFlushIo): boolean { + if (error instanceof JmapMethodError) return RETRY_LATER.has(error.type); + return !io.isRefusal(error); +} + +/** Copies in Drafts by Message-ID, for the ones asked about. More than the pages hold can only cost a spare copy. */ +async function copiesInDrafts(io: DraftFlushIo, jmap: string, since: number): Promise> { + const boxes = await io.request([['Mailbox/get', { accountId: jmap, properties: ['id', 'role'] }, 'm']], MAIL_USING); + const drafts = (responseFor(boxes, 'm')[1].list as Array<{ id: string; role?: string | null }>).find((box) => box.role === 'drafts'); + const found = new Map(); + if (!drafts) return found; + const filter = { operator: 'AND', conditions: [{ inMailbox: drafts.id }, { after: new Date(since - CLOCK_SKEW_MARGIN_MS).toISOString() }] }; + for (let page = 0; page < MAX_PAGES; page += 1) { + const response = await io.request([ + ['Email/query', { accountId: jmap, filter, sort: [{ property: 'receivedAt', isAscending: false }], position: page * PAGE, limit: PAGE }, 'q'], + ['Email/get', { accountId: jmap, '#ids': { resultOf: 'q', name: 'Email/query', path: '/ids' }, properties: ['id', 'messageId'] }, 'g'], + ], MAIL_USING); + const ids = responseFor(response, 'q')[1].ids as string[]; + for (const email of responseFor(response, 'g')[1].list as Array<{ id: string; messageId?: string[] | null }>) { + for (const messageId of email.messageId ?? []) found.set(bare(messageId), email.id); + } + if (ids.length < PAGE) break; + } + return found; +} + +async function readReplaced(io: DraftFlushIo, row: PendingDraft): Promise> { + const out: Record = {}; + const byAccount = new Map(); + for (const target of row.replaces) byAccount.set(target.jmap, [...(byAccount.get(target.jmap) ?? []), target.id]); + for (const [jmap, ids] of byAccount) { + const response = await io.request([['Email/get', { accountId: jmap, ids, properties: EMAIL_LIST_PROPERTIES }, 'r']], MAIL_USING); + const list = responseFor(response, 'r')[1].list as EmailObject[]; + for (const id of ids) out[id] = list.find((email) => email.id === id) ?? null; + } + return out; +} + +/** Reading the new message back can't fail the flush: it's on the server. */ +async function readCreated(io: DraftFlushIo, jmap: string, id: string): Promise { + try { + const response = await io.request([['Email/get', { + accountId: jmap, + ids: [id], + properties: [...EMAIL_LIST_PROPERTIES, ...EMAIL_BODY_PROPERTIES], + fetchAllBodyValues: true, + maxBodyValueBytes: 1_000_000, + }, 'n']], MAIL_USING); + return (responseFor(response, 'n')[1].list as EmailObject[])[0] ?? null; + } catch { + return null; + } +} + +type RowOutcome = 'next' | 'stop'; + +async function flushRow(io: DraftFlushIo, row: PendingDraft, copies: Map | null): Promise { + if (!(await io.store.claim(row.localId))) return 'next'; + // Each step knows what a failure means: before the create nothing has left. + let serverId = copies?.get(bare(row.messageId)); + let server: Record; + const stay = async (status: 'waiting' | 'uncertain') => { + await io.store.failed(row.localId, status, undefined, status === 'waiting' && row.firstAttemptAt === null); + return 'stop' as const; + }; + const refuse = async (error: DraftError) => { + await io.store.failed(row.localId, 'refused', error); + io.onRefused(row.subject, error); + return 'next' as const; + }; + try { + server = await readReplaced(io, row); + } catch (error) { + if (isOutage(error, io)) return stay(row.firstAttemptAt === null ? 'waiting' : 'uncertain'); + return refuse({ text: errorText(error) }); + } + if (!serverId) { + const uploaded = new Map(); + try { + for (const blobId of new Set(heldBlobsOf(row.create))) { + const real = await io.upload(row.jmap, blobId); + if (real === null) return refuse({ code: 'file_missing' }); + uploaded.set(blobId, real); + } + } catch (error) { + if (isOutage(error, io)) return stay(row.firstAttemptAt === null ? 'waiting' : 'uncertain'); + return refuse({ text: errorText(error) }); + } + const creationId = 'draft'; + let result: Record>; + try { + const response = await io.request([['Email/set', { + accountId: row.jmap, + create: { [creationId]: withUploadedBlobs(row.create, uploaded) }, + }, 'c']], MAIL_USING); + [, result] = responseFor(response, 'c'); + } catch (error) { + // It went out: from here on it may have arrived. + if (isOutage(error, io)) return stay('uncertain'); + return refuse({ text: errorText(error) }); + } + const refusal = result.notCreated?.[creationId]; + if (refusal) { + if (RETRY_LATER.has(refusal.type ?? '')) return stay('uncertain'); + return refuse({ type: refusal.type, text: refusal.description ?? refusal.type }); + } + serverId = result.created?.[creationId]?.id; + if (!serverId) return stay('uncertain'); + } + const email = await readCreated(io, row.jmap, serverId); + const { conflicts } = await io.store.created({ localId: row.localId, seq: row.seq, jmap: row.jmap, messageId: row.messageId, serverId, email, server }); + for (const subject of conflicts) io.onConflict(subject); + io.onSent(); + return 'next'; +} + +/** True when all went through to an answer, false when the run stopped on an outage. */ +export async function flushDrafts(io: DraftFlushIo): Promise { + const { rows, ghosts } = await io.store.read(); + if (rows.length === 0 && ghosts.length === 0) return true; + const accounts = new Set([...rows.map((row) => row.jmap), ...ghosts.map((entry) => entry.jmap)]); + for (const jmap of accounts) { + const ours = rows.filter((row) => row.jmap === jmap); + const theirGhosts = ghosts.filter((entry) => entry.jmap === jmap); + const attempts = [...ours.filter((row) => row.firstAttemptAt !== null).map((row) => row.firstAttemptAt as number), ...theirGhosts.map((entry) => entry.at)]; + let copies: Map | null = null; + if (attempts.length > 0) { + try { + copies = await copiesInDrafts(io, jmap, Math.min(...attempts)); + } catch { + // Can't look: nothing may go that might go twice. + return false; + } + const looked = theirGhosts.map((entry) => entry.messageId); + const found = looked.flatMap((messageId) => copies?.get(bare(messageId)) ?? []); + if (looked.length > 0) await io.store.ghostsLooked(jmap, looked, found); + } + for (const row of ours) { + if ((await flushRow(io, row, copies)) === 'stop') return false; + } + } + return true; +} diff --git a/lib/store/drafts/local-email.ts b/lib/store/drafts/local-email.ts new file mode 100644 index 000000000..7e4134469 --- /dev/null +++ b/lib/store/drafts/local-email.ts @@ -0,0 +1,135 @@ +/** + * What a draft saved on this device looks like to everyone who reads it: the + * Email/set create the client wrote, turned into the list properties and the + * body a server would answer with. The composer reads it back straight after a + * save and matches attachments by name and size. + */ + +import { isLocalBlobId } from '../outbox/items'; +import type { StorageAdapter } from '../storage'; + +export type DraftCreate = Record; + +interface CreatePart { + partId?: string; + blobId?: string; + type?: string; + name?: string | null; + disposition?: string | null; + cid?: string | null; + subParts?: CreatePart[]; +} + +interface BodyPart extends CreatePart { + size: number; +} + +/** A held file's size is known. A part of a kept message's is too. Else the server knows, and the list shows nothing. */ +function blobSize(db: StorageAdapter, jmap: string, blobId: string): number { + const row = isLocalBlobId(blobId) + ? db.get<{ size: number }>('SELECT size FROM upload WHERE local_id = ?', [blobId]) + : db.get<{ size: number }>('SELECT size FROM email_part WHERE jmap = ? AND blob_id = ? ORDER BY size DESC LIMIT 1', [jmap, blobId]); + return Number(row?.size ?? 0); +} + +function leaves(part: CreatePart): CreatePart[] { + return part.subParts ? part.subParts.flatMap(leaves) : [part]; +} + +/** Every blob part a create names, wherever it put them. */ +export function blobPartsOf(create: DraftCreate): CreatePart[] { + const listed = (create.attachments ?? []) as CreatePart[]; + const tree = create.bodyStructure ? leaves(create.bodyStructure as CreatePart) : []; + return [...listed, ...tree].filter((part) => typeof part.blobId === 'string'); +} + +export function heldBlobsOf(create: DraftCreate): string[] { + return blobPartsOf(create).map((part) => part.blobId as string).filter(isLocalBlobId); +} + +/** The create with the blob ids a flush got for its held files. */ +export function withUploadedBlobs(create: DraftCreate, uploaded: ReadonlyMap): DraftCreate { + const swap = (part: CreatePart): CreatePart => ({ + ...part, + ...(part.blobId && uploaded.has(part.blobId) ? { blobId: uploaded.get(part.blobId) } : {}), + ...(part.subParts ? { subParts: part.subParts.map(swap) } : {}), + }); + const out: DraftCreate = { ...create }; + if (Array.isArray(create.attachments)) out.attachments = (create.attachments as CreatePart[]).map(swap); + if (create.bodyStructure) out.bodyStructure = swap(create.bodyStructure as CreatePart); + return out; +} + +function previewOf(text: string): string { + return text.replace(/\s+/g, ' ').trim().slice(0, 256); +} + +function plainFromHtml(html: string): string { + return html.replace(/<(style|script)\b[^<]*(?:<(?!\/\1>)[^<]*)*<\/\1>/gi, ' ').replace(/<[^<>]*>/g, ' '); +} + +function headersOf(create: DraftCreate): Array<{ name: string; value: string }> { + const out: Array<{ name: string; value: string }> = []; + for (const [key, value] of Object.entries(create)) { + const match = /^header:([^:]+):asText$/.exec(key); + if (match && typeof value === 'string') out.push({ name: match[1], value: ` ${value}` }); + } + return out; +} + +/** + * The message as it's answered from the store. `id` also serves as thread id and + * blob id: nothing else has one yet. The raw message doesn't exist until the + * server made it, so an export finds no raw copy to read. + */ +export function localEmailFrom(db: StorageAdapter, jmap: string, id: string, create: DraftCreate, messageId: string, now: number): { list: Record & { id: string }; body: Record } { + const values = (create.bodyValues ?? {}) as Record; + const valueOf = (part: CreatePart) => values[part.partId ?? '']?.value ?? ''; + const asBodyPart = (part: CreatePart, type: string): BodyPart => ({ partId: part.partId, type: part.type ?? type, size: valueOf(part).length }); + const tree = create.bodyStructure ? leaves(create.bodyStructure as CreatePart) : []; + const textParts = (tree.length ? tree.filter((part) => part.partId && part.type === 'text/plain') : (create.textBody ?? []) as CreatePart[]).map((part) => asBodyPart(part, 'text/plain')); + const htmlParts = (tree.length ? tree.filter((part) => part.partId && part.type === 'text/html') : (create.htmlBody ?? []) as CreatePart[]).map((part) => asBodyPart(part, 'text/html')); + const attachments: BodyPart[] = blobPartsOf(create).map((part) => ({ + partId: part.blobId, + blobId: part.blobId, + type: part.type ?? 'application/octet-stream', + name: part.name ?? null, + size: blobSize(db, jmap, part.blobId as string), + disposition: part.disposition ?? 'attachment', + cid: part.cid ?? null, + })); + const text = textParts.map(valueOf).join('\n') || plainFromHtml(htmlParts.map(valueOf).join('\n')); + const size = [...textParts, ...htmlParts, ...attachments].reduce((sum, part) => sum + part.size, 0); + const at = new Date(now).toISOString(); + const list = { + id, + threadId: id, + blobId: id, + mailboxIds: create.mailboxIds, + keywords: create.keywords, + size, + receivedAt: at, + sentAt: at, + from: create.from ?? [], + to: create.to ?? [], + cc: create.cc ?? [], + subject: typeof create.subject === 'string' ? create.subject : '', + preview: previewOf(text), + hasAttachment: attachments.some((part) => part.disposition !== 'inline'), + } as Record & { id: string }; + const body: Record = { + bcc: create.bcc ?? [], + replyTo: create.replyTo ?? null, + messageId: [messageId], + inReplyTo: create.inReplyTo ?? null, + references: create.references ?? null, + headers: headersOf(create), + textBody: textParts.length ? textParts : htmlParts, + // A plain text draft lists its text part as the HTML body too, as a server does (RFC 8621 4.1.4). + htmlBody: htmlParts.length ? htmlParts : textParts, + attachments, + bodyStructure: create.bodyStructure ?? { type: 'multipart/mixed', subParts: [...textParts, ...htmlParts, ...attachments] }, + bodyValues: Object.fromEntries(Object.entries(values).map(([partId, value]) => [partId, { value: value.value ?? '', isEncodingProblem: false, isTruncated: false }])), + }; + return { list, body }; +} diff --git a/lib/store/drafts/pending.ts b/lib/store/drafts/pending.ts new file mode 100644 index 000000000..d890ad3f0 --- /dev/null +++ b/lib/store/drafts/pending.ts @@ -0,0 +1,264 @@ +/** + * Drafts saved on this device. createDraft on the client saves by creating a new + * version and then destroying the one before, and that's how it reaches this + * file: a create hands out a stand-in id, and the destroy that follows names the + * new version in `replacedBy`. Each draft therefore has one pending row however + * often it's saved. A version on the server stays there, hidden here, until the + * new one has arrived (flush.ts). + * + * A version that may already have reached the server (its Message-ID was handed + * over, and the answer got lost) leaves a ghost behind when it's replaced. + */ + +import { isStandInId, newStandInId } from '@/lib/jmap/stand-in-ids'; +import type { MethodContext, RecordOverlay, StoreContext } from '../area'; +import { listObjects } from '../engine/core-schema'; +import { deleteEmail, storeBody, upsertEmail, type EmailObject } from '../mail/records'; +import { localEmail, nextSeq, pendingRow, queueDestroy, queueUpdate, savePatch, EMAIL_TYPE } from '../outbox/email-set'; +import { applyPatch, normalizePatch, patchProblem, stateOf, type Patch, type RecordState } from '../outbox/patches'; +import type { StorageAdapter } from '../storage'; +import { localEmailFrom, type DraftCreate } from './local-email'; + +export interface ReplacedDraft { + jmap: string; + id: string; + base: RecordState | null; + subject: string | null; +} + +export interface DraftRow { + local_id: string; + jmap: string; + seq: number; + subject: string | null; + create_json: string; + message_id: string; + replaces: string; + status: DraftStatus; + sending_since: number | null; + first_attempt_at: number | null; + error: string | null; + created_at: number; +} + +/** + * - waiting: goes up by itself when the server answers; + * - sending: on its way (a tab took it); + * - uncertain: the connection broke after it may have arrived. Looked up first; + * - refused: the server said no. Stays here until it's edited or tried again. + */ +export type DraftStatus = 'waiting' | 'sending' | 'uncertain' | 'refused'; + +export interface DraftError { + code?: 'file_missing'; + type?: string; + text?: string; +} + +interface SetError { type: string; description?: string } + +const notADraft = (): SetError => ({ type: 'serverUnavailable', description: 'This needs the server' }); + +function rowOf(db: StorageAdapter, localId: string): DraftRow | undefined { + return db.get('SELECT * FROM draft_pending WHERE local_id = ?', [localId]); +} + +export function replacesOf(row: Pick): ReplacedDraft[] { + return JSON.parse(row.replaces) as ReplacedDraft[]; +} + +/** A message of the Drafts folder with the draft keyword, in a request that doesn't also send it. */ +function isDraftCreate(ctx: MethodContext, jmap: string, create: DraftCreate): boolean { + if (ctx.calls.some(([name]) => name === 'EmailSubmission/set')) return false; + if (!(create.keywords as Record | undefined)?.$draft) return false; + const boxes = Object.keys((create.mailboxIds ?? {}) as object); + if (boxes.length === 0) return false; + const drafts = listObjects<{ id: string; role?: string | null }>(ctx.db, jmap, 'Mailbox').filter((box) => box.role === 'drafts').map((box) => box.id); + return boxes.every((box) => drafts.includes(box)); +} + +function ghost(db: StorageAdapter, jmap: string, messageId: string, at: number): void { + db.run('INSERT OR REPLACE INTO draft_ghost (jmap, message_id, at) VALUES (?, ?, ?)', [jmap, messageId, at]); +} + +function currentGen(db: StorageAdapter, jmap: string): number { + return Number(db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM email WHERE jmap = ?', [jmap])?.gen ?? 0); +} + +/** Writes the row lists and search read. A held file isn't something sync could fetch, and there's no raw message yet. */ +function writeLocalRow(db: StorageAdapter, jmap: string, id: string, create: DraftCreate, messageId: string, now: number): { size: number } { + const { list, body } = localEmailFrom(db, jmap, id, create, messageId, now); + const rowid = upsertEmail(db, jmap, list, currentGen(db, jmap)); + storeBody(db, rowid, JSON.stringify(list), body); + db.run("DELETE FROM email_part WHERE email_rowid = ? AND (raw = 1 OR blob_id LIKE 'bulwark-local-%')", [rowid]); + return { size: Number(list.size) }; +} + +export function createLocalDraft(ctx: MethodContext, jmap: string, create: DraftCreate, networkTried: boolean): Record | SetError & { refused: true } { + if (!isDraftCreate(ctx, jmap, create)) return { ...notADraft(), refused: true }; + const id = newStandInId(); + const now = ctx.now(); + const messageId = (create.messageId as string[] | undefined)?.[0] ?? `${id}@bulwark.invalid`; + const stored: DraftCreate = { ...create, messageId: [messageId] }; + const { size } = writeLocalRow(ctx.db, jmap, id, stored, messageId, now); + ctx.db.run( + `INSERT INTO draft_pending (local_id, jmap, seq, subject, create_json, message_id, replaces, status, first_attempt_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, '[]', ?, ?, ?)`, + [id, jmap, nextSeq(ctx.db), typeof create.subject === 'string' ? create.subject : '', JSON.stringify(stored), messageId, networkTried ? 'uncertain' : 'waiting', networkTried ? now : null, now], + ); + return { id, blobId: id, threadId: id, size }; +} + +/** A flag or a move on a draft that only exists here: it goes into the create. */ +export function updateLocalDraft(ctx: Pick, jmap: string, id: string, rawPatch: Patch): SetError | null { + const row = rowOf(ctx.db, id); + if (!row) { + const real = mappedId(ctx.db, id); + return real ? queueUpdate(ctx.db, jmap, real, rawPatch, ctx.now()) : { type: 'notFound' }; + } + const problem = patchProblem(rawPatch); + if (problem) return problem; + const local = localEmail(ctx.db, row.jmap, id); + const create = JSON.parse(row.create_json) as DraftCreate; + const after = applyPatch(create, normalizePatch(rawPatch)); + if (Object.keys(after.mailboxIds ?? {}).length === 0) return { type: 'invalidProperties', description: 'A message has to stay in a folder' }; + ctx.db.run('UPDATE draft_pending SET create_json = ?, seq = ? WHERE local_id = ?', [JSON.stringify(after), nextSeq(ctx.db), id]); + if (local) upsertEmail(ctx.db, row.jmap, { ...local.email, keywords: after.keywords, mailboxIds: after.mailboxIds }, local.gen); + return null; +} + +/** What `id` is now on the server, for a stand-in that made it there. */ +function mappedId(db: StorageAdapter, id: string): string | null { + return db.get<{ server_id: string }>('SELECT server_id FROM draft_id_map WHERE local_id = ?', [id])?.server_id ?? null; +} + +export function resolveStandIns(db: StorageAdapter, ids: readonly string[]): Record { + return Object.fromEntries(ids.map((id) => [id, mappedId(db, id)])); +} + +function queueRemoval(db: StorageAdapter, target: ReplacedDraft, now: number): void { + const state = target.base ? JSON.stringify(target.base) : null; + savePatch(db, { jmap: target.jmap, record_id: target.id, op: 'destroy', patch: '{}', base: state, server: state, subject: target.subject }, now); +} + +/** + * Takes a pending row away: `into` carries on what it replaced, else the user + * discarded or sent the draft and what it replaced goes too. + */ +function dropPending(ctx: Pick, row: DraftRow, into: DraftRow | null): void { + const { db } = ctx; + const replaced = replacesOf(row); + if (into) { + db.run('UPDATE draft_pending SET replaces = ? WHERE local_id = ?', [JSON.stringify([...replacesOf(into), ...replaced]), into.local_id]); + } else { + for (const target of replaced) queueRemoval(db, target, ctx.now()); + } + if (row.first_attempt_at !== null) ghost(db, row.jmap, row.message_id, Number(row.first_attempt_at)); + db.run('DELETE FROM draft_pending WHERE local_id = ?', [row.local_id]); + deleteEmail(db, row.jmap, row.local_id); +} + +/** + * Email/set destroy of a stand-in, or of any message when a new version on the + * device replaces it. False when it's an ordinary delete for the outbox to queue. + */ +export function destroyForDraft(ctx: Pick, jmap: string, id: string, replacedBy: string | undefined): boolean { + const { db } = ctx; + const into = replacedBy ? rowOf(db, replacedBy) ?? null : null; + if (isStandInId(id)) { + const row = rowOf(db, id); + if (row) { + dropPending(ctx, row, into); + return true; + } + const real = mappedId(db, id); + if (!real) return true; // Gone already: that's the same result. + if (into) return destroyForDraft(ctx, jmap, real, replacedBy); + queueDestroy(db, jmap, real, ctx.now()); + return true; + } + if (!into) return false; + // The version on the server stays there until the new one has arrived. + const local = localEmail(db, jmap, id); + const waiting = pendingRow(db, jmap, id); + const base = local ? stateOf(local.email) : waiting?.base ? JSON.parse(waiting.base) as RecordState : null; + const entry: ReplacedDraft = { jmap, id, base, subject: typeof local?.email.subject === 'string' ? local.email.subject : waiting?.subject ?? null }; + db.run('UPDATE draft_pending SET replaces = ? WHERE local_id = ?', [JSON.stringify([...replacesOf(into), entry]), into.local_id]); + db.run('DELETE FROM outbox_patch WHERE jmap = ? AND type = ? AND record_id = ?', [jmap, EMAIL_TYPE, id]); + deleteEmail(db, jmap, id); + return true; +} + +/** A send naming a draft that's still on the device: the send takes care of it. */ +function draftsWithQueuedSend(db: StorageAdapter): Set { + const rows = db.all<{ payload: string }>("SELECT payload FROM outbox WHERE status IN ('waiting', 'sending', 'uncertain', 'held') AND kind = 'email'"); + const out = new Set(); + for (const row of rows) { + const draft = (JSON.parse(row.payload) as { item: { args: unknown[] } }).item.args[7]; + if (typeof draft === 'string' && isStandInId(draft)) out.add(draft); + } + return out; +} + +export function allPending(db: StorageAdapter): DraftRow[] { + return db.all('SELECT * FROM draft_pending ORDER BY seq ASC'); +} + +export function pendingForFlush(db: StorageAdapter): DraftRow[] { + const held = draftsWithQueuedSend(db); + return allPending(db).filter((row) => row.status !== 'refused' && !held.has(row.local_id)); +} + +/** + * Sync must not bring back a version that a draft here replaces, and the folder + * counts stay the server's with what waits here added when they're read. + */ +export const draftsOverlay: RecordOverlay = (ctx, jmap, type, records) => { + if (type !== 'Email' && type !== 'Thread' && type !== 'Mailbox') return records; + const rows = ctx.db.all>('SELECT replaces FROM draft_pending'); + if (rows.length === 0) return records; + const replaced = rows.flatMap(replacesOf).filter((target) => target.jmap === jmap); + if (type === 'Email') { + const hidden = new Set(replaced.map((target) => target.id)); + return records.filter((record) => !hidden.has(record.id)); + } + if (type === 'Thread') { + const hidden = new Set(replaced.map((target) => target.id)); + return (records as Array<{ id: string; emailIds: string[] }>).map((thread) => ({ ...thread, emailIds: thread.emailIds.filter((id) => !hidden.has(id)) })); + } + const deltas = new Map(); + const bump = (box: string, total: number, unread: number) => { + const delta = deltas.get(box) ?? { total: 0, unread: 0 }; + deltas.set(box, { total: delta.total + total, unread: delta.unread + unread }); + }; + for (const row of allPending(ctx.db)) { + if (row.jmap !== jmap) continue; + const create = JSON.parse(row.create_json) as RecordState; + for (const box of Object.keys(create.mailboxIds ?? {})) bump(box, 1, create.keywords?.$seen ? 0 : 1); + } + for (const target of replaced) { + for (const box of Object.keys(target.base?.mailboxIds ?? {})) bump(box, -1, target.base?.keywords?.$seen ? 0 : -1); + } + if (deltas.size === 0) return records; + return (records as Array<{ id: string; totalEmails?: unknown; unreadEmails?: unknown }>).map((box) => { + const delta = deltas.get(box.id); + if (!delta) return box; + const count = (value: unknown, by: number) => (typeof value === 'number' ? Math.max(0, value + by) : value); + return { ...box, totalEmails: count(box.totalEmails, delta.total), unreadEmails: count(box.unreadEmails, delta.unread) }; + }); +}; + +/** After a create went through: the real message takes the stand-in's place. */ +export function replaceLocalRow(db: StorageAdapter, row: DraftRow, serverId: string, email: EmailObject | null): void { + const local = localEmail(db, row.jmap, row.local_id); + deleteEmail(db, row.jmap, row.local_id); + const gen = currentGen(db, row.jmap); + if (email) { + const { bodyValues, textBody, htmlBody, attachments, bodyStructure, headers, bcc, replyTo, messageId, inReplyTo, references, sender, ...list } = email; + const rowid = upsertEmail(db, row.jmap, list as EmailObject, gen); + storeBody(db, rowid, JSON.stringify(list), { bodyValues, textBody, htmlBody, attachments, bodyStructure, headers, bcc, replyTo, messageId, inReplyTo, references, sender }); + return; + } + // It's there, but reading it back failed. Sync brings the text when it can. + if (local) upsertEmail(db, row.jmap, { ...local.email, id: serverId, threadId: serverId, blobId: serverId }, gen); +} diff --git a/lib/store/drafts/schema.ts b/lib/store/drafts/schema.ts new file mode 100644 index 000000000..8411b539f --- /dev/null +++ b/lib/store/drafts/schema.ts @@ -0,0 +1,58 @@ +import type { Migration } from '../storage'; + +export const DRAFT_MIGRATIONS: readonly Migration[] = [ + { + id: 400, + name: 'drafts', + sql: ` + -- A draft saved on this device and not on the server yet. One row per + -- draft: a later save takes over from the row before it. + CREATE TABLE draft_pending ( + local_id TEXT PRIMARY KEY, + jmap TEXT NOT NULL, + seq INTEGER NOT NULL, + subject TEXT, + -- The create as the client wrote it, with its fixed Message-ID. + create_json TEXT NOT NULL, + message_id TEXT NOT NULL, + -- Versions on the server this one replaces: destroyed after it's there. + replaces TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL DEFAULT 'waiting', + sending_since INTEGER, + -- Set once it may have reached the server: then it's looked up first. + first_attempt_at INTEGER, + error TEXT, + created_at INTEGER NOT NULL + ) WITHOUT ROWID; + -- What a stand-in id became. The page may hold one for a long time. + CREATE TABLE draft_id_map ( + local_id TEXT PRIMARY KEY, + jmap TEXT NOT NULL, + server_id TEXT NOT NULL, + at INTEGER NOT NULL + ) WITHOUT ROWID; + -- A version that may have reached the server and has been replaced or + -- discarded here since. Looked up and removed when the server is back. + CREATE TABLE draft_ghost ( + jmap TEXT NOT NULL, + message_id TEXT NOT NULL, + at INTEGER NOT NULL, + PRIMARY KEY (jmap, message_id) + ) WITHOUT ROWID; + `, + }, + { + id: 401, + name: 'held files in chunks', + sql: ` + ALTER TABLE upload ADD COLUMN complete INTEGER NOT NULL DEFAULT 1; + -- Binding a whole file grows the WASM heap by its size, and that never shrinks. + CREATE TABLE upload_chunk ( + local_id TEXT NOT NULL, + seq INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (local_id, seq) + ); + `, + }, +]; diff --git a/lib/store/engine/account-engine.ts b/lib/store/engine/account-engine.ts index df9452248..6c70dc5e2 100644 --- a/lib/store/engine/account-engine.ts +++ b/lib/store/engine/account-engine.ts @@ -37,6 +37,7 @@ export class AccountEngine { private scope: StoreScope; private readonly transport: JmapTransport; private readonly syncers: readonly Syncer[]; + private readonly areas: readonly StoreArea[]; private readonly methods: ReadonlyMap; private readonly calls = new Map(); private readonly overlays: readonly RecordOverlay[]; @@ -58,7 +59,9 @@ export class AccountEngine { this.session = options.session; this.scope = options.scope; this.transport = options.transport; - this.syncers = options.areas.flatMap((area) => area.syncers); + this.areas = options.areas; + const syncers = options.areas.flatMap((area) => area.syncers); + this.syncers = [...syncers.filter((syncer) => !syncer.low), ...syncers.filter((syncer) => syncer.low)]; this.methods = buildMethodTable(options.areas); this.overlays = options.areas.flatMap((area) => (area.overlay ? [area.overlay] : [])); for (const area of options.areas) { @@ -75,6 +78,12 @@ export class AccountEngine { // An offline start has no way to probe the server, so the last answer stays. this.invertedKeywordSort = options.invertedKeywordSort ?? metaGet(this.db, 'inverted_keyword_sort') === '1'; if (options.invertedKeywordSort !== undefined) metaSet(this.db, 'inverted_keyword_sort', options.invertedKeywordSort ? '1' : '0'); + this.applyScope(); + } + + private applyScope(): void { + const ctx = this.storeContext(); + for (const area of this.areas) area.scopeChanged?.(ctx); } /** The session kept from the last time, for a start without a network. */ @@ -94,6 +103,7 @@ export class AccountEngine { // The user may just have made room by keeping less. this.storageBlockedUntil = 0; metaSet(this.db, 'scope', JSON.stringify(scope)); + this.applyScope(); } setInvertedKeywordSort(inverted: boolean): void { @@ -225,15 +235,21 @@ export class AccountEngine { } private syncContext(): SyncContext { + const scope = () => this.scope; return { account: this.account, db: this.db, session: this.session, - scope: this.scope, + // A pass can take long, and a tighter policy applies to the rest of it. + get scope() { + return scope(); + }, transport: this.transport, now: this.now, emit: this.emitter, stopped: () => this.closed || !this.online, + syncWanted: () => this.again, + pause: this.sleep, syncState: (jmap, type) => this.syncState(jmap, type), setSyncState: (jmap, type, row) => this.setSyncState(jmap, type, row), overlay: (jmap, type, records) => this.overlay(jmap, type, records), diff --git a/lib/store/engine/responder.ts b/lib/store/engine/responder.ts index be03dc5db..c01b2d932 100644 --- a/lib/store/engine/responder.ts +++ b/lib/store/engine/responder.ts @@ -56,12 +56,12 @@ export function buildMethodTable(areas: readonly StoreArea[]): Map, + ctx: Omit, methods: ReadonlyMap, calls: readonly MethodCall[], ): Promise { const done: MethodResponse[] = []; - const full: MethodContext = { ...ctx, createdIds: new Map() }; + const full: MethodContext = { ...ctx, createdIds: new Map(), calls }; let answered = 0; for (const call of calls) { const [name, rawArgs, callId] = call; diff --git a/lib/store/engine/transport.ts b/lib/store/engine/transport.ts index cc55ff300..dfa58058e 100644 --- a/lib/store/engine/transport.ts +++ b/lib/store/engine/transport.ts @@ -43,6 +43,11 @@ export class TransportError extends Error { export interface JmapTransport { request(calls: MethodCall[], using: string[]): Promise; download(jmap: string, blobId: string, type: string, name: string): Promise; + /** + * The same, piece by piece, so a large file isn't held in memory. An error + * thrown by `sink` ends the download and comes out as it is. + */ + downloadStream?(jmap: string, blobId: string, type: string, name: string, sink: (part: Uint8Array) => void): Promise; upload(jmap: string, data: Uint8Array, type: string): Promise<{ blobId: string; type: string; size: number }>; } diff --git a/lib/store/mail/attachments.ts b/lib/store/mail/attachments.ts new file mode 100644 index 000000000..8bd154075 --- /dev/null +++ b/lib/store/mail/attachments.ts @@ -0,0 +1,295 @@ +/** + * The files of kept mail: attachments, inline images (so a message offline shows + * them) and the raw message an export reads (decision in + * local-data/desktop-offline-decisions.md). Fetched last in a pass, newest mail + * first, one at a time and paced, and a new pass that's asked for goes first. + * + * What's kept is bounded by the scope: nothing with `attachments` off, nothing + * over `maxAttachmentBytes`, and `maxTotalBytes` in all. At the total a newer + * message's file pushes out files of older messages, oldest first. Never the + * files of a pinned message. Files held for a draft or a send are in `upload`, + * pinned files in the files area, and neither is touched here. + * + * A file comes in as a stream where the transport can, written in 1 MB chunks + * that each commit on their own, and is held nowhere whole. Its row in `blob` + * comes last, so a broken download leaves nothing readable. + */ + +import type { StoreContext, SyncContext, Syncer } from '../area'; +import { TransportError } from '../engine/transport'; +import type { StorageAdapter } from '../storage'; +import { BLOB_CHUNK_BYTES, dropBlob, putBlob } from './records'; + +/** Keeps well under the server's 1000 requests a minute, which the user's own clicks count against too. */ +const PACE_MS = 200; +const PAGE = 20; +/** + * Parts in a row that don't fit before a pass stops looking. Each check reads the + * kept list, and older mail only has less to push out. + */ +const MOST_PASSED_IN_A_ROW = 50; +const PINNED = '$pinned'; +const RAW_TYPE = 'message/rfc822'; + +class DownloadStopped extends Error {} + +class DownloadTooLarge extends Error {} + +interface Part { + blob_id: string; + size: number; + type: string | null; + name: string | null; + received_at: number; +} + +function usedBytes(db: StorageAdapter): number { + return Number(db.get<{ n: number | null }>('SELECT SUM(size) AS n FROM blob')?.n ?? 0); +} + +const NOT_PINNED = `NOT EXISTS ( + SELECT 1 FROM email_part p JOIN email_keyword k ON k.email_rowid = p.email_rowid + WHERE p.jmap = b.jmap AND p.blob_id = b.blob_id AND k.keyword = '${PINNED}' +)`; + +/** Whether `size` more can fit, counting what could be pushed out for mail received at `at`. */ +function couldFit(db: StorageAdapter, cap: number, size: number, at: number): boolean { + const used = usedBytes(db); + if (used + size <= cap) return true; + const older = Number(db.get<{ n: number | null }>(`SELECT SUM(b.size) AS n FROM blob b WHERE b.at < ? AND ${NOT_PINNED}`, [at])?.n ?? 0); + return used - older + size <= cap; +} + +/** Pushes out files of mail older than `at`, oldest first, until `size` more fits or nothing is left to push out. */ +function pushOut(db: StorageAdapter, cap: number, size: number, at: number): void { + let used = usedBytes(db); + while (used + size > cap) { + const oldest = db.all<{ jmap: string; blob_id: string; size: number }>(`SELECT b.jmap, b.blob_id, b.size FROM blob b WHERE b.at < ? AND ${NOT_PINNED} ORDER BY b.at ASC LIMIT 50`, [at]); + if (oldest.length === 0) return; + for (const row of oldest) { + if (used + size <= cap) break; + dropBlob(db, row.jmap, row.blob_id); + used -= Number(row.size); + } + } +} + +/** Makes room for `size` more. False, with nothing pushed out, when it can't. */ +function makeRoom(db: StorageAdapter, cap: number, size: number, at: number): boolean { + if (!couldFit(db, cap, size, at)) return false; + pushOut(db, cap, size, at); + return true; +} + +/** A tighter scope applies at once, not at the next sync. */ +export function applyAttachmentScope(ctx: Pick): void { + const { db, scope } = ctx; + db.transaction(() => { + if (!scope.attachments) { + db.run('DELETE FROM blob_chunk'); + db.run('DELETE FROM blob'); + return; + } + for (const row of db.all<{ jmap: string; blob_id: string }>('SELECT jmap, blob_id FROM blob WHERE size > ?', [scope.maxAttachmentBytes])) { + dropBlob(db, row.jmap, row.blob_id); + } + pushOut(db, scope.maxTotalBytes, 0, Number.MAX_SAFE_INTEGER); + }); +} + +/** A kept file, piece by piece into a Blob, which the browser can keep out of this worker's memory. */ +export function keptBlob(ctx: Pick, jmap: string, blobId: string): { type: string | null; size: number; blob: Blob } | null { + if (!ctx.scope.attachments) return null; + const row = ctx.db.get<{ type: string | null; size: number; data: Uint8Array }>('SELECT type, size, data FROM blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + if (!row) return null; + const type = row.type ?? ''; + // A file from before chunks has its bytes in the row; a new one has none there. + let blob = new Blob(row.data?.byteLength ? [row.data as BlobPart] : [], { type }); + for (let seq = 0; ; seq += 1) { + const chunk = ctx.db.get<{ data: Uint8Array }>('SELECT data FROM blob_chunk WHERE jmap = ? AND blob_id = ? AND seq = ?', [jmap, blobId, seq]); + if (!chunk) break; + // Each step only refers to the last Blob, so one chunk at a time is in memory here. + blob = new Blob([blob, chunk.data as BlobPart], { type }); + } + return { type: row.type, size: Number(row.size), blob }; +} + +/** A file the user opened online: kept if it belongs to a kept message and the scope allows it. */ +export function recordKeptBlob(ctx: Pick, jmap: string, blobId: string, type: string | null, data: Uint8Array): boolean { + const { db, scope } = ctx; + const owner = db.get<{ received_at: number }>('SELECT received_at FROM email_part WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + if (!owner || !scope.attachments || data.byteLength > scope.maxAttachmentBytes) return false; + const at = Number(owner.received_at); + return db.transaction(() => { + if (!makeRoom(db, scope.maxTotalBytes, data.byteLength, at)) return false; + putBlob(db, jmap, blobId, type, data, at); + return true; + }); +} + +export type RawRefusal = 'attachments_off' | 'not_kept'; + +/** + * A whole message as the server has it, by `Email.blobId`, for an offline + * export. It stays a refusal rather than something rebuilt: a rebuilt message + * would lose its signatures and exact bytes. + */ +export function rawBlob(ctx: Pick, jmap: string, blobId: string): { ok: true; type: string; size: number; blob: Blob } | { ok: false; reason: RawRefusal } { + if (!ctx.scope.attachments) return { ok: false, reason: 'attachments_off' }; + const isRaw = ctx.db.get('SELECT 1 AS x FROM email_part WHERE jmap = ? AND blob_id = ? AND raw = 1', [jmap, blobId]); + const kept = isRaw ? keptBlob(ctx, jmap, blobId) : null; + if (!kept) return { ok: false, reason: 'not_kept' }; + return { ok: true, type: RAW_TYPE, size: kept.size, blob: new Blob([kept.blob], { type: RAW_TYPE }) }; +} + +/** How many messages (of one folder, or all) an export could write right now. */ +export function rawStatus(ctx: Pick, jmap: string, mailboxId?: string): { total: number; kept: number } { + const inFolder = mailboxId ? 'JOIN email_mailbox m ON m.email_rowid = e.rowid AND m.mailbox_id = ?' : ''; + const params = mailboxId ? [mailboxId, jmap] : [jmap]; + const total = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM email e ${inFolder} WHERE e.jmap = ?`, params)?.n ?? 0); + if (!ctx.scope.attachments) return { total, kept: 0 }; + const kept = Number(ctx.db.get<{ n: number }>( + `SELECT COUNT(DISTINCT e.rowid) AS n FROM email e ${inFolder} + JOIN email_part p ON p.email_rowid = e.rowid AND p.raw = 1 + JOIN blob b ON b.jmap = p.jmap AND b.blob_id = p.blob_id + WHERE e.jmap = ?`, + params, + )?.n ?? 0); + return { total, kept }; +} + +function clearChunks(db: StorageAdapter, jmap: string, blobId: string): void { + db.run('DELETE FROM blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); +} + +/** Takes a body piece by piece and writes it in chunks, each committed on its own. */ +function chunkWriter(ctx: SyncContext, jmap: string, blobId: string) { + const buffer = new Uint8Array(BLOB_CHUNK_BYTES); + let filled = 0; + let seq = 0; + let total = 0; + const write = () => { + if (filled === 0) return; + ctx.db.run('INSERT INTO blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, buffer.subarray(0, filled)]); + seq += 1; + filled = 0; + }; + return { + push(part: Uint8Array): void { + total += part.length; + if (total > ctx.scope.maxAttachmentBytes) throw new DownloadTooLarge(); + for (let at = 0; at < part.length;) { + const take = Math.min(part.length - at, BLOB_CHUNK_BYTES - filled); + buffer.set(part.subarray(at, at + take), filled); + filled += take; + at += take; + if (filled === BLOB_CHUNK_BYTES) { + if (ctx.stopped() || !ctx.scope.attachments) throw new DownloadStopped(); + write(); + } + } + }, + /** The last piece goes in the transaction that files it, so a small file costs one commit. */ + writeLast: write, + get total(): number { + return total; + }, + }; +} + +function refuse(db: StorageAdapter, jmap: string, blobId: string): void { + db.run('UPDATE email_part SET refused = 1 WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); +} + +type Fetched = 'kept' | 'skipped' | 'stopped'; + +async function fetchOne(ctx: SyncContext, jmap: string, part: Part): Promise { + const { db, transport } = ctx; + const blobId = String(part.blob_id); + const type = part.type ?? 'application/octet-stream'; + const name = part.name ?? 'attachment'; + clearChunks(db, jmap, blobId); + const writer = chunkWriter(ctx, jmap, blobId); + try { + if (transport.downloadStream) await transport.downloadStream(jmap, blobId, type, name, (piece) => writer.push(piece)); + else writer.push(await transport.download(jmap, blobId, type, name)); + } catch (error) { + clearChunks(db, jmap, blobId); + if (error instanceof DownloadStopped) return 'stopped'; + if (error instanceof DownloadTooLarge) { + refuse(db, jmap, blobId); + return 'skipped'; + } + if (error instanceof TransportError && error.kind === 'http') { + refuse(db, jmap, blobId); + return 'skipped'; + } + // The connection works and this transfer broke: the next one may be fine. + if (error instanceof TransportError && (error as TransportError & { interrupted?: boolean }).interrupted) return 'skipped'; + throw error; + } + const at = Number(part.received_at); + const size = writer.total; + return db.transaction((): Fetched => { + // Its message may have gone while this was on its way, or the rules changed. + const wanted = ctx.scope.attachments && db.get('SELECT 1 AS x FROM email_part WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + if (!wanted || !makeRoom(db, ctx.scope.maxTotalBytes, size, at)) { + clearChunks(db, jmap, blobId); + return 'skipped'; + } + writer.writeLast(); + db.run( + `INSERT INTO blob (jmap, blob_id, type, size, at, data) VALUES (?, ?, ?, ?, ?, x'') + ON CONFLICT (jmap, blob_id) DO UPDATE SET type = excluded.type, size = excluded.size, at = excluded.at, data = excluded.data`, + [jmap, blobId, part.type, size, at], + ); + return 'kept'; + }); +} + +async function fetchKept(ctx: SyncContext, jmap: string): Promise { + const { db } = ctx; + // What a closed app or a stopped pass left unfinished. + db.run('DELETE FROM blob_chunk WHERE jmap = ? AND NOT EXISTS (SELECT 1 FROM blob b WHERE b.jmap = blob_chunk.jmap AND b.blob_id = blob_chunk.blob_id)', [jmap]); + applyAttachmentScope(ctx); + if (!ctx.scope.attachments) return; + // Rows passed over in this pass stay in the list, so the next page starts past them. + let passed = 0; + let passedInARow = 0; + let done = 0; + for (;;) { + const parts = db.all( + `SELECT p.blob_id, p.size, p.type, p.name, p.received_at FROM email_part p + WHERE p.jmap = ? AND p.refused = 0 AND p.size <= ? + AND NOT EXISTS (SELECT 1 FROM blob b WHERE b.jmap = p.jmap AND b.blob_id = p.blob_id) + ORDER BY p.received_at DESC, p.raw ASC, p.blob_id ASC LIMIT ? OFFSET ?`, + [jmap, ctx.scope.maxAttachmentBytes, PAGE, passed], + ); + if (parts.length === 0) return; + for (const part of parts) { + if (ctx.stopped() || ctx.syncWanted()) return; + // The same file a second time in this page: kept by now. + if (db.get('SELECT 1 AS x FROM blob WHERE jmap = ? AND blob_id = ?', [jmap, String(part.blob_id)])) continue; + if (!couldFit(db, ctx.scope.maxTotalBytes, Number(part.size), Number(part.received_at))) { + passed += 1; + passedInARow += 1; + if (passedInARow >= MOST_PASSED_IN_A_ROW) return; + continue; + } + passedInARow = 0; + const outcome = await fetchOne(ctx, jmap, part); + if (outcome === 'stopped') return; + if (outcome === 'skipped') passed += 1; + done += 1; + ctx.emit({ type: 'progress', account: ctx.account, area: 'mail', phase: 'attachments', done, total: null }); + await ctx.pause(PACE_MS); + } + } +} + +export const attachmentSyncer: Syncer = { + name: 'attachments', + capability: 'urn:ietf:params:jmap:mail', + low: true, + run: fetchKept, +}; diff --git a/lib/store/mail/calls.ts b/lib/store/mail/calls.ts index 429b55dd5..3091fd1bb 100644 --- a/lib/store/mail/calls.ts +++ b/lib/store/mail/calls.ts @@ -1,7 +1,8 @@ /** What the page asks the mail part of the store outside JMAP. */ import type { AreaCall, StoreContext } from '../area'; -import { putBlob, storeBody, upsertEmail, type EmailObject } from './records'; +import { keptBlob, rawBlob, rawStatus, recordKeptBlob } from './attachments'; +import { storeBody, upsertEmail, type EmailObject } from './records'; /** * A message the user opened online: kept with its text if the store doesn't @@ -34,15 +35,11 @@ function recordOpened(ctx: StoreContext, params: { jmap: string; emails: EmailOb /** A file of a kept message the user opened. Nothing else is kept: not an export, not a stray file. */ function recordBlob(ctx: StoreContext, params: { jmap: string; blobId: string; type: string | null; data: Uint8Array }): boolean { - const owner = ctx.db.get<{ received_at: number }>('SELECT received_at FROM email_part WHERE jmap = ? AND blob_id = ?', [params.jmap, params.blobId]); - if (!owner) return false; - putBlob(ctx.db, params.jmap, params.blobId, params.type, params.data, Number(owner.received_at)); - return true; + return recordKeptBlob(ctx, params.jmap, params.blobId, params.type, params.data); } -function getBlob(ctx: StoreContext, params: { jmap: string; blobId: string }): { type: string | null; data: Uint8Array } | null { - const row = ctx.db.get<{ type: string | null; data: Uint8Array }>('SELECT type, data FROM blob WHERE jmap = ? AND blob_id = ?', [params.jmap, params.blobId]); - return row ? { type: row.type, data: row.data } : null; +function getBlob(ctx: StoreContext, params: { jmap: string; blobId: string }): { type: string | null; size: number; blob: Blob } | null { + return keptBlob(ctx, params.jmap, params.blobId); } /** How much of the disk this account's file takes. */ @@ -56,6 +53,10 @@ export const MAIL_CALLS: Readonly> = { 'mail.recordOpened': recordOpened, 'mail.recordBlob': recordBlob, 'mail.getBlob': getBlob, + /** A whole message for an offline export, or why not. */ + 'mail.rawBlob': (ctx, params: { jmap: string; blobId: string }) => rawBlob(ctx, params.jmap, params.blobId), + /** How many messages an export could write now, so it can say how many it can't. */ + 'mail.rawStatus': (ctx, params: { jmap: string; mailboxId?: string }) => rawStatus(ctx, params.jmap, params.mailboxId), 'mail.usage': usage, /** The session kept with the account, for a start from the local copy. */ 'mail.session': (ctx) => ctx.session, diff --git a/lib/store/mail/index.ts b/lib/store/mail/index.ts index aa0ba6e6b..a8be8481a 100644 --- a/lib/store/mail/index.ts +++ b/lib/store/mail/index.ts @@ -1,4 +1,5 @@ import type { StoreArea } from '../area'; +import { applyAttachmentScope, attachmentSyncer } from './attachments'; import { MAIL_CALLS } from './calls'; import { MAIL_METHODS } from './methods'; import { MAIL_MIGRATIONS } from './schema'; @@ -9,6 +10,7 @@ export const mailArea: StoreArea = { name: 'mail', migrations: [...MAIL_MIGRATIONS, ...SEARCH_MIGRATIONS], methods: MAIL_METHODS, - syncers: [mailSyncer, identitySyncer, quotaSyncer], + syncers: [mailSyncer, identitySyncer, quotaSyncer, attachmentSyncer], calls: MAIL_CALLS, + scopeChanged: applyAttachmentScope, }; diff --git a/lib/store/mail/records.ts b/lib/store/mail/records.ts index 147b5d5f2..f2f78412a 100644 --- a/lib/store/mail/records.ts +++ b/lib/store/mail/records.ts @@ -85,12 +85,21 @@ export function upsertEmail(db: StorageAdapter, jmap: string, email: EmailObject return rowid; } +/** Kept files are stored in pieces of this size. */ +export const BLOB_CHUNK_BYTES = 1024 * 1024; + +export function dropBlob(db: StorageAdapter, jmap: string, blobId: string): void { + db.run('DELETE FROM blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); + db.run('DELETE FROM blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); +} + +/** A file that's here as a whole already. */ export function putBlob(db: StorageAdapter, jmap: string, blobId: string, type: string | null, data: Uint8Array, at: number): void { - db.run( - `INSERT INTO blob (jmap, blob_id, type, size, at, data) VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT (jmap, blob_id) DO UPDATE SET type = excluded.type, size = excluded.size, at = excluded.at, data = excluded.data`, - [jmap, blobId, type, data.byteLength, at, data], - ); + dropBlob(db, jmap, blobId); + for (let from = 0, seq = 0; from < data.byteLength; from += BLOB_CHUNK_BYTES, seq += 1) { + db.run('INSERT INTO blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, data.subarray(from, from + BLOB_CHUNK_BYTES)]); + } + db.run("INSERT INTO blob (jmap, blob_id, type, size, at, data) VALUES (?, ?, ?, ?, ?, x'')", [jmap, blobId, type, data.byteLength, at]); } export function deleteEmail(db: StorageAdapter, jmap: string, id: string): void { @@ -225,24 +234,48 @@ function forgetParts(db: StorageAdapter, rowid: number): void { const parts = db.all<{ jmap: string; blob_id: string }>('SELECT jmap, blob_id FROM email_part WHERE email_rowid = ?', [rowid]); db.run('DELETE FROM email_part WHERE email_rowid = ?', [rowid]); for (const part of parts) { - db.run( - 'DELETE FROM blob WHERE jmap = ? AND blob_id = ? AND NOT EXISTS (SELECT 1 FROM email_part p WHERE p.jmap = ? AND p.blob_id = ?)', - [part.jmap, part.blob_id, part.jmap, part.blob_id], - ); + if (db.get('SELECT 1 AS x FROM email_part WHERE jmap = ? AND blob_id = ?', [part.jmap, part.blob_id])) continue; + dropBlob(db, part.jmap, part.blob_id); } } -function rememberParts(db: StorageAdapter, rowid: number, attachments: unknown): void { +interface MimePart { + blobId?: unknown; + size?: unknown; + type?: unknown; + name?: unknown; + disposition?: unknown; + subParts?: MimePart[]; +} + +function leafParts(part: MimePart): MimePart[] { + return part.subParts ? part.subParts.flatMap(leafParts) : [part]; +} + +/** + * The parts a message needs as files: its attachments, and whatever else the + * body shows. Stalwart 1.0 doesn't list inline images under `attachments`, so + * those come from the body lists and the MIME tree. Text parts are in bodyValues. + */ +function fileParts(body: Record): MimePart[] { + const listed = (key: string) => (Array.isArray(body[key]) ? body[key] as MimePart[] : []); + const shown = [...listed('textBody'), ...listed('htmlBody'), ...(body.bodyStructure ? leafParts(body.bodyStructure as MimePart) : [])] + .filter((part) => !(typeof part.type === 'string' && part.type.startsWith('text/')) || part.disposition === 'attachment'); + return [...listed('attachments'), ...shown]; +} + +function rememberParts(db: StorageAdapter, rowid: number, list: Record, body: Record): void { const row = db.get<{ jmap: string; received_at: number }>('SELECT jmap, received_at FROM email WHERE rowid = ?', [rowid]); - if (!row || !Array.isArray(attachments)) return; - for (const part of attachments as Array<{ blobId?: unknown; size?: unknown; type?: unknown; name?: unknown }>) { - if (typeof part.blobId !== 'string') continue; - db.run( - `INSERT INTO email_part (jmap, blob_id, email_rowid, received_at, size, type, name) VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (jmap, blob_id, email_rowid) DO NOTHING`, - [row.jmap, part.blobId, rowid, row.received_at, typeof part.size === 'number' ? part.size : 0, typeof part.type === 'string' ? part.type : null, typeof part.name === 'string' ? part.name : null], - ); + if (!row) return; + const remember = (blobId: string, size: unknown, type: unknown, name: unknown, raw: number) => db.run( + `INSERT INTO email_part (jmap, blob_id, email_rowid, received_at, size, type, name, raw) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (jmap, blob_id, email_rowid) DO NOTHING`, + [row.jmap, blobId, rowid, row.received_at, typeof size === 'number' ? size : 0, typeof type === 'string' ? type : null, typeof name === 'string' ? name : null, raw], + ); + for (const part of fileParts(body)) { + if (typeof part.blobId === 'string') remember(part.blobId, part.size, part.type, part.name, 0); } + if (typeof list.blobId === 'string') remember(list.blobId, list.size, 'message/rfc822', 'message.eml', 1); } export function storeBody(db: StorageAdapter, rowid: number, listJson: string, full: Record): void { @@ -250,7 +283,7 @@ export function storeBody(db: StorageAdapter, rowid: number, listJson: string, f const body: Record = {}; for (const [key, value] of Object.entries(full)) if (!(key in list)) body[key] = value; const hadBody = Number(db.get<{ body_state: number }>('SELECT body_state FROM email WHERE rowid = ?', [rowid])?.body_state) === 1; - rememberParts(db, rowid, body.attachments); + rememberParts(db, rowid, list, body); db.run('INSERT INTO email_body (email_rowid, json) VALUES (?, ?) ON CONFLICT (email_rowid) DO UPDATE SET json = excluded.json', [rowid, JSON.stringify(body)]); db.run('UPDATE email SET body_state = 1 WHERE rowid = ?', [rowid]); // A first body is one insert. Only a body stored again has an old row to replace. diff --git a/lib/store/mail/schema.ts b/lib/store/mail/schema.ts index 64def1243..13b7cfc1c 100644 --- a/lib/store/mail/schema.ts +++ b/lib/store/mail/schema.ts @@ -92,4 +92,22 @@ export const MAIL_MIGRATIONS: readonly Migration[] = [ ) WITHOUT ROWID; `, }, + // In the drafts range: this came with drafts and attachments. + { + id: 410, + name: 'kept files in chunks, raw messages', + sql: ` + -- Binding a whole file grows the WASM heap by its size, and that never shrinks. + -- A file counts as kept once its row in blob exists. + CREATE TABLE blob_chunk ( + jmap TEXT NOT NULL, + blob_id TEXT NOT NULL, + seq INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (jmap, blob_id, seq) + ); + -- 1 for the whole message (Email.blobId), for an export. + ALTER TABLE email_part ADD COLUMN raw INTEGER NOT NULL DEFAULT 0; + `, + }, ]; diff --git a/lib/store/mail/sync.ts b/lib/store/mail/sync.ts index ebe43fecc..c636d6f0a 100644 --- a/lib/store/mail/sync.ts +++ b/lib/store/mail/sync.ts @@ -10,14 +10,13 @@ import type { SyncContext, Syncer } from '../area'; import { deleteObject, putObject } from '../engine/core-schema'; import { runChangesLoop } from '../engine/changes'; -import { JmapMethodError, responseFor, TransportError, type MethodCall } from '../engine/transport'; +import { JmapMethodError, responseFor, type MethodCall } from '../engine/transport'; import { deleteEmail, deleteEmailRow, EMAIL_BODY_PROPERTIES, EMAIL_LIST_PROPERTIES, MAIL_USING, - putBlob, storeBody, upsertEmail, type EmailObject, @@ -113,7 +112,8 @@ async function startFullList(ctx: SyncContext, jmap: string): Promise { for (;;) { if (ctx.stopped()) return false; - const unseen = ctx.db.all<{ rowid: number; id: string }>('SELECT rowid, id FROM email WHERE jmap = ? AND gen < ? LIMIT ?', [jmap, gen, PAGE]); + // A draft saved on this device isn't on the server yet: it has no say in this. + const unseen = ctx.db.all<{ rowid: number; id: string }>("SELECT rowid, id FROM email WHERE jmap = ? AND gen < ? AND id NOT LIKE 'local-%' LIMIT ?", [jmap, gen, PAGE]); if (unseen.length === 0) return true; const response = await ctx.transport.request([['Email/get', { accountId: jmap, @@ -349,45 +349,6 @@ async function backfillBodies(ctx: SyncContext, jmap: string): Promise { } } -// ---- Attachments ---- - -/** - * The files of kept messages, newest first, within what the user chose: one - * at a time, each in its own transaction, so a reload loses at most one. - */ -async function fetchAttachments(ctx: SyncContext, jmap: string): Promise { - if (!ctx.scope.attachments) return; - let used = Number(ctx.db.get<{ n: number | null }>('SELECT SUM(size) AS n FROM blob')?.n ?? 0); - for (;;) { - if (ctx.stopped()) return; - const parts = ctx.db.all<{ blob_id: string; size: number; type: string | null; name: string | null; received_at: number }>( - `SELECT p.blob_id, p.size, p.type, p.name, p.received_at FROM email_part p - WHERE p.jmap = ? AND p.refused = 0 AND p.size <= ? - AND NOT EXISTS (SELECT 1 FROM blob b WHERE b.jmap = p.jmap AND b.blob_id = p.blob_id) - ORDER BY p.received_at DESC LIMIT 20`, - [jmap, ctx.scope.maxAttachmentBytes], - ); - if (parts.length === 0) return; - for (const part of parts) { - if (ctx.stopped()) return; - const blobId = String(part.blob_id); - if (used + Number(part.size) > ctx.scope.maxTotalBytes) return; - let data: Uint8Array; - try { - data = await ctx.transport.download(jmap, blobId, part.type ?? 'application/octet-stream', part.name ?? 'attachment'); - } catch (error) { - if (error instanceof TransportError && error.kind === 'http') { - ctx.db.run('UPDATE email_part SET refused = 1 WHERE jmap = ? AND blob_id = ?', [jmap, blobId]); - continue; - } - throw error; - } - putBlob(ctx.db, jmap, blobId, part.type, data, Number(part.received_at)); - used += data.byteLength; - } - } -} - // ---- Identities and quotas ---- const IDENTITY_USING = ['urn:ietf:params:jmap:core', 'urn:ietf:params:jmap:mail', 'urn:ietf:params:jmap:submission']; @@ -477,7 +438,5 @@ export const mailSyncer: Syncer = { await syncThreads(ctx, jmap); if (ctx.stopped()) return; await backfillBodies(ctx, jmap); - if (ctx.stopped()) return; - await fetchAttachments(ctx, jmap); }, }; diff --git a/lib/store/outbox/calls.ts b/lib/store/outbox/calls.ts index d34e38949..a87c9be15 100644 --- a/lib/store/outbox/calls.ts +++ b/lib/store/outbox/calls.ts @@ -1,6 +1,7 @@ /** What the page asks the outbox. Each call is one SQLite transaction, so two tabs can't interleave. */ import type { AreaCall, StoreContext } from '../area'; +import { heldBlobsOf, type DraftCreate } from '../drafts/local-email'; import { metaGet, metaSet } from '../engine/core-schema'; import { deleteEmail, upsertEmail, type EmailObject } from '../mail/records'; import { nextSeq, queueUpdate, type PatchRow } from './email-set'; @@ -130,40 +131,102 @@ function settleStale(ctx: StoreContext, params: { before: number }): number { ).changes; } -/** Removes an entry and the files it held. */ +/** Every held file a waiting message or a draft on this device still needs. */ +function neededFiles(ctx: StoreContext): Set { + const forSends = ctx.db.all('SELECT * FROM outbox').flatMap((row) => heldBlobIds(toEntry(row).item)); + const forDrafts = ctx.db.all<{ create_json: string }>('SELECT create_json FROM draft_pending').flatMap((row) => heldBlobsOf(JSON.parse(row.create_json) as DraftCreate)); + return new Set([...forSends, ...forDrafts]); +} + +function dropHeld(ctx: StoreContext, blobId: string): void { + ctx.db.run('DELETE FROM upload_chunk WHERE local_id = ?', [blobId]); + ctx.db.run('DELETE FROM upload WHERE local_id = ?', [blobId]); +} + +/** Removes an entry and the files only it held. A draft may share one with the message it became. */ function remove(ctx: StoreContext, params: { id: number }): void { ctx.db.transaction(() => { const row = ctx.db.get('SELECT * FROM outbox WHERE id = ?', [params.id]); if (!row) return; - for (const blobId of heldBlobIds(toEntry(row).item)) ctx.db.run('DELETE FROM upload WHERE local_id = ?', [blobId]); ctx.db.run('DELETE FROM outbox WHERE id = ?', [params.id]); + const needed = neededFiles(ctx); + for (const blobId of heldBlobIds(toEntry(row).item)) if (!needed.has(blobId)) dropHeld(ctx, blobId); }); } // ---- Files attached offline ---- -function holdFile(ctx: StoreContext, params: { jmap?: string; name: string; type: string; data: Uint8Array; localId?: string }): { blobId: string; size: number; type: string } { - const blobId = params.localId ?? `${LOCAL_BLOB_PREFIX}${crypto.randomUUID()}`; +/** The size the page cuts a file into on its way here. */ +export const HOLD_CHUNK_BYTES = 1024 * 1024; + +/** A file that comes in pieces: nothing can read it until the last one is in. */ +function holdStart(ctx: StoreContext, params: { jmap?: string; name: string; type: string; size: number }): { blobId: string } { + const blobId = `${LOCAL_BLOB_PREFIX}${crypto.randomUUID()}`; ctx.db.run( - 'INSERT OR REPLACE INTO upload (local_id, jmap, type, name, size, data, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)', - [blobId, params.jmap ?? null, params.type, params.name, params.data.byteLength, params.data, ctx.now()], + "INSERT INTO upload (local_id, jmap, type, name, size, data, created_at, complete) VALUES (?, ?, ?, ?, ?, x'', ?, 0)", + [blobId, params.jmap ?? null, params.type, params.name, params.size, ctx.now()], ); - return { blobId, size: params.data.byteLength, type: params.type || 'application/octet-stream' }; + return { blobId }; } -function heldFile(ctx: StoreContext, params: { blobId: string }): { name: string; type: string; data: Uint8Array } | null { - const row = ctx.db.get<{ name: string | null; type: string | null; data: Uint8Array }>('SELECT name, type, data FROM upload WHERE local_id = ?', [params.blobId]); - return row ? { name: row.name ?? 'attachment', type: row.type ?? 'application/octet-stream', data: row.data } : null; +function holdChunk(ctx: StoreContext, params: { blobId: string; seq: number; data: Uint8Array }): void { + ctx.db.run('INSERT OR REPLACE INTO upload_chunk (local_id, seq, data) VALUES (?, ?, ?)', [params.blobId, params.seq, params.data]); } -/** Files held for no entry (attached offline to a message that was then discarded). SEAM: the drafts agent keeps its drafts' files here too. */ +function holdEnd(ctx: StoreContext, params: { blobId: string }): { blobId: string; size: number; type: string } { + ctx.db.run('UPDATE upload SET complete = 1 WHERE local_id = ?', [params.blobId]); + const row = ctx.db.get<{ size: number; type: string | null }>('SELECT size, type FROM upload WHERE local_id = ?', [params.blobId]); + return { blobId: params.blobId, size: Number(row?.size ?? 0), type: row?.type || 'application/octet-stream' }; +} + +function holdAbort(ctx: StoreContext, params: { blobId: string }): void { + ctx.db.transaction(() => dropHeld(ctx, params.blobId)); +} + +/** A file that's here as a whole already: the old database's, on its move here. */ +function holdFile(ctx: StoreContext, params: { jmap?: string; name: string; type: string; data: Uint8Array; localId?: string }): { blobId: string; size: number; type: string } { + return ctx.db.transaction(() => { + const blobId = params.localId ?? `${LOCAL_BLOB_PREFIX}${crypto.randomUUID()}`; + dropHeld(ctx, blobId); + ctx.db.run( + "INSERT INTO upload (local_id, jmap, type, name, size, data, created_at) VALUES (?, ?, ?, ?, ?, x'', ?)", + [blobId, params.jmap ?? null, params.type, params.name, params.data.byteLength, ctx.now()], + ); + for (let at = 0, seq = 0; at < params.data.byteLength; at += HOLD_CHUNK_BYTES, seq += 1) { + ctx.db.run('INSERT INTO upload_chunk (local_id, seq, data) VALUES (?, ?, ?)', [blobId, seq, params.data.subarray(at, at + HOLD_CHUNK_BYTES)]); + } + return { blobId, size: params.data.byteLength, type: params.type || 'application/octet-stream' }; + }); +} + +/** Piece by piece into a Blob, which the browser can keep out of this worker's memory. */ +function heldFile(ctx: StoreContext, params: { blobId: string }): { name: string; type: string; size: number; blob: Blob } | null { + const row = ctx.db.get<{ name: string | null; type: string | null; size: number; data: Uint8Array }>('SELECT name, type, size, data FROM upload WHERE local_id = ? AND complete = 1', [params.blobId]); + if (!row) return null; + const type = row.type || 'application/octet-stream'; + // A file from before chunks has its bytes in the row; a new one has none there. + let blob = new Blob(row.data?.byteLength ? [row.data as BlobPart] : [], { type }); + for (let seq = 0; ; seq += 1) { + const chunk = ctx.db.get<{ data: Uint8Array }>('SELECT data FROM upload_chunk WHERE local_id = ? AND seq = ?', [params.blobId, seq]); + if (!chunk) break; + // Each step only refers to the last Blob, so one chunk at a time is in memory here. + blob = new Blob([blob, chunk.data as BlobPart], { type }); + } + return { name: row.name ?? 'attachment', type, size: Number(row.size), blob }; +} + +/** + * Files nothing needs any more: attached to a message that was then discarded, + * or sent or saved long enough ago. Not at once, because an open composer may + * still show one. + */ function dropOrphans(ctx: StoreContext, params: { olderThan: number }): number { return ctx.db.transaction(() => { - const needed = new Set(ctx.db.all('SELECT * FROM outbox').flatMap((row) => heldBlobIds(toEntry(row).item))); + const needed = neededFiles(ctx); let dropped = 0; for (const row of ctx.db.all<{ local_id: string }>('SELECT local_id FROM upload WHERE created_at < ?', [params.olderThan])) { if (needed.has(String(row.local_id))) continue; - ctx.db.run('DELETE FROM upload WHERE local_id = ?', [row.local_id]); + dropHeld(ctx, String(row.local_id)); dropped += 1; } return dropped; @@ -274,6 +337,10 @@ export const OUTBOX_CALLS: Readonly> = { 'outbox.settleStale': settleStale, 'outbox.remove': remove, 'outbox.holdFile': holdFile, + 'outbox.holdStart': holdStart, + 'outbox.holdChunk': holdChunk, + 'outbox.holdEnd': holdEnd, + 'outbox.holdAbort': holdAbort, 'outbox.heldFile': heldFile, 'outbox.dropOrphans': dropOrphans, 'outbox.pendingPatches': pendingPatches, diff --git a/lib/store/outbox/email-set.ts b/lib/store/outbox/email-set.ts index 5451e1c21..51e2bae25 100644 --- a/lib/store/outbox/email-set.ts +++ b/lib/store/outbox/email-set.ts @@ -4,7 +4,10 @@ * same way. Answers carry no state: a made-up one would reach the server later. */ +import { isStandInId } from '@/lib/jmap/stand-in-ids'; import { MethodError, type MethodContext, type MethodHandler, type RecordOverlay } from '../area'; +import { createLocalDraft, destroyForDraft, updateLocalDraft } from '../drafts/pending'; +import type { DraftCreate } from '../drafts/local-email'; import { metaGet, metaSet } from '../engine/core-schema'; import { deleteEmail, upsertEmail, type EmailObject } from '../mail/records'; import type { StorageAdapter } from '../storage'; @@ -24,6 +27,7 @@ export interface PatchRow { } const EMAIL = 'Email'; +export const EMAIL_TYPE = EMAIL; /** The next place in the order changes and sends were made. */ export function nextSeq(db: StorageAdapter): number { @@ -32,11 +36,11 @@ export function nextSeq(db: StorageAdapter): number { return next; } -function pendingRow(db: StorageAdapter, jmap: string, id: string): PatchRow | undefined { +export function pendingRow(db: StorageAdapter, jmap: string, id: string): PatchRow | undefined { return db.get('SELECT * FROM outbox_patch WHERE jmap = ? AND type = ? AND record_id = ?', [jmap, EMAIL, id]); } -function savePatch(db: StorageAdapter, row: Omit, now: number): void { +export function savePatch(db: StorageAdapter, row: Omit, now: number): void { db.run( `INSERT INTO outbox_patch (jmap, type, record_id, op, patch, base, server, subject, seq, queued_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -45,7 +49,7 @@ function savePatch(db: StorageAdapter, row: Omit('SELECT json, gen FROM email WHERE jmap = ? AND id = ?', [jmap, id]); return row ? { email: JSON.parse(row.json) as EmailObject, gen: Number(row.gen) } : null; } @@ -90,13 +94,16 @@ export function queueDestroy(db: StorageAdapter, jmap: string, id: string, now: if (local) deleteEmail(db, jmap, id); } -/** - * SEAM for the drafts agent: Email/set create is theirs. Until it lands every - * create is refused per message, so createDraft throws and the composer shows - * its "draft not saved" state, as it does today. - */ -function createEmails(_ctx: MethodContext, _jmap: string, create: Record): Record { - return Object.fromEntries(Object.keys(create).map((id) => [id, { type: 'serverUnavailable', description: 'Saving a draft needs the server' }])); +/** A draft stays on this device until it can go up (lib/store/drafts). Any other new message needs the server. */ +function createEmails(ctx: MethodContext, jmap: string, create: Record, networkTried: boolean): { created: Record; notCreated: Record } { + const created: Record = {}; + const notCreated: Record = {}; + for (const [creationId, email] of Object.entries(create)) { + const { refused, ...answer } = createLocalDraft(ctx, jmap, email as DraftCreate, networkTried); + if (refused) notCreated[creationId] = answer; + else created[creationId] = answer; + } + return { created, notCreated }; } export const emailSet: MethodHandler = (ctx, args) => { @@ -107,14 +114,15 @@ export const emailSet: MethodHandler = (ctx, args) => { const result: Record = { accountId: jmap }; ctx.db.transaction(() => { if (args.create && Object.keys(args.create as object).length > 0) { - result.created = {}; - result.notCreated = createEmails(ctx, jmap, args.create as Record); + const { created, notCreated } = createEmails(ctx, jmap, args.create as Record, args.networkTried === true); + result.created = created; + if (Object.keys(notCreated).length > 0) result.notCreated = notCreated; } if (args.update) { const updated: Record = {}; const notUpdated: Record = {}; for (const [id, patch] of Object.entries(args.update as Record)) { - const problem = queueUpdate(ctx.db, jmap, id, patch, ctx.now()); + const problem = isStandInId(id) ? updateLocalDraft(ctx, jmap, id, patch) : queueUpdate(ctx.db, jmap, id, patch, ctx.now()); if (problem) notUpdated[id] = problem; else updated[id] = null; } @@ -122,7 +130,10 @@ export const emailSet: MethodHandler = (ctx, args) => { if (Object.keys(notUpdated).length > 0) result.notUpdated = notUpdated; } if (Array.isArray(args.destroy)) { - for (const id of args.destroy as string[]) queueDestroy(ctx.db, jmap, id, ctx.now()); + const replacedBy = typeof args.replacedBy === 'string' ? args.replacedBy : undefined; + for (const id of args.destroy as string[]) { + if (!destroyForDraft(ctx, jmap, id, replacedBy)) queueDestroy(ctx.db, jmap, id, ctx.now()); + } result.destroyed = args.destroy; } }); diff --git a/lib/store/outbox/index.ts b/lib/store/outbox/index.ts index 4d65885cb..6c812f8ba 100644 --- a/lib/store/outbox/index.ts +++ b/lib/store/outbox/index.ts @@ -1,13 +1,19 @@ -import type { StoreArea } from '../area'; +import type { RecordOverlay, StoreArea } from '../area'; +import { DRAFT_CALLS } from '../drafts/calls'; +import { draftsOverlay } from '../drafts/pending'; +import { DRAFT_MIGRATIONS } from '../drafts/schema'; import { OUTBOX_CALLS } from './calls'; import { emailSet, outboxOverlay } from './email-set'; import { OUTBOX_MIGRATIONS } from './schema'; +const overlay: RecordOverlay = (ctx, jmap, type, records) => draftsOverlay(ctx, jmap, type, outboxOverlay(ctx, jmap, type, records)); + +/** Drafts answer the same Email/set and share the queue's order, so they plug in here. */ export const outboxArea: StoreArea = { name: 'outbox', - migrations: OUTBOX_MIGRATIONS, + migrations: [...OUTBOX_MIGRATIONS, ...DRAFT_MIGRATIONS], methods: { 'Email/set': emailSet }, syncers: [], - calls: OUTBOX_CALLS, - overlay: outboxOverlay, + calls: { ...OUTBOX_CALLS, ...DRAFT_CALLS }, + overlay, }; diff --git a/lib/store/outbox/items.ts b/lib/store/outbox/items.ts index 259c11877..1a4c63c62 100644 --- a/lib/store/outbox/items.ts +++ b/lib/store/outbox/items.ts @@ -89,7 +89,8 @@ export type ConflictReason = 'changed' | 'gone' | 'refused'; export interface ConflictSummary { subject: string; - op: 'update' | 'destroy'; + /** `draft`: a draft saved here met a version changed elsewhere. Both are kept. */ + op: 'update' | 'destroy' | 'draft'; reason: ConflictReason; /** The server's word when it refused. */ detail?: string; diff --git a/locales/ar/common.json b/locales/ar/common.json index ae8e0aa58..b71c6f9bb 100644 --- a/locales/ar/common.json +++ b/locales/ar/common.json @@ -5756,6 +5756,25 @@ "toast": "تغييرات أُجريت دون اتصال ولم تُطبَّق: {count}. يوضح صندوق الصادر أيها.", "storage_failed": "لم تعد هناك مساحة على هذا الجهاز للبريد دون اتصال. أفرغ مساحة أو احتفظ بأقل من الإعدادات." }, + "offlineDrafts": { + "title": "المسودات على هذا الجهاز", + "description": "هذه المسودات محفوظة على هذا الجهاز. ستنتقل إلى الخادم تلقائيًا فور إمكان الوصول إليه.", + "saved_on_device": "محفوظة على هذا الجهاز", + "status_waiting": "في انتظار الخادم", + "status_sending": "جارٍ الحفظ على الخادم", + "status_uncertain": "جارٍ التحقق من وصولها إلى الخادم", + "status_refused": "رفض الخادم هذه المسودة: {error}", + "file_missing": "لم يعد ملف مرفق موجودًا على هذا الجهاز", + "open": "فتح", + "open_label": "فتح {subject}", + "retry": "إعادة المحاولة", + "retry_label": "إعادة محاولة {subject}", + "toast_refused": "تعذر حفظ المسودة {subject} على الخادم. ستبقى على هذا الجهاز.", + "toast_conflict": "تم تغيير المسودة {subject} على جهاز آخر. يتم الاحتفاظ بالنسختين.", + "conflict_changed": "تم التغيير على جهاز آخر. النسختان في المسودات.", + "conflict_gone": "تم التغيير أو الإزالة على جهاز آخر. نسختك في المسودات.", + "banner_refused": "المسودات التي رفضها الخادم: {count}" + }, "deep_link": { "message_not_found": "لم تعد هذه الرسالة متاحة.", "thread_not_found": "لم تعد هذه المحادثة متاحة.", diff --git a/locales/ca/common.json b/locales/ca/common.json index ce62318b7..c1cbfe6df 100644 --- a/locales/ca/common.json +++ b/locales/ca/common.json @@ -5756,6 +5756,25 @@ "toast": "Canvis fets sense connexió que no s'han aplicat: {count}. La safata de sortida indica quins.", "storage_failed": "Aquest dispositiu no té més espai per al correu sense connexió. Allibereu espai o conserveu-ne menys a la configuració." }, + "offlineDrafts": { + "title": "Esborranys en aquest dispositiu", + "description": "Aquests esborranys estan desats en aquest dispositiu. Aniran al servidor sols tan bon punt s'hi pugui connectar.", + "saved_on_device": "Desat en aquest dispositiu", + "status_waiting": "Esperant el servidor", + "status_sending": "Desant al servidor", + "status_uncertain": "Comprovant si ha arribat al servidor", + "status_refused": "El servidor ha rebutjat aquest esborrany: {error}", + "file_missing": "Un fitxer adjunt ja no és en aquest dispositiu", + "open": "Obre", + "open_label": "Obre {subject}", + "retry": "Torna-ho a provar", + "retry_label": "Torna a provar {subject}", + "toast_refused": "L'esborrany {subject} no s'ha pogut desar al servidor. Es queda en aquest dispositiu.", + "toast_conflict": "L'esborrany {subject} s'ha canviat en un altre dispositiu. Es conserven les dues versions.", + "conflict_changed": "Canviat en un altre dispositiu. Les dues versions són a Esborranys.", + "conflict_gone": "Canviat o eliminat en un altre dispositiu. La vostra versió és a Esborranys.", + "banner_refused": "Esborranys rebutjats pel servidor: {count}" + }, "deep_link": { "message_not_found": "Aquest missatge ja no està disponible.", "thread_not_found": "Aquesta conversa ja no està disponible.", diff --git a/locales/cs/common.json b/locales/cs/common.json index a37357845..81fe6034b 100644 --- a/locales/cs/common.json +++ b/locales/cs/common.json @@ -5756,6 +5756,25 @@ "toast": "Nepoužité změny provedené offline: {count}. Které to jsou, ukazuje pošta k odeslání.", "storage_failed": "Na tomto zařízení už není místo pro poštu offline. Uvolněte místo nebo v nastavení ponechte méně." }, + "offlineDrafts": { + "title": "Koncepty v tomto zařízení", + "description": "Tyto koncepty jsou uložené v tomto zařízení. Na server se odešlou samy, jakmile bude dostupný.", + "saved_on_device": "Uloženo v tomto zařízení", + "status_waiting": "Čeká na server", + "status_sending": "Ukládá se na server", + "status_uncertain": "Ověřuje se, zda dorazil na server", + "status_refused": "Server tento koncept odmítl: {error}", + "file_missing": "Přiložený soubor už v tomto zařízení není", + "open": "Otevřít", + "open_label": "Otevřít {subject}", + "retry": "Zkusit znovu", + "retry_label": "Zkusit znovu: {subject}", + "toast_refused": "Koncept {subject} se nepodařilo uložit na server. Zůstává v tomto zařízení.", + "toast_conflict": "Koncept {subject} byl změněn v jiném zařízení. Obě verze zůstávají zachovány.", + "conflict_changed": "Změněno v jiném zařízení. Obě verze jsou v Konceptech.", + "conflict_gone": "Změněno nebo odstraněno v jiném zařízení. Vaše verze je v Konceptech.", + "banner_refused": "Koncepty odmítnuté serverem: {count}" + }, "deep_link": { "message_not_found": "Tato zpráva už není dostupná.", "thread_not_found": "Tato konverzace už není dostupná.", diff --git a/locales/da/common.json b/locales/da/common.json index b024c725c..eb6bfcddf 100644 --- a/locales/da/common.json +++ b/locales/da/common.json @@ -5756,6 +5756,25 @@ "toast": "Ændringer lavet offline, der ikke blev gennemført: {count}. Udbakken viser hvilke.", "storage_failed": "Denne enhed har ikke mere plads til offlinemail. Frigør plads, eller behold mindre i indstillingerne." }, + "offlineDrafts": { + "title": "Kladder på denne enhed", + "description": "Disse kladder er gemt på denne enhed. De sendes selv til serveren, så snart den kan nås.", + "saved_on_device": "Gemt på denne enhed", + "status_waiting": "Venter på serveren", + "status_sending": "Gemmer på serveren", + "status_uncertain": "Tjekker, om den nåede serveren", + "status_refused": "Serveren afviste denne kladde: {error}", + "file_missing": "En vedhæftet fil er ikke længere på denne enhed", + "open": "Åbn", + "open_label": "Åbn {subject}", + "retry": "Prøv igen", + "retry_label": "Prøv {subject} igen", + "toast_refused": "Kladden {subject} kunne ikke gemmes på serveren. Den bliver på denne enhed.", + "toast_conflict": "Kladden {subject} blev ændret på en anden enhed. Begge versioner beholdes.", + "conflict_changed": "Ændret på en anden enhed. Begge versioner ligger i Kladder.", + "conflict_gone": "Ændret eller fjernet på en anden enhed. Din version ligger i Kladder.", + "banner_refused": "Kladder afvist af serveren: {count}" + }, "deep_link": { "message_not_found": "Denne besked er ikke længere tilgængelig.", "thread_not_found": "Denne samtale er ikke længere tilgængelig.", diff --git a/locales/de/common.json b/locales/de/common.json index 2dc411640..4e85d0bc2 100644 --- a/locales/de/common.json +++ b/locales/de/common.json @@ -5756,6 +5756,25 @@ "toast": "Offline gemachte Änderungen wurden nicht übernommen: {count}. Welche, steht im Postausgang.", "storage_failed": "Auf diesem Gerät ist kein Platz mehr für Offline-E-Mails. Geben Sie Speicher frei oder behalten Sie in den Einstellungen weniger." }, + "offlineDrafts": { + "title": "Entwürfe auf diesem Gerät", + "description": "Diese Entwürfe sind auf diesem Gerät gespeichert. Sie gehen von selbst an den Server, sobald er erreichbar ist.", + "saved_on_device": "Auf diesem Gerät gespeichert", + "status_waiting": "Wartet auf den Server", + "status_sending": "Wird auf dem Server gespeichert", + "status_uncertain": "Es wird geprüft, ob er den Server erreicht hat", + "status_refused": "Der Server hat diesen Entwurf abgelehnt: {error}", + "file_missing": "Eine angehängte Datei ist nicht mehr auf diesem Gerät", + "open": "Öffnen", + "open_label": "{subject} öffnen", + "retry": "Erneut versuchen", + "retry_label": "{subject} erneut versuchen", + "toast_refused": "Der Entwurf {subject} konnte nicht auf dem Server gespeichert werden. Er bleibt auf diesem Gerät.", + "toast_conflict": "Der Entwurf {subject} wurde auf einem anderen Gerät geändert. Beide Fassungen bleiben erhalten.", + "conflict_changed": "Auf einem anderen Gerät geändert. Beide Fassungen liegen in den Entwürfen.", + "conflict_gone": "Auf einem anderen Gerät geändert oder entfernt. Ihre Fassung liegt in den Entwürfen.", + "banner_refused": "Vom Server abgelehnte Entwürfe: {count}" + }, "deep_link": { "message_not_found": "Diese Nachricht ist nicht mehr verfügbar.", "thread_not_found": "Diese Unterhaltung ist nicht mehr verfügbar.", diff --git a/locales/en/common.json b/locales/en/common.json index fdb1eac13..fb4d1658c 100644 --- a/locales/en/common.json +++ b/locales/en/common.json @@ -5756,6 +5756,25 @@ "toast": "Changes you made offline weren't applied: {count}. The outbox shows which.", "storage_failed": "This device has no room left for offline mail. Free up space, or keep less in Settings." }, + "offlineDrafts": { + "title": "Drafts on this device", + "description": "These drafts are saved on this device. They go to the server by themselves once it can be reached.", + "saved_on_device": "Saved on this device", + "status_waiting": "Waiting for the server", + "status_sending": "Saving to the server", + "status_uncertain": "Checking whether it reached the server", + "status_refused": "The server refused this draft: {error}", + "file_missing": "A file attached to it is no longer on this device", + "open": "Open", + "open_label": "Open {subject}", + "retry": "Try again", + "retry_label": "Try {subject} again", + "toast_refused": "The draft {subject} couldn't be saved on the server. It stays on this device.", + "toast_conflict": "The draft {subject} was changed on another device. Both versions are kept.", + "conflict_changed": "Changed on another device. Both versions are in Drafts.", + "conflict_gone": "Changed or removed on another device. Your version is in Drafts.", + "banner_refused": "Drafts the server refused: {count}" + }, "deep_link": { "message_not_found": "This message is no longer available.", "thread_not_found": "This conversation is no longer available.", diff --git a/locales/es/common.json b/locales/es/common.json index 7b90943c5..36bbe848a 100644 --- a/locales/es/common.json +++ b/locales/es/common.json @@ -5756,6 +5756,25 @@ "toast": "Cambios hechos sin conexión que no se aplicaron: {count}. La bandeja de salida indica cuáles.", "storage_failed": "Este dispositivo no tiene espacio para el correo sin conexión. Libera espacio o guarda menos en Ajustes." }, + "offlineDrafts": { + "title": "Borradores en este dispositivo", + "description": "Estos borradores están guardados en este dispositivo. Irán al servidor por sí solos en cuanto se pueda conectar con él.", + "saved_on_device": "Guardado en este dispositivo", + "status_waiting": "Esperando al servidor", + "status_sending": "Guardando en el servidor", + "status_uncertain": "Comprobando si llegó al servidor", + "status_refused": "El servidor rechazó este borrador: {error}", + "file_missing": "Un archivo adjunto ya no está en este dispositivo", + "open": "Abrir", + "open_label": "Abrir {subject}", + "retry": "Reintentar", + "retry_label": "Reintentar {subject}", + "toast_refused": "El borrador {subject} no se pudo guardar en el servidor. Se queda en este dispositivo.", + "toast_conflict": "El borrador {subject} se cambió en otro dispositivo. Se conservan las dos versiones.", + "conflict_changed": "Cambiado en otro dispositivo. Las dos versiones están en Borradores.", + "conflict_gone": "Cambiado o eliminado en otro dispositivo. Su versión está en Borradores.", + "banner_refused": "Borradores rechazados por el servidor: {count}" + }, "deep_link": { "message_not_found": "Este mensaje ya no está disponible.", "thread_not_found": "Esta conversación ya no está disponible.", diff --git a/locales/et/common.json b/locales/et/common.json index a6b5b5eb2..716b282ad 100644 --- a/locales/et/common.json +++ b/locales/et/common.json @@ -5756,6 +5756,25 @@ "toast": "Võrguühenduseta tehtud rakendamata muudatused: {count}. Väljundkaust näitab, millised.", "storage_failed": "Selles seadmes pole enam ruumi võrguühenduseta kirjade jaoks. Vabastage ruumi või hoidke seadetes vähem." }, + "offlineDrafts": { + "title": "Mustandid selles seadmes", + "description": "Need mustandid on salvestatud selles seadmes. Need lähevad serverisse ise, kui server on kättesaadav.", + "saved_on_device": "Salvestatud selles seadmes", + "status_waiting": "Ootab serverit", + "status_sending": "Salvestamine serverisse", + "status_uncertain": "Kontrollimine, kas see jõudis serverisse", + "status_refused": "Server keeldus sellest mustandist: {error}", + "file_missing": "Manustatud faili pole enam selles seadmes", + "open": "Ava", + "open_label": "Ava {subject}", + "retry": "Proovi uuesti", + "retry_label": "Proovi uuesti: {subject}", + "toast_refused": "Mustandit {subject} ei saanud serverisse salvestada. See jääb sellesse seadmesse.", + "toast_conflict": "Mustandit {subject} muudeti teises seadmes. Mõlemad versioonid jäävad alles.", + "conflict_changed": "Muudetud teises seadmes. Mõlemad versioonid on kaustas Mustandid.", + "conflict_gone": "Muudetud või eemaldatud teises seadmes. Teie versioon on kaustas Mustandid.", + "banner_refused": "Serveri tagasi lükatud mustandid: {count}" + }, "deep_link": { "message_not_found": "See sõnum ei ole enam saadaval.", "thread_not_found": "See vestlus ei ole enam saadaval.", diff --git a/locales/fa/common.json b/locales/fa/common.json index 270c6c3a8..9778450e8 100644 --- a/locales/fa/common.json +++ b/locales/fa/common.json @@ -5756,6 +5756,25 @@ "toast": "تغییرات آفلاین که اعمال نشدند: {count}. صندوق خروجی نشان می‌دهد کدام‌ها.", "storage_failed": "در این دستگاه جایی برای ایمیل آفلاین نمانده است. فضا را آزاد کنید یا در تنظیمات مقدار کمتری نگه دارید." }, + "offlineDrafts": { + "title": "پیش‌نویس‌های این دستگاه", + "description": "این پیش‌نویس‌ها روی این دستگاه ذخیره شده‌اند. به‌محض دسترسی به سرور، خودکار به آن می‌روند.", + "saved_on_device": "روی این دستگاه ذخیره شد", + "status_waiting": "در انتظار سرور", + "status_sending": "در حال ذخیره در سرور", + "status_uncertain": "در حال بررسی رسیدن آن به سرور", + "status_refused": "سرور این پیش‌نویس را رد کرد: {error}", + "file_missing": "یک فایل پیوست دیگر روی این دستگاه نیست", + "open": "باز کردن", + "open_label": "باز کردن {subject}", + "retry": "تلاش دوباره", + "retry_label": "تلاش دوباره برای {subject}", + "toast_refused": "پیش‌نویس {subject} در سرور ذخیره نشد. روی این دستگاه می‌ماند.", + "toast_conflict": "پیش‌نویس {subject} در دستگاه دیگری تغییر کرد. هر دو نسخه نگه داشته می‌شوند.", + "conflict_changed": "در دستگاه دیگری تغییر کرد. هر دو نسخه در پیش‌نویس‌ها هستند.", + "conflict_gone": "در دستگاه دیگری تغییر کرد یا حذف شد. نسخه شما در پیش‌نویس‌ها است.", + "banner_refused": "پیش‌نویس‌های ردشده توسط سرور: {count}" + }, "deep_link": { "message_not_found": "این پیام دیگر در دسترس نیست.", "thread_not_found": "این گفتگو دیگر در دسترس نیست.", diff --git a/locales/fr/common.json b/locales/fr/common.json index dae94c9c7..4e5d3bdc9 100644 --- a/locales/fr/common.json +++ b/locales/fr/common.json @@ -5756,6 +5756,25 @@ "toast": "Des modifications faites hors ligne n'ont pas été appliquées : {count}. La boîte d'envoi indique lesquelles.", "storage_failed": "Cet appareil n'a plus de place pour le courrier hors ligne. Libérez de l'espace ou conservez moins dans les paramètres." }, + "offlineDrafts": { + "title": "Brouillons sur cet appareil", + "description": "Ces brouillons sont enregistrés sur cet appareil. Ils partent seuls vers le serveur dès qu'il est joignable.", + "saved_on_device": "Enregistré sur cet appareil", + "status_waiting": "En attente du serveur", + "status_sending": "Enregistrement sur le serveur", + "status_uncertain": "Vérification de son arrivée sur le serveur", + "status_refused": "Le serveur a refusé ce brouillon : {error}", + "file_missing": "Un fichier joint n'est plus sur cet appareil", + "open": "Ouvrir", + "open_label": "Ouvrir {subject}", + "retry": "Réessayer", + "retry_label": "Réessayer {subject}", + "toast_refused": "Le brouillon {subject} n'a pas pu être enregistré sur le serveur. Il reste sur cet appareil.", + "toast_conflict": "Le brouillon {subject} a été modifié sur un autre appareil. Les deux versions sont conservées.", + "conflict_changed": "Modifié sur un autre appareil. Les deux versions sont dans Brouillons.", + "conflict_gone": "Modifié ou supprimé sur un autre appareil. Votre version est dans Brouillons.", + "banner_refused": "Brouillons refusés par le serveur : {count}" + }, "deep_link": { "message_not_found": "Ce message n'est plus disponible.", "thread_not_found": "Cette conversation n'est plus disponible.", diff --git a/locales/he/common.json b/locales/he/common.json index 6053b7fc7..c2653773a 100644 --- a/locales/he/common.json +++ b/locales/he/common.json @@ -5757,6 +5757,25 @@ "toast": "שינויים במצב לא מקוון שלא הוחלו: {count}. דואר יוצא מראה אילו.", "storage_failed": "אין יותר מקום במכשיר הזה לדואר לא מקוון. יש לפנות מקום או לשמור פחות בהגדרות." }, + "offlineDrafts": { + "title": "טיוטות במכשיר זה", + "description": "הטיוטות האלה שמורות במכשיר זה. הן יעברו לשרת מעצמן ברגע שאפשר יהיה להגיע אליו.", + "saved_on_device": "נשמר במכשיר זה", + "status_waiting": "ממתין לשרת", + "status_sending": "שומר בשרת", + "status_uncertain": "בודק אם הגיע לשרת", + "status_refused": "השרת דחה את הטיוטה: {error}", + "file_missing": "קובץ מצורף כבר לא נמצא במכשיר זה", + "open": "פתיחה", + "open_label": "פתיחת {subject}", + "retry": "ניסיון חוזר", + "retry_label": "ניסיון חוזר עבור {subject}", + "toast_refused": "לא ניתן היה לשמור את הטיוטה {subject} בשרת. היא נשארת במכשיר זה.", + "toast_conflict": "הטיוטה {subject} שונתה במכשיר אחר. שתי הגרסאות נשמרות.", + "conflict_changed": "שונה במכשיר אחר. שתי הגרסאות נמצאות בטיוטות.", + "conflict_gone": "שונה או הוסר במכשיר אחר. הגרסה שלך נמצאת בטיוטות.", + "banner_refused": "טיוטות שהשרת דחה: {count}" + }, "deep_link": { "message_not_found": "הודעה זו אינה זמינה עוד.", "thread_not_found": "שיחה זו אינה זמינה עוד.", diff --git a/locales/hu/common.json b/locales/hu/common.json index 07cca9946..040c5ba4b 100644 --- a/locales/hu/common.json +++ b/locales/hu/common.json @@ -5756,6 +5756,25 @@ "toast": "Offline végzett, nem alkalmazott módosítások: {count}. A postázandók mappa mutatja, melyek.", "storage_failed": "Ezen az eszközön nincs több hely az offline leveleknek. Szabadítson fel helyet, vagy tartson meg kevesebbet a beállításokban." }, + "offlineDrafts": { + "title": "Piszkozatok ezen az eszközön", + "description": "Ezek a piszkozatok ezen az eszközön vannak mentve. Maguktól a szerverre kerülnek, amint az elérhető.", + "saved_on_device": "Mentve ezen az eszközön", + "status_waiting": "Várakozás a szerverre", + "status_sending": "Mentés a szerverre", + "status_uncertain": "Annak ellenőrzése, hogy elérte-e a szervert", + "status_refused": "A szerver elutasította ezt a piszkozatot: {error}", + "file_missing": "Egy csatolt fájl már nincs ezen az eszközön", + "open": "Megnyitás", + "open_label": "{subject} megnyitása", + "retry": "Újra", + "retry_label": "{subject} újrapróbálása", + "toast_refused": "A(z) {subject} piszkozatot nem sikerült a szerverre menteni. Ezen az eszközön marad.", + "toast_conflict": "A(z) {subject} piszkozat egy másik eszközön módosult. Mindkét változat megmarad.", + "conflict_changed": "Másik eszközön módosult. Mindkét változat a Piszkozatok között van.", + "conflict_gone": "Másik eszközön módosult vagy törlődött. Az Ön változata a Piszkozatok között van.", + "banner_refused": "A szerver által elutasított piszkozatok: {count}" + }, "deep_link": { "message_not_found": "Ez az üzenet már nem érhető el.", "thread_not_found": "Ez a beszélgetés már nem érhető el.", diff --git a/locales/it/common.json b/locales/it/common.json index 398c64b80..be0943c4e 100644 --- a/locales/it/common.json +++ b/locales/it/common.json @@ -5756,6 +5756,25 @@ "toast": "Modifiche fatte offline non applicate: {count}. La posta in uscita indica quali.", "storage_failed": "Questo dispositivo non ha più spazio per la posta offline. Libera spazio o conservane meno nelle impostazioni." }, + "offlineDrafts": { + "title": "Bozze su questo dispositivo", + "description": "Queste bozze sono salvate su questo dispositivo. Passano da sole al server appena è raggiungibile.", + "saved_on_device": "Salvata su questo dispositivo", + "status_waiting": "In attesa del server", + "status_sending": "Salvataggio sul server", + "status_uncertain": "Verifica dell'arrivo sul server", + "status_refused": "Il server ha rifiutato questa bozza: {error}", + "file_missing": "Un file allegato non è più su questo dispositivo", + "open": "Apri", + "open_label": "Apri {subject}", + "retry": "Riprova", + "retry_label": "Riprova {subject}", + "toast_refused": "Non è stato possibile salvare la bozza {subject} sul server. Resta su questo dispositivo.", + "toast_conflict": "La bozza {subject} è stata modificata su un altro dispositivo. Vengono conservate entrambe le versioni.", + "conflict_changed": "Modificata su un altro dispositivo. Entrambe le versioni sono in Bozze.", + "conflict_gone": "Modificata o rimossa su un altro dispositivo. La tua versione è in Bozze.", + "banner_refused": "Bozze rifiutate dal server: {count}" + }, "deep_link": { "message_not_found": "Questo messaggio non è più disponibile.", "thread_not_found": "Questa conversazione non è più disponibile.", diff --git a/locales/ja/common.json b/locales/ja/common.json index 7f5688704..950a1c258 100644 --- a/locales/ja/common.json +++ b/locales/ja/common.json @@ -5756,6 +5756,25 @@ "toast": "オフライン中の変更で適用されなかったもの: {count} 件。送信トレイで確認できます。", "storage_failed": "このデバイスにはオフラインメール用の空き容量がありません。空き容量を増やすか、設定で保存する量を減らしてください。" }, + "offlineDrafts": { + "title": "このデバイスの下書き", + "description": "これらの下書きはこのデバイスに保存されています。サーバーに接続できるようになると自動で送られます。", + "saved_on_device": "このデバイスに保存しました", + "status_waiting": "サーバーを待っています", + "status_sending": "サーバーに保存しています", + "status_uncertain": "サーバーに届いたか確認しています", + "status_refused": "サーバーがこの下書きを拒否しました: {error}", + "file_missing": "添付ファイルがこのデバイスにありません", + "open": "開く", + "open_label": "{subject} を開く", + "retry": "再試行", + "retry_label": "{subject} を再試行", + "toast_refused": "下書き {subject} をサーバーに保存できませんでした。このデバイスに残ります。", + "toast_conflict": "下書き {subject} は別のデバイスで変更されました。両方のバージョンを残します。", + "conflict_changed": "別のデバイスで変更されました。両方のバージョンが下書きにあります。", + "conflict_gone": "別のデバイスで変更または削除されました。あなたのバージョンは下書きにあります。", + "banner_refused": "サーバーが拒否した下書き: {count}" + }, "deep_link": { "message_not_found": "このメッセージは利用できなくなりました。", "thread_not_found": "この会話は利用できなくなりました。", diff --git a/locales/ko/common.json b/locales/ko/common.json index 4c06d6b3f..68dac51bf 100644 --- a/locales/ko/common.json +++ b/locales/ko/common.json @@ -5756,6 +5756,25 @@ "toast": "오프라인에서 변경했지만 적용되지 않은 항목: {count}개. 보낼 편지함에서 확인할 수 있습니다.", "storage_failed": "이 기기에 오프라인 메일을 저장할 공간이 없습니다. 공간을 확보하거나 설정에서 보관량을 줄이세요." }, + "offlineDrafts": { + "title": "이 기기의 임시 보관함", + "description": "이 임시 보관 메일은 이 기기에 저장되어 있습니다. 서버에 연결되면 자동으로 올라갑니다.", + "saved_on_device": "이 기기에 저장됨", + "status_waiting": "서버 기다리는 중", + "status_sending": "서버에 저장하는 중", + "status_uncertain": "서버에 도착했는지 확인하는 중", + "status_refused": "서버가 이 임시 보관 메일을 거부했습니다: {error}", + "file_missing": "첨부 파일이 더 이상 이 기기에 없습니다", + "open": "열기", + "open_label": "{subject} 열기", + "retry": "다시 시도", + "retry_label": "{subject} 다시 시도", + "toast_refused": "임시 보관 메일 {subject}을(를) 서버에 저장하지 못했습니다. 이 기기에 남아 있습니다.", + "toast_conflict": "임시 보관 메일 {subject}이(가) 다른 기기에서 변경되었습니다. 두 버전을 모두 보관합니다.", + "conflict_changed": "다른 기기에서 변경되었습니다. 두 버전 모두 임시 보관함에 있습니다.", + "conflict_gone": "다른 기기에서 변경되었거나 삭제되었습니다. 내 버전은 임시 보관함에 있습니다.", + "banner_refused": "서버가 거부한 임시 보관 메일: {count}" + }, "deep_link": { "message_not_found": "이 메시지를 더 이상 사용할 수 없습니다.", "thread_not_found": "이 대화를 더 이상 사용할 수 없습니다.", diff --git a/locales/lv/common.json b/locales/lv/common.json index 8ebac9b3b..a534d445c 100644 --- a/locales/lv/common.json +++ b/locales/lv/common.json @@ -5756,6 +5756,25 @@ "toast": "Bezsaistē veiktās nepiemērotās izmaiņas: {count}. Izsūtne parāda, kuras.", "storage_failed": "Šajā ierīcē vairs nav vietas bezsaistes pastam. Atbrīvojiet vietu vai iestatījumos glabājiet mazāk." }, + "offlineDrafts": { + "title": "Melnraksti šajā ierīcē", + "description": "Šie melnraksti ir saglabāti šajā ierīcē. Tie paši nonāks serverī, tiklīdz tas būs sasniedzams.", + "saved_on_device": "Saglabāts šajā ierīcē", + "status_waiting": "Gaida serveri", + "status_sending": "Saglabā serverī", + "status_uncertain": "Pārbauda, vai tas sasniedza serveri", + "status_refused": "Serveris noraidīja šo melnrakstu: {error}", + "file_missing": "Pievienotais fails vairs nav šajā ierīcē", + "open": "Atvērt", + "open_label": "Atvērt {subject}", + "retry": "Mēģināt vēlreiz", + "retry_label": "Mēģināt vēlreiz: {subject}", + "toast_refused": "Melnrakstu {subject} neizdevās saglabāt serverī. Tas paliek šajā ierīcē.", + "toast_conflict": "Melnraksts {subject} tika mainīts citā ierīcē. Abas versijas tiek saglabātas.", + "conflict_changed": "Mainīts citā ierīcē. Abas versijas ir mapē Melnraksti.", + "conflict_gone": "Mainīts vai noņemts citā ierīcē. Jūsu versija ir mapē Melnraksti.", + "banner_refused": "Servera noraidītie melnraksti: {count}" + }, "deep_link": { "message_not_found": "Šī ziņa vairs nav pieejama.", "thread_not_found": "Šī sarakste vairs nav pieejama.", diff --git a/locales/mn/common.json b/locales/mn/common.json index 43edeb367..6bad44312 100644 --- a/locales/mn/common.json +++ b/locales/mn/common.json @@ -5756,6 +5756,25 @@ "toast": "Офлайн хийсэн, хэрэгжээгүй өөрчлөлтүүд: {count}. Аль нь болохыг Илгээх хайрцгаас харна уу.", "storage_failed": "Энэ төхөөрөмжид офлайн захидалд зай үлдсэнгүй. Зай чөлөөлөх эсвэл тохиргооноос цөөнийг хадгална уу." }, + "offlineDrafts": { + "title": "Энэ төхөөрөмж дээрх ноорог", + "description": "Эдгээр ноорог энэ төхөөрөмж дээр хадгалагдсан. Сервертэй холбогдмогц өөрсдөө илгээгдэнэ.", + "saved_on_device": "Энэ төхөөрөмж дээр хадгалагдсан", + "status_waiting": "Серверийг хүлээж байна", + "status_sending": "Сервер дээр хадгалж байна", + "status_uncertain": "Сервертэй хүрсэн эсэхийг шалгаж байна", + "status_refused": "Сервер энэ ноорогийг татгалзлаа: {error}", + "file_missing": "Хавсаргасан файл энэ төхөөрөмж дээр байхгүй болсон", + "open": "Нээх", + "open_label": "{subject}-г нээх", + "retry": "Дахин оролдох", + "retry_label": "{subject}-г дахин оролдох", + "toast_refused": "{subject} ноорогийг сервер дээр хадгалж чадсангүй. Энэ төхөөрөмж дээр үлдэнэ.", + "toast_conflict": "{subject} ноорог өөр төхөөрөмж дээр өөрчлөгдсөн. Хоёр хувилбарыг хоёуланг нь хадгална.", + "conflict_changed": "Өөр төхөөрөмж дээр өөрчлөгдсөн. Хоёр хувилбар хоёулаа Ноорог хавтсанд байна.", + "conflict_gone": "Өөр төхөөрөмж дээр өөрчлөгдсөн эсвэл устгагдсан. Таны хувилбар Ноорог хавтсанд байна.", + "banner_refused": "Серверийн татгалзсан ноорог: {count}" + }, "deep_link": { "message_not_found": "Энэ зурвас дахин ашиглах боломжгүй болсон.", "thread_not_found": "Энэ харилцан яриа дахин ашиглах боломжгүй болсон.", diff --git a/locales/nb/common.json b/locales/nb/common.json index 267f73a56..4c355a0a0 100644 --- a/locales/nb/common.json +++ b/locales/nb/common.json @@ -5756,6 +5756,25 @@ "toast": "Endringer gjort uten nett som ikke ble utført: {count}. Utboksen viser hvilke.", "storage_failed": "Denne enheten har ikke mer plass til e-post uten nett. Frigjør plass, eller behold mindre i innstillingene." }, + "offlineDrafts": { + "title": "Utkast på denne enheten", + "description": "Disse utkastene er lagret på denne enheten. De går til serveren av seg selv så snart den kan nås.", + "saved_on_device": "Lagret på denne enheten", + "status_waiting": "Venter på serveren", + "status_sending": "Lagrer på serveren", + "status_uncertain": "Sjekker om det nådde serveren", + "status_refused": "Serveren avviste dette utkastet: {error}", + "file_missing": "En vedlagt fil er ikke lenger på denne enheten", + "open": "Åpne", + "open_label": "Åpne {subject}", + "retry": "Prøv igjen", + "retry_label": "Prøv {subject} igjen", + "toast_refused": "Utkastet {subject} kunne ikke lagres på serveren. Det blir på denne enheten.", + "toast_conflict": "Utkastet {subject} ble endret på en annen enhet. Begge versjonene beholdes.", + "conflict_changed": "Endret på en annen enhet. Begge versjonene ligger i Utkast.", + "conflict_gone": "Endret eller fjernet på en annen enhet. Din versjon ligger i Utkast.", + "banner_refused": "Utkast avvist av serveren: {count}" + }, "deep_link": { "message_not_found": "Denne meldingen er ikke lenger tilgjengelig.", "thread_not_found": "Denne samtalen er ikke lenger tilgjengelig.", diff --git a/locales/nl/common.json b/locales/nl/common.json index 1cf1cabbb..0691d534d 100644 --- a/locales/nl/common.json +++ b/locales/nl/common.json @@ -5756,6 +5756,25 @@ "toast": "Offline gemaakte wijzigingen niet doorgevoerd: {count}. Het Postvak UIT laat zien welke.", "storage_failed": "Dit apparaat heeft geen ruimte meer voor offline mail. Maak ruimte vrij of bewaar minder via Instellingen." }, + "offlineDrafts": { + "title": "Concepten op dit apparaat", + "description": "Deze concepten zijn op dit apparaat opgeslagen. Ze gaan vanzelf naar de server zodra die bereikbaar is.", + "saved_on_device": "Opgeslagen op dit apparaat", + "status_waiting": "Wacht op de server", + "status_sending": "Wordt op de server opgeslagen", + "status_uncertain": "Controleren of het de server heeft bereikt", + "status_refused": "De server heeft dit concept geweigerd: {error}", + "file_missing": "Een bijgevoegd bestand staat niet meer op dit apparaat", + "open": "Openen", + "open_label": "{subject} openen", + "retry": "Opnieuw proberen", + "retry_label": "{subject} opnieuw proberen", + "toast_refused": "Het concept {subject} kon niet op de server worden opgeslagen. Het blijft op dit apparaat.", + "toast_conflict": "Het concept {subject} is op een ander apparaat gewijzigd. Beide versies blijven bewaard.", + "conflict_changed": "Gewijzigd op een ander apparaat. Beide versies staan in Concepten.", + "conflict_gone": "Gewijzigd of verwijderd op een ander apparaat. Uw versie staat in Concepten.", + "banner_refused": "Door de server geweigerde concepten: {count}" + }, "deep_link": { "message_not_found": "Dit bericht is niet meer beschikbaar.", "thread_not_found": "Dit gesprek is niet meer beschikbaar.", diff --git a/locales/pl/common.json b/locales/pl/common.json index 7067bbaaa..4d611b157 100644 --- a/locales/pl/common.json +++ b/locales/pl/common.json @@ -5756,6 +5756,25 @@ "toast": "Niezastosowane zmiany wprowadzone offline: {count}. Skrzynka nadawcza pokazuje które.", "storage_failed": "Na tym urządzeniu nie ma już miejsca na pocztę offline. Zwolnij miejsce lub zachowuj mniej w ustawieniach." }, + "offlineDrafts": { + "title": "Wersje robocze na tym urządzeniu", + "description": "Te wersje robocze są zapisane na tym urządzeniu. Trafią na serwer same, gdy tylko będzie dostępny.", + "saved_on_device": "Zapisano na tym urządzeniu", + "status_waiting": "Czeka na serwer", + "status_sending": "Zapisywanie na serwerze", + "status_uncertain": "Sprawdzanie, czy dotarła na serwer", + "status_refused": "Serwer odrzucił tę wersję roboczą: {error}", + "file_missing": "Załączonego pliku nie ma już na tym urządzeniu", + "open": "Otwórz", + "open_label": "Otwórz {subject}", + "retry": "Spróbuj ponownie", + "retry_label": "Spróbuj ponownie: {subject}", + "toast_refused": "Nie udało się zapisać wersji roboczej {subject} na serwerze. Zostaje na tym urządzeniu.", + "toast_conflict": "Wersja robocza {subject} została zmieniona na innym urządzeniu. Obie wersje zostają zachowane.", + "conflict_changed": "Zmieniono na innym urządzeniu. Obie wersje są w Wersjach roboczych.", + "conflict_gone": "Zmieniono lub usunięto na innym urządzeniu. Twoja wersja jest w Wersjach roboczych.", + "banner_refused": "Wersje robocze odrzucone przez serwer: {count}" + }, "deep_link": { "message_not_found": "Ta wiadomość nie jest już dostępna.", "thread_not_found": "Ta rozmowa nie jest już dostępna.", diff --git a/locales/pt/common.json b/locales/pt/common.json index 4bce29dd5..69e9384eb 100644 --- a/locales/pt/common.json +++ b/locales/pt/common.json @@ -5756,6 +5756,25 @@ "toast": "Alterações feitas offline não aplicadas: {count}. A caixa de saída mostra quais.", "storage_failed": "Este dispositivo não tem espaço para o correio offline. Liberte espaço ou guarde menos nas definições." }, + "offlineDrafts": { + "title": "Rascunhos neste dispositivo", + "description": "Estes rascunhos estão guardados neste dispositivo. Vão para o servidor por si assim que for possível contactá-lo.", + "saved_on_device": "Guardado neste dispositivo", + "status_waiting": "A aguardar o servidor", + "status_sending": "A guardar no servidor", + "status_uncertain": "A verificar se chegou ao servidor", + "status_refused": "O servidor recusou este rascunho: {error}", + "file_missing": "Um ficheiro anexado já não está neste dispositivo", + "open": "Abrir", + "open_label": "Abrir {subject}", + "retry": "Tentar novamente", + "retry_label": "Tentar {subject} novamente", + "toast_refused": "Não foi possível guardar o rascunho {subject} no servidor. Fica neste dispositivo.", + "toast_conflict": "O rascunho {subject} foi alterado noutro dispositivo. As duas versões são mantidas.", + "conflict_changed": "Alterado noutro dispositivo. As duas versões estão em Rascunhos.", + "conflict_gone": "Alterado ou removido noutro dispositivo. A sua versão está em Rascunhos.", + "banner_refused": "Rascunhos recusados pelo servidor: {count}" + }, "deep_link": { "message_not_found": "Esta mensagem não está mais disponível.", "thread_not_found": "Esta conversa não está mais disponível.", diff --git a/locales/ro/common.json b/locales/ro/common.json index 17abb3f3a..29a438e86 100644 --- a/locales/ro/common.json +++ b/locales/ro/common.json @@ -5756,6 +5756,25 @@ "toast": "Modificări făcute offline care nu au fost aplicate: {count}. Căsuța de trimitere arată care.", "storage_failed": "Acest dispozitiv nu mai are loc pentru e-mailul offline. Eliberați spațiu sau păstrați mai puțin din setări." }, + "offlineDrafts": { + "title": "Ciorne pe acest dispozitiv", + "description": "Aceste ciorne sunt salvate pe acest dispozitiv. Ajung singure pe server imediat ce poate fi contactat.", + "saved_on_device": "Salvat pe acest dispozitiv", + "status_waiting": "Se așteaptă serverul", + "status_sending": "Se salvează pe server", + "status_uncertain": "Se verifică dacă a ajuns pe server", + "status_refused": "Serverul a refuzat această ciornă: {error}", + "file_missing": "Un fișier atașat nu mai este pe acest dispozitiv", + "open": "Deschide", + "open_label": "Deschide {subject}", + "retry": "Încearcă din nou", + "retry_label": "Încearcă din nou: {subject}", + "toast_refused": "Ciorna {subject} nu a putut fi salvată pe server. Rămâne pe acest dispozitiv.", + "toast_conflict": "Ciorna {subject} a fost modificată pe alt dispozitiv. Ambele versiuni sunt păstrate.", + "conflict_changed": "Modificată pe alt dispozitiv. Ambele versiuni sunt în Ciorne.", + "conflict_gone": "Modificată sau eliminată pe alt dispozitiv. Versiunea dvs. este în Ciorne.", + "banner_refused": "Ciorne refuzate de server: {count}" + }, "deep_link": { "message_not_found": "Acest mesaj nu mai este disponibil.", "thread_not_found": "Această conversație nu mai este disponibilă.", diff --git a/locales/ru/common.json b/locales/ru/common.json index 9d50c9b45..9beefcbec 100644 --- a/locales/ru/common.json +++ b/locales/ru/common.json @@ -5756,6 +5756,25 @@ "toast": "Изменения, внесённые офлайн и не применённые: {count}. Какие именно, видно в исходящих.", "storage_failed": "На этом устройстве не осталось места для офлайн-почты. Освободите место или храните меньше в настройках." }, + "offlineDrafts": { + "title": "Черновики на этом устройстве", + "description": "Эти черновики сохранены на этом устройстве. Они сами уйдут на сервер, как только он станет доступен.", + "saved_on_device": "Сохранено на этом устройстве", + "status_waiting": "Ожидание сервера", + "status_sending": "Сохранение на сервере", + "status_uncertain": "Проверка, дошёл ли он до сервера", + "status_refused": "Сервер отклонил этот черновик: {error}", + "file_missing": "Вложенного файла больше нет на этом устройстве", + "open": "Открыть", + "open_label": "Открыть {subject}", + "retry": "Повторить", + "retry_label": "Повторить: {subject}", + "toast_refused": "Черновик {subject} не удалось сохранить на сервере. Он остаётся на этом устройстве.", + "toast_conflict": "Черновик {subject} изменён на другом устройстве. Сохранены обе версии.", + "conflict_changed": "Изменён на другом устройстве. Обе версии находятся в черновиках.", + "conflict_gone": "Изменён или удалён на другом устройстве. Ваша версия находится в черновиках.", + "banner_refused": "Черновики, отклонённые сервером: {count}" + }, "deep_link": { "message_not_found": "Это письмо больше недоступно.", "thread_not_found": "Эта переписка больше недоступна.", diff --git a/locales/sk/common.json b/locales/sk/common.json index 13b2e0ae2..6882ac32d 100644 --- a/locales/sk/common.json +++ b/locales/sk/common.json @@ -5756,6 +5756,25 @@ "toast": "Nepoužité zmeny urobené offline: {count}. Ktoré to sú, ukazuje pošta na odoslanie.", "storage_failed": "V tomto zariadení už nie je miesto na poštu offline. Uvoľnite miesto alebo v nastaveniach ponechajte menej." }, + "offlineDrafts": { + "title": "Koncepty v tomto zariadení", + "description": "Tieto koncepty sú uložené v tomto zariadení. Na server sa odošlú samy, hneď ako bude dostupný.", + "saved_on_device": "Uložené v tomto zariadení", + "status_waiting": "Čaká na server", + "status_sending": "Ukladá sa na server", + "status_uncertain": "Overuje sa, či dorazil na server", + "status_refused": "Server tento koncept odmietol: {error}", + "file_missing": "Priložený súbor už v tomto zariadení nie je", + "open": "Otvoriť", + "open_label": "Otvoriť {subject}", + "retry": "Skúsiť znova", + "retry_label": "Skúsiť znova: {subject}", + "toast_refused": "Koncept {subject} sa nepodarilo uložiť na server. Zostáva v tomto zariadení.", + "toast_conflict": "Koncept {subject} bol zmenený v inom zariadení. Obe verzie zostávajú zachované.", + "conflict_changed": "Zmenené v inom zariadení. Obe verzie sú v Konceptoch.", + "conflict_gone": "Zmenené alebo odstránené v inom zariadení. Vaša verzia je v Konceptoch.", + "banner_refused": "Koncepty odmietnuté serverom: {count}" + }, "deep_link": { "message_not_found": "Táto správa už nie je dostupná.", "thread_not_found": "Táto konverzácia už nie je dostupná.", diff --git a/locales/th/common.json b/locales/th/common.json index a1cb466d9..a5759dd23 100644 --- a/locales/th/common.json +++ b/locales/th/common.json @@ -2101,6 +2101,25 @@ "toast": "การเปลี่ยนแปลงขณะออฟไลน์ที่ไม่ได้นำไปใช้: {count} ดูรายการได้ในกล่องขาออก", "storage_failed": "อุปกรณ์นี้ไม่มีที่ว่างสำหรับอีเมลออฟไลน์แล้ว โปรดเพิ่มพื้นที่ว่างหรือเก็บให้น้อยลงในการตั้งค่า" }, + "offlineDrafts": { + "title": "ฉบับร่างในอุปกรณ์นี้", + "description": "ฉบับร่างเหล่านี้บันทึกไว้ในอุปกรณ์นี้ และจะส่งไปยังเซิร์ฟเวอร์เองเมื่อเชื่อมต่อได้", + "saved_on_device": "บันทึกในอุปกรณ์นี้แล้ว", + "status_waiting": "กำลังรอเซิร์ฟเวอร์", + "status_sending": "กำลังบันทึกไปยังเซิร์ฟเวอร์", + "status_uncertain": "กำลังตรวจสอบว่าถึงเซิร์ฟเวอร์แล้วหรือไม่", + "status_refused": "เซิร์ฟเวอร์ปฏิเสธฉบับร่างนี้: {error}", + "file_missing": "ไฟล์แนบไม่อยู่ในอุปกรณ์นี้แล้ว", + "open": "เปิด", + "open_label": "เปิด {subject}", + "retry": "ลองอีกครั้ง", + "retry_label": "ลอง {subject} อีกครั้ง", + "toast_refused": "บันทึกฉบับร่าง {subject} ไปยังเซิร์ฟเวอร์ไม่ได้ ฉบับร่างยังอยู่ในอุปกรณ์นี้", + "toast_conflict": "ฉบับร่าง {subject} ถูกเปลี่ยนในอุปกรณ์อื่น เก็บไว้ทั้งสองฉบับ", + "conflict_changed": "เปลี่ยนในอุปกรณ์อื่น ทั้งสองฉบับอยู่ในฉบับร่าง", + "conflict_gone": "เปลี่ยนหรือลบในอุปกรณ์อื่น ฉบับของคุณอยู่ในฉบับร่าง", + "banner_refused": "ฉบับร่างที่เซิร์ฟเวอร์ปฏิเสธ: {count}" + }, "plugin_pages": { "loading": "กำลังโหลดหน้าเว็บ...", "not_available_title": "หน้านี้ไม่พร้อมใช้งาน", diff --git a/locales/tr/common.json b/locales/tr/common.json index 839601428..138426136 100644 --- a/locales/tr/common.json +++ b/locales/tr/common.json @@ -5756,6 +5756,25 @@ "toast": "Çevrimdışı yapılan ve uygulanmayan değişiklikler: {count}. Giden kutusu hangileri olduğunu gösterir.", "storage_failed": "Bu cihazda çevrimdışı e-posta için yer kalmadı. Yer açın veya Ayarlar'dan daha azını saklayın." }, + "offlineDrafts": { + "title": "Bu cihazdaki taslaklar", + "description": "Bu taslaklar bu cihaza kaydedildi. Sunucuya ulaşılır ulaşılmaz kendiliğinden gönderilir.", + "saved_on_device": "Bu cihaza kaydedildi", + "status_waiting": "Sunucu bekleniyor", + "status_sending": "Sunucuya kaydediliyor", + "status_uncertain": "Sunucuya ulaşıp ulaşmadığı denetleniyor", + "status_refused": "Sunucu bu taslağı reddetti: {error}", + "file_missing": "Ekli bir dosya artık bu cihazda değil", + "open": "Aç", + "open_label": "{subject} öğesini aç", + "retry": "Yeniden dene", + "retry_label": "{subject} için yeniden dene", + "toast_refused": "{subject} taslağı sunucuya kaydedilemedi. Bu cihazda kalıyor.", + "toast_conflict": "{subject} taslağı başka bir cihazda değiştirildi. İki sürüm de saklanıyor.", + "conflict_changed": "Başka bir cihazda değiştirildi. İki sürüm de Taslaklar'da.", + "conflict_gone": "Başka bir cihazda değiştirildi veya kaldırıldı. Sizin sürümünüz Taslaklar'da.", + "banner_refused": "Sunucunun reddettiği taslaklar: {count}" + }, "deep_link": { "message_not_found": "Bu ileti artık kullanılamıyor.", "thread_not_found": "Bu sohbet artık kullanılamıyor.", diff --git a/locales/uk/common.json b/locales/uk/common.json index e9ab2cd21..f1988c446 100644 --- a/locales/uk/common.json +++ b/locales/uk/common.json @@ -5756,6 +5756,25 @@ "toast": "Зміни, внесені офлайн і не застосовані: {count}. Які саме, видно у вихідних.", "storage_failed": "На цьому пристрої не залишилося місця для офлайн-пошти. Звільніть місце або зберігайте менше в налаштуваннях." }, + "offlineDrafts": { + "title": "Чернетки на цьому пристрої", + "description": "Ці чернетки збережено на цьому пристрої. Вони самі підуть на сервер, щойно він стане доступним.", + "saved_on_device": "Збережено на цьому пристрої", + "status_waiting": "Очікування сервера", + "status_sending": "Збереження на сервері", + "status_uncertain": "Перевірка, чи дійшла вона до сервера", + "status_refused": "Сервер відхилив цю чернетку: {error}", + "file_missing": "Вкладеного файлу більше немає на цьому пристрої", + "open": "Відкрити", + "open_label": "Відкрити {subject}", + "retry": "Повторити", + "retry_label": "Повторити: {subject}", + "toast_refused": "Чернетку {subject} не вдалося зберегти на сервері. Вона залишається на цьому пристрої.", + "toast_conflict": "Чернетку {subject} змінено на іншому пристрої. Збережено обидві версії.", + "conflict_changed": "Змінено на іншому пристрої. Обидві версії є в чернетках.", + "conflict_gone": "Змінено або видалено на іншому пристрої. Ваша версія є в чернетках.", + "banner_refused": "Чернетки, відхилені сервером: {count}" + }, "deep_link": { "message_not_found": "Цей лист більше недоступний.", "thread_not_found": "Ця розмова більше недоступна.", diff --git a/locales/vi/common.json b/locales/vi/common.json index 2c242d359..d59eda55f 100644 --- a/locales/vi/common.json +++ b/locales/vi/common.json @@ -5768,6 +5768,25 @@ "toast": "Thay đổi khi ngoại tuyến chưa được áp dụng: {count}. Hộp thư đi cho biết là những thay đổi nào.", "storage_failed": "Thiết bị này không còn chỗ cho thư ngoại tuyến. Hãy giải phóng dung lượng hoặc giữ ít hơn trong Cài đặt." }, + "offlineDrafts": { + "title": "Bản nháp trên thiết bị này", + "description": "Các bản nháp này được lưu trên thiết bị này. Chúng sẽ tự chuyển lên máy chủ ngay khi kết nối được.", + "saved_on_device": "Đã lưu trên thiết bị này", + "status_waiting": "Đang chờ máy chủ", + "status_sending": "Đang lưu lên máy chủ", + "status_uncertain": "Đang kiểm tra xem đã đến máy chủ chưa", + "status_refused": "Máy chủ đã từ chối bản nháp này: {error}", + "file_missing": "Một tệp đính kèm không còn trên thiết bị này", + "open": "Mở", + "open_label": "Mở {subject}", + "retry": "Thử lại", + "retry_label": "Thử lại {subject}", + "toast_refused": "Không thể lưu bản nháp {subject} lên máy chủ. Bản nháp vẫn ở trên thiết bị này.", + "toast_conflict": "Bản nháp {subject} đã được thay đổi trên thiết bị khác. Cả hai phiên bản đều được giữ lại.", + "conflict_changed": "Đã thay đổi trên thiết bị khác. Cả hai phiên bản đều ở trong Bản nháp.", + "conflict_gone": "Đã thay đổi hoặc xóa trên thiết bị khác. Phiên bản của bạn ở trong Bản nháp.", + "banner_refused": "Bản nháp bị máy chủ từ chối: {count}" + }, "deep_link": { "message_not_found": "Thư này không còn nữa.", "thread_not_found": "Cuộc trò chuyện này không còn tồn tại nữa.", diff --git a/locales/zh-TW/common.json b/locales/zh-TW/common.json index dc8d0f4bf..c49cc86cb 100644 --- a/locales/zh-TW/common.json +++ b/locales/zh-TW/common.json @@ -5756,6 +5756,25 @@ "toast": "離線時的變更未被套用：{count} 項。可在寄件匣中查看。", "storage_failed": "此裝置已沒有空間存放離線郵件。請釋出空間，或在設定中減少保留的內容。" }, + "offlineDrafts": { + "title": "此裝置上的草稿", + "description": "這些草稿儲存在此裝置上。一旦能連線到伺服器，就會自動上傳。", + "saved_on_device": "已儲存在此裝置上", + "status_waiting": "正在等待伺服器", + "status_sending": "正在儲存到伺服器", + "status_uncertain": "正在檢查是否已到達伺服器", + "status_refused": "伺服器拒絕了此草稿：{error}", + "file_missing": "附件已不在此裝置上", + "open": "開啟", + "open_label": "開啟 {subject}", + "retry": "重試", + "retry_label": "重試 {subject}", + "toast_refused": "草稿 {subject} 無法儲存到伺服器。它仍保留在此裝置上。", + "toast_conflict": "草稿 {subject} 已在其他裝置上變更。兩個版本都會保留。", + "conflict_changed": "已在其他裝置上變更。兩個版本都在草稿中。", + "conflict_gone": "已在其他裝置上變更或刪除。您的版本在草稿中。", + "banner_refused": "伺服器拒絕的草稿：{count}" + }, "deep_link": { "message_not_found": "已無法使用此郵件。", "thread_not_found": "已無法使用此對話串。", diff --git a/locales/zh/common.json b/locales/zh/common.json index bc7da9471..9e7630bab 100644 --- a/locales/zh/common.json +++ b/locales/zh/common.json @@ -5756,6 +5756,25 @@ "toast": "离线时的更改未被应用：{count} 项。可在发件箱中查看。", "storage_failed": "此设备已没有空间存放离线邮件。请释放空间，或在设置中减少保留的内容。" }, + "offlineDrafts": { + "title": "此设备上的草稿", + "description": "这些草稿保存在此设备上。一旦能连接服务器，就会自动上传。", + "saved_on_device": "已保存在此设备上", + "status_waiting": "正在等待服务器", + "status_sending": "正在保存到服务器", + "status_uncertain": "正在检查是否已到达服务器", + "status_refused": "服务器拒绝了此草稿：{error}", + "file_missing": "附件已不在此设备上", + "open": "打开", + "open_label": "打开 {subject}", + "retry": "重试", + "retry_label": "重试 {subject}", + "toast_refused": "草稿 {subject} 无法保存到服务器。它仍保留在此设备上。", + "toast_conflict": "草稿 {subject} 已在其他设备上更改。两个版本都会保留。", + "conflict_changed": "已在其他设备上更改。两个版本都在草稿中。", + "conflict_gone": "已在其他设备上更改或删除。您的版本在草稿中。", + "banner_refused": "服务器拒绝的草稿：{count}" + }, "deep_link": { "message_not_found": "此邮件已不可用。", "thread_not_found": "此会话已不可用。", From 5dc0f05a707e367955b10f6bfbbd8db5d4c5ea51 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:16:32 +0100 Subject: [PATCH 2/7] fix: count a draft with a waiting send once when signing out --- components/offline/__tests__/outbox-drafts.test.tsx | 1 - lib/store/__tests__/drafts.test.ts | 6 ++++-- lib/store/drafts/calls.ts | 8 +++----- lib/store/drafts/pending.ts | 6 ++++++ lib/store/outbox/calls.ts | 4 ++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/components/offline/__tests__/outbox-drafts.test.tsx b/components/offline/__tests__/outbox-drafts.test.tsx index 6bb708538..6113196d3 100644 --- a/components/offline/__tests__/outbox-drafts.test.tsx +++ b/components/offline/__tests__/outbox-drafts.test.tsx @@ -22,7 +22,6 @@ const draft = (overrides: Partial = {}): LocalDraftRow => ({ jmap: 'j1', subject: 'Only here', status: 'waiting', - hasFiles: false, createdAt: Date.UTC(2026, 10, 3), ...overrides, }); diff --git a/lib/store/__tests__/drafts.test.ts b/lib/store/__tests__/drafts.test.ts index 31fa738a8..b24c3031c 100644 --- a/lib/store/__tests__/drafts.test.ts +++ b/lib/store/__tests__/drafts.test.ts @@ -580,10 +580,12 @@ describe('sending a draft that\'s on the device', () => { }); await flush(); expect(t.server.drafts()).toEqual([]); - expect(await list()).toHaveLength(1); + expect((await get(id)).list).toHaveLength(1); + // It's that message, in the outbox: not counted a second time as unsent. + expect(await list()).toEqual([]); await t.engine.areaCall('drafts.discard', { jmap: JMAP_ACCOUNT, id }); - expect(await list()).toEqual([]); + expect((await get(id)).notFound).toEqual([id]); }); it('goes up as a draft if the send fails, so nothing depends on the outbox', async () => { diff --git a/lib/store/drafts/calls.ts b/lib/store/drafts/calls.ts index 3378c36ed..a6a678f37 100644 --- a/lib/store/drafts/calls.ts +++ b/lib/store/drafts/calls.ts @@ -5,8 +5,8 @@ import { upsertEmail, type EmailObject } from '../mail/records'; import { savePatch, EMAIL_TYPE } from '../outbox/email-set'; import type { ConflictSummary } from '../outbox/items'; import { applyPatch, destroyConflicts, type RecordState } from '../outbox/patches'; -import { blobPartsOf, type DraftCreate } from './local-email'; -import { allPending, destroyForDraft, pendingForFlush, replaceLocalRow, replacesOf, resolveStandIns, type DraftError, type DraftRow, type DraftStatus } from './pending'; +import type { DraftCreate } from './local-email'; +import { destroyForDraft, listedDrafts, pendingForFlush, replaceLocalRow, replacesOf, resolveStandIns, type DraftError, type DraftRow, type DraftStatus } from './pending'; export interface PendingDraft { localId: string; @@ -33,7 +33,6 @@ export interface LocalDraftSummary { subject: string; status: DraftStatus; error?: DraftError; - hasFiles: boolean; createdAt: number; } @@ -67,12 +66,11 @@ function read(ctx: StoreContext, params: { before: number }): { rows: PendingDra } function list(ctx: StoreContext): LocalDraftSummary[] { - return allPending(ctx.db).map((row) => ({ + return listedDrafts(ctx.db).map((row) => ({ localId: row.local_id, jmap: row.jmap, subject: row.subject ?? '', status: row.status, - hasFiles: blobPartsOf(JSON.parse(row.create_json) as DraftCreate).length > 0, createdAt: Number(row.created_at), ...(row.error ? { error: JSON.parse(row.error) as DraftError } : {}), })); diff --git a/lib/store/drafts/pending.ts b/lib/store/drafts/pending.ts index d890ad3f0..bbdaef0b2 100644 --- a/lib/store/drafts/pending.ts +++ b/lib/store/drafts/pending.ts @@ -204,6 +204,12 @@ export function allPending(db: StorageAdapter): DraftRow[] { return db.all('SELECT * FROM draft_pending ORDER BY seq ASC'); } +/** The drafts that are only here and nothing else speaks for. One a send names is that message, in the outbox. */ +export function listedDrafts(db: StorageAdapter): DraftRow[] { + const held = draftsWithQueuedSend(db); + return allPending(db).filter((row) => !held.has(row.local_id)); +} + export function pendingForFlush(db: StorageAdapter): DraftRow[] { const held = draftsWithQueuedSend(db); return allPending(db).filter((row) => row.status !== 'refused' && !held.has(row.local_id)); diff --git a/lib/store/outbox/calls.ts b/lib/store/outbox/calls.ts index a87c9be15..2496b3e6d 100644 --- a/lib/store/outbox/calls.ts +++ b/lib/store/outbox/calls.ts @@ -156,8 +156,8 @@ function remove(ctx: StoreContext, params: { id: number }): void { // ---- Files attached offline ---- -/** The size the page cuts a file into on its way here. */ -export const HOLD_CHUNK_BYTES = 1024 * 1024; +/** The same size the page cuts a file into on its way here. */ +const HOLD_CHUNK_BYTES = 1024 * 1024; /** A file that comes in pieces: nothing can read it until the last one is in. */ function holdStart(ctx: StoreContext, params: { jmap?: string; name: string; type: string; size: number }): { blobId: string } { From f5c97c19974e6ed0b450261e486ab017cb18bc90 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:25:28 +0100 Subject: [PATCH 3/7] test: write a draft offline with a file, and read a kept file offline, on both servers --- integration/tests/22-ent-offline.spec.ts | 159 ++++++++++++++++++++++- 1 file changed, 158 insertions(+), 1 deletion(-) diff --git a/integration/tests/22-ent-offline.spec.ts b/integration/tests/22-ent-offline.spec.ts index ff4dcd3cf..a546a3cef 100644 --- a/integration/tests/22-ent-offline.spec.ts +++ b/integration/tests/22-ent-offline.spec.ts @@ -2,6 +2,7 @@ import { test, expect, type Browser, type BrowserContext, type Page } from '@pla import { execSync } from 'node:child_process'; import { ENT, + ENT_PASSWORD, adminContext, deliver, getPolicy, @@ -12,7 +13,7 @@ import { shot, userPage, } from './helpers/ent'; -import { accountSwitcher } from './helpers/app'; +import { accountSwitcher, openFolder } from './helpers/app'; import { JMAP_URL } from './helpers/config'; /** @@ -27,6 +28,9 @@ import { JMAP_URL } from './helpers/config'; * - a change the server can no longer take shows as changed on another device; * - the Trash emptied offline is emptied on the server after; * - a send whose response got lost is looked up and goes out once; + * - a draft written offline with a file survives a reload without a network, + * is on the server once after reconnecting, with the file, and can be sent; + * - a kept message shows its inline image and gives its attachment offline; * - a sign-out the user starts asks first when unsent mail would be lost, * then wipes the device; the idle sign-out wipes it and the login page * names the lost message; @@ -432,6 +436,159 @@ test('a send whose response got lost goes out once', async ({ browser }) => { await context.close(); }); +interface ServerDraft { id: string; subject: string; blobId: string; attachments: Array<{ blobId: string; name: string; size: number }> } + +/** The account's Drafts folder as the server has it, for one subject. */ +async function serverDrafts(account: string, subject: string): Promise { + const client = await jmapAs(account); + const boxes = await client.request([['Mailbox/get', { accountId: client.accountId, properties: ['id', 'role'] }, '0']]); + const drafts = (boxes.methodResponses[0][1].list as Array<{ id: string; role: string | null }>).find((box) => box.role === 'drafts'); + const r = await client.request([ + ['Email/query', { accountId: client.accountId, filter: { inMailbox: drafts?.id }, limit: 200 }, '0'], + ['Email/get', { accountId: client.accountId, '#ids': { resultOf: '0', name: 'Email/query', path: '/ids' }, properties: ['id', 'subject', 'blobId', 'attachments'] }, '1'], + ]); + return (r.methodResponses[1][1].list as ServerDraft[]).filter((email) => email.subject === subject); +} + +test('a draft written offline with a file survives a reload, is on the server once after reconnecting, and can be sent', async ({ browser }) => { + const subject = `Offline draft ${RUN}`; + const fileText = `Attached offline ${RUN}`; + const page = await userPage(browser, ENT.alice); + await enableOffline(page); + await toMail(page); + const context = page.context(); + await context.setOffline(true); + + await compose(page, ENT.bob, subject, `First words ${RUN}`); + const composer = page.locator('[data-testid="email-composer"]'); + await composer.locator('input[type="file"]').first().setInputFiles({ name: 'offline-note.txt', mimeType: 'text/plain', buffer: Buffer.from(fileText) }); + await expect(composer).toContainText('offline-note.txt', { timeout: 15_000 }); + await expect(composer.getByText('Saved on this device').first()).toBeVisible({ timeout: 30_000 }); + await shot(page, 'f-offline-draft'); + // Saved again and again: still one draft in the end. + for (const word of [' second', ' third', ' last']) { + await composer.locator('[contenteditable="true"]').first().click(); + await page.keyboard.press('Control+End'); + await page.keyboard.type(word); + await expect(composer.getByTestId('composer-save-status').first()).toHaveAttribute('data-status', 'saved', { timeout: 30_000 }); + await page.waitForTimeout(2500); + } + await page.getByTestId('composer-close').first().click(); + await expect(composer).toBeHidden({ timeout: 30_000 }); + expect(await serverDrafts(ENT.alice, subject)).toHaveLength(0); + + // A start without a network: it's in Drafts, with its text and its file. + await page.reload(); + await expect(page.locator('[data-testid="email-list-item"]').first()).toBeVisible({ timeout: 60_000 }); + await openFolder(page, { role: 'drafts' }); + const row = page.locator('[data-testid="email-list-item"], [data-thread-keys]', { hasText: subject }).first(); + await expect(row).toBeVisible({ timeout: 60_000 }); + await row.click(); + await page.getByTestId('edit-draft').click(); + await expect(composer).toBeVisible({ timeout: 20_000 }); + await expect(composer.locator('[contenteditable="true"]').first()).toContainText('last', { timeout: 20_000 }); + await expect(composer).toContainText('offline-note.txt'); + await page.getByTestId('composer-close').first().click(); + await expect(composer).toBeHidden({ timeout: 30_000 }); + expect(await serverDrafts(ENT.alice, subject)).toHaveLength(0); + + // Back online: one draft, with the latest text and the file. + await context.setOffline(false); + await expect.poll(async () => (await serverDrafts(ENT.alice, subject)).length, { timeout: 180_000, intervals: [2000] }).toBe(1); + await page.waitForTimeout(5000); + const [draft] = await serverDrafts(ENT.alice, subject); + expect(await serverDrafts(ENT.alice, subject)).toHaveLength(1); + expect(draft.attachments.map((part) => part.name)).toEqual(['offline-note.txt']); + expect(await rawMessage(ENT.alice, draft.attachments[0].blobId)).toContain(fileText); + expect(await rawMessage(ENT.alice, draft.blobId)).toContain('last'); + + // And it goes out from there, with the file. + await page.goto('/mail'); + await expect(page.locator('[data-testid="email-list-item"]').first()).toBeVisible({ timeout: 60_000 }); + await openFolder(page, { role: 'drafts' }); + const again = page.locator('[data-testid="email-list-item"], [data-thread-keys]', { hasText: subject }).first(); + await expect(again).toBeVisible({ timeout: 60_000 }); + await again.click(); + await page.getByTestId('edit-draft').click(); + await expect(composer).toContainText('offline-note.txt', { timeout: 20_000 }); + await send(page); + await expect(composer).toBeHidden({ timeout: 30_000 }); + const got = await waitForMessage(ENT.bob, (m) => m.subject === subject); + expect(await rawMessage(ENT.bob, got.blobId)).toContain('offline-note.txt'); + await expect.poll(async () => (await serverDrafts(ENT.alice, subject)).length, { timeout: 30_000 }).toBe(0); + await context.close(); +}); + +const ONE_PIXEL = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M/wHwAEAQH/7yMK/gAAAABJRU5ErkJggg==', 'base64'); + +async function uploadAs(account: string, data: Buffer, type: string): Promise { + const client = await jmapAs(account); + const res = await fetch(`${JMAP_URL}/jmap/upload/${client.accountId}/`, { + method: 'POST', + headers: { 'Content-Type': type, Authorization: 'Basic ' + Buffer.from(`${account}:${ENT_PASSWORD}`).toString('base64') }, + body: new Uint8Array(data), + }); + if (!res.ok) throw new Error(`upload: ${res.status}`); + return (await res.json() as { blobId: string }).blobId; +} + +test('a kept message shows its inline image and gives its attachment offline', async ({ browser }) => { + const subject = `Offline files ${RUN}`; + const fileText = `Kept attachment ${RUN}`; + const client = await jmapAs(ENT.alice); + const inbox = await client.mailboxByRole('inbox'); + const image = await uploadAs(ENT.alice, ONE_PIXEL, 'image/png'); + const file = await uploadAs(ENT.alice, Buffer.from(fileText), 'application/octet-stream'); + const made = await client.request([['Email/set', { + accountId: client.accountId, + create: { + m: { + mailboxIds: { [inbox!.id]: true }, + from: [{ email: ENT.bob }], + to: [{ email: ENT.alice }], + subject, + bodyValues: { t: { value: `Text ${RUN}` }, h: { value: `

Html ${RUN}

pixel` } }, + bodyStructure: { + type: 'multipart/mixed', + subParts: [ + { + type: 'multipart/alternative', + subParts: [ + { partId: 't', type: 'text/plain' }, + { type: 'multipart/related', subParts: [{ partId: 'h', type: 'text/html' }, { blobId: image, type: 'image/png', name: 'pixel.png', disposition: 'inline', cid: 'pixel@example.org' }] }, + ], + }, + { blobId: file, type: 'application/octet-stream', name: 'kept.bin', disposition: 'attachment' }, + ], + }, + }, + }, + }, '0']]); + expect(made.methodResponses[0][1].created?.m?.id, JSON.stringify(made.methodResponses[0][1].notCreated)).toBeTruthy(); + + const page = await userPage(browser, ENT.alice); + await enableOffline(page); + await toMail(page); + // Files come last in a pass, and yield to anything newer. This message's come first of them. + await page.waitForTimeout(10_000); + await page.context().setOffline(true); + await openMessage(page, subject); + + const pixel = page.frameLocator('iframe').locator('img[alt="pixel"]').first(); + // A broken image is 0 wide until the app's placeholder comes in, so ask where it came from. + await expect.poll(() => pixel.evaluate((img: HTMLImageElement) => (img.src.startsWith('blob:') ? img.naturalWidth : 0)).catch(() => 0), { timeout: 30_000 }).toBe(1); + + const chip = page.locator('[data-testid="attachment"][data-attachment-name="kept.bin"]').first(); + await expect(chip).toBeVisible({ timeout: 20_000 }); + const download = page.waitForEvent('download', { timeout: 30_000 }); + await chip.locator('button[title="Download"]').click(); + const saved = await (await download).createReadStream(); + const chunks: Buffer[] = []; + for await (const chunk of saved) chunks.push(chunk as Buffer); + expect(Buffer.concat(chunks).toString()).toBe(fileText); + await page.context().close(); +}); + test('signing out asks first when unsent mail would be lost, then wipes the device', async ({ browser }) => { const subject = `Unsent at sign-out ${RUN}`; const page = await userPage(browser, ENT.alice); From 4fd82437ad72b1c75d7915311fd5a440b3eafa8c Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:06:17 +0100 Subject: [PATCH 4/7] fix: keep attachments alive through a flush and never destroy a copy that isn't ours Review of the offline drafts found these. - Every new message gets new part ids and old ones die with it, so an open composer failed with `blobNotFound` after a flush. The client and the flush re-read the draft and re-point parts once. The fake now gives new part ids, which is why `drafts.test.ts` and `drafts-client.test.ts` compare file contents instead of blob ids. - A Message-ID alone could destroy another client's message. A copy now has to be solely in Drafts, with `$draft`, close to the attempt, with the same subject and size, and what the lookup saw is the destroy's base. Ghosts ride on the replacing row and only queue once it's up. `drafts.created` no longer queues a copy that arrived for a replaced row, so its test now expects the next flush to do it. - Turning offline off asks about device-only drafts too. - A kept file can't read back empty: chunks of a kept file are left alone, and `keptBlob` and `heldFile` refuse pieces that don't add up. - A flush stopped by a passing failure tries again, the banner shows waiting drafts, and under the flush lock a `sending` row counts as left by a dead tab at once. A lookup refused for good no longer stops the outbox. - `FetchTransport` streams downloads and reports a broken transfer as interrupted, so one bad file doesn't end the pass. - Raw messages come after every file, only into free room, and a file needing room pushes them out first. - An old version in another login stays until the new one is up. An online send of an unflushed stand-in still removes its submission. --- .../__tests__/composer-draft-login.test.tsx | 19 ++ components/email/email-composer.tsx | 8 +- .../__tests__/offline-settings.test.tsx | 24 ++ .../offline/__tests__/outbox-drafts.test.tsx | 4 +- components/offline/offline-banner.tsx | 14 +- components/offline/offline-settings.tsx | 2 +- lib/jmap/client.ts | 86 ++++- lib/jmap/outgoing-body.ts | 32 ++ lib/offline/__tests__/drafts-flush.test.ts | 103 +++++- lib/offline/flush.ts | 38 ++- lib/offline/queue.ts | 5 + lib/store/__tests__/attachments.test.ts | 99 ++++++ lib/store/__tests__/drafts-client.test.ts | 88 +++++- lib/store/__tests__/drafts.test.ts | 294 +++++++++++++++++- lib/store/__tests__/helpers/fake-drafts.ts | 81 ++++- lib/store/__tests__/transport-real.test.ts | 116 +++++++ lib/store/drafts/calls.ts | 92 ++++-- lib/store/drafts/flush.ts | 172 ++++++++-- lib/store/drafts/local-email.ts | 2 +- lib/store/drafts/pending.ts | 56 +++- lib/store/drafts/schema.ts | 25 +- lib/store/engine/account-engine.ts | 2 + lib/store/engine/transport.ts | 164 ++++++++-- lib/store/mail/attachments.ts | 72 +++-- lib/store/outbox/calls.ts | 2 + locales/ar/common.json | 3 +- locales/ca/common.json | 3 +- locales/cs/common.json | 3 +- locales/da/common.json | 3 +- locales/de/common.json | 3 +- locales/en/common.json | 3 +- locales/es/common.json | 3 +- locales/et/common.json | 3 +- locales/fa/common.json | 3 +- locales/fr/common.json | 3 +- locales/he/common.json | 3 +- locales/hu/common.json | 3 +- locales/it/common.json | 3 +- locales/ja/common.json | 3 +- locales/ko/common.json | 3 +- locales/lv/common.json | 3 +- locales/mn/common.json | 3 +- locales/nb/common.json | 3 +- locales/nl/common.json | 3 +- locales/pl/common.json | 3 +- locales/pt/common.json | 3 +- locales/ro/common.json | 3 +- locales/ru/common.json | 3 +- locales/sk/common.json | 3 +- locales/th/common.json | 3 +- locales/tr/common.json | 3 +- locales/uk/common.json | 3 +- locales/vi/common.json | 3 +- locales/zh-TW/common.json | 3 +- locales/zh/common.json | 3 +- 55 files changed, 1496 insertions(+), 194 deletions(-) create mode 100644 lib/store/__tests__/transport-real.test.ts diff --git a/components/email/__tests__/composer-draft-login.test.tsx b/components/email/__tests__/composer-draft-login.test.tsx index 0e8ef5923..7f6e1a4b3 100644 --- a/components/email/__tests__/composer-draft-login.test.tsx +++ b/components/email/__tests__/composer-draft-login.test.tsx @@ -4,6 +4,12 @@ import React from 'react'; import { EmailComposer, type ComposerDraftData } from '../email-composer'; import { useAuthStore } from '@/stores/auth-store'; +const offlineQueue = vi.hoisted(() => ({ noteDraftReplacedElsewhere: vi.fn(async () => {}) })); +vi.mock('@/lib/offline/queue', async (importOriginal) => ({ + ...(await importOriginal()), + noteDraftReplacedElsewhere: offlineQueue.noteDraftReplacedElsewhere, +})); + // ─── Heavy component mocks (mirrors composer-close-guard.test.tsx) ──────────── vi.mock('@/components/email/rich-text-editor', () => ({ @@ -293,6 +299,19 @@ describe('a draft and the login it was saved with', () => { expect(onDiscardDraft).toHaveBeenCalledWith('draft-b2', { accountId: undefined, localAccountId: 'login-b' }); }); + it('leaves the old version in its own login while the new one is only on the device', async () => { + const composer = open(DRAFT); + await composer.editSubject('Hello', 'Hello 2'); + (clientB.createDraft as ReturnType).mockReset(); + (clientB.createDraft as ReturnType).mockResolvedValue('local-standin'); + composer.switchFrom('login-b::id-b'); + await composer.editSubject('Hello 2', 'Hello 3'); + + expect(clientB.createDraft).toHaveBeenCalledTimes(1); + expect(clientA.deleteEmail).not.toHaveBeenCalled(); + expect(offlineQueue.noteDraftReplacedElsewhere).toHaveBeenCalledWith('login-b', 'local-standin', { login: 'login-a', jmap: undefined, id: 'draft-a2' }); + }); + it('keeps replacing versions through createDraft while the login stays the same', async () => { const composer = open({ ...DRAFT, draftLocalAccountId: 'login-a' }); diff --git a/components/email/email-composer.tsx b/components/email/email-composer.tsx index c12ae34ad..0679e889f 100644 --- a/components/email/email-composer.tsx +++ b/components/email/email-composer.tsx @@ -90,6 +90,7 @@ import { DirectoryPickerDialog } from "@/components/contacts/directory-picker-di import type { DirectoryEntry } from "@/stores/contact-store"; import type { IJMAPClient } from "@/lib/jmap/client-interface"; import { isStandInId } from "@/lib/jmap/stand-in-ids"; +import { noteDraftReplacedElsewhere } from "@/lib/offline/queue"; import type { FileNode } from "@/lib/jmap/types"; import { useContactStore, getContactDisplayName, getContactPrimaryEmail } from "@/stores/contact-store"; import { renderTemplateWithPlugins, notifyTemplateApplied } from "@/stores/template-store"; @@ -2859,7 +2860,12 @@ export function EmailComposer({ { priority, accountId: draftAccountId, previousDraftAccountId: replacedElsewhere ? undefined : previousDraftAccountId, classification: classificationRef.current.draftChoice }, ); if (replacedElsewhere) { - await removeDraftVersion(savedDraft.draftId!, previousDraftLogin, previousDraftAccountId); + if (isStandInId(savedDraftId) && savingLogin) { + // The new version is only on this device: the old one stays until it's up. + await noteDraftReplacedElsewhere(savingLogin, savedDraftId, { login: previousDraftLogin!, jmap: previousDraftAccountId, id: savedDraft.draftId! }); + } else { + await removeDraftVersion(savedDraft.draftId!, previousDraftLogin, previousDraftAccountId); + } } // Update the ref synchronously so a queued save sees the new id and diff --git a/components/offline/__tests__/offline-settings.test.tsx b/components/offline/__tests__/offline-settings.test.tsx index 73e13bbbf..806b61ef9 100644 --- a/components/offline/__tests__/offline-settings.test.tsx +++ b/components/offline/__tests__/offline-settings.test.tsx @@ -67,6 +67,30 @@ describe('Settings > Offline', () => { expect(screen.queryByTestId('offline-switch')).toBeNull(); }); + it('asks before turning off when a draft is only on this device, and counts both lists', () => { + useOfflineStore.setState({ + enabled: ['acc'], + localDrafts: [{ account: 'acc', localId: 'local-1', jmap: 'j', subject: 'Only here', status: 'waiting', createdAt: 0 }], + outbox: [{ id: 1, account: 'acc', createdAt: 0, subject: 'Queued', to: [], status: 'waiting' }], + }); + render(); + fireEvent.click(screen.getByTestId('offline-switch')); + expect(offline.disableOffline).not.toHaveBeenCalled(); + expect(screen.getByText('off_confirm_title')).toBeInTheDocument(); + useOfflineStore.setState({ localDrafts: [], outbox: [] }); + }); + + it('asks for one draft alone', () => { + useOfflineStore.setState({ + enabled: ['acc'], + localDrafts: [{ account: 'acc', localId: 'local-1', jmap: 'j', subject: 'Only here', status: 'waiting', createdAt: 0 }], + }); + render(); + fireEvent.click(screen.getByTestId('offline-switch')); + expect(offline.disableOffline).not.toHaveBeenCalled(); + useOfflineStore.setState({ localDrafts: [] }); + }); + it('lists the outbox', () => { useOfflineStore.setState({ outbox: [{ id: 1, account: 'acc', createdAt: Date.now(), subject: 'Quarterly', to: ['b@example.org'], status: 'failed', error: { text: 'Rejected' } }] }); render(); diff --git a/components/offline/__tests__/outbox-drafts.test.tsx b/components/offline/__tests__/outbox-drafts.test.tsx index 6113196d3..498bb9546 100644 --- a/components/offline/__tests__/outbox-drafts.test.tsx +++ b/components/offline/__tests__/outbox-drafts.test.tsx @@ -112,12 +112,14 @@ describe('drafts that are only on this device', () => { expect(rows[1].textContent).toContain('conflict_gone'); }); - it('puts a refused one in the banner while online, and not one that only waits', () => { + it('says in the banner while online that drafts wait, and which were refused', () => { useOfflineStore.setState({ localDrafts: [draft()] }); const { rerender } = render(); expect(screen.queryByTestId('offline-banner-drafts')).toBeNull(); + expect(screen.getByTestId('offline-banner-drafts-waiting').textContent).toContain('banner_waiting'); act(() => useOfflineStore.setState({ localDrafts: [draft({ status: 'refused' })] })); rerender(); expect(screen.getByTestId('offline-banner-drafts').textContent).toContain('banner_refused'); + expect(screen.queryByTestId('offline-banner-drafts-waiting')).toBeNull(); }); }); diff --git a/components/offline/offline-banner.tsx b/components/offline/offline-banner.tsx index e87b70e3a..7f37f1377 100644 --- a/components/offline/offline-banner.tsx +++ b/components/offline/offline-banner.tsx @@ -26,11 +26,12 @@ export function OfflineBanner() { const waiting = useOfflineStore((s) => s.outbox.length); const conflicts = useOfflineStore((s) => s.conflicts.length); const refusedDrafts = useOfflineStore((s) => s.localDrafts.filter((draft) => draft.status === 'refused').length); + const waitingDrafts = useOfflineStore((s) => s.localDrafts.filter((draft) => draft.status !== 'refused').length); const tConflicts = useTranslations('offlineConflicts'); const tDrafts = useTranslations('offlineDrafts'); const [open, setOpen] = useState(false); const offline = browserOffline || serverDown; - if (IS_LITE || !active || (!offline && waiting === 0 && conflicts === 0 && refusedDrafts === 0 && !open)) return null; + if (IS_LITE || !active || (!offline && waiting === 0 && conflicts === 0 && refusedDrafts === 0 && waitingDrafts === 0 && !open)) return null; return ( <>
)} + {waitingDrafts > 0 && ( + + )} {refusedDrafts > 0 && (