starred/webmail
1
0
Fork 0
mirror of https://github.com/bulwarkmail/webmail.git synced 2026-11-04 16:14:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: merge offline drafts and kept attachments

Browse Source
This commit is contained in:
Linus Rath
2026-11-03 11:46:03 +01:00
parent 2c08b10fef df58110989
commit 8512e68d12
81 changed files with 5811 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/jmap/client-interface.ts
+1
View File
@@ -553,6 +553,7 @@ export interface IJMAPClient {
accountId?: string;
onProgress?: (loaded: number, total: number) => void;
signal?: AbortSignal;
noHold?: boolean;
},
): Promise<{ blobId: string; size: number; type: string }>;
getBlobDownloadUrl(blobId: string, name?: string, type?: string, accountId?: string): string;
lib/jmap/client.ts
+122 -42
View File
@@ -6,7 +6,8 @@ import { IncompleteDeleteError } from "./delete-error";
import { downloadAllowed, saveBlobAs, type DownloadKind } from "@/lib/download-guard";
import { ArchiveEmailsError } from "./archive-error";
import { generateUUID } from '@/lib/utils';
import { standInsIn, withRealIds, withStandInsBack } from './stand-in-ids';
import { isStandInId, standInsIn, withRealIds, withStandInsBack } from './stand-in-ids';
import { isLocalBlobId } from '@/lib/store/outbox/items';
import type { Attachment, Email, Mailbox, MailboxRights, StateChange, AccountStates, CollectionChanges, ShareNotification, BusyPeriod, CalendarParticipantIdentity, CalendarEventNotification, Thread, Identity, EmailAddress, ContactCard, AddressBook, AddressBookRights, VacationResponse, Calendar, CalendarComponentType, CalendarRights, CalendarEvent, CalendarEventFilter, CalendarTask, CreateCalendarOptions, FileNode, FileNodeFilter, FileNodeRights, Principal, PrincipalAvailability, PrincipalListing, PushSubscription, EmailPushConfig, DeliveryStatus, EmailSubmission, RejectedRecipient, ScheduledEmail, SendEmailResult, SharedAccount } from "./types";
import type { SieveScript, SieveCapabilities } from "./sieve-types";
import type { BlobBatch, CalendarEventUpdateOptions, CalendarQueryOptions, FileNodeListings, IJMAPClient, KeywordDiscoveryResult, KeywordInfo, KeywordMigration, SearchEmailsResult, SearchRequestOptions, UploadedEmailImport, UploadedEmailImportResult } from "./client-interface";
@@ -54,7 +55,7 @@ import { toStorageQuota, type JMAPQuota, type StorageQuota } from "@/lib/quota";
import { messagePriorityHeaders, type MessagePriority } from "@/lib/message-priority";
import { customHeaderProperties, withLabelHeaders } from "@/lib/outgoing/custom-headers";
import { draftLabelProperty } from "@/lib/classification/labels";
import { attachmentParts, HTML_PART_ID, relatedBodyStructure, TEXT_PART_ID, type OutgoingBodyPart } from "./outgoing-body";
import { attachmentParts, HTML_PART_ID, relatedBodyStructure, repointAttachments, TEXT_PART_ID, type OutgoingBodyPart, type SavedPart } from "./outgoing-body";
import { addBodyMediaToAttachments } from "./body-media-attachments";
// Cap for the follow-up Email/get issued when a displayed body part comes
@@ -2224,8 +2225,10 @@ export class JMAPClient implements IJMAPClient {
* `networkOnly`: only the server's own word counts. A send or a change the
* outbox hands over must never be answered by the device: the device would
* queue it again, or answer for the server.
* `deviceOnly`: the other way round. What belongs to a draft saved on the
* device stays with it, even if the server came back in between.
*/
private async request(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean; skipOfflineRecord?: boolean; networkOnly?: boolean }): Promise<JMAPResponse> {
private async request(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean; skipOfflineRecord?: boolean; networkOnly?: boolean; deviceOnly?: boolean }): Promise<JMAPResponse> {
if (!this.apiUrl) {
throw new Error('Not connected. Call connect() first.');
}
@@ -2235,32 +2238,30 @@ export class JMAPClient implements IJMAPClient {
// browser is offline or the network fails. Without offline mode no hooks
// are set and this is the plain network request.
const offline = getOfflineHooks();
if (this.offlineOnly || offline?.skipNetwork(this)) {
// A stand-in id from an event or a draft created offline, still held by an
// open dialog or a list: the server knows it by its real id, or not yet at
// all. The store files what reached the server under the real id too.
const standIns = offline?.resolveStandIns ? standInsIn(methodCalls) : [];
let realIds: Record<string, string> | null = null;
let onDeviceOnly = false;
if (standIns.length > 0) {
const resolved = await offline!.resolveStandIns!(this, standIns);
const known = Object.entries(resolved).filter((entry): entry is [string, string] => !!entry[1]);
if (known.length > 0) {
realIds = Object.fromEntries(known);
methodCalls = withRealIds(methodCalls, realIds);
}
onDeviceOnly = standIns.some((id) => !resolved[id]);
}
const asAsked = (response: JMAPResponse) => (realIds ? withStandInsBack(response, realIds) : response);
if (this.offlineOnly || opts?.deviceOnly || onDeviceOnly || offline?.skipNetwork(this)) {
const local = offline && !opts?.networkOnly ? await offline.answer(this, methodCalls) : null;
if (local) {
this.lastAnswerFromDevice = true;
return local as JMAPResponse;
return asAsked(local as JMAPResponse);
}
throw new OfflineUnavailableError();
}
// A stand-in id from an event created offline, still held by an open dialog
// or a list: the server knows it by its real id, or not yet at all.
const standIns = offline?.resolveStandIns ? standInsIn(methodCalls) : [];
let realIds: Record<string, string> | null = null;
if (standIns.length > 0) {
const resolved = await offline!.resolveStandIns!(this, standIns);
if (standIns.some((id) => !resolved[id])) {
const local = opts?.networkOnly ? null : await offline!.answer(this, methodCalls);
if (local) {
this.lastAnswerFromDevice = true;
return local as JMAPResponse;
}
throw new OfflineUnavailableError();
}
realIds = resolved as Record<string, string>;
methodCalls = withRealIds(methodCalls, realIds);
}
const asAsked = (response: JMAPResponse) => (realIds ? withStandInsBack(response, realIds) : response);
let answer: JMAPResponse;
try {
answer = await this.networkRequest(methodCalls, using, opts);
@@ -2268,7 +2269,7 @@ export class JMAPClient implements IJMAPClient {
// Only when the server is really gone: a refusal without CORS
// headers looks the same to the browser and must stay an error.
if (offline && isNetworkFailure(error) && await offline.serverGone(this, error)) {
const local = opts?.networkOnly ? null : await offline.answer(this, methodCalls);
const local = opts?.networkOnly ? null : await offline.answer(this, methodCalls, { networkTried: true });
if (local) {
this.lastAnswerFromDevice = true;
return asAsked(local as JMAPResponse);
@@ -5115,9 +5116,23 @@ export class JMAPClient implements IJMAPClient {
throw new Error('No drafts mailbox found');
}
attachments = await this.withPartsOfFlushedDraft(attachments, draftId, options?.previousDraftAccountId);
// Files attached offline go up now if the server answers. If it doesn't,
// the draft stays on the device with them, as when offline.
const hooks = getOfflineHooks();
if (attachments?.some(att => isLocalBlobId(att.blobId)) && hooks?.resolveHeldAttachments && !this.offlineOnly && !hooks.skipNetwork(this)) {
try {
attachments = await hooks.resolveHeldAttachments(this, attachments, draftAccountId);
} catch (error) {
if (!(error instanceof Error) || error.name !== 'SendNotStartedError') throw error;
}
}
const emailId = `draft-${Date.now()}`;
interface EmailDraft {
messageId: string[];
from: { name?: string; email: string }[];
to: { name?: string; email: string }[];
cc?: { name?: string; email: string }[];
@@ -5141,6 +5156,9 @@ export class JMAPClient implements IJMAPClient {
const draftInReplyTo = inReplyTo?.map(stripMessageIdBrackets).filter(Boolean);
const draftReferences = references?.map(stripMessageIdBrackets).filter(Boolean);
const emailData: EmailDraft = {
// Fixed here: when the connection breaks on the way, this is how the
// device later finds out whether the server took the draft after all.
messageId: [generateMessageId(parseRecipientString(fromEmail || this.username).email)],
from: [{ ...(sanitizedFromName ? { name: sanitizedFromName } : {}), email: fromEmail || this.username }],
// "Name <addr>" must be split into the two JMAP fields: storing the whole
// mailbox as the address makes the server treat the display name as part
@@ -5167,14 +5185,17 @@ export class JMAPClient implements IJMAPClient {
// Images the HTML shows by cid: go into a multipart/related beside it
// (#1451), so the draft and the mail sent from it display them.
const draftStructure = relatedBodyStructure(htmlBody, attachments);
if (draftStructure) {
delete emailData.textBody;
delete emailData.htmlBody;
emailData.bodyStructure = draftStructure;
} else if (attachments?.length) {
emailData.attachments = attachmentParts(attachments);
}
const putParts = (parts: typeof attachments) => {
const draftStructure = relatedBodyStructure(htmlBody, parts);
if (draftStructure) {
delete emailData.textBody;
delete emailData.htmlBody;
emailData.bodyStructure = draftStructure;
} else if (parts?.length) {
emailData.attachments = attachmentParts(parts);
}
};
putParts(attachments);
// Destroy the previous draft version only AFTER the replacement was
// created (#849). A combined `Email/set { create, destroy }` processes
@@ -5182,12 +5203,23 @@ export class JMAPClient implements IJMAPClient {
// previous version (re-opened draft with attachments) and fails with
// blobNotFound, the destroy still went through - deleting the last good
// copy of the draft. Worst case now is an orphaned old version.
const response = await this.request([
const createVersion = () => this.request([
["Email/set", {
accountId: draftAccountId,
create: { [emailId]: emailData },
}, "0"],
]);
let response = await createVersion();
// A part that died with the message it belonged to: its file is in the draft
// under another id now. Once, with the parts the draft has.
if (draftId && response.methodResponses?.[0]?.[1]?.notCreated?.[emailId]?.type === 'blobNotFound') {
const fresh = await this.withPartsOfDraft(attachments, draftId, options?.previousDraftAccountId);
if (fresh !== attachments) {
attachments = fresh;
putParts(fresh);
response = await createVersion();
}
}
if (response.methodResponses?.[0]?.[0] === "Email/set") {
const result = response.methodResponses[0][1];
@@ -5205,9 +5237,12 @@ export class JMAPClient implements IJMAPClient {
if (draftId) {
try {
// A version saved on the device replaces the old one when it
// reaches the server, not before: the old one stays there until then.
const onDevice = isStandInId(newDraftId);
const destroyResponse = await this.request([
["Email/set", { accountId: options?.previousDraftAccountId || this.accountId, destroy: [draftId] }, "0"],
]);
["Email/set", { accountId: options?.previousDraftAccountId || this.accountId, destroy: [draftId], ...(onDevice ? { replacedBy: newDraftId } : {}) }, "0"],
], undefined, onDevice ? { deviceOnly: true } : undefined);
const destroyResult = destroyResponse.methodResponses?.[0]?.[1];
if (destroyResult?.notDestroyed) {
console.warn('Failed to destroy old draft:', destroyResult.notDestroyed);
@@ -5225,6 +5260,33 @@ export class JMAPClient implements IJMAPClient {
throw new Error('Failed to save draft');
}
/** `attachments` pointed at the parts the draft has now. The same list when nothing changes. */
private async withPartsOfDraft<T extends { blobId: string; name: string; size?: number }>(attachments: T[] | undefined, draftId: string, accountId: string | undefined): Promise<T[] | undefined> {
if (!attachments?.some((attachment) => !isLocalBlobId(attachment.blobId))) return attachments;
let email: Pick<Email, "attachments" | "textBody" | "htmlBody"> | undefined;
try {
const response = await this.request([["Email/get", { accountId: accountId || this.accountId, ids: [draftId], properties: ["attachments", "textBody", "htmlBody"] }, "0"]]);
email = response.methodResponses?.[0]?.[1]?.list?.[0];
} catch (error) {
debug.warn('email', 'Could not read the parts of the draft:', error);
}
if (!email) return attachments;
const parts: SavedPart[] = addBodyMediaToAttachments(email).attachments ?? [];
const fresh = repointAttachments(attachments, parts, (attachment) => isLocalBlobId(attachment.blobId));
return fresh.some((attachment, i) => attachment !== attachments[i]) ? fresh : attachments;
}
/**
* A stand-in that has reached the server: the draft was created anew there, so
* parts the page got from an older version may be gone.
*/
private async withPartsOfFlushedDraft<T extends { blobId: string; name: string; size?: number }>(attachments: T[] | undefined, draftId: string | undefined, accountId: string | undefined): Promise<T[] | undefined> {
const offline = getOfflineHooks();
if (!draftId || !isStandInId(draftId) || !offline?.resolveStandIns || this.offlineOnly || offline.skipNetwork(this)) return attachments;
const resolved = await offline.resolveStandIns(this, [draftId]);
return resolved[draftId] ? this.withPartsOfDraft(attachments, draftId, accountId) : attachments;
}
async sendEmail(
to: string[],
subject: string,
@@ -5245,9 +5307,12 @@ export class JMAPClient implements IJMAPClient {
// one (see createDraft). `messageId` and `existingCopyId` come from the
// offline outbox: a Message-ID fixed when the message was queued, and the
// copy already on the server that only needs submitting (lib/store/outbox/dedupe.ts).
options?: SubmissionSendOptions & { draftAccountId?: string; messageId?: string; existingCopyId?: string }
options?: SubmissionSendOptions & { draftAccountId?: string; messageId?: string; existingCopyId?: string; partsRepointed?: boolean }
): Promise<SendEmailResult> {
const holdForSeconds = delayedUntil ? this.validateDelayedUntil(delayedUntil) : undefined;
if (!options?.partsRepointed && !options?.existingCopyId) {
attachments = await this.beforeSubmission(() => this.withPartsOfFlushedDraft(attachments, draftId, options?.draftAccountId));
}
const emailId = `send-${Date.now()}`;
const targetAccountId = await this.beforeSubmission(() => this.resolveSendAccountId(fromEmail));
const mboxResp = await this.beforeSubmission(() => this.request([
@@ -5525,6 +5590,14 @@ export class JMAPClient implements IJMAPClient {
if (failure) {
console.error('[sendEmail] JMAP method error:', failure);
await this.discardUnsentCopy(draftCopyId, targetAccountId);
// A part that died with its message: nothing went out, so once more with
// the parts the draft has now.
if (failure.type === 'blobNotFound' && draftId && !options?.partsRepointed) {
const fresh = await this.withPartsOfDraft(attachments, draftId, options?.draftAccountId);
if (fresh !== attachments) {
return this.sendEmail(to, subject, body, cc, bcc, identityId, fromEmail, draftId, fromName, htmlBody, fresh, inReplyTo, references, delayedUntil, envelopeMailFrom, { ...options, partsRepointed: true });
}
}
const propsHint = failure.properties?.length ? ` (properties: ${failure.properties.join(', ')})` : '';
throw jmapError(`${failure.description || `Failed to send email: ${failure.type}`}${propsHint}`, failure);
}
@@ -5638,12 +5711,16 @@ export class JMAPClient implements IJMAPClient {
// a filing warning rather than a failed send (#849). The submission
// record rides along, and nothing about it may turn into a warning.
if (draftId && createdEmailId) {
// A draft still on the device is answered by the device, which can't
// remove a record on the server: that one goes on its own.
const recordAlongside = submissionToDestroy && !isStandInId(draftId) ? submissionToDestroy : undefined;
if (submissionToDestroy && !recordAlongside) await this.destroyEmailSubmissions([submissionToDestroy], submissionAccountId);
try {
const cleanupCalls: JMAPMethodCall[] = [
["Email/set", { accountId: options?.draftAccountId || this.accountId, destroy: [draftId] }, "0"],
];
if (submissionToDestroy) {
cleanupCalls.push(submissionDestroyCall([submissionToDestroy], submissionAccountId, "1"));
if (recordAlongside) {
cleanupCalls.push(submissionDestroyCall([recordAlongside], submissionAccountId, "1"));
}
const destroyResponse = await this.request(cleanupCalls);
const destroyResult = destroyResponse.methodResponses?.find(([, , callId]) => callId === "0")?.[1];
@@ -5652,7 +5729,7 @@ export class JMAPClient implements IJMAPClient {
const first = Object.values(destroyResult.notDestroyed as Record<string, { type?: string; description?: string }>)[0];
filingError = filingError ?? (first?.description || first?.type || 'old draft cleanup failed');
}
if (submissionToDestroy) readSubmissionDestroy(destroyResponse, "1");
if (recordAlongside) readSubmissionDestroy(destroyResponse, "1");
} catch (err) {
console.error('[sendEmail] old draft cleanup failed:', err);
filingError = filingError ?? 'old draft cleanup failed';
@@ -6315,6 +6392,8 @@ export class JMAPClient implements IJMAPClient {
accountId?: string;
onProgress?: (loaded: number, total: number) => void;
signal?: AbortSignal;
/** A file already held on this device that's on its way up: an outage is an error, not a second copy. */
noHold?: boolean;
},
): Promise<{ blobId: string; size: number; type: string }> {
const opts =
@@ -6337,7 +6416,7 @@ export class JMAPClient implements IJMAPClient {
// is uploaded when the message it belongs to is sent from the outbox.
const offline = getOfflineHooks();
if (this.offlineOnly || offline?.skipNetwork(this)) {
const held = offline ? await offline.holdUpload(this, file, targetAccountId) : null;
const held = offline && !opts.noHold ? await offline.holdUpload(this, file, targetAccountId) : null;
if (held) return held;
throw new OfflineUnavailableError();
}
@@ -6372,7 +6451,7 @@ export class JMAPClient implements IJMAPClient {
// The server is gone though the browser says it's online (#523): the
// file waits on this device as when offline. Asking whether it is
// gone tells an outage from a refusal (a too-large file, a quota).
if (offline && !opts.signal?.aborted && await offline.serverGone(this)) {
if (offline && !opts.noHold && !opts.signal?.aborted && await offline.serverGone(this)) {
const held = await offline.holdUpload(this, file, targetAccountId);
if (held) return held;
}
@@ -6553,7 +6632,8 @@ export class JMAPClient implements IJMAPClient {
private async offlineAwareDownload(url: string, blobId: string, accountId: string | undefined): Promise<Response | Blob> {
const offline = getOfflineHooks();
const account = accountId || this.accountId;
if (this.offlineOnly || offline?.skipNetwork(this)) {
// A file attached offline only exists on this device, whatever the network does.
if (this.offlineOnly || isLocalBlobId(blobId) || offline?.skipNetwork(this)) {
const local = offline ? await offline.blob(this, blobId, account) : null;
if (local) return local;
throw new OfflineUnavailableError();
lib/jmap/offline-hooks.ts
+11 -2
View File
@@ -29,8 +29,12 @@ export interface OfflineClientRef {
export interface OfflineHooks {
/** Answer without trying the network: the browser says it is offline (or the client was started offline). */
skipNetwork(client: OfflineClientRef): boolean;
/** An answer from the device for a request the network couldn't take, or null. */
answer(client: OfflineClientRef, methodCalls: OfflineMethodCall[]): Promise<OfflineResponse | null>;
/**
* An answer from the device for a request the network couldn't take, or null.
* `networkTried`: the request went out and its connection broke, so the server
* may have taken it even though no answer came.
*/
answer(client: OfflineClientRef, methodCalls: OfflineMethodCall[], options?: { networkTried?: boolean }): Promise<OfflineResponse | null>;
/** What the server answered, for the cache. */
record(client: OfflineClientRef, methodCalls: OfflineMethodCall[], response: OfflineResponse): void;
/** A request reached the server (true) or failed on the network (false). */
@@ -61,6 +65,11 @@ export interface OfflineHooks {
* ids as they are.
*/
resolveStandIns?(client: OfflineClientRef, ids: string[]): Promise<Record<string, string | null>>;
/**
* Files attached offline, uploaded now that the server answers, for a save
* that names them. Throws SendNotStartedError when the upload hits an outage.
*/
resolveHeldAttachments?<T extends { blobId: string }>(client: OfflineClientRef, attachments: T[], accountId?: string): Promise<T[]>;
/** The user opened this file: the local store keeps recently opened files. */
fileOpened?(client: OfflineClientRef, accountId: string, id: string): void;
}
lib/jmap/outgoing-body.ts
+32
View File
@@ -40,6 +40,38 @@ export function attachmentParts(attachments: OutgoingAttachment[]): OutgoingBody
return attachments.map(attachmentPart);
}
/** A part of a saved message, as Email/get lists it. */
export interface SavedPart {
blobId?: string | null;
name?: string | null;
size?: number;
}
/**
* The attachments of a message pointed at the parts of its saved version, by name
* and size. Every new message gets new part ids, and old ones die with their
* message. One whose size isn't known goes by name. `keep` leaves one as it is.
*/
export function repointAttachments<T extends { blobId: string; name: string; size?: number }>(
attachments: T[],
parts: readonly SavedPart[],
keep: (attachment: T) => boolean,
): T[] {
const free = parts.filter((part) => part.blobId);
const take = (test: (part: SavedPart) => boolean): SavedPart | undefined => {
const at = free.findIndex(test);
return at < 0 ? undefined : free.splice(at, 1)[0];
};
// What's still alive keeps its part, so a file of the same name can't take it.
const alive = attachments.filter((attachment) => !keep(attachment) && take((part) => part.blobId === attachment.blobId));
return attachments.map((attachment) => {
if (keep(attachment) || alive.includes(attachment)) return attachment;
const found = take((part) => part.name === attachment.name && part.size === attachment.size)
?? (attachment.size ? undefined : take((part) => part.name === attachment.name));
return found ? { ...attachment, blobId: found.blobId as string } : attachment;
});
}
function referencedByHtml(html: string, cid: string): boolean {
const ref = `cid:${normalizeCid(cid)}`.toLowerCase();
const lower = html.toLowerCase();
lib/jmap/stand-in-ids.ts
+11 -7
View File
@@ -1,7 +1,8 @@
/**
* Stand-in ids: what the local store hands out for an event created offline.
* Stand-in ids: what the local store hands out for an event or a draft created
* offline.
*
* A page can hold one long after the event reached the server: an open dialog,
* A page can hold one long after the record reached the server: an open dialog,
* a list not refetched yet, a plugin. So the client swaps them for the real ids
* on the way to the network, and swaps them back in the answer, because callers
* look up the outcome under the id they asked about.
@@ -19,14 +20,17 @@ export function newStandInId(): string {
return `${STAND_IN_PREFIX}${crypto.randomUUID()}`;
}
const SET_METHODS = new Set(['CalendarEvent/set', 'Email/set']);
const GET_METHODS = new Set(['CalendarEvent/get', 'Email/get']);
/** The stand-in ids a request names. */
export function standInsIn(calls: readonly OfflineMethodCall[]): string[] {
const found = new Set<string>();
for (const [name, args] of calls) {
if (name === 'CalendarEvent/set') {
if (SET_METHODS.has(name)) {
for (const id of Object.keys((args.update ?? {}) as object)) if (isStandInId(id)) found.add(id);
for (const id of (args.destroy ?? []) as string[]) if (isStandInId(id)) found.add(id);
} else if (name === 'CalendarEvent/get') {
} else if (GET_METHODS.has(name)) {
for (const id of (args.ids ?? []) as string[]) if (isStandInId(id)) found.add(id);
}
}
@@ -42,13 +46,13 @@ export function withRealIds<T extends OfflineMethodCall>(calls: readonly T[], re
const rename = (id: string) => real[id] ?? id;
return calls.map((call) => {
const [name, args, callId] = call;
if (name === 'CalendarEvent/set') {
if (SET_METHODS.has(name)) {
const next = { ...args };
if (args.update) next.update = renamedKeys(args.update as Record<string, unknown>, rename);
if (args.destroy) next.destroy = (args.destroy as string[]).map(rename);
return [name, next, callId] as unknown as T;
}
if (name === 'CalendarEvent/get' && args.ids) {
if (GET_METHODS.has(name) && args.ids) {
return [name, { ...args, ids: (args.ids as string[]).map(rename) }, callId] as unknown as T;
}
return call;
@@ -62,7 +66,7 @@ export function withStandInsBack<T extends OfflineResponse>(response: T, real: R
return {
...response,
methodResponses: response.methodResponses.map(([name, result, callId]) => {
if (name !== 'CalendarEvent/set' && name !== 'CalendarEvent/get') return [name, result, callId];
if (!SET_METHODS.has(name) && !GET_METHODS.has(name)) return [name, result, callId];
const next = { ...result };
for (const key of ['updated', 'notUpdated', 'notDestroyed']) {
if (next[key]) next[key] = renamedKeys(next[key], rename);
lib/offline/__tests__/drafts-flush.test.ts
+323
View File
@@ -0,0 +1,323 @@
import 'fake-indexeddb/auto';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { useAuthStore } from '@/stores/auth-store';
import { useAccountStore } from '@/stores/account-store';
import { usePolicyStore } from '@/stores/policy-store';
import { DEFAULT_ORGANISATION_POLICY } from '@/lib/admin/organisation-policy';
import { setStoreBackend } from '@/lib/store/client/api';
import { FakeJmapDrafts } from '@/lib/store/__tests__/helpers/fake-drafts';
import { JMAP_ACCOUNT } from '@/lib/store/__tests__/helpers/fake-jmap';
import { InProcessStore } from '@/lib/store/__tests__/helpers/in-process-store';
import { consumeLost } from '../lost';
import { offlineNotify, refreshQueueState, setOfflineNotifier, startOfflineMode, unsentCount, wipeAccount, wipeEverything } from '../offline-mode';
import { flushQueue, forgetOrphanCheck } from '../flush';
import { heldFile, holdFile, noteDraftReplacedElsewhere, queueSend, readLocalDrafts, retryLocalDraft } from '../queue';
import { forgetLegacyMove } from '../migrate-legacy';
import { useOfflineStore, writeEnabled } from '../state';
import type { JMAPClient } from '@/lib/jmap/client';
let server: FakeJmapDrafts;
let store: InProcessStore;
function fakeClient(overrides: Record<string, unknown> = {}) {
return {
getServerUrl: () => 'https://mail.example.org',
getUsername: () => 'alice@example.org',
getAccountId: () => JMAP_ACCOUNT,
isOfflineOnly: () => false,
getSessionSnapshot: () => null,
getAuthHeader: () => 'Basic x',
keywordSortInverted: () => null,
sendEmail: vi.fn(async () => ({ scheduled: false, emailId: 'sent-1', emailSubmissionId: 'sub-1' })),
uploadBlob: vi.fn(async (file: File) => server.upload(JMAP_ACCOUNT, new Uint8Array(await file.arrayBuffer()))),
requestNetworkOnly: vi.fn((calls: never) => server.request(calls)),
resolveSendAccountId: vi.fn(async () => JMAP_ACCOUNT),
submissionAccountFor: () => JMAP_ACCOUNT,
pruneEmailSubmissions: vi.fn(async () => {}),
...overrides,
};
}
let stop: () => void = () => {};
async function saveOffline(subject: string, attachments?: Array<Record<string, unknown>>): Promise<string> {
const answer = await store.answer('acc', [['Email/set', {
accountId: JMAP_ACCOUNT,
create: {
c: {
messageId: [`${subject}@example.org`],
subject,
keywords: { $seen: true, $draft: true },
mailboxIds: { drafts: true },
bodyValues: { 1: { value: 'text' } },
textBody: [{ partId: '1' }],
...(attachments ? { attachments } : {}),
},
},
}, 's']]);
return answer?.methodResponses[0][1].created.c.id as string;
}
beforeEach(async () => {
server = new FakeJmapDrafts({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'drafts', role: 'drafts' }, { id: 'sent', role: 'sent' }] });
server.now = Date.now();
store = new InProcessStore(server);
setStoreBackend(store);
forgetLegacyMove();
forgetOrphanCheck();
localStorage.clear();
useOfflineStore.setState({ enabled: [], outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [], browserOffline: false, downServers: [], startedOffline: false });
usePolicyStore.setState({ loaded: true, subject: null, policy: { ...usePolicyStore.getState().policy, organisation: DEFAULT_ORGANISATION_POLICY } });
useAccountStore.setState({ accounts: [{ id: 'acc', label: 'alice', serverUrl: 'https://mail.example.org', username: 'alice@example.org' } as never] });
const client = fakeClient();
useAuthStore.setState({ client: client as never, activeAccountId: 'acc' });
writeEnabled(['acc']);
stop = startOfflineMode();
await store.sync('acc');
});
afterEach(async () => {
stop();
await wipeEverything();
setStoreBackend(null);
});
describe('flushQueue and drafts', () => {
it('uploads a draft\'s files, creates it, tells the page, and leaves nothing behind', async () => {
const client = fakeClient();
const held = await holdFile('acc', new File(['hello'], 'note.txt', { type: 'text/plain' }), 'note.txt');
await saveOffline('With a file', [{ blobId: held.blobId, type: 'text/plain', name: 'note.txt' }]);
const onDraftSent = vi.fn();
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true, onDraftSent });
expect(server.drafts().map((email) => email.subject)).toEqual(['With a file']);
expect(client.uploadBlob).toHaveBeenCalledWith(expect.any(File), { accountId: JMAP_ACCOUNT, noHold: true });
expect(onDraftSent).toHaveBeenCalledTimes(1);
expect(await readLocalDrafts('acc')).toEqual([]);
});
it('holds a large file in pieces and reads it back whole', async () => {
const size = 2 * 1024 * 1024 + 3;
const held = await holdFile('acc', new File([new Uint8Array(size).fill(4)], 'big.bin'), 'big.bin');
expect(held.size).toBe(size);
const area = vi.spyOn(store, 'area');
const file = await heldFile('acc', held.blobId);
expect(file?.name).toBe('big.bin');
expect(file?.size).toBe(size);
expect(new Uint8Array(await (file as File).arrayBuffer()).every((value) => value === 4)).toBe(true);
area.mockRestore();
});
it('tells the user about a refused draft, keeps it, and lists it with its error', async () => {
const client = fakeClient();
server.refuseCreate = { type: 'overQuota', description: 'No room' };
const id = await saveOffline('Too much');
const notify = vi.fn();
await flushQueue('acc', client as unknown as JMAPClient, { notify, allowed: () => true });
expect(notify).toHaveBeenCalledWith({ kind: 'draft-refused', subject: 'Too much', error: { type: 'overQuota', text: 'No room' } });
await refreshQueueState();
expect(useOfflineStore.getState().localDrafts).toMatchObject([{ account: 'acc', localId: id, status: 'refused' }]);
server.refuseCreate = null;
await retryLocalDraft('acc', id);
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(server.drafts()).toHaveLength(1);
});
it('stops on an outage and carries on with what waited after it', async () => {
const client = fakeClient();
await saveOffline('One');
server.offline = true;
const sendEmail = client.sendEmail;
const entry = await queueSend('acc', { subject: 'Queued', to: [] }, { kind: 'email', args: [['b@example.org'], 'Queued', 'x'] });
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(sendEmail).not.toHaveBeenCalled();
server.offline = false;
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(server.drafts()).toHaveLength(1);
expect(sendEmail).toHaveBeenCalledTimes(1);
expect(entry).toBeGreaterThan(0);
});
it('sends a draft kept here through the send path with its files, and the draft doesn\'t go up as well', async () => {
const held = await holdFile('acc', new File(['hello'], 'note.txt', { type: 'text/plain' }), 'note.txt');
const id = await saveOffline('To send', [{ blobId: held.blobId, type: 'text/plain', name: 'note.txt' }]);
const client = fakeClient({
// As the client does: the draft goes once the message is out.
sendEmail: vi.fn(async () => {
await store.answer('acc', [['Email/set', { accountId: JMAP_ACCOUNT, destroy: [id] }, 'd']]);
return { scheduled: false, emailId: 'sent-1', emailSubmissionId: 'sub-1' };
}),
});
await queueSend('acc', { subject: 'To send', to: [] }, {
kind: 'email',
args: [['b@example.org'], 'To send', 'x', undefined, undefined, 'i', 'alice@example.org', id, undefined, undefined, [{ blobId: held.blobId, name: 'note.txt', type: 'text/plain', size: 5 }]],
});
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(client.sendEmail).toHaveBeenCalledTimes(1);
expect(((client.sendEmail as ReturnType<typeof vi.fn>).mock.calls[0] as unknown[])[10]).toMatchObject([{ blobId: server.uploads[0].blobId }]);
expect(server.drafts()).toEqual([]);
expect(await readLocalDrafts('acc')).toEqual([]);
});
it('drops the draft of a send the lookup found went out', async () => {
const id = await saveOffline('Went out');
const client = fakeClient();
server.addEmail({ id: 'in-sent', subject: 'Went out', mailboxIds: { sent: true }, messageId: ['fixed@example.org'], receivedAt: new Date().toISOString() });
const entryId = await queueSend('acc', { subject: 'Went out', to: [] }, {
kind: 'email',
args: [['b@example.org'], 'Went out', 'x', undefined, undefined, 'i', 'alice@example.org', id],
}, undefined, 'fixed@example.org');
await store.area('acc', 'outbox.claim', { id: entryId });
await store.area('acc', 'outbox.update', { id: entryId, status: 'uncertain' });
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(await readLocalDrafts('acc')).toEqual([]);
expect(server.drafts()).toEqual([]);
});
it('cleans held files nothing needs, only long after', async () => {
const client = fakeClient();
const orphan = await holdFile('acc', new File(['a'], 'a.txt'), 'a.txt');
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(await heldFile('acc', orphan.blobId)).not.toBeNull();
const now = Date.now();
const clock = vi.spyOn(Date, 'now').mockReturnValue(now + 8 * 24 * 60 * 60 * 1000);
await flushQueue('acc', client as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
clock.mockRestore();
expect(await heldFile('acc', orphan.blobId)).toBeNull();
});
});
describe('a notice before the host is there', () => {
it('waits for it, and comes once', async () => {
vi.resetModules();
const fresh = await import('../offline-mode');
fresh.offlineNotify({ kind: 'store-reset' });
const show = vi.fn();
fresh.setOfflineNotifier(show);
expect(show).toHaveBeenCalledTimes(1);
expect(show).toHaveBeenCalledWith({ kind: 'store-reset' });
fresh.setOfflineNotifier(vi.fn());
expect(show).toHaveBeenCalledTimes(1);
vi.resetModules();
});
it('goes straight to a host that\'s there', () => {
const show = vi.fn();
setOfflineNotifier(show);
offlineNotify({ kind: 'store-reset' });
expect(show).toHaveBeenCalledTimes(1);
});
});
describe('what a sign-out asks about', () => {
it('counts a draft only on this device', async () => {
await saveOffline('Only here');
expect(await unsentCount('acc')).toMatchObject({ messages: 1 });
});
it('counts a draft a waiting send names once, as that send', async () => {
const id = await saveOffline('Being sent');
await queueSend('acc', { subject: 'Being sent', to: [] }, { kind: 'email', args: [['b@example.org'], 'Being sent', 'x', undefined, undefined, 'i', 'alice@example.org', id] });
expect(await unsentCount('acc')).toMatchObject({ messages: 1 });
});
});
describe('a wipe and local drafts', () => {
it('names them as lost when the user didn\'t agree', async () => {
// A second account stays, so the wipe doesn't take the whole store.
await saveOffline('Only here');
await refreshQueueState();
consumeLost();
await wipeAccount('acc');
expect(consumeLost()).toEqual(['Only here']);
});
});
describe('a flush that stops', () => {
it('tries again soon when a passing failure stopped it', async () => {
await saveOffline('One');
server.offline = true;
const retryLater = vi.fn();
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true, retryLater });
expect(retryLater).toHaveBeenCalled();
});
it('doesn\'t when nothing stopped it', async () => {
await saveOffline('One');
const retryLater = vi.fn();
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true, retryLater });
expect(retryLater).not.toHaveBeenCalled();
});
});
describe('a reload in the middle of a flush', () => {
const nav = navigator as Navigator & { locks?: unknown };
afterEach(() => {
Object.defineProperty(nav, 'locks', { configurable: true, value: undefined });
});
async function leftSending() {
const id = await saveOffline('Left on the way');
expect(await store.area('acc', 'drafts.claim', { localId: id })).toBe(true);
return id;
}
it('goes on at once under the flush lock: nobody else can be sending it', async () => {
Object.defineProperty(nav, 'locks', { configurable: true, value: { request: (_name: string, run: () => Promise<unknown>) => run() } });
await leftSending();
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(server.drafts()).toHaveLength(1);
});
it('waits the minute without Web Locks, because another tab may be on it', async () => {
await leftSending();
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(server.drafts()).toEqual([]);
});
});
describe('an old version in another login', () => {
function clients() {
const other = { ...fakeClient(), deleteEmail: vi.fn(async () => {}) };
useAuthStore.setState({ getClientForAccount: ((id: string) => (id === 'other' ? other : undefined)) as never });
return other;
}
it('stays until the new version is on the server, then goes through its own login', async () => {
const other = clients();
server.addEmail({ id: 'R', subject: 'Elsewhere', mailboxIds: { drafts: true }, keywords: { $draft: true } });
const id = await saveOffline('New');
await noteDraftReplacedElsewhere('acc', id, { login: 'other', jmap: JMAP_ACCOUNT, id: 'R' });
expect(other.deleteEmail).not.toHaveBeenCalled();
server.refuseCreate = { type: 'overQuota' };
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(other.deleteEmail).not.toHaveBeenCalled();
server.refuseCreate = null;
await retryLocalDraft('acc', id);
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(other.deleteEmail).toHaveBeenCalledWith('R', JMAP_ACCOUNT);
});
it('stays when it\'s no longer a draft there', async () => {
const other = clients();
server.addEmail({ id: 'R', subject: 'Sent elsewhere', mailboxIds: { drafts: true }, keywords: {} });
const id = await saveOffline('New');
await noteDraftReplacedElsewhere('acc', id, { login: 'other', jmap: JMAP_ACCOUNT, id: 'R' });
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(other.deleteEmail).not.toHaveBeenCalled();
});
it('carries over to the next save, and waits for that login to be there', async () => {
useAuthStore.setState({ getClientForAccount: (() => undefined) as never });
server.addEmail({ id: 'R', subject: 'Elsewhere', mailboxIds: { drafts: true }, keywords: { $draft: true } });
const first = await saveOffline('First');
await noteDraftReplacedElsewhere('acc', first, { login: 'other', jmap: JMAP_ACCOUNT, id: 'R' });
const second = await saveOffline('Second');
await store.answer('acc', [['Email/set', { accountId: JMAP_ACCOUNT, destroy: [first], replacedBy: second }, 'd']]);
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(server.emails.has('R')).toBe(true);
const other = clients();
await flushQueue('acc', fakeClient() as unknown as JMAPClient, { notify: vi.fn(), allowed: () => true });
expect(other.deleteEmail).toHaveBeenCalledWith('R', JMAP_ACCOUNT);
});
});
lib/offline/__tests__/migrate-legacy.test.ts
+2 -2
View File
@@ -121,9 +121,9 @@ describe('moving the old outbox into the store', () => {
expect(outbox[0].messageId).toMatch(/@example\.org$/);
expect(outbox[1].messageId).toBeUndefined();
expect(outbox[3].messageId).toBe('raw-1@example.org');
const file = await store.area<{ name: string; data: Uint8Array }>('acc', 'outbox.heldFile', { blobId: 'bulwark-local-raw' });
const file = await store.area<{ name: string; blob: Blob }>('acc', 'outbox.heldFile', { blobId: 'bulwark-local-raw' });
expect(file.name).toBe('message.eml');
expect(new TextDecoder().decode(file.data)).toBe(RAW);
expect(await file.blob.text()).toBe(RAW);
expect(await oldExists()).toBe(false);
expect(store.has('stranger')).toBe(false);
});
lib/offline/flush.ts
+71 -5
View File
@@ -29,11 +29,15 @@
import type { JMAPClient } from '@/lib/jmap/client';
import { isNetworkFailure } from '@/lib/jmap/offline-hooks';
import type { SendEmailResult } from '@/lib/jmap/types';
import { isStandInId } from '@/lib/jmap/stand-in-ids';
import { useAuthStore } from '@/stores/auth-store';
import type { DraftElsewhere } from '@/lib/store/drafts/calls';
import { getStore } from '@/lib/store/client/api';
import { flushDrafts, type DraftFlushIo } from '@/lib/store/drafts/flush';
import { isStorageFailure } from '@/lib/store/storage';
import type { MethodCall } from '@/lib/store/engine/transport';
import { decideRetry, type LookupIo, type RetryDecision } from '@/lib/store/outbox/dedupe';
import { flushPatches, type PatchIo, type PatchOutcome, type PendingPatch } from '@/lib/store/outbox/flush-patches';
import { flushPatches, isRequestRefusal, type PatchIo, type PatchOutcome, type PendingPatch } from '@/lib/store/outbox/flush-patches';
import {
claimForSending,
heldFile,
@@ -56,6 +60,8 @@ export interface FlushDeps {
onScheduled?: (entry: OutboxEntry, result: SendEmailResult) => void;
/** A send's connection broke though the server may be fine: look it up again soon. */
retryLater?: () => void;
/** A draft saved on this device reached the server: lists and open pages read again. */
onDraftSent?: () => void;
}
/** A message that just went may not show in a lookup yet, which would send it twice. */
@@ -113,7 +119,7 @@ async function uploadHeld(account: string, client: JMAPClient, blobId: string):
if (!file) throw new OutboxPrepError('file_missing');
let uploaded: string;
try {
uploaded = (await client.uploadBlob(file)).blobId;
uploaded = (await client.uploadBlob(file, { noHold: true })).blobId;
} catch (error) {
if (isNetworkFailure(error)) throw notStartedError('upload failed');
throw error;
@@ -167,8 +173,8 @@ async function sendEntry(account: string, client: JMAPClient, entry: OutboxEntry
/** Sends what waits for an account, one tab at a time. */
export async function flushQueue(account: string, client: JMAPClient, deps: FlushDeps): Promise<void> {
const run = () => flushLocked(account, client, deps);
const locks = typeof navigator !== 'undefined' ? (navigator as Navigator & { locks?: LockManager }).locks : undefined;
const run = () => flushLocked(account, client, deps, Boolean(locks?.request));
if (locks?.request) {
await locks.request(`bulwark-offline-flush:${account}`, run);
return;
@@ -200,6 +206,52 @@ export async function settleStaleSends(account?: string, now = Date.now()): Prom
await Promise.allSettled(accounts.map((id) => getStore().area(id, 'outbox.settleStale', { before: now - SENDING_STALE_MS })));
}
/** The drafts saved on this device. False when the run stopped on an outage. */
function sendDrafts(account: string, client: JMAPClient, deps: FlushDeps, locked: boolean): Promise<boolean> {
const store = getStore();
const io: DraftFlushIo = {
store: {
// Under the lock, what says it's on its way was left by a tab that's gone.
read: () => store.area(account, 'drafts.read', { before: locked ? Date.now() + 1 : Date.now() - SENDING_STALE_MS }),
claim: (localId) => store.area(account, 'drafts.claim', { localId }),
failed: (localId, status, error, neverSent) => store.area(account, 'drafts.failed', { localId, status, error, neverSent }),
created: (result) => store.area(account, 'drafts.created', result),
ghostsLooked: (jmap, messageIds, found) => store.area(account, 'drafts.ghostsLooked', { jmap, messageIds, found }),
},
request: (calls, using) => client.requestNetworkOnly(calls, using),
upload: async (jmap, blobId) => {
const file = await heldFile(account, blobId);
return file ? (await client.uploadBlob(file, { accountId: jmap, noHold: true })).blobId : null;
},
// An upload's refusal has no JMAP problem document, only its status.
isRefusal: (error) => isRequestRefusal(error) || (error instanceof Error && /^Failed to upload file: (400|403|404|413|422)\b/.test(error.message)),
onRefused: (subject, error) => deps.notify({ kind: 'draft-refused', subject, error }),
onConflict: (subject) => deps.notify({ kind: 'draft-conflict', subject }),
onSent: () => deps.onDraftSent?.(),
};
return flushDrafts(io);
}
/**
* Old versions in another login, now that their replacement is up. Best effort,
* and only while still a draft there: an old copy is the worst that can come of it.
*/
async function removeVersionsElsewhere(account: string): Promise<void> {
const store = getStore();
for (const row of await store.area<DraftElsewhere[]>(account, 'drafts.elsewhereReady')) {
const other = useAuthStore.getState().getClientForAccount?.(row.login);
if (!other) continue;
try {
const response = await other.requestNetworkOnly([['Email/get', { accountId: row.jmap ?? other.getAccountId(), ids: [row.recordId], properties: ['keywords'] }, '0']]);
const email = response.methodResponses?.[0]?.[1]?.list?.[0] as { keywords?: Record<string, boolean> } | undefined;
if (email?.keywords?.$draft) await other.deleteEmail(row.recordId, row.jmap ?? undefined);
await store.area(account, 'drafts.elsewhereDone', { id: row.id });
} catch {
// That login's server may be down too. It waits.
}
}
}
/** The changes queued before `upToSeq` (all, with null). False when the run stopped on an outage. */
async function sendPatches(account: string, client: JMAPClient, deps: FlushDeps, upToSeq: number | null): Promise<boolean> {
const pending = await getStore().area<PendingPatch[]>(account, 'outbox.pendingPatches', { upToSeq });
@@ -232,8 +284,14 @@ async function lookUp(client: JMAPClient, entry: OutboxEntry): Promise<RetryDeci
async function dropOldDraft(client: JMAPClient, entry: OutboxEntry): Promise<void> {
if (entry.item.kind !== 'email' || typeof entry.item.args[7] !== 'string') return;
const options = entry.item.args[15] as { draftAccountId?: string } | undefined;
const jmap = options?.draftAccountId || client.getAccountId();
try {
await client.requestNetworkOnly([['Email/set', { accountId: options?.draftAccountId || client.getAccountId(), destroy: [entry.item.args[7]] }, '0']]);
if (isStandInId(entry.item.args[7])) {
// Never reached the server: it only has to stop going there.
await getStore().area(entry.account, 'drafts.discard', { jmap, id: entry.item.args[7] });
return;
}
await client.requestNetworkOnly([['Email/set', { accountId: jmap, destroy: [entry.item.args[7]] }, '0']]);
} catch {
// An orphan in Drafts, nothing worse.
}
@@ -278,11 +336,19 @@ async function settleUncertain(account: string, client: JMAPClient, deps: FlushD
return true;
}
async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps): Promise<void> {
async function flushLocked(account: string, client: JMAPClient, deps: FlushDeps, locked: boolean): Promise<void> {
if (flushing.has(account) || !deps.allowed()) return;
flushing.add(account);
try {
await settleStaleSends(account);
// A draft goes before the changes that may be waiting: what it replaces
// only starts to go once it's there.
if (!(await sendDrafts(account, client, deps, locked))) {
// Nothing else starts it again if the server only stumbled.
if (!stillOffline()) deps.retryLater?.();
return;
}
await removeVersionsElsewhere(account);
if (!(await settleUncertain(account, client, deps))) return;
// In the order they were made: a reply's $answered, queued after the
// reply, goes after it.
lib/offline/offline-mode.ts
+94 -43
View File
@@ -31,6 +31,7 @@ import { getSendGuardSource, guardState, guardsSending } from '@/lib/plugin-send
import { emitPushStateChange } from '@/lib/push-events';
import { requestOutboxSync } from '@/lib/pwa-notify/outbox-sync';
import { STORAGE_ERROR_AREA } from '@/lib/store/area';
import type { DraftError } from '@/lib/store/drafts/pending';
import { configureStore, getStore, type StorePageHooks } from '@/lib/store/client/api';
import { purgeStoreFiles } from '@/lib/store/client/purge';
import type { StoreSession } from '@/lib/store/engine/transport';
@@ -40,7 +41,7 @@ import { useEmailStore } from '@/stores/email-store';
import { usePolicyStore, whenPolicySettled } from '@/stores/policy-store';
import { onBeforeSignOut, onFullSignOut } from '@/lib/sign-out-cleanup';
import { generateMessageId } from '@/lib/jmap/client';
import { SendNotStartedError, setOfflineHooks, type OfflineClientRef, type OfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks';
import { isNetworkFailure, SendNotStartedError, setOfflineHooks, type OfflineClientRef, type OfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks';
import type { SendEmailResult } from '@/lib/jmap/types';
import { setOutboxHooks, type OutboxHooks } from './outbox-hooks';
import {
@@ -48,6 +49,7 @@ import {
holdFile,
isLocalBlobId,
queueSend,
readLocalDrafts,
readQueue,
type OutboxEntry,
type OutboxError,
@@ -68,6 +70,8 @@ export type OfflineNotice =
| { kind: 'held'; subject: string }
| { kind: 'lost'; subjects: string[] }
| { kind: 'conflict'; count: number }
| { kind: 'draft-refused'; subject: string; error: DraftError }
| { kind: 'draft-conflict'; subject: string }
| { kind: 'storage-failed' }
| { kind: 'store-unavailable' }
| { kind: 'store-reset' }
@@ -77,11 +81,23 @@ export type OfflineNotice =
| { kind: 'forbidden' }
| { kind: 'back-online-reload' };
let notify: (notice: OfflineNotice) => void = () => {};
let notifier: ((notice: OfflineNotice) => void) | null = null;
/**
* The store opens at a start from the local copy, before the host component has
* mounted. What it has to say by then, like "this store couldn't be read", waits.
*/
const earlyNotices: OfflineNotice[] = [];
const MOST_EARLY_NOTICES = 10;
function notify(notice: OfflineNotice): void {
if (notifier) notifier(notice);
else if (earlyNotices.length < MOST_EARLY_NOTICES) earlyNotices.push(notice);
}
/** Where the notices go (the host component turns them into toasts). */
export function setOfflineNotifier(fn: (notice: OfflineNotice) => void): void {
notify = fn;
notifier = fn;
for (const notice of earlyNotices.splice(0)) fn(notice);
}
export function offlineNotify(notice: OfflineNotice): void {
@@ -253,12 +269,16 @@ const hooks: OfflineHooks = {
if (typeof navigator === 'undefined' || navigator.onLine !== false) return false;
return offlineActive(accountForClient(client));
},
async answer(client, methodCalls) {
async answer(client, methodCalls, options) {
const account = accountForClient(client);
if (!offlineActive(account) || !account) return null;
let answered: OfflineResponse | null = null;
try {
answered = await getStore().answer(account, methodCalls);
// A draft whose connection broke may be on the server already.
const calls = options?.networkTried
? methodCalls.map((call): OfflineMethodCall => (call[0] === 'Email/set' && call[1].create ? [call[0], { ...call[1], networkTried: true }, call[2]] : call))
: methodCalls;
answered = await getStore().answer(account, calls);
} catch {
// The store couldn't start or lost its owner: as if nothing were kept.
}
@@ -272,7 +292,12 @@ const hooks: OfflineHooks = {
async resolveStandIns(client, ids) {
const account = accountForClient(client);
if (!offlineActive(account) || !account) return {};
return getStore().area<Record<string, string | null>>(account, 'calendar.resolveStandIns', { ids }).catch(() => ({}));
const ask = (call: string) => getStore().area<Record<string, string | null>>(account, call, { ids }).catch(() => ({} as Record<string, string | null>));
const [drafts, events] = await Promise.all([ask('drafts.resolveStandIns'), ask('calendar.resolveStandIns')]);
return Object.fromEntries(ids.map((id) => [id, events[id] ?? drafts[id] ?? null]));
},
resolveHeldAttachments(client, attachments, accountId) {
return resolveHeld(client, attachments, accountId);
},
fileOpened(client, accountId, id) {
const account = accountForClient(client);
@@ -333,7 +358,8 @@ const hooks: OfflineHooks = {
// An attachment, or a file from the Files area.
const kept = await getStore().area<KeptBlob | null>(account, 'mail.getBlob', { jmap, blobId }).catch(() => null)
?? await getStore().area<KeptBlob | null>(account, 'files.blob', { jmap, blobId }).catch(() => null);
return kept ? new Blob([kept.data as BlobPart], { type: kept.type || 'application/octet-stream' }) : null;
if (!kept) return null;
return new Blob(kept.blob ? [kept.blob] : [kept.data as BlobPart], { type: kept.type || 'application/octet-stream' });
},
recordBlob(client, blobId, jmap, blob) {
const account = accountForClient(client);
@@ -351,7 +377,31 @@ const hooks: OfflineHooks = {
},
};
interface KeptBlob { type: string | null; data: Uint8Array }
interface KeptBlob { type: string | null; blob?: Blob; data?: Uint8Array }
/** Files attached offline, uploaded now that the server answers. An outage here means nothing went anywhere. */
async function resolveHeld<T extends { blobId: string }>(client: OfflineClientRef, attachments: T[], accountId?: string): Promise<T[]> {
const account = accountForClient(client);
if (!account || !attachments.some((a) => isLocalBlobId(a.blobId))) return attachments;
const real = onlineClientFor(account);
if (!real) return attachments;
const out: T[] = [];
for (const attachment of attachments) {
if (!isLocalBlobId(attachment.blobId)) { out.push(attachment); continue; }
const file = await heldFile(account, attachment.blobId);
if (!file) throw Object.assign(new Error('A file attached offline is no longer on this device'), { name: 'OfflineFileMissingError' });
try {
const { blobId } = await real.uploadBlob(file, { accountId, noHold: true });
// Held again after all: that's no upload.
if (isLocalBlobId(blobId)) throw new SendNotStartedError();
out.push({ ...attachment, blobId });
} catch (error) {
if (isNetworkFailure(error)) throw new SendNotStartedError(error);
throw error;
}
}
return out;
}
/** Attachments and images opened while online are kept up to this size. */
const MAX_RECORDED_BLOB_BYTES = 5 * 1024 * 1024;
@@ -448,22 +498,8 @@ const outbox: OutboxHooks = {
notify({ kind: 'queued', subject: summary.subject });
return queuedResult(id);
},
async resolveHeldAttachments(client, attachments) {
const account = accountForClient(client);
if (!account || !attachments.some((a) => isLocalBlobId(a.blobId))) return attachments;
const real = onlineClientFor(account);
if (!real) return attachments;
const out = [];
for (const attachment of attachments) {
if (!isLocalBlobId(attachment.blobId)) { out.push(attachment); continue; }
const file = await heldFile(account, attachment.blobId);
if (!file) throw Object.assign(new Error('A file attached offline is no longer on this device'), { name: 'OfflineFileMissingError' });
const { blobId } = await real.uploadBlob(file);
// The server went away just now, and offline mode held the file again.
if (isLocalBlobId(blobId)) throw new SendNotStartedError();
out.push({ ...attachment, blobId });
}
return out;
resolveHeldAttachments(client, attachments) {
return resolveHeld(client, attachments);
},
};
@@ -500,13 +536,13 @@ export async function refreshQueueState(): Promise<void> {
// Nothing is kept for an account that hasn't it on; and opening the
// store would create it (on the sign-in page, after a sign-out).
if (visibleAccounts().length === 0) {
useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [] });
useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [] });
return;
}
try {
// Sends of a tab closed on the way: uncertain, so they can be looked up.
await settleStaleSends();
const { failed, patches, outbox: entries, conflicts } = await readQueue();
const [{ failed, patches, outbox: entries, conflicts }, drafts] = await Promise.all([readQueue(), readLocalDrafts()]);
if (failed.length > 0) noteStoreUnavailable();
const rows: OutboxRow[] = entries.map((entry) => ({
id: entry.id,
@@ -518,8 +554,10 @@ export async function refreshQueueState(): Promise<void> {
error: entry.error,
}));
// What an account's store can't tell right now is still there.
const unread = useOfflineStore.getState().outbox.filter((row) => failed.includes(row.account));
useOfflineStore.setState({ outbox: [...unread, ...rows], pendingChanges: patches, conflicts });
const shown = useOfflineStore.getState();
const unread = shown.outbox.filter((row) => failed.includes(row.account));
const unreadDrafts = shown.localDrafts.filter((row) => failed.includes(row.account));
useOfflineStore.setState({ outbox: [...unread, ...rows], pendingChanges: patches, conflicts, localDrafts: [...unreadDrafts, ...drafts] });
if (rows.some((row) => row.status === 'sending')) checkStaleSendsLater();
} catch {
// The store is being deleted, or can't start.
@@ -614,6 +652,13 @@ const AREA_FLUSHES = [
{ call: 'contacts.flush', types: ['AddressBook', 'ContactCard'] },
];
/** The page reads these types again. */
function emitTypesChanged(client: NonNullable<ReturnType<typeof onlineClientFor>>, types: string[]): void {
const role = (useAuthStore.getState().client as unknown) === client ? 'active' : 'background';
const changed = { [client.getAccountId()]: Object.fromEntries(types.map((type) => [type, 'flushed'])) };
emitPushStateChange({ '@type': 'StateChange', changed } as never, client, role);
}
async function flushAreas(account: string, client: NonNullable<ReturnType<typeof onlineClientFor>>): Promise<void> {
for (const { call, types } of AREA_FLUSHES) {
// Contacts may not be there in this build: nothing to flush then.
@@ -621,9 +666,7 @@ async function flushAreas(account: string, client: NonNullable<ReturnType<typeof
if (!tally) continue;
const conflicts = Array.isArray(tally.conflicts) ? tally.conflicts.length : tally.conflicts ?? 0;
if (!tally.sent && !conflicts) continue;
const role = (useAuthStore.getState().client as unknown) === client ? 'active' : 'background';
const changed = { [client.getAccountId()]: Object.fromEntries(types.map((type) => [type, 'flushed'])) };
emitPushStateChange({ '@type': 'StateChange', changed } as never, client, role);
emitTypesChanged(client, types);
}
}
@@ -654,6 +697,7 @@ export async function flushAll(): Promise<void> {
stricterNow,
onScheduled: offerUndo,
retryLater: () => scheduleFlushAll(LOOKUP_AGAIN_MS),
onDraftSent: () => emitTypesChanged(client, ['Email', 'Mailbox', 'Thread']),
}).catch(() => {});
await flushAreas(account, client);
}
@@ -699,13 +743,14 @@ export function confirmLoss(scope: string): void {
}
async function unsentSubjects(scope: string): Promise<string[]> {
let rows = useOfflineStore.getState().outbox;
const known = useOfflineStore.getState();
let rows: Array<{ account: string; subject: string }> = [...known.outbox, ...known.localDrafts];
// Reading an account's store would create it if it never had one.
if (scope !== 'all' && !useOfflineStore.getState().enabled.includes(scope)) return rows.filter((row) => row.account === scope).map((row) => row.subject);
if (scope !== 'all' && !known.enabled.includes(scope)) return rows.filter((row) => row.account === scope).map((row) => row.subject);
try {
rows = (await readQueue(scope === 'all' ? undefined : scope)).outbox.map((entry) => ({
id: entry.id, account: entry.account, createdAt: entry.createdAt, subject: entry.summary.subject, to: entry.summary.to, status: entry.status,
}));
const wanted = scope === 'all' ? undefined : scope;
const [{ outbox: entries }, drafts] = await Promise.all([readQueue(wanted), readLocalDrafts(wanted)]);
rows = [...entries.map((entry) => ({ account: entry.account, subject: entry.summary.subject })), ...drafts];
} catch {
// No store: what the state last saw.
}
@@ -715,19 +760,24 @@ async function unsentSubjects(scope: string): Promise<string[]> {
/** How long a sign-out waits for the store's word on what would be lost. */
const UNSENT_READ_MS = 2_000;
/** What a sign-out of `scope` would delete: messages, and edits the server hasn't got. */
/**
* What a sign-out of `scope` would delete: messages, drafts only on this device,
* and edits the server hasn't got. A draft a waiting send names isn't listed as a
* draft, so it counts once.
*/
export async function unsentCount(scope: string, waitMs = UNSENT_READ_MS): Promise<{ messages: number; changes: number }> {
const shown = useOfflineStore.getState();
const known = { messages: shown.outbox.filter((row) => scope === 'all' || row.account === scope).length, changes: shown.pendingChanges };
const known = { messages: [...shown.outbox, ...shown.localDrafts].filter((row) => scope === 'all' || row.account === scope).length, changes: shown.pendingChanges };
const accounts = (scope === 'all' ? visibleAccounts() : [scope]).filter((account) => shown.enabled.includes(account));
if (accounts.length === 0) return known;
const pendingIn = (account: string, area: string) => getStore().area<number>(account, `${area}.pending`).catch(() => 0);
const read = (async () => {
const { failed, outbox, patches } = await readQueue(scope === 'all' ? undefined : scope);
const wanted = scope === 'all' ? undefined : scope;
const [{ failed, outbox, patches }, drafts] = await Promise.all([readQueue(wanted), readLocalDrafts(wanted)]);
const edits = await Promise.all(accounts.flatMap((account) => [pendingIn(account, 'calendar'), pendingIn(account, 'contacts')]));
return {
// An account that can't answer counts with what it last said.
messages: outbox.length + shown.outbox.filter((row) => failed.includes(row.account)).length,
messages: outbox.length + drafts.length + [...shown.outbox, ...shown.localDrafts].filter((row) => failed.includes(row.account)).length,
changes: patches + edits.reduce((sum, count) => sum + count, 0),
};
})().catch(() => known);
@@ -757,7 +807,8 @@ export async function wipeAccount(account: string): Promise<void> {
syncTimers.delete(account);
// Waited for a sign-in that never came: whoever is here now mustn't read its subjects.
if (isEvicted(account)) named.add(account);
if (useOfflineStore.getState().enabled.includes(account) || useOfflineStore.getState().outbox.some((row) => row.account === account)) {
const before = useOfflineStore.getState();
if (before.enabled.includes(account) || before.outbox.some((row) => row.account === account) || before.localDrafts.some((row) => row.account === account)) {
await nameLost(account, useAuthStore.getState().isAuthenticated);
}
named.delete(account);
@@ -797,7 +848,7 @@ export async function wipeEverything(): Promise<void> {
clearHandoffs();
confirmed.clear();
named.clear();
useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [], usageBytes: 0, syncing: [], lastSyncedAt: {} });
useOfflineStore.setState({ outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [], usageBytes: 0, syncing: [], lastSyncedAt: {} });
}
/** Whether the policy of an account is known well enough to decide on it. */
@@ -1063,7 +1114,7 @@ onFullSignOut(({ keepLocalStore }) => {
if (!keepLocalStore) getStore().stop();
confirmed.clear();
named.clear();
useOfflineStore.setState((s) => ({ enabled: keepLocalStore ? s.enabled : [], outbox: [], pendingChanges: 0, conflicts: [], usageBytes: 0, syncing: [], lastSyncedAt: {}, startedOffline: false }));
useOfflineStore.setState((s) => ({ enabled: keepLocalStore ? s.enabled : [], outbox: [], pendingChanges: 0, conflicts: [], localDrafts: [], usageBytes: 0, syncing: [], lastSyncedAt: {}, startedOffline: false }));
});
// Every sign-out the user didn't agree to lose mail at (the session limits,
lib/offline/queue.ts
+40 -4
View File
@@ -8,6 +8,7 @@
import { requestOutboxSync } from '@/lib/pwa-notify/outbox-sync';
import { getStore } from '@/lib/store/client/api';
import type { LocalDraftSummary } from '@/lib/store/drafts/calls';
import type { ConflictSummary, OutboxError, OutboxStatus, OutboxSummary, PolicyFingerprint, SendItem, StoredConflict, StoredOutboxEntry } from '@/lib/store/outbox/items';
import { visibleAccounts } from './state';
@@ -77,20 +78,55 @@ export function removeOutboxEntry(entry: Pick<OutboxEntry, 'account' | 'id'>): P
return getStore().area(entry.account, 'outbox.remove', { id: entry.id });
}
export interface LocalDraftRow extends LocalDraftSummary {
account: string;
}
/** Drafts that are only on this device, oldest first. */
export async function readLocalDrafts(account?: string): Promise<LocalDraftRow[]> {
const accounts = account ? [account] : visibleAccounts();
// An account in trouble is what readQueue reports. Its drafts are still there.
const settled = await Promise.allSettled(accounts.map(async (id) => ((await getStore().area<LocalDraftSummary[] | undefined>(id, 'drafts.list')) ?? []).map((row) => ({ ...row, account: id }))));
return settled.flatMap((result) => (result.status === 'fulfilled' ? result.value : []));
}
/** An old version in another login: removed through it once the draft here is on the server. */
export function noteDraftReplacedElsewhere(account: string, localId: string, version: { login: string; jmap?: string; id: string }): Promise<void> {
return getStore().area(account, 'drafts.noteElsewhere', { localId, ...version });
}
/** Asks the server again about a draft it refused. */
export function retryLocalDraft(account: string, localId: string): Promise<void> {
return getStore().area(account, 'drafts.retry', { localId });
}
export function dismissConflict(account: string, id: number): Promise<void> {
return getStore().area(account, 'outbox.dismissConflict', { id });
}
// ---- Files attached while offline ----
/** A file on its way to the store is cut into pieces of this size, so it's never in memory as a whole. */
const HOLD_CHUNK_BYTES = 1024 * 1024;
/** Keeps a file under a stand-in blob id until its message is sent. */
export async function holdFile(account: string, file: Blob, name: string, jmap?: string): Promise<{ blobId: string; size: number; type: string }> {
const data = new Uint8Array(await file.arrayBuffer());
return getStore().area(account, 'outbox.holdFile', { jmap, name, type: file.type, data });
const store = getStore();
const { blobId } = await store.area<{ blobId: string }>(account, 'outbox.holdStart', { jmap, name, type: file.type, size: file.size });
try {
for (let at = 0, seq = 0; at < file.size; at += HOLD_CHUNK_BYTES, seq += 1) {
const data = new Uint8Array(await file.slice(at, at + HOLD_CHUNK_BYTES).arrayBuffer());
await store.area(account, 'outbox.holdChunk', { blobId, seq, data });
}
return await store.area(account, 'outbox.holdEnd', { blobId });
} catch (error) {
await store.area(account, 'outbox.holdAbort', { blobId }).catch(() => {});
throw error;
}
}
export async function heldFile(account: string, blobId: string): Promise<File | null> {
const held = await getStore().area<{ name: string; type: string; data: Uint8Array } | null>(account, 'outbox.heldFile', { blobId });
const held = await getStore().area<{ name: string; type: string; blob: Blob } | null>(account, 'outbox.heldFile', { blobId });
if (!held) return null;
return new File([held.data as BlobPart], held.name, { type: held.type });
return new File([held.blob], held.name, { type: held.type });
}
lib/offline/state.ts
+7 -1
View File
@@ -8,7 +8,7 @@
import { create } from 'zustand';
import { DEFAULT_SCOPE, type StoreScope } from '@/lib/store/area';
import type { ConflictEntry, OutboxEntry, OutboxError } from './queue';
import type { ConflictEntry, LocalDraftRow, OutboxEntry, OutboxError } from './queue';
/** localStorage: the local account ids that keep mail on this device. */
export const OFFLINE_ACCOUNTS_KEY = 'bulwark-offline-accounts';
@@ -43,6 +43,10 @@ interface OfflineState {
pendingChanges: number;
/** Changes the server refused, or that met a message changed on another device. */
conflicts: ConflictEntry[];
/** Drafts that are only on this device: on their way up, or refused by the server. */
localDrafts: LocalDraftRow[];
/** A local draft the user asked to open, for the mail page to pick up. */
draftToOpen: { account: string; jmap: string; id: string } | null;
usageBytes: number;
/** Accounts whose mail is being brought onto the device right now. */
syncing: string[];
@@ -69,6 +73,8 @@ export const useOfflineStore = create<OfflineState>()(() => ({
outbox: [],
pendingChanges: 0,
conflicts: [],
localDrafts: [],
draftToOpen: null,
usageBytes: 0,
syncing: [],
lastSyncedAt: {},
lib/store/__tests__/attachments.test.ts
+534
View File
@@ -0,0 +1,534 @@
// @vitest-environment node
import { describe, expect, it } from 'vitest';
import { DEFAULT_SCOPE, type StoreScope } from '../area';
import { AccountEngine } from '../engine/account-engine';
import { TransportError } from '../engine/transport';
import { mailArea } from '../mail';
import { outboxArea } from '../outbox';
import { FakeJmapDrafts } from './helpers/fake-drafts';
import { FAKE_SESSION, JMAP_ACCOUNT, type FakeEmail } from './helpers/fake-jmap';
import { openMemoryStore } from './helpers/node-adapter';
const SCOPE: StoreScope = { ...DEFAULT_SCOPE, days: null };
const RAW = 'message/rfc822';
/** Hands out files in pieces, so a test can tell a stream from a whole download. */
class Files extends FakeJmapDrafts {
readonly downloads: string[] = [];
readonly rawDownloads: string[] = [];
readonly files = new Map<string, Uint8Array>();
gone = new Set<string>();
breakAfter: { blobId: string; bytes: number } | null = null;
pieces = 0;
onDownload: ((blobId: string) => void) | null = null;
private record(blobId: string, type: string): Uint8Array {
if (type === RAW) this.rawDownloads.push(blobId);
else this.downloads.push(blobId);
this.onDownload?.(blobId);
if (this.gone.has(blobId)) throw new TransportError('http', 'Gone', 404);
return this.files.get(blobId) ?? new Uint8Array(type === RAW ? [9, 9] : [1, 2, 3]);
}
override async download(_jmap?: string, blobId?: string, type?: string): Promise<Uint8Array> {
return this.record(String(blobId), String(type));
}
async downloadStream(_jmap: string, blobId: string, type: string, _name: string, sink: (part: Uint8Array) => void): Promise<void> {
const data = this.record(blobId, type);
const step = 256 * 1024;
for (let at = 0; at < data.length; at += step) {
if (this.breakAfter?.blobId === blobId && at >= this.breakAfter.bytes) throw new TransportError('network', 'The download broke off');
this.pieces += 1;
sink(data.subarray(at, at + step));
}
}
}
function engineFor(server: Files, scope: StoreScope = SCOPE, areas = [mailArea]) {
const db = openMemoryStore();
const engine = new AccountEngine({
account: 'a@fake.test',
db,
session: FAKE_SESSION,
transport: server,
areas,
scope,
now: () => Date.UTC(2026, 10, 3),
sleep: async () => {},
});
return { engine, db };
}
const kept = (engine: AccountEngine, blobId: string) => engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId }) as Promise<{ type: string; size: number; blob: Blob } | null>;
function withFile(server: Files, id: string, blobId: string, size = 3, day = 1, extra: Partial<FakeEmail> = {}): FakeEmail {
server.files.set(blobId, new Uint8Array(size).fill(7));
return server.addEmail({
id,
size: 100_000_000,
receivedAt: new Date(Date.UTC(2026, 9, day)).toISOString(),
attachments: [{ partId: '2', blobId, name: `${blobId}.pdf`, type: 'application/pdf', size }],
...extra,
});
}
describe('what\'s fetched', () => {
it('goes newest mail first, after the rest of the pass', async () => {
const server = new Files();
withFile(server, 'old', 'b-old', 3, 1);
withFile(server, 'new', 'b-new', 3, 20);
withFile(server, 'mid', 'b-mid', 3, 10);
const { engine } = engineFor(server);
await engine.requestSync();
expect(server.downloads).toEqual(['b-new', 'b-mid', 'b-old']);
});
it('runs after every other syncer, and a newer pass goes first', async () => {
const server = new Files();
withFile(server, 'one', 'b-1', 3, 2);
withFile(server, 'two', 'b-2', 3, 1);
const order: string[] = [];
const later = { name: 'later', migrations: [], methods: {}, syncers: [{ name: 'later', capability: 'urn:ietf:params:jmap:mail', run: async () => { order.push('later'); } }] };
const { engine } = engineFor(server, SCOPE, [mailArea, later]);
let wanted = false;
server.onDownload = () => {
order.push('file');
// Asked for while a file comes in: the next file waits for that pass.
if (!wanted) {
wanted = true;
void engine.requestSync();
}
};
await engine.requestSync();
expect(order[0]).toBe('later');
// later, file 1, then the wanted pass: later again before file 2.
expect(order.slice(0, 3)).toEqual(['later', 'file', 'later']);
expect(server.downloads).toEqual(['b-1', 'b-2']);
});
it('includes inline images that only the body lists, as Stalwart 1.0 answers', async () => {
const server = new Files();
server.addEmail({
id: 'm',
attachments: [],
htmlBody: [{ partId: '1', type: 'text/html' }, { partId: '2', blobId: 'b-inline', type: 'image/png', name: 'i.png', size: 3, cid: 'i', disposition: 'inline' }] as never,
bodyStructure: {
type: 'multipart/related',
subParts: [{ partId: '1', blobId: 'b-text', type: 'text/html', size: 10 }, { partId: '3', blobId: 'b-tree', type: 'image/gif', size: 3, cid: 'j', disposition: 'inline' }],
} as never,
});
const { engine } = engineFor(server);
await engine.requestSync();
expect(server.downloads.sort()).toEqual(['b-inline', 'b-tree']);
expect(await kept(engine, 'b-inline')).toMatchObject({ type: 'image/png' });
});
it('comes through the stream in chunks that are each kept, and is readable whole', async () => {
const server = new Files();
const size = 2 * 1024 * 1024 + 5;
withFile(server, 'm', 'b-big', size);
server.files.get('b-big')?.fill(3);
const { engine, db } = engineFor(server);
await engine.requestSync();
expect(server.pieces).toBeGreaterThan(3);
expect(Number(db.get<{ n: number }>("SELECT COUNT(*) AS n FROM blob_chunk WHERE blob_id = 'b-big'")?.n)).toBe(3);
const file = await kept(engine, 'b-big');
expect(file?.size).toBe(size);
const bytes = new Uint8Array(await (file as { blob: Blob }).blob.arrayBuffer());
expect(bytes.length).toBe(size);
expect(bytes.every((value) => value === 3)).toBe(true);
});
it('leaves nothing readable when a download breaks, and tries again next time', async () => {
const server = new Files();
withFile(server, 'm', 'b-big', 2 * 1024 * 1024);
server.breakAfter = { blobId: 'b-big', bytes: 1024 * 1024 + 1 };
const { engine, db } = engineFor(server);
await engine.requestSync();
expect(await kept(engine, 'b-big')).toBeNull();
expect(db.get("SELECT 1 AS x FROM blob_chunk WHERE blob_id = 'b-big'")).toBeUndefined();
server.breakAfter = null;
await engine.requestSync();
expect((await kept(engine, 'b-big'))?.size).toBe(2 * 1024 * 1024);
});
it('drops a file larger than it said, and doesn\'t fetch it again', async () => {
const server = new Files();
withFile(server, 'm', 'b-liar', 3);
server.files.set('b-liar', new Uint8Array(5000));
const { engine } = engineFor(server, { ...SCOPE, maxAttachmentBytes: 1000 });
await engine.requestSync();
await engine.requestSync();
expect(await kept(engine, 'b-liar')).toBeNull();
expect(server.downloads).toEqual(['b-liar']);
});
});
describe('the cap', () => {
it('pushes out older mail\'s files for a newer message\'s', async () => {
const server = new Files();
withFile(server, 'old', 'b-old', 40, 1);
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
expect(await kept(engine, 'b-old')).not.toBeNull();
withFile(server, 'new', 'b-new', 40, 20);
await engine.requestSync();
expect(await kept(engine, 'b-new')).not.toBeNull();
expect(await kept(engine, 'b-old')).toBeNull();
});
it('doesn\'t fetch an older file back that a newer one pushed out', async () => {
const server = new Files();
withFile(server, 'old', 'b-old', 40, 1);
withFile(server, 'new', 'b-new', 40, 20);
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
await engine.requestSync();
expect(server.downloads).toEqual(['b-new']);
});
it('doesn\'t let a file of older mail the user opened push out a newer one', async () => {
const server = new Files();
withFile(server, 'new', 'b-new', 40, 20);
server.addEmail({ id: 'old', size: 100_000_000, receivedAt: new Date(Date.UTC(2026, 9, 1)).toISOString(), attachments: [{ partId: '2', blobId: 'b-old', size: 40, type: 'image/png' }] });
server.gone.add('b-old');
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
expect(await engine.areaCall('mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-old', type: null, data: new Uint8Array(40) })).toBe(false);
expect(await kept(engine, 'b-new')).not.toBeNull();
});
it('keeps a smaller older file that still fits past one that doesn\'t', async () => {
const server = new Files();
withFile(server, 'new', 'b-new', 40, 20);
withFile(server, 'mid', 'b-mid', 40, 10);
withFile(server, 'old', 'b-old', 10, 1);
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
expect(server.downloads).toEqual(['b-new', 'b-old']);
});
it('stops looking after a long run of files that don\'t fit, instead of reading the kept list for each', async () => {
const server = new Files();
withFile(server, 'fills', 'b-fills', 50, 28);
for (let n = 0; n < 60; n += 1) withFile(server, `no-${n}`, `b-no-${n}`, 40, 27);
withFile(server, 'small-old', 'b-small-old', 5, 1);
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
expect(server.downloads).toEqual(['b-fills']);
});
it('never pushes out the files of a pinned message', async () => {
const server = new Files();
withFile(server, 'pinned', 'b-pinned', 40, 1, { keywords: { $pinned: true } });
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
withFile(server, 'new', 'b-new', 40, 20);
await engine.requestSync();
expect(await kept(engine, 'b-pinned')).not.toBeNull();
expect(await kept(engine, 'b-new')).toBeNull();
});
it('never touches a file held for a draft', async () => {
const server = new Files();
withFile(server, 'old', 'b-old', 40, 1);
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 }, [mailArea, outboxArea]);
const held = await engine.areaCall('outbox.holdFile', { name: 'mine', type: '', data: new Uint8Array(100) }) as { blobId: string };
await engine.requestSync();
withFile(server, 'new', 'b-new', 40, 20);
await engine.requestSync();
expect(await kept(engine, 'b-old')).toBeNull();
expect(await engine.areaCall('outbox.heldFile', { blobId: held.blobId })).not.toBeNull();
});
it('applies a lowered total at once, oldest first', async () => {
const server = new Files();
withFile(server, 'old', 'b-old', 40, 1);
withFile(server, 'new', 'b-new', 40, 20);
const { engine } = engineFor(server);
await engine.requestSync();
engine.setScope({ ...SCOPE, maxTotalBytes: 50 });
expect(await kept(engine, 'b-old')).toBeNull();
expect(await kept(engine, 'b-new')).not.toBeNull();
});
it('applies a lowered size limit at once', async () => {
const server = new Files();
withFile(server, 'big', 'b-big', 40, 1);
withFile(server, 'small', 'b-small', 4, 2);
const { engine } = engineFor(server);
await engine.requestSync();
engine.setScope({ ...SCOPE, maxAttachmentBytes: 10 });
expect(await kept(engine, 'b-big')).toBeNull();
expect(await kept(engine, 'b-small')).not.toBeNull();
});
});
describe('a policy forbidding attachments', () => {
it('fetches nothing, drops what\'s kept at once, and gives nothing out', async () => {
const server = new Files();
withFile(server, 'm', 'b-1');
const { engine, db } = engineFor(server);
await engine.requestSync();
expect(await kept(engine, 'b-1')).not.toBeNull();
engine.setScope({ ...SCOPE, attachments: false });
expect(db.get('SELECT 1 AS x FROM blob')).toBeUndefined();
expect(db.get('SELECT 1 AS x FROM blob_chunk')).toBeUndefined();
expect(await kept(engine, 'b-1')).toBeNull();
server.downloads.length = 0;
server.rawDownloads.length = 0;
server.addEmail({ id: 'more', attachments: [{ partId: '2', blobId: 'b-2', size: 3, type: 'application/pdf' }] });
await engine.requestSync();
expect(server.downloads).toEqual([]);
expect(server.rawDownloads).toEqual([]);
});
it('refuses a file the user opened online too', async () => {
const server = new Files();
server.addEmail({ id: 'm', attachments: [{ partId: '2', blobId: 'b-1', size: 3, type: 'image/png' }] });
const { engine } = engineFor(server, { ...SCOPE, attachments: false });
await engine.requestSync();
expect(await engine.areaCall('mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-1', type: null, data: new Uint8Array([1]) })).toBe(false);
});
it('applies when the store opens with it, without waiting for a sync', async () => {
const server = new Files();
withFile(server, 'm', 'b-1');
const { engine, db } = engineFor(server);
await engine.requestSync();
expect(db.get('SELECT 1 AS x FROM blob')).toBeTruthy();
new AccountEngine({ account: 'a@fake.test', db, session: FAKE_SESSION, transport: server, areas: [mailArea], scope: { ...SCOPE, attachments: false }, sleep: async () => {} });
expect(db.get('SELECT 1 AS x FROM blob')).toBeUndefined();
});
it('keeps a file the user attached to a draft: it\'s theirs, not a copy of mail', async () => {
const server = new Files();
const { engine } = engineFor(server, { ...SCOPE, attachments: false }, [mailArea, outboxArea]);
const held = await engine.areaCall('outbox.holdFile', { name: 'mine', type: '', data: new Uint8Array(3) }) as { blobId: string };
await engine.requestSync();
engine.setScope({ ...SCOPE, attachments: false });
expect(await engine.areaCall('outbox.heldFile', { blobId: held.blobId })).not.toBeNull();
});
});
describe('raw messages for an export', () => {
it('are kept with the attachments, and handed out by Email.blobId', async () => {
const server = new Files();
server.addEmail({ id: 'm', blobId: 'raw-m', size: 2 });
server.files.set('raw-m', new TextEncoder().encode('From: a\r\n\r\nx'));
const { engine } = engineFor(server);
await engine.requestSync();
expect(server.rawDownloads).toEqual(['raw-m']);
const raw = await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-m' }) as { ok: true; type: string; blob: Blob };
expect(raw.ok).toBe(true);
expect(raw.type).toBe(RAW);
expect(await raw.blob.text()).toBe('From: a\r\n\r\nx');
expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT })).toEqual({ total: 1, kept: 1 });
});
it('come after the files of the same message', async () => {
const server = new Files();
const order: string[] = [];
server.onDownload = (blobId) => order.push(blobId);
withFile(server, 'm', 'b-1', 3, 1, { blobId: 'raw-m', size: 3 });
const { engine } = engineFor(server);
await engine.requestSync();
expect(order).toEqual(['b-1', 'raw-m']);
});
it('are refused plainly with attachments off, and counted as not there', async () => {
const server = new Files();
server.addEmail({ id: 'm', blobId: 'raw-m', mailboxIds: { inbox: true } });
const { engine } = engineFor(server, { ...SCOPE, attachments: false });
await engine.requestSync();
expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-m' })).toEqual({ ok: false, reason: 'attachments_off' });
expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT })).toEqual({ total: 1, kept: 0 });
});
it('are refused when they aren\'t kept, whatever else is', async () => {
const server = new Files();
server.addEmail({ id: 'm', blobId: 'raw-m', size: 5_000, attachments: [{ partId: '2', blobId: 'b-1', size: 3, type: 'image/png' }] });
const { engine } = engineFor(server, { ...SCOPE, maxAttachmentBytes: 1000 });
await engine.requestSync();
expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-m' })).toEqual({ ok: false, reason: 'not_kept' });
// An attachment isn't a whole message.
expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-1' })).toEqual({ ok: false, reason: 'not_kept' });
expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT })).toEqual({ total: 1, kept: 0 });
});
it('count per folder for an export of one', async () => {
const server = new Files({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'other', name: 'Other' }] });
server.addEmail({ id: 'a', blobId: 'raw-a', size: 2, mailboxIds: { inbox: true } });
server.addEmail({ id: 'b', blobId: 'raw-b', size: 2, mailboxIds: { other: true } });
server.gone.add('raw-b');
const { engine } = engineFor(server);
await engine.requestSync();
expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT, mailboxId: 'inbox' })).toEqual({ total: 1, kept: 1 });
expect(await engine.areaCall('mail.rawStatus', { jmap: JMAP_ACCOUNT, mailboxId: 'other' })).toEqual({ total: 1, kept: 0 });
});
it('don\'t exist for a draft on the device', async () => {
const server = new Files({ mailboxes: [{ id: 'drafts', role: 'drafts' }] });
const { engine, db } = engineFor(server, SCOPE, [mailArea, outboxArea]);
await engine.requestSync();
const answer = await engine.answer([['Email/set', { create: { c: { keywords: { $draft: true }, mailboxIds: { drafts: true }, bodyValues: { 1: { value: 'x' } }, textBody: [{ partId: '1' }] } } }, 's']]);
const id = answer?.methodResponses[0][1].created.c.id as string;
expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: id })).toEqual({ ok: false, reason: 'not_kept' });
expect(db.get('SELECT 1 AS x FROM email_part')).toBeUndefined();
});
});
describe('a kept file read back', () => {
it('isn\'t emptied by a download of the same file that the user opening it overtook', async () => {
const server = new Files();
withFile(server, 'm', 'b-1', 2 * 1024 * 1024 + 5);
server.files.get('b-1')?.fill(3);
const { engine } = engineFor(server);
server.onDownload = () => {
server.onDownload = null;
void engine.areaCall('mail.recordBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-1', type: 'application/pdf', data: new Uint8Array(2 * 1024 * 1024 + 5).fill(3) });
};
await engine.requestSync();
const file = await kept(engine, 'b-1');
expect(file?.size).toBe(2 * 1024 * 1024 + 5);
expect((file as { blob: Blob }).blob.size).toBe(2 * 1024 * 1024 + 5);
});
it('doesn\'t lose what a kept file has when the next pass looks at it', async () => {
const server = new Files();
withFile(server, 'm', 'b-1', 300);
const { engine } = engineFor(server);
await engine.requestSync();
server.addEmail({ id: 'n', attachments: [{ partId: '2', blobId: 'b-1', size: 300, type: 'application/pdf', name: 'same.pdf' }] });
await engine.requestSync();
expect((await kept(engine, 'b-1'))?.blob.size).toBe(300);
});
it('reads as not kept when its pieces don\'t add up, and is fetched again', async () => {
const server = new Files();
withFile(server, 'm', 'b-1', 300);
const { engine, db } = engineFor(server);
await engine.requestSync();
db.run("DELETE FROM blob_chunk WHERE blob_id = 'b-1'");
expect(await kept(engine, 'b-1')).toBeNull();
server.downloads.length = 0;
await engine.requestSync();
expect(server.downloads).toEqual(['b-1']);
expect((await kept(engine, 'b-1'))?.blob.size).toBe(300);
});
});
describe('raw messages come last', () => {
it('after every file, even of older mail', async () => {
const server = new Files();
const order: string[] = [];
server.onDownload = (blobId) => order.push(blobId);
server.addEmail({ id: 'new', blobId: 'raw-new', size: 3, receivedAt: new Date(Date.UTC(2026, 9, 20)).toISOString() });
withFile(server, 'old', 'b-old', 3, 1, { blobId: 'raw-old', size: 3 });
const { engine } = engineFor(server);
await engine.requestSync();
expect(order).toEqual(['b-old', 'raw-new', 'raw-old']);
});
it('never push out a file, however old', async () => {
const server = new Files();
withFile(server, 'old', 'b-old', 40, 1);
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
const raw = new Uint8Array(40);
server.files.set('raw-new', raw);
server.addEmail({ id: 'new', blobId: 'raw-new', size: 40, receivedAt: new Date(Date.UTC(2026, 9, 20)).toISOString() });
await engine.requestSync();
expect(await kept(engine, 'b-old')).not.toBeNull();
expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-new' })).toEqual({ ok: false, reason: 'not_kept' });
});
it('go first when a file needs the room, whatever their age', async () => {
const server = new Files();
server.files.set('raw-newest', new Uint8Array(40));
server.addEmail({ id: 'newest', blobId: 'raw-newest', size: 40, receivedAt: new Date(Date.UTC(2026, 9, 25)).toISOString() });
withFile(server, 'old', 'b-old', 10, 1);
const { engine } = engineFor(server, { ...SCOPE, maxTotalBytes: 60 });
await engine.requestSync();
expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-newest' })).toMatchObject({ ok: true });
withFile(server, 'mid', 'b-mid', 30, 10);
await engine.requestSync();
expect(await kept(engine, 'b-mid')).not.toBeNull();
expect(await kept(engine, 'b-old')).not.toBeNull();
expect(await engine.areaCall('mail.rawBlob', { jmap: JMAP_ACCOUNT, blobId: 'raw-newest' })).toEqual({ ok: false, reason: 'not_kept' });
});
});
describe('the pace', () => {
it('is slower for raw messages than for files', async () => {
const server = new Files();
withFile(server, 'm', 'b-1', 3, 1, { blobId: 'raw-m', size: 3 });
const waits: number[] = [];
const engine = new AccountEngine({
account: 'a@fake.test',
db: openMemoryStore(),
session: FAKE_SESSION,
transport: server,
areas: [mailArea],
scope: SCOPE,
sleep: async (ms) => { waits.push(ms); },
});
await engine.requestSync();
expect(waits).toEqual([200, 1000]);
});
});
describe('what a server error means', () => {
it('doesn\'t give up on a file for a passing error, and stops for this pass', async () => {
const server = new Files();
withFile(server, 'new', 'b-new', 3, 20);
withFile(server, 'old', 'b-old', 3, 1);
server.onDownload = (blobId) => {
if (blobId === 'b-new') throw new TransportError('http', 'Busy', 500);
};
const { engine, db } = engineFor(server);
await engine.requestSync();
expect(server.downloads).toEqual(['b-new']);
expect(db.get<{ refused: number }>("SELECT refused FROM email_part WHERE blob_id = 'b-new'")?.refused).toBe(0);
server.onDownload = null;
await engine.requestSync();
expect(await kept(engine, 'b-new')).not.toBeNull();
expect(await kept(engine, 'b-old')).not.toBeNull();
});
it('gives up on one the server says won\'t come, and goes on', async () => {
for (const status of [403, 404, 410]) {
const server = new Files();
withFile(server, 'new', 'b-new', 3, 20);
withFile(server, 'old', 'b-old', 3, 1);
server.onDownload = (blobId) => {
if (blobId === 'b-new') throw new TransportError('http', 'No', status);
};
const { engine, db } = engineFor(server);
await engine.requestSync();
expect(db.get<{ refused: number }>("SELECT refused FROM email_part WHERE blob_id = 'b-new'")?.refused).toBe(1);
expect(await kept(engine, 'b-old')).not.toBeNull();
}
});
});
describe('a transfer that breaks', () => {
it('doesn\'t end the pass: the next file still comes, and this one is tried again', async () => {
const server = new Files();
withFile(server, 'new', 'b-new', 300, 20);
withFile(server, 'old', 'b-old', 300, 1);
server.onDownload = (blobId) => {
if (blobId === 'b-new') throw new TransportError('network', 'The download broke off', undefined, undefined, undefined, true);
};
const { engine } = engineFor(server);
await engine.requestSync();
expect(await kept(engine, 'b-old')).not.toBeNull();
expect(await kept(engine, 'b-new')).toBeNull();
server.onDownload = null;
await engine.requestSync();
expect(await kept(engine, 'b-new')).not.toBeNull();
});
});
lib/store/__tests__/drafts-client.test.ts
+383
View File
@@ -0,0 +1,383 @@
/**
* The real JMAPClient's createDraft, getEmail and blob calls against the store
* while offline and a fake server online, through a stubbed fetch: what the
* composer does around a save.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { JMAPClient } from '@/lib/jmap/client';
import { setOfflineHooks, type OfflineMethodCall, type OfflineResponse } from '@/lib/jmap/offline-hooks';
import { DEFAULT_SCOPE } from '../area';
import type { DraftGhost, PendingDraft } from '../drafts/calls';
import { flushDrafts } from '../drafts/flush';
import { AccountEngine } from '../engine/account-engine';
import { TransportError, type MethodCall } from '../engine/transport';
import { mailArea } from '../mail';
import { outboxArea } from '../outbox';
import { flushPatches, type PatchOutcome, type PendingPatch } from '../outbox/flush-patches';
import { FakeJmapDrafts } from './helpers/fake-drafts';
import { JMAP_ACCOUNT } from './helpers/fake-jmap';
import { openMemoryStore } from './helpers/node-adapter';
const NOW = Date.UTC(2026, 10, 3);
const MAIL = 'urn:ietf:params:jmap:mail';
const SUBMISSION = 'urn:ietf:params:jmap:submission';
const STALWART = 'urn:stalwart:jmap';
const SESSION = {
capabilities: { 'urn:ietf:params:jmap:core': { maxObjectsInGet: 500, maxObjectsInSet: 500 }, [MAIL]: {}, [SUBMISSION]: {}, [STALWART]: {} },
accounts: { [JMAP_ACCOUNT]: { name: 'a@example.org', isPersonal: true, accountCapabilities: { [MAIL]: {}, [SUBMISSION]: {}, [STALWART]: {} } } },
primaryAccounts: { [MAIL]: JMAP_ACCOUNT },
username: 'a@example.org',
apiUrl: 'https://fake.test/api',
downloadUrl: 'https://fake.test/d/{accountId}/{blobId}/{name}?type={type}',
uploadUrl: 'https://fake.test/u/{accountId}',
eventSourceUrl: '',
state: 's',
};
function json(body: unknown): Response {
return new Response(JSON.stringify(body), { status: 200, headers: { 'Content-Type': 'application/json' } });
}
let server: FakeJmapDrafts;
let engine: AccountEngine;
let online: boolean;
/** The server comes back right after the next create was answered on the device. */
let backAfterCreate: boolean;
let spy: ReturnType<typeof vi.spyOn>;
let client: JMAPClient;
/** What reached the network. */
let wire: MethodCall[];
beforeEach(async () => {
server = new FakeJmapDrafts({ mailboxes: [{ id: 'inbox', role: 'inbox' }, { id: 'drafts', role: 'drafts' }, { id: 'sent', role: 'sent' }] });
server.now = NOW;
engine = new AccountEngine({
account: 'a@fake.test',
db: openMemoryStore(),
session: SESSION as never,
transport: server,
areas: [mailArea, outboxArea],
scope: { ...DEFAULT_SCOPE, days: null, attachments: false },
now: () => NOW,
sleep: async () => {},
});
online = true;
backAfterCreate = false;
wire = [];
spy = vi.spyOn(globalThis, 'fetch');
spy.mockImplementation(async (input: RequestInfo | URL, init?: RequestInit) => {
const url = String(input);
if (!online) throw new TypeError('Failed to fetch');
if (url.includes('/u/')) {
const uploaded = await server.upload(JMAP_ACCOUNT, new Uint8Array(await (init?.body as Blob).arrayBuffer()));
return json(uploaded);
}
if (url.includes('/d/')) return new Response(new Uint8Array([9]), { status: 200 });
if (!init?.body) return json(SESSION);
const calls = (JSON.parse(String(init.body)) as { methodCalls: MethodCall[] }).methodCalls;
wire.push(...calls);
return json(await server.request(calls));
});
setOfflineHooks({
skipNetwork: () => !online,
answer: async (_client, calls: OfflineMethodCall[], options) => {
const marked = options?.networkTried
? calls.map((call): OfflineMethodCall => (call[0] === 'Email/set' && call[1].create ? [call[0], { ...call[1], networkTried: true }, call[2]] : call))
: calls;
const answer = await engine.answer(marked);
if (backAfterCreate && calls.some(([name, args]) => name === 'Email/set' && args.create)) online = true;
return answer as OfflineResponse | null;
},
record: () => {},
noteNetwork: () => {},
serverGone: async () => true,
blob: async (_client, blobId) => {
const held = await engine.areaCall('outbox.heldFile', { blobId }) as { blob: Blob } | null;
return held?.blob ?? null;
},
recordBlob: () => {},
holdUpload: async (_client, file) => engine.areaCall('outbox.holdFile', { name: file.name, type: file.type, data: new Uint8Array(await file.arrayBuffer()) }) as Promise<{ blobId: string; size: number; type: string }>,
resolveStandIns: (_client, ids) => engine.areaCall('drafts.resolveStandIns', { ids }) as Promise<Record<string, string | null>>,
resolveHeldAttachments: async (_client, attachments) => {
const out = [];
for (const attachment of attachments) {
const held = await engine.areaCall('outbox.heldFile', { blobId: attachment.blobId }) as { blob: Blob } | null;
out.push(held ? { ...attachment, blobId: (await server.upload(JMAP_ACCOUNT, new Uint8Array(await held.blob.arrayBuffer()))).blobId } : attachment);
}
return out;
},
});
await engine.requestSync();
client = new JMAPClient('https://fake.test', 'a@example.org', 'pw');
await client.connect();
wire.length = 0;
});
afterEach(() => {
spy.mockRestore();
setOfflineHooks(null);
});
const saveDraft = (text: string, previous?: string, attachments?: Array<{ blobId: string; name: string; type: string; size: number }>) =>
client.createDraft(['b@example.org'], 'Subject', text, undefined, undefined, undefined, 'a@example.org', previous, attachments);
async function flushAll(): Promise<void> {
await flushDrafts({
store: {
read: () => engine.areaCall('drafts.read', { before: NOW - 60_000 }) as Promise<{ rows: PendingDraft[]; ghosts: DraftGhost[] }>,
claim: (localId) => engine.areaCall('drafts.claim', { localId }) as Promise<boolean>,
failed: async (localId, status, error, neverSent) => { await engine.areaCall('drafts.failed', { localId, status, error, neverSent }); },
created: (result) => engine.areaCall('drafts.created', result) as Promise<{ conflicts: string[] }>,
ghostsLooked: async (jmap, messageIds, found) => { await engine.areaCall('drafts.ghostsLooked', { jmap, messageIds, found }); },
},
request: (calls) => server.request(calls),
upload: async (jmap, blobId) => {
const held = await engine.areaCall('outbox.heldFile', { blobId }) as { blob: Blob } | null;
return held ? (await server.upload(jmap, new Uint8Array(await held.blob.arrayBuffer()))).blobId : null;
},
isRefusal: (error) => error instanceof TransportError && error.kind === 'http',
onRefused: () => {},
onConflict: () => {},
onSent: () => {},
});
const pending = await engine.areaCall('outbox.pendingPatches', {}) as PendingPatch[];
await flushPatches({ request: (calls) => server.request(calls) }, pending, async (outcomes: PatchOutcome[]) => {
await engine.areaCall('outbox.patchResults', { outcomes });
});
}
describe('createDraft offline', () => {
it('saves on the device, and the composer can read the draft straight back', async () => {
online = false;
const id = await saveDraft('Written offline');
expect(id).toMatch(/^local-/);
expect(server.drafts()).toEqual([]);
const email = await client.getEmail(id);
expect(email?.id).toBe(id);
expect(email?.bodyValues?.['1']?.value).toBe('Written offline');
expect(email?.to?.[0]?.email).toBe('b@example.org');
});
it('merges saves, so a hundred leave one draft on the server with the latest text', async () => {
online = false;
let id = await saveDraft('Save 0');
for (let n = 1; n < 100; n += 1) id = await saveDraft(`Save ${n}`, id);
online = true;
await flushAll();
expect(server.drafts()).toHaveLength(1);
expect((server.drafts()[0].bodyValues as Record<string, { value: string }>)['1'].value).toBe('Save 99');
});
it('leaves the old version alone on the server, even when it comes back between the two steps', async () => {
const real = await saveDraft('Online version');
online = false;
backAfterCreate = true;
wire.length = 0;
const id = await saveDraft('Offline version', real);
expect(id).toMatch(/^local-/);
expect(online).toBe(true);
expect(server.emails.has(real)).toBe(true);
expect(wire.filter(([name, args]) => name === 'Email/set' && args.destroy)).toEqual([]);
await flushAll();
expect(server.drafts().map((email) => (email.bodyValues as Record<string, { value: string }>)['1'].value)).toEqual(['Offline version']);
});
it('gives each create a Message-ID of its own, online too, for finding it after a lost answer', async () => {
await saveDraft('One');
const create = wire.find(([name]) => name === 'Email/set')?.[1].create as Record<string, { messageId: string[] }>;
expect(Object.values(create)[0].messageId[0]).toMatch(/@example\.org$/);
});
it('marks a create whose connection broke, so the flush looks before it creates', async () => {
const original = spy.getMockImplementation();
spy.mockImplementation(async (input: RequestInfo | URL, init?: RequestInit) => {
const body = init?.body ? String(init.body) : '';
if (body.includes('"Email/set"')) throw new TypeError('Failed to fetch');
return original!(input, init);
});
const id = await saveDraft('On the way');
spy.mockImplementation(original!);
expect(id).toMatch(/^local-/);
expect(await engine.areaCall('drafts.list', {})).toMatchObject([{ status: 'uncertain' }]);
});
});
describe('after a flush the page still holds the stand-in', () => {
it('reads the real message by it, and the answer says what the caller asked', async () => {
online = false;
const id = await saveDraft('Text');
online = true;
await flushAll();
const real = server.drafts()[0].id;
wire.length = 0;
const email = await client.getEmail(id);
expect(email?.id).toBe(real);
expect((wire.find(([name]) => name === 'Email/get')?.[1].ids as string[])).toEqual([real]);
});
it('replaces the real message on the next save, online', async () => {
online = false;
const id = await saveDraft('First');
online = true;
await flushAll();
const next = await saveDraft('Second', id);
expect(server.drafts().map((email) => email.id)).toEqual([next]);
});
it('also finds it offline again', async () => {
online = false;
const id = await saveDraft('First');
online = true;
await flushAll();
await engine.requestSync();
online = false;
const email = await client.getEmail(id);
expect(email?.bodyValues?.['1']?.value).toBe('First');
});
it('replaces the real message on the next save, offline, after the new one is up', async () => {
online = false;
const id = await saveDraft('First');
online = true;
await flushAll();
const real = server.drafts()[0].id;
online = false;
await saveDraft('Second', id);
expect(server.emails.has(real)).toBe(true);
online = true;
await flushAll();
expect(server.emails.has(real)).toBe(false);
expect((server.drafts()[0].bodyValues as Record<string, { value: string }>)['1'].value).toBe('Second');
});
});
describe('before the flush, with the server back', () => {
it('answers a read of the stand-in from the device, not the server', async () => {
online = false;
const id = await saveDraft('Text');
online = true;
wire.length = 0;
expect((await client.getEmail(id))?.bodyValues?.['1']?.value).toBe('Text');
expect(wire.filter(([name]) => name === 'Email/get')).toEqual([]);
});
it('replaces the stand-in with a save that goes straight to the server', async () => {
online = false;
const id = await saveDraft('Offline');
online = true;
const real = await saveDraft('Online', id);
expect(real).not.toMatch(/^local-/);
await flushAll();
expect(server.drafts().map((email) => email.id)).toEqual([real]);
expect(await engine.areaCall('drafts.list', {})).toEqual([]);
});
it('uploads files attached offline before the save goes to the server', async () => {
online = false;
const { blobId } = await client.uploadBlob(new File(['hello'], 'note.txt', { type: 'text/plain' }));
expect(blobId).toMatch(/^bulwark-local-/);
const id = await saveDraft('With a file', undefined, [{ blobId, name: 'note.txt', type: 'text/plain', size: 5 }]);
online = true;
await saveDraft('With a file, later', id, [{ blobId, name: 'note.txt', type: 'text/plain', size: 5 }]);
const [draft] = server.drafts();
expect(new TextDecoder().decode(server.bytesOf((draft.attachments as Array<{ blobId: string }>)[0].blobId))).toBe('hello');
});
});
describe('a file attached offline', () => {
it('stays readable and downloadable, offline and once the server is back', async () => {
online = false;
const { blobId } = await client.uploadBlob(new File(['hello'], 'note.txt', { type: 'text/plain' }));
expect(await (await client.fetchBlob(blobId, 'note.txt', 'text/plain')).text()).toBe('hello');
online = true;
expect(await (await client.fetchBlob(blobId, 'note.txt', 'text/plain')).text()).toBe('hello');
expect(spy.mock.calls.filter(([url]: [unknown]) => String(url).includes('/d/'))).toEqual([]);
});
it('isn\'t held a second time when its own upload hits an outage', async () => {
const holdUpload = vi.fn();
const hooks = { skipNetwork: () => false, answer: async () => null, record: () => {}, noteNetwork: () => {}, serverGone: async () => true, blob: async () => null, recordBlob: () => {}, holdUpload };
setOfflineHooks(hooks as never);
online = false;
await expect(client.uploadBlob(new File(['x'], 'x.txt'), { noHold: true })).rejects.toThrow();
expect(holdUpload).not.toHaveBeenCalled();
});
});
describe('an open composer\'s attachment after a flush', () => {
const FILE = { name: 'note.txt', type: 'text/plain', size: 5 };
/** A draft with a file on the server, edited offline with its part, and flushed: that part is dead now. */
async function flushedWithOldPart() {
const { blobId: uploaded } = await client.uploadBlob(new File(['hello'], 'note.txt', { type: 'text/plain' }));
const first = await saveDraft('Online', undefined, [{ ...FILE, blobId: uploaded }]);
const part = (server.emails.get(first)?.attachments as Array<{ blobId: string }>)[0].blobId;
// The upload's id would still work: only the part is left in the composer.
server.blobs.delete(uploaded);
online = false;
const standIn = await saveDraft('Offline', first, [{ ...FILE, blobId: part }]);
online = true;
await flushAll();
expect(server.emails.has(first)).toBe(false);
expect(server.bytesOf(part)).toBeUndefined();
return { standIn, part };
}
const fileOnServer = () => new TextDecoder().decode(server.bytesOf((server.drafts()[0].attachments as Array<{ blobId: string }>)[0].blobId));
it('survives the next save, which reads the real draft through the map', async () => {
const { standIn, part } = await flushedWithOldPart();
const saved = await saveDraft('Later', standIn, [{ ...FILE, blobId: part }]);
expect(server.drafts().map((email) => email.id)).toEqual([saved]);
expect(fileOnServer()).toBe('hello');
});
it('survives the next send', async () => {
const { standIn, part } = await flushedWithOldPart();
const result = await client.sendEmail(['b@example.org'], 'Subject', 'Text', undefined, undefined, undefined, 'a@example.org', standIn, undefined, undefined, [{ ...FILE, blobId: part }]);
expect(result.emailId).toBeTruthy();
expect(server.drafts()).toEqual([]);
expect((server.emails.get(result.emailId as string)?.attachments as unknown[])).toHaveLength(1);
});
it('retries once when the server says a part is gone under a real id too', async () => {
const { part } = await flushedWithOldPart();
const real = server.drafts()[0].id;
const saved = await saveDraft('Later', real, [{ ...FILE, blobId: part }]);
expect(server.drafts().map((email) => email.id)).toEqual([saved]);
expect(fileOnServer()).toBe('hello');
});
it('retries a send that says a part is gone under a real id', async () => {
const { part } = await flushedWithOldPart();
const real = server.drafts()[0].id;
const result = await client.sendEmail(['b@example.org'], 'Subject', 'Text', undefined, undefined, undefined, 'a@example.org', real, undefined, undefined, [{ ...FILE, blobId: part }]);
expect(result.emailId).toBeTruthy();
});
it('still fails when no part of that name and size is there', async () => {
const { standIn, part } = await flushedWithOldPart();
await expect(saveDraft('Later', standIn, [{ name: 'other.txt', type: 'text/plain', size: 5, blobId: part }])).rejects.toThrow(/blobNotFound/);
});
it('asks again at most once', async () => {
const { part } = await flushedWithOldPart();
const real = server.drafts()[0].id;
server.refuseCreate = { type: 'blobNotFound' };
wire.length = 0;
await expect(saveDraft('Later', real, [{ ...FILE, blobId: part }])).rejects.toThrow(/blobNotFound/);
expect(wire.filter(([name, args]) => name === 'Email/set' && args.create).length).toBeLessThanOrEqual(2);
});
});
describe('an online send of a draft still on the device', () => {
it('drops the draft here and still removes the submission record on the server', async () => {
online = false;
const id = await saveDraft('Only here');
online = true;
const result = await client.sendEmail(['b@example.org'], 'Subject', 'Text', undefined, undefined, undefined, 'a@example.org', id);
expect(result.emailId).toBeTruthy();
expect(await engine.areaCall('drafts.list', {})).toEqual([]);
expect(server.submissionsDestroyed()).toHaveLength(1);
});
});
lib/store/__tests__/drafts.test.ts
+881
View File
@@ -0,0 +1,881 @@
// @vitest-environment node
import { beforeEach, describe, expect, it } from 'vitest';
import { DEFAULT_SCOPE } from '../area';
import type { DraftError } from '../drafts/pending';
import type { LocalDraftSummary, PendingDraft, DraftGhost } from '../drafts/calls';
import { flushDrafts, type DraftFlushIo } from '../drafts/flush';
import { AccountEngine } from '../engine/account-engine';
import { TransportError, type MethodCall } from '../engine/transport';
import { mailArea } from '../mail';
import { outboxArea } from '../outbox';
import { flushPatches, type PatchOutcome, type PendingPatch } from '../outbox/flush-patches';
import type { StoredConflict } from '../outbox/items';
import { FakeJmapDrafts } from './helpers/fake-drafts';
import { FAKE_SESSION, JMAP_ACCOUNT, type FakeEmail } from './helpers/fake-jmap';
import { openMemoryStore } from './helpers/node-adapter';
const NOW = Date.UTC(2026, 10, 3);
const BOXES = [
{ id: 'inbox', name: 'Inbox', role: 'inbox', totalEmails: 0, unreadEmails: 0 },
{ id: 'drafts', name: 'Drafts', role: 'drafts', totalEmails: 0, unreadEmails: 0 },
{ id: 'trash', name: 'Trash', role: 'trash', totalEmails: 0, unreadEmails: 0 },
];
function setup() {
const server = new FakeJmapDrafts({ mailboxes: BOXES.map((box) => ({ ...box })) });
server.now = NOW;
const db = openMemoryStore();
const engine = new AccountEngine({
account: 'a@fake.test',
db,
session: FAKE_SESSION,
transport: server,
areas: [mailArea, outboxArea],
scope: { ...DEFAULT_SCOPE, days: null, attachments: false },
now: () => NOW,
sleep: async () => {},
});
return { server, db, engine };
}
let t: ReturnType<typeof setup>;
beforeEach(async () => {
t = setup();
await t.engine.requestSync();
});
interface SaveOptions {
subject?: string;
text?: string;
previous?: string;
attachments?: Array<Record<string, unknown>>;
networkTried?: boolean;
messageId?: string;
}
let nextMessage = 0;
/** What createDraft sends: a new version, then the old one goes. */
async function save(options: SaveOptions = {}): Promise<string> {
nextMessage += 1;
const answer = await t.engine.answer([['Email/set', {
accountId: JMAP_ACCOUNT,
...(options.networkTried ? { networkTried: true } : {}),
create: {
c1: {
messageId: [options.messageId ?? `draft-${nextMessage}@example.org`],
from: [{ email: 'a@example.org' }],
to: [{ email: 'b@example.org' }],
subject: options.subject ?? 'A draft',
keywords: { $seen: true, $draft: true },
mailboxIds: { drafts: true },
bodyValues: { 1: { value: options.text ?? 'Text' } },
textBody: [{ partId: '1' }],
...(options.attachments ? { attachments: options.attachments } : {}),
},
},
}, 's']]);
const result = answer?.methodResponses[0][1];
const id = result.created?.c1?.id as string;
if (!id) throw new Error(`not created: ${JSON.stringify(result.notCreated)}`);
if (options.previous) {
await t.engine.answer([['Email/set', { accountId: JMAP_ACCOUNT, destroy: [options.previous], replacedBy: id }, 'd']]);
}
return id;
}
async function get(id: string, properties?: string[]) {
const answer = await t.engine.answer([['Email/get', { accountId: JMAP_ACCOUNT, ids: [id], fetchAllBodyValues: true, ...(properties ? { properties } : {}) }, 'g']]);
return answer?.methodResponses[0][1] as { list: Array<Record<string, never>>; notFound: string[] };
}
async function draftBox(): Promise<string[]> {
const answer = await t.engine.answer([['Email/query', { accountId: JMAP_ACCOUNT, filter: { inMailbox: 'drafts' } }, 'q']]);
return answer?.methodResponses[0][1].ids;
}
const list = () => t.engine.areaCall('drafts.list', {}) as Promise<LocalDraftSummary[]>;
const conflicts = async () => (await t.engine.areaCall('outbox.read', {}) as { conflicts: StoredConflict[] }).conflicts;
interface Run {
refused: Array<[string, DraftError]>;
conflicted: string[];
sent: number;
}
/** The flush as the page runs it, with the file upload on the fake server. */
function ioFor(run: Run): DraftFlushIo {
const { engine, server } = t;
return {
store: {
read: () => engine.areaCall('drafts.read', { before: NOW - 60_000 }) as Promise<{ rows: PendingDraft[]; ghosts: DraftGhost[] }>,
claim: (localId) => engine.areaCall('drafts.claim', { localId }) as Promise<boolean>,
failed: async (localId, status, error, neverSent) => { await engine.areaCall('drafts.failed', { localId, status, error, neverSent }); },
created: (result) => engine.areaCall('drafts.created', result) as Promise<{ conflicts: string[] }>,
ghostsLooked: async (jmap, messageIds, found) => { await engine.areaCall('drafts.ghostsLooked', { jmap, messageIds, found }); },
},
request: (calls: MethodCall[]) => server.request(calls),
upload: async (jmap, blobId) => {
const held = await engine.areaCall('outbox.heldFile', { blobId }) as { blob: Blob } | null;
if (!held) return null;
return (await server.upload(jmap, new Uint8Array(await held.blob.arrayBuffer()))).blobId;
},
isRefusal: (error) => error instanceof TransportError && error.kind === 'http',
onRefused: (subject, error) => run.refused.push([subject, error]),
onConflict: (subject) => run.conflicted.push(subject),
onSent: () => { run.sent += 1; },
};
}
async function flush(): Promise<Run & { finished: boolean }> {
const run: Run = { refused: [], conflicted: [], sent: 0 };
const finished = await flushDrafts(ioFor(run));
return { ...run, finished };
}
/** And what the flush left in the usual queue: the old versions go here. */
async function flushQueue(): Promise<boolean> {
const pending = await t.engine.areaCall('outbox.pendingPatches', {}) as PendingPatch[];
return flushPatches({ request: (calls) => t.server.request(calls) }, pending, async (outcomes: PatchOutcome[]) => {
await t.engine.areaCall('outbox.patchResults', { outcomes });
});
}
function serverDraft(subject: string, id = 'R') {
t.server.addEmail({ id, subject, mailboxIds: { drafts: true }, keywords: { $seen: true, $draft: true } });
}
async function hold(text: string, name = 'note.txt'): Promise<string> {
const held = await t.engine.areaCall('outbox.holdFile', { name, type: 'text/plain', data: new TextEncoder().encode(text) }) as { blobId: string };
return held.blobId;
}
describe('a draft saved offline', () => {
it('gets a stand-in id and answers as a server would, without going to the server', async () => {
const requests = t.server.requests;
const id = await save({ subject: 'Offline', text: 'Written on a train' });
expect(id).toMatch(/^local-/);
expect(t.server.requests).toBe(requests);
expect(await draftBox()).toEqual([id]);
const [email] = (await get(id)).list;
expect(email.subject).toBe('Offline');
expect(email.keywords).toEqual({ $seen: true, $draft: true });
expect(email.preview).toBe('Written on a train');
expect((email.bodyValues as Record<string, { value: string }>)['1'].value).toBe('Written on a train');
expect(email.textBody).toEqual([{ partId: '1', type: 'text/plain', size: 18 }]);
});
it('adds itself to the folder count', async () => {
await save();
const answer = await t.engine.answer([['Mailbox/get', {}, 'm']]);
const drafts = (answer?.methodResponses[0][1].list as Array<{ id: string; totalEmails: number }>).find((box) => box.id === 'drafts');
expect(drafts?.totalEmails).toBe(1);
});
it('can be found by its words', async () => {
const id = await save({ text: 'quarterly figures' });
const answer = await t.engine.answer([['Email/query', { filter: { text: 'quarterly' } }, 'q']]);
expect(answer?.methodResponses[0][1].ids).toEqual([id]);
});
it('reads headers back the way the composer does', async () => {
const answer = await t.engine.answer([['Email/set', {
create: { c: { keywords: { $draft: true }, mailboxIds: { drafts: true }, subject: 's', 'header:X-Priority:asText': '1 (Highest)', bodyValues: { 1: { value: 'x' } }, textBody: [{ partId: '1' }] } },
}, 's']]);
const id = answer?.methodResponses[0][1].created.c.id as string;
const [email] = (await get(id, ['header:X-Priority:asText'])).list;
expect(email['header:X-Priority:asText']).toBe('1 (Highest)');
});
it('survives a full resync, which asks the server about what it didn\'t list', async () => {
const id = await save();
t.server.addEmail({ id: 'new-mail', subject: 'Trigger' });
t.server.truncateLog('Email');
t.server.calls.length = 0;
await t.engine.requestSync();
// It did list everything again, and asked about what it didn't see.
expect(t.server.calls.some(([name]) => name === 'Email/query')).toBe(true);
expect((await get(id)).list).toHaveLength(1);
});
it('takes a flag and a move, which go up with it', async () => {
const id = await save();
await t.engine.answer([['Email/set', { update: { [id]: { 'keywords/$flagged': true } } }, 'u']]);
expect((await get(id, ['keywords'])).list[0].keywords).toEqual({ $seen: true, $draft: true, $flagged: true });
await flush();
expect(t.server.drafts()[0].keywords).toEqual({ $seen: true, $draft: true, $flagged: true });
});
it('is refused as a draft when the same request sends it, or when it isn\'t in Drafts', async () => {
const sendLike = await t.engine.answer([
['Email/set', { create: { c: { keywords: { $draft: true }, mailboxIds: { drafts: true } } } }, '0'],
['EmailSubmission/set', { create: { s: { emailId: '#c' } } }, '1'],
]);
expect(sendLike?.methodResponses[0][1].notCreated.c.type).toBe('serverUnavailable');
const elsewhere = await t.engine.answer([['Email/set', { create: { c: { keywords: { $draft: true }, mailboxIds: { inbox: true } } } }, '0']]);
expect(elsewhere?.methodResponses[0][1].notCreated.c.type).toBe('serverUnavailable');
const noKeyword = await t.engine.answer([['Email/set', { create: { c: { keywords: {}, mailboxIds: { drafts: true } } } }, '0']]);
expect(noKeyword?.methodResponses[0][1].notCreated.c.type).toBe('serverUnavailable');
expect(await list()).toEqual([]);
});
});
describe('saving again', () => {
it('leaves one pending draft after a hundred saves, and one on the server with the latest text', async () => {
let id = await save({ text: 'Save 0' });
for (let n = 1; n < 100; n += 1) id = await save({ text: `Save ${n}`, previous: id });
expect(await list()).toHaveLength(1);
expect(await draftBox()).toEqual([id]);
const { finished, sent } = await flush();
expect(finished).toBe(true);
expect(sent).toBe(1);
expect(t.server.drafts()).toHaveLength(1);
expect((t.server.drafts()[0].bodyValues as Record<string, { value: string }>)['1'].value).toBe('Save 99');
expect(await list()).toEqual([]);
});
it('removes the older stand-in from everything that reads', async () => {
const first = await save();
const second = await save({ previous: first });
expect((await get(first)).notFound).toEqual([first]);
expect((await get(second)).list).toHaveLength(1);
});
});
describe('a stand-in in a destroy', () => {
it('discards the draft, and nothing reaches the server', async () => {
const id = await save();
const answer = await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]);
expect(answer?.methodResponses[0][1].destroyed).toEqual([id]);
expect((await get(id)).notFound).toEqual([id]);
await flush();
expect(t.server.drafts()).toEqual([]);
expect(await t.engine.areaCall('outbox.pendingPatches', {})).toEqual([]);
});
it('after the flush goes to the real message, by the map', async () => {
const id = await save();
await flush();
const real = t.server.drafts()[0].id;
expect(await t.engine.areaCall('drafts.resolveStandIns', { ids: [id, 'local-unknown'] })).toEqual({ [id]: real, 'local-unknown': null });
await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]);
await flushQueue();
expect(t.server.emails.has(real)).toBe(false);
});
});
describe('a draft from the server edited offline', () => {
beforeEach(async () => {
serverDraft('On the server');
await t.engine.requestSync();
});
it('hides the old version here and leaves it on the server until the new one has arrived', async () => {
const id = await save({ subject: 'Edited', previous: 'R' });
expect(await draftBox()).toEqual([id]);
expect(t.server.emails.has('R')).toBe(true);
expect(await t.engine.areaCall('outbox.pendingPatches', {})).toEqual([]);
// A sync doesn't bring it back.
t.server.updateEmail('R', { keywords: { $seen: true, $draft: true } });
t.server.coll.Email.push('updated', 'R');
await t.engine.requestSync();
expect(await draftBox()).toEqual([id]);
});
it('creates the new version first and destroys the old one after', async () => {
await save({ subject: 'Edited', previous: 'R' });
await flush();
expect(t.server.emails.has('R')).toBe(true);
await flushQueue();
expect(t.server.log.filter((line) => /^(create|destroy)/.test(line))).toEqual(['create srv1', 'destroy R']);
expect(t.server.drafts().map((email) => email.subject)).toEqual(['Edited']);
});
it('keeps both when the destroy fails: nothing is lost, and it goes again', async () => {
await save({ subject: 'Edited', previous: 'R' });
await flush();
t.server.offline = true;
expect(await flushQueue()).toBe(false);
t.server.offline = false;
expect(t.server.drafts()).toHaveLength(2);
await flushQueue();
expect(t.server.drafts().map((email) => email.subject)).toEqual(['Edited']);
});
it('keeps the old version when the create fails', async () => {
t.server.refuseCreate = { type: 'overQuota' };
await save({ subject: 'Edited', previous: 'R' });
await flush();
await flushQueue();
expect(t.server.emails.has('R')).toBe(true);
});
it('carries what an earlier save replaced through later saves', async () => {
const first = await save({ text: 'One', previous: 'R' });
await save({ text: 'Two', previous: first });
await flush();
await flushQueue();
expect(t.server.drafts()).toHaveLength(1);
expect((t.server.drafts()[0].bodyValues as Record<string, { value: string }>)['1'].value).toBe('Two');
});
it('removes the old version too when the new one is discarded before it went up', async () => {
const id = await save({ previous: 'R' });
await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]);
await flushQueue();
expect(t.server.drafts()).toEqual([]);
});
it('keeps both and says so when another device changed it meanwhile', async () => {
await save({ subject: 'Mine', previous: 'R' });
t.server.updateEmail('R', { keywords: { $seen: true, $draft: true, $flagged: true } });
const { conflicted } = await flush();
await flushQueue();
expect(conflicted).toEqual(['On the server']);
expect(t.server.drafts().map((email) => email.subject).sort()).toEqual(['Mine', 'On the server']);
expect((await conflicts())[0].summary).toMatchObject({ op: 'draft', reason: 'changed', subject: 'On the server' });
// The other version is back in view: nothing else would bring it.
expect((await draftBox()).sort()).toEqual(['R', 'srv1'].sort());
});
it('keeps ours and says so when another device replaced or removed it', async () => {
await save({ subject: 'Mine', previous: 'R' });
t.server.destroyEmail('R');
t.server.addEmail({ id: 'R2', subject: 'Theirs', mailboxIds: { drafts: true }, keywords: { $seen: true, $draft: true } });
const { conflicted } = await flush();
await flushQueue();
expect(conflicted).toEqual(['On the server']);
expect(t.server.drafts().map((email) => email.subject).sort()).toEqual(['Mine', 'Theirs']);
expect((await conflicts())[0].summary).toMatchObject({ op: 'draft', reason: 'gone' });
});
});
describe('a lost response', () => {
it('doesn\'t make a second draft: the copy is found by its Message-ID', async () => {
const id = await save({ messageId: 'fixed@example.org' });
t.server.loseAnswerOf = 'Email/set';
const first = await flush();
expect(first.finished).toBe(false);
expect(t.server.drafts()).toHaveLength(1);
expect((await list())[0].status).toBe('uncertain');
// It stays readable under its stand-in meanwhile.
expect((await get(id)).list).toHaveLength(1);
const second = await flush();
expect(second.finished).toBe(true);
expect(t.server.drafts()).toHaveLength(1);
expect(await list()).toEqual([]);
expect(await t.engine.areaCall('drafts.resolveStandIns', { ids: [id] })).toEqual({ [id]: t.server.drafts()[0].id });
});
it('creates it when the lookup finds that it never arrived', async () => {
await save();
t.server.offline = true;
await flush();
t.server.offline = false;
await flush();
expect(t.server.drafts()).toHaveLength(1);
});
it('removes the copy of a version replaced while the answer was missing', async () => {
const first = await save({ text: 'Old', messageId: 'old@example.org' });
t.server.loseAnswerOf = 'Email/set';
await flush();
await save({ text: 'New', previous: first });
await flush();
await flushQueue();
expect(t.server.drafts()).toHaveLength(1);
expect((t.server.drafts()[0].bodyValues as Record<string, { value: string }>)['1'].value).toBe('New');
expect(await t.engine.areaCall('drafts.read', { before: 0 })).toMatchObject({ ghosts: [] });
});
it('looks a create up that came to the store after its own request broke', async () => {
t.server.addEmail({ id: 'landed', subject: 'A draft', mailboxIds: { drafts: true }, keywords: { $draft: true }, messageId: ['tried@example.org'], receivedAt: new Date(NOW).toISOString() });
const id = await save({ messageId: 'tried@example.org', networkTried: true });
expect((await list())[0].status).toBe('uncertain');
await flush();
expect(t.server.drafts().map((email) => email.id)).toEqual(['landed']);
expect(await t.engine.areaCall('drafts.resolveStandIns', { ids: [id] })).toEqual({ [id]: 'landed' });
});
it('counts a tab closed on the way as uncertain', async () => {
const id = await save({ messageId: 'closed@example.org' });
expect(await t.engine.areaCall('drafts.claim', { localId: id })).toBe(true);
await t.engine.areaCall('drafts.read', { before: NOW + 1 });
expect((await list())[0].status).toBe('uncertain');
});
it('removes a copy that arrives after the draft was replaced on the way, only once the new one is up', async () => {
const id = await save({ text: 'Old', messageId: 'late@example.org' });
await t.engine.areaCall('drafts.claim', { localId: id });
const next = await save({ text: 'New', previous: id });
t.server.addEmail({ id: 'late', subject: 'A draft', mailboxIds: { drafts: true }, keywords: { $seen: true, $draft: true }, messageId: ['late@example.org'], size: 3, receivedAt: new Date(NOW).toISOString() });
await t.engine.areaCall('drafts.created', { localId: id, seq: 1, jmap: JMAP_ACCOUNT, messageId: 'late@example.org', serverId: 'late', email: null, server: {}, ghostCopies: [] });
expect(await t.engine.areaCall('outbox.pendingPatches', {})).toEqual([]);
expect((await get(next)).list).toHaveLength(1);
await flush();
await flushQueue();
expect(t.server.emails.has('late')).toBe(false);
expect((t.server.drafts()[0].bodyValues as Record<string, { value: string }>)['1'].value).toBe('New');
});
});
describe('a refused draft', () => {
it('stays on the device with its error, and the composer can still open it', async () => {
t.server.refuseCreate = { type: 'overQuota', description: 'No room left' };
const id = await save({ subject: 'Too much', text: 'Keep me' });
const { refused, finished } = await flush();
expect(finished).toBe(true);
expect(refused).toEqual([['Too much', { type: 'overQuota', text: 'No room left' }]]);
expect(await list()).toMatchObject([{ localId: id, status: 'refused', error: { type: 'overQuota', text: 'No room left' } }]);
expect((await get(id)).list[0].bodyValues).toMatchObject({ 1: { value: 'Keep me' } });
expect(await draftBox()).toEqual([id]);
});
it('isn\'t asked about again by itself, and goes up when the user tries again', async () => {
t.server.refuseCreate = { type: 'overQuota' };
const id = await save();
await flush();
t.server.refuseCreate = null;
await flush();
expect(t.server.drafts()).toEqual([]);
await t.engine.areaCall('drafts.retry', { localId: id });
await flush();
expect(t.server.drafts()).toHaveLength(1);
expect(await list()).toEqual([]);
});
it('goes up again once it\'s edited', async () => {
t.server.refuseCreate = { type: 'overQuota' };
const id = await save({ text: 'One' });
await flush();
t.server.refuseCreate = null;
await save({ text: 'Two', previous: id });
await flush();
expect((t.server.drafts()[0].bodyValues as Record<string, { value: string }>)['1'].value).toBe('Two');
});
it('says so when the request itself is refused for good', async () => {
const id = await save();
const original = t.server.request.bind(t.server);
t.server.request = async () => { throw new TransportError('http', 'Too large', 413); };
const { refused } = await flush();
t.server.request = original;
expect(refused).toHaveLength(1);
expect((await list())[0]).toMatchObject({ localId: id, status: 'refused' });
});
});
describe('files attached offline', () => {
it('are shown on the draft with their size, stay readable, and go up on the flush', async () => {
const blobId = await hold('hello');
const id = await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt', disposition: 'attachment' }] });
const [email] = (await get(id)).list;
expect(email.hasAttachment).toBe(true);
expect(email.attachments).toMatchObject([{ blobId, name: 'note.txt', size: 5, type: 'text/plain', disposition: 'attachment' }]);
const file = await t.engine.areaCall('outbox.heldFile', { blobId }) as { name: string; blob: Blob };
expect(await file.blob.text()).toBe('hello');
await flush();
const [draft] = t.server.drafts();
expect(t.server.uploads).toHaveLength(1);
const [part] = draft.attachments as Array<{ blobId: string; size: number }>;
expect(part.size).toBe(5);
expect(new TextDecoder().decode(t.server.bytesOf(part.blobId))).toBe('hello');
});
it('go up before the create that names them', async () => {
const blobId = await hold('hello');
await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt' }] });
await flush();
expect(t.server.log.slice(0, 2)).toEqual(['upload up1', 'create srv1']);
});
it('come in pieces, and a broken one leaves nothing readable', async () => {
const started = await t.engine.areaCall('outbox.holdStart', { name: 'big.bin', type: '', size: 6 }) as { blobId: string };
await t.engine.areaCall('outbox.holdChunk', { blobId: started.blobId, seq: 0, data: new Uint8Array([1, 2, 3]) });
expect(await t.engine.areaCall('outbox.heldFile', { blobId: started.blobId })).toBeNull();
await t.engine.areaCall('outbox.holdChunk', { blobId: started.blobId, seq: 1, data: new Uint8Array([4, 5, 6]) });
await t.engine.areaCall('outbox.holdEnd', { blobId: started.blobId });
const file = await t.engine.areaCall('outbox.heldFile', { blobId: started.blobId }) as { blob: Blob };
expect([...new Uint8Array(await file.blob.arrayBuffer())]).toEqual([1, 2, 3, 4, 5, 6]);
await t.engine.areaCall('outbox.holdAbort', { blobId: started.blobId });
expect(await t.engine.areaCall('outbox.heldFile', { blobId: started.blobId })).toBeNull();
expect(t.db.get('SELECT 1 AS x FROM upload_chunk')).toBeUndefined();
});
it('wait when the upload hits an outage, without counting as maybe sent', async () => {
const blobId = await hold('hello');
await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt' }] });
t.server.failUpload = true;
const { finished } = await flush();
expect(finished).toBe(false);
expect(t.server.drafts()).toEqual([]);
expect((await list())[0].status).toBe('waiting');
});
it('refuse the draft when a file is gone from the device', async () => {
const blobId = await hold('hello');
const id = await save({ attachments: [{ blobId, type: 'text/plain', name: 'note.txt' }] });
await t.engine.areaCall('outbox.holdAbort', { blobId });
const { refused } = await flush();
expect(refused).toEqual([['A draft', { code: 'file_missing' }]]);
expect((await list())[0]).toMatchObject({ localId: id, status: 'refused', error: { code: 'file_missing' } });
});
it('count as needed for as long as a draft or a send names them', async () => {
const forDraft = await hold('a');
const forNothing = await hold('b');
const id = await save({ attachments: [{ blobId: forDraft, type: 'text/plain', name: 'a' }] });
expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW + 1 })).toBe(1);
expect(await t.engine.areaCall('outbox.heldFile', { blobId: forNothing })).toBeNull();
expect(await t.engine.areaCall('outbox.heldFile', { blobId: forDraft })).not.toBeNull();
await t.engine.answer([['Email/set', { destroy: [id] }, 'd']]);
expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW + 1 })).toBe(1);
});
it('survive removing the send that shared them with a draft', async () => {
const blobId = await hold('a');
await save({ attachments: [{ blobId, type: 'text/plain', name: 'a' }] });
const entry = await t.engine.areaCall('outbox.queueSend', {
summary: { subject: 's', to: [] },
item: { kind: 'email', args: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, [{ blobId }]] },
}) as number;
await t.engine.areaCall('outbox.remove', { id: entry });
expect(await t.engine.areaCall('outbox.heldFile', { blobId })).not.toBeNull();
});
it('are found inside a body structure too, as a draft with inline images writes them', async () => {
const blobId = await hold('png');
const answer = await t.engine.answer([['Email/set', {
create: {
c: {
keywords: { $draft: true },
mailboxIds: { drafts: true },
bodyValues: { text: { value: 'text' }, html: { value: '<img src="cid:i">' } },
bodyStructure: {
type: 'multipart/alternative',
subParts: [
{ partId: 'text', type: 'text/plain' },
{ type: 'multipart/related', subParts: [{ partId: 'html', type: 'text/html' }, { blobId, type: 'image/png', name: 'i.png', disposition: 'inline', cid: 'i' }] },
],
},
},
},
}, 's']]);
const id = answer?.methodResponses[0][1].created.c.id as string;
const [email] = (await get(id)).list;
expect(email.hasAttachment).toBe(false);
expect(email.attachments).toMatchObject([{ blobId, cid: 'i', disposition: 'inline', size: 3 }]);
expect(email.htmlBody).toMatchObject([{ partId: 'html', type: 'text/html' }]);
await flush();
const leaf = (t.server.calls.find(([name, args]) => name === 'Email/set' && args.create)?.[1].create as Record<string, { bodyStructure: { subParts: Array<{ subParts: Array<{ blobId?: string }> }> } }>).draft.bodyStructure.subParts[1].subParts[1];
expect(leaf.blobId).toBe(t.server.uploads[0].blobId);
});
});
describe('sending a draft that\'s on the device', () => {
it('leaves the draft alone while the send waits, and the send takes care of it', async () => {
const id = await save({ previous: undefined });
await t.engine.areaCall('outbox.queueSend', {
summary: { subject: 's', to: [] },
item: { kind: 'email', args: [['b@example.org'], 's', 'x', undefined, undefined, undefined, undefined, id] },
});
await flush();
expect(t.server.drafts()).toEqual([]);
expect((await get(id)).list).toHaveLength(1);
// It's that message, in the outbox: not counted a second time as unsent.
expect(await list()).toEqual([]);
await t.engine.areaCall('drafts.discard', { jmap: JMAP_ACCOUNT, id });
expect((await get(id)).notFound).toEqual([id]);
});
it('goes up as a draft if the send fails, so nothing depends on the outbox', async () => {
const id = await save();
const entry = await t.engine.areaCall('outbox.queueSend', {
summary: { subject: 's', to: [] },
item: { kind: 'email', args: [['b@example.org'], 's', 'x', undefined, undefined, undefined, undefined, id] },
}) as number;
await t.engine.areaCall('outbox.update', { id: entry, status: 'failed', error: { text: 'refused' } });
await flush();
expect(t.server.drafts()).toHaveLength(1);
});
});
const valueOf = (email: FakeEmail) => (email.bodyValues as Record<string, { value: string }>)['1'].value;
describe('a draft naming a part that died with its message', () => {
function partOnServer(id: string, part: string, bytes: string) {
t.server.parts.set(part, new TextEncoder().encode(bytes));
t.server.addEmail({
id,
subject: 'On the server',
mailboxIds: { drafts: true },
keywords: { $seen: true, $draft: true },
attachments: [{ partId: part, blobId: part, name: 'a.txt', type: 'text/plain', size: bytes.length }] as never,
});
}
it('is pointed at the part of the version it replaces, and goes up', async () => {
partOnServer('R', 'part-live', 'abc');
await t.engine.requestSync();
await save({ previous: 'R', attachments: [{ blobId: 'part-dead', type: 'text/plain', name: 'a.txt' }] });
const { refused, finished } = await flush();
expect(refused).toEqual([]);
expect(finished).toBe(true);
await flushQueue();
const [draft] = t.server.drafts();
expect(t.server.drafts()).toHaveLength(1);
expect(new TextDecoder().decode(t.server.bytesOf((draft.attachments as Array<{ blobId: string }>)[0].blobId))).toBe('abc');
});
it('keeps two files of one name apart: one that\'s alive keeps its part', async () => {
t.server.parts.set('p-short', new Uint8Array(3));
t.server.parts.set('p-long', new Uint8Array(9));
t.server.addEmail({
id: 'R',
subject: 'Two',
mailboxIds: { drafts: true },
keywords: { $draft: true },
attachments: [
{ partId: 'p-short', blobId: 'p-short', name: 'a.txt', type: 'text/plain', size: 3 },
{ partId: 'p-long', blobId: 'p-long', name: 'a.txt', type: 'text/plain', size: 9 },
] as never,
});
await t.engine.requestSync();
await save({ previous: 'R', attachments: [{ blobId: 'p-long', type: 'text/plain', name: 'a.txt' }, { blobId: 'dead', type: 'text/plain', name: 'a.txt' }] });
await flush();
const parts = t.server.drafts().find((email) => email.id !== 'R')?.attachments as Array<{ size: number }>;
expect(parts.map((part) => part.size).sort()).toEqual([3, 9]);
});
it('stays here with its error when nothing matches', async () => {
partOnServer('R', 'part-live', 'abc');
await t.engine.requestSync();
const id = await save({ previous: 'R', attachments: [{ blobId: 'part-dead', type: 'text/plain', name: 'other.txt' }] });
const { refused } = await flush();
expect(refused[0][1]).toMatchObject({ type: 'blobNotFound' });
expect((await list())[0]).toMatchObject({ localId: id, status: 'refused' });
expect(t.server.emails.has('R')).toBe(true);
});
});
describe('a version that may have reached the server, once replaced', () => {
async function lostThenReplaced(text = 'New') {
const first = await save({ text: 'Old', messageId: 'old@example.org' });
t.server.loseAnswerOf = 'Email/set';
await flush();
return { first, copy: t.server.drafts()[0].id, next: () => save({ text, previous: first }) };
}
it('is removed when it\'s the copy this device made', async () => {
const { next } = await lostThenReplaced();
await next();
await flush();
await flushQueue();
expect(t.server.drafts().map(valueOf)).toEqual(['New']);
});
it('is left alone when another client saved again under that Message-ID', async () => {
const { copy, next } = await lostThenReplaced();
t.server.destroyEmail(copy);
t.server.addEmail({ id: 'theirs', subject: 'Theirs now', mailboxIds: { drafts: true }, keywords: { $draft: true }, messageId: ['old@example.org'], size: 3, receivedAt: new Date(NOW).toISOString() });
await next();
await flush();
await flushQueue();
expect(t.server.emails.has('theirs')).toBe(true);
expect(t.server.drafts()).toHaveLength(2);
});
it('is left alone when it was received long after the attempt', async () => {
const { copy, next } = await lostThenReplaced();
t.server.updateEmail(copy, { receivedAt: new Date(NOW + 6 * 60 * 60 * 1000).toISOString() });
await next();
await flush();
await flushQueue();
expect(t.server.emails.has(copy)).toBe(true);
});
it('is left alone when its size doesn\'t fit', async () => {
const { copy, next } = await lostThenReplaced();
t.server.updateEmail(copy, { size: 5_000_000 });
await next();
await flush();
await flushQueue();
expect(t.server.emails.has(copy)).toBe(true);
});
it('is left alone when it\'s in another folder too, or no longer a draft', async () => {
const { copy, next } = await lostThenReplaced();
t.server.updateEmail(copy, { mailboxIds: { drafts: true, inbox: true } });
await next();
await flush();
await flushQueue();
expect(t.server.emails.has(copy)).toBe(true);
t = setup();
await t.engine.requestSync();
const again = await lostThenReplaced();
t.server.updateEmail(again.copy, { keywords: { $seen: true } });
await again.next();
await flush();
await flushQueue();
expect(t.server.emails.has(again.copy)).toBe(true);
});
it('is left alone when it moves between the lookup and the destroy', async () => {
const { copy, next } = await lostThenReplaced();
await next();
await flush();
t.server.updateEmail(copy, { mailboxIds: { trash: true } });
await flushQueue();
expect(t.server.emails.has(copy)).toBe(true);
expect((t.server.emails.get(copy)?.mailboxIds as Record<string, boolean>).trash).toBe(true);
});
it('isn\'t touched while the newer version is refused, and goes once that one is up', async () => {
const { copy, next } = await lostThenReplaced();
const newer = await next();
t.server.refuseCreate = { type: 'overQuota' };
await flush();
expect(await t.engine.areaCall('outbox.pendingPatches', {})).toEqual([]);
await flushQueue();
expect(t.server.emails.has(copy)).toBe(true);
t.server.refuseCreate = null;
await t.engine.areaCall('drafts.retry', { localId: newer });
await flush();
await flushQueue();
expect(t.server.emails.has(copy)).toBe(false);
expect(t.server.drafts().map(valueOf)).toEqual(['New']);
});
it('carries through later saves', async () => {
const { copy, first } = await lostThenReplaced();
const second = await save({ text: 'Two', previous: first });
await save({ text: 'Three', previous: second });
await flush();
await flushQueue();
expect(t.server.emails.has(copy)).toBe(false);
expect(t.server.drafts().map(valueOf)).toEqual(['Three']);
});
it('is looked up and removed under the same rules when the draft is discarded', async () => {
const { first, copy } = await lostThenReplaced();
await t.engine.answer([['Email/set', { destroy: [first] }, 'd']]);
await flush();
await flushQueue();
expect(t.server.emails.has(copy)).toBe(false);
t = setup();
await t.engine.requestSync();
const again = await lostThenReplaced();
t.server.updateEmail(again.copy, { subject: 'Renamed elsewhere' });
await t.engine.answer([['Email/set', { destroy: [again.first] }, 'd']]);
await flush();
await flushQueue();
expect(t.server.emails.has(again.copy)).toBe(true);
});
it('isn\'t taken for this draft\'s own copy when another client saved under its Message-ID', async () => {
const id = await save({ text: 'Mine', messageId: 'same@example.org' });
t.server.loseAnswerOf = 'Email/set';
await flush();
const copy = t.server.drafts()[0].id;
t.server.destroyEmail(copy);
t.server.addEmail({ id: 'theirs', subject: 'Theirs now', mailboxIds: { drafts: true }, keywords: { $draft: true }, messageId: ['same@example.org'], size: 3, receivedAt: new Date(NOW).toISOString() });
await flush();
expect(t.server.emails.has('theirs')).toBe(true);
expect(t.server.drafts().map((email) => email.subject).sort()).toEqual(['A draft', 'Theirs now']);
const resolved = await t.engine.areaCall('drafts.resolveStandIns', { ids: [id] }) as Record<string, string>;
expect(resolved[id]).not.toBe('theirs');
});
});
describe('a version the device doesn\'t keep, replaced', () => {
it('goes after the new one when it\'s a plain draft, checked against what the server says then', async () => {
t.server.addEmail({ id: 'R', subject: 'Not kept', mailboxIds: { drafts: true }, keywords: { $draft: true } });
await save({ subject: 'Mine', previous: 'R' });
await flush();
await flushQueue();
expect(t.server.emails.has('R')).toBe(false);
});
it('stays, and the user is told, when it\'s no longer a plain draft', async () => {
t.server.addEmail({ id: 'R', subject: 'Moved', mailboxIds: { trash: true }, keywords: { $draft: true } });
const { conflicted } = await (async () => {
await save({ subject: 'Mine', previous: 'R' });
return flush();
})();
await flushQueue();
expect(t.server.emails.has('R')).toBe(true);
expect(conflicted).toEqual(['Moved']);
});
it('stays when it moves between the read and the destroy', async () => {
t.server.addEmail({ id: 'R', subject: 'Moves', mailboxIds: { drafts: true }, keywords: { $draft: true } });
await save({ subject: 'Mine', previous: 'R' });
await flush();
t.server.updateEmail('R', { mailboxIds: { trash: true } });
await flushQueue();
expect(t.server.emails.has('R')).toBe(true);
});
});
describe('a lookup that fails', () => {
function queryFails(error: Error) {
const original = t.server.request.bind(t.server);
t.server.request = async (calls) => {
if (calls.some(([name]) => name === 'Email/query')) throw error;
return original(calls);
};
return () => { t.server.request = original; };
}
it('waits on an outage', async () => {
const first = await save({ messageId: 'old@example.org' });
t.server.loseAnswerOf = 'Email/set';
await flush();
const restore = queryFails(new TransportError('network', 'gone'));
const { finished } = await flush();
restore();
expect(finished).toBe(false);
expect((await list())[0].localId).toBe(first);
});
it('counts as no copies when the server refuses it for good, so the rest can go', async () => {
await save({ messageId: 'old@example.org' });
t.server.loseAnswerOf = 'Email/set';
await flush();
const restore = queryFails(new TransportError('http', 'Refused', 400));
const { finished } = await flush();
restore();
expect(finished).toBe(true);
expect(await list()).toEqual([]);
});
it('drops a ghost it can\'t ever look up', async () => {
const first = await save({ messageId: 'old@example.org' });
t.server.loseAnswerOf = 'Email/set';
await flush();
const copy = t.server.drafts()[0].id;
await t.engine.answer([['Email/set', { destroy: [first] }, 'd']]);
const restore = queryFails(new TransportError('http', 'Refused', 400));
await flush();
restore();
expect(await t.engine.areaCall('drafts.read', { before: 0 })).toMatchObject({ ghosts: [] });
expect(t.server.emails.has(copy)).toBe(true);
});
});
describe('a held file whose pieces don\'t add up', () => {
it('reads as gone, not as an empty file', async () => {
const blobId = await hold('hello');
t.db.run('DELETE FROM upload_chunk WHERE local_id = ?', [blobId]);
expect(await t.engine.areaCall('outbox.heldFile', { blobId })).toBeNull();
});
});
lib/store/__tests__/helpers/fake-drafts.ts
+177
View File
@@ -0,0 +1,177 @@
/**
* FakeJmapMail with what drafts talk to: Email/set create, uploads, blob
* downloads and a send. Follows Stalwart: every new message gets new part ids,
* and a part's id dies with its message, so a create naming one gets
* `blobNotFound`. An upload's id stays usable. Two creates with one Message-ID
* are both accepted, and each create makes a new message.
*/
import { TransportError, type JmapResponse, type MethodCall } from '../../engine/transport';
import { type FakeEmail } from './fake-jmap';
import { FakeJmapMail } from './fake-jmap-mail';
interface Part {
partId?: string;
blobId?: string;
type?: string;
name?: string;
disposition?: string;
cid?: string;
size?: number;
subParts?: Part[];
}
export class FakeJmapDrafts extends FakeJmapMail {
/** What was uploaded, by blob id. */
readonly blobs = new Map<string, Uint8Array>();
/** The bytes of the parts of messages, by part id. */
readonly parts = new Map<string, Uint8Array>();
readonly uploads: Array<{ blobId: string; size: number }> = [];
/** The order things happened in, for tests about order. */
readonly log: string[] = [];
/** Every create is refused with this. */
refuseCreate: { type: string; description?: string } | null = null;
/** The server does the work of the next request naming this method, and the answer never arrives. */
loseAnswerOf: string | null = null;
/** The next upload fails like a dropped connection. */
failUpload = false;
now = Date.UTC(2026, 10, 3);
private nextCreated = 1;
private nextBlob = 1;
private nextPart = 1;
private createdInRequest = new Map<string, string>();
private readonly destroyedSubmissions: string[] = [];
drafts(): FakeEmail[] {
return [...this.emails.values()].filter((email) => (email.mailboxIds as Record<string, boolean>).drafts);
}
submissionsDestroyed(): readonly string[] {
return this.destroyedSubmissions;
}
private knows(blobId: string): boolean {
if (this.blobs.has(blobId)) return true;
return [...this.emails.values()].some((email) => (email.attachments as Part[] | undefined)?.some((part) => part.blobId === blobId));
}
/** The bytes behind an upload or a part still alive. */
bytesOf(blobId: string): Uint8Array | undefined {
return this.blobs.get(blobId) ?? (this.knows(blobId) ? this.parts.get(blobId) : undefined);
}
private partsOf(create: Record<string, unknown>): Part[] {
const leaves = (part: Part): Part[] => (part.subParts ? part.subParts.flatMap(leaves) : [part]);
return [...((create.attachments ?? []) as Part[]), ...(create.bodyStructure ? leaves(create.bodyStructure as Part) : [])].filter((part) => part.blobId);
}
private createOne(create: Record<string, unknown>): { id: string; blobId: string; threadId: string; size: number } | { type: string; description?: string } {
if (this.refuseCreate) return this.refuseCreate;
const named = this.partsOf(create);
if (named.some((part) => !this.knows(part.blobId as string))) return { type: 'blobNotFound' };
const id = `srv${this.nextCreated++}`;
const values = (create.bodyValues ?? {}) as Record<string, { value: string }>;
const attachments = named.map((part) => {
const data = this.bytesOf(part.blobId as string) ?? new Uint8Array();
const fresh = `part${this.nextPart++}`;
this.parts.set(fresh, data);
return { ...part, partId: fresh, blobId: fresh, size: data.length };
});
const size = Object.values(values).reduce((sum, value) => sum + value.value.length, 0) + attachments.reduce((sum, part) => sum + part.size, 0);
this.addEmail({
id,
threadId: `t-${id}`,
mailboxIds: create.mailboxIds as Record<string, boolean>,
keywords: create.keywords as Record<string, boolean>,
subject: String(create.subject ?? ''),
from: create.from as never,
to: create.to as never,
messageId: create.messageId as never,
receivedAt: new Date(this.now).toISOString(),
size,
blobId: `raw-${id}`,
preview: '',
hasAttachment: named.length > 0,
textBody: (create.textBody ?? []) as never,
htmlBody: (create.htmlBody ?? create.textBody ?? []) as never,
bodyValues: Object.fromEntries(Object.entries(values).map(([key, value]) => [key, { value: value.value, isTruncated: false, isEncodingProblem: false }])) as never,
attachments: attachments as never,
});
this.log.push(`create ${id}`);
return { id, blobId: `raw-${id}`, threadId: `t-${id}`, size };
}
private submissionSet(args: Record<string, unknown>): Record<string, unknown> {
const created: Record<string, unknown> = {};
const notCreated: Record<string, unknown> = {};
for (const [key, create] of Object.entries((args.create ?? {}) as Record<string, { emailId: string }>)) {
const emailId = create.emailId.startsWith('#') ? this.createdInRequest.get(create.emailId.slice(1)) : create.emailId;
if (!emailId || !this.emails.has(emailId)) {
notCreated[key] = { type: 'invalidProperties' };
continue;
}
created[key] = { id: this.addSubmission(emailId, 'final').id, undoStatus: 'final' };
this.log.push(`submit ${emailId}`);
const email = this.emails.get(emailId) as FakeEmail;
const keywords = { ...(email.keywords as Record<string, boolean>) };
delete keywords.$draft;
this.updateEmail(emailId, { keywords, mailboxIds: { sent: true } });
}
const destroyed = (args.destroy ?? []) as string[];
for (const id of destroyed) {
this.submissions.delete(id);
this.destroyedSubmissions.push(id);
}
return {
accountId: args.accountId,
...(args.create ? { created } : {}),
...(Object.keys(notCreated).length > 0 ? { notCreated } : {}),
...(args.destroy ? { destroyed } : {}),
};
}
protected override handle(name: string, args: Record<string, unknown>): Record<string, unknown> {
if (name === 'EmailSubmission/set') return this.submissionSet(args);
if (name !== 'Email/set') return super.handle(name, args);
const created: Record<string, unknown> = {};
const notCreated: Record<string, unknown> = {};
for (const [creationId, create] of Object.entries((args.create ?? {}) as Record<string, Record<string, unknown>>)) {
const answer = this.createOne(create);
if ('id' in answer) {
created[creationId] = answer;
this.createdInRequest.set(creationId, answer.id);
} else {
notCreated[creationId] = answer;
}
}
for (const id of (args.destroy ?? []) as string[]) this.log.push(`destroy ${id}`);
return {
...super.handle(name, args),
...(args.create ? { created } : {}),
...(Object.keys(notCreated).length > 0 ? { notCreated } : {}),
};
}
override async request(calls: MethodCall[]): Promise<JmapResponse> {
this.createdInRequest = new Map();
const answer = await super.request(calls);
if (this.loseAnswerOf && calls.some(([name]) => name === this.loseAnswerOf)) {
this.loseAnswerOf = null;
throw new TransportError('network', 'The connection broke');
}
return answer;
}
override async upload(_jmap?: string, data?: Uint8Array): Promise<{ blobId: string; type: string; size: number }> {
if (this.offline || this.failUpload) {
this.failUpload = false;
throw new TransportError('network', 'Failed to fetch');
}
const blobId = `up${this.nextBlob++}`;
const bytes = data ?? new Uint8Array();
this.blobs.set(blobId, bytes);
this.uploads.push({ blobId, size: bytes.length });
this.log.push(`upload ${blobId}`);
return { blobId, type: 'application/octet-stream', size: bytes.length };
}
}
lib/store/__tests__/helpers/in-process-store.ts
+15 -2
View File
@@ -19,6 +19,19 @@ const NO_SERVER: JmapTransport = {
upload: async () => ({ blobId: 'x', type: '', size: 0 }),
};
/**
* What crosses to the worker and back is a copy, except a Blob: that goes by
* reference there, and jsdom's Blob can't be cloned here.
*/
function acrossWorker<T>(value: T): T {
if (value instanceof Blob) return value;
if (Array.isArray(value)) return value.map(acrossWorker) as T;
if (value && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype) {
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, acrossWorker(item)])) as T;
}
return structuredClone(value);
}
export class InProcessStore implements StoreBackend {
readonly engines = new Map<string, AccountEngine>();
readonly dbs = new Map<string, StorageAdapter>();
@@ -71,8 +84,8 @@ export class InProcessStore implements StoreBackend {
async area<T = unknown>(account: string, name: string, params?: unknown): Promise<T> {
if (this.failArea?.(name)) throw new Error(`${name} failed`);
const result = await this.engine(account).areaCall(name, structuredClone(params));
return (result === undefined ? undefined : structuredClone(result)) as T;
const result = await this.engine(account).areaCall(name, acrossWorker(params));
return (result === undefined ? undefined : acrossWorker(result)) as T;
}
async sync(account: string): Promise<void> {
lib/store/__tests__/mail-answers.test.ts
+5 -3
View File
@@ -406,7 +406,9 @@ describe('attachments', () => {
class Files extends FakeJmapMail {
downloads: string[] = [];
missing = new Set<string>();
override async download(_jmap?: string, blobId?: string): Promise<Uint8Array> {
override async download(_jmap?: string, blobId?: string, type?: string): Promise<Uint8Array> {
// Raw messages are kept too since exports read them; these tests are about attachments.
if (type === 'message/rfc822') return new Uint8Array([9]);
this.downloads.push(String(blobId));
if (this.offline) throw new TransportError('network', 'Failed to fetch');
if (this.missing.has(String(blobId))) throw new TransportError('http', 'Gone', 404);
@@ -426,9 +428,9 @@ describe('attachments', () => {
const { engine, db } = engineFor(server, { ...EVERYTHING, maxAttachmentBytes: 1000 });
await engine.requestSync();
expect(server.downloads).toEqual(['b-small']);
const stored = await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-small' }) as { type: string; data: Uint8Array };
const stored = await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'b-small' }) as { type: string; blob: Blob };
expect(stored.type).toBe('application/pdf');
expect([...stored.data]).toEqual([1, 2, 3]);
expect([...new Uint8Array(await stored.blob.arrayBuffer())]).toEqual([1, 2, 3]);
server.destroyEmail('small');
await engine.requestSync();
lib/store/__tests__/mail-sync.test.ts
+2 -1
View File
@@ -13,7 +13,8 @@ import { mailArea } from '../mail';
import { FAKE_SESSION, FakeJmap, JMAP_ACCOUNT } from './helpers/fake-jmap';
import { openFileStore, openMemoryStore } from './helpers/node-adapter';
const EVERYTHING = { ...DEFAULT_SCOPE, days: null };
// No files: raw messages would add a commit per message, and these tests are about mail.
const EVERYTHING = { ...DEFAULT_SCOPE, days: null, attachments: false };
const INBOX = { inMailbox: 'inbox' };
function engineFor(server: FakeJmap, db = openMemoryStore(), emit?: (e: StoreEvent) => void) {
lib/store/__tests__/outbox.test.ts
+2 -12
View File
@@ -6,7 +6,6 @@ import { TransportError, type MethodCall } from '../engine/transport';
import { mailArea } from '../mail';
import { outboxArea } from '../outbox';
import { CLOCK_SKEW_MARGIN_MS, decideRetry, LOOKUP_TRUSTED_MS, type LookupArgs } from '../outbox/dedupe';
import { addHeldFileSource } from '../outbox/calls';
import { flushPatches, type PatchOutcome, type PendingPatch } from '../outbox/flush-patches';
import type { StoredConflict, StoredOutboxEntry } from '../outbox/items';
import { mergePatches, patchConflicts } from '../outbox/patches';
@@ -485,9 +484,9 @@ describe('sends', () => {
const held = await t.engine.areaCall('outbox.holdFile', { name: 'note.txt', type: 'text/plain', data }) as { blobId: string; size: number };
expect(held).toMatchObject({ size: 5, type: 'text/plain' });
expect(held.blobId).toMatch(/^bulwark-local-/);
const file = await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId }) as { name: string; data: Uint8Array };
const file = await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId }) as { name: string; blob: Blob };
expect(file.name).toBe('note.txt');
expect(new TextDecoder().decode(file.data)).toBe('hello');
expect(await file.blob.text()).toBe('hello');
const id = await t.engine.areaCall('outbox.queueSend', { summary, item: { kind: 'raw', blobId: held.blobId, identityId: 'i' } }) as number;
await t.engine.areaCall('outbox.remove', { id });
expect(await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId })).toBeNull();
@@ -495,15 +494,6 @@ describe('sends', () => {
expect((await read()).entries).toEqual([]);
});
it('leaves a file alone that another area holds', async () => {
const held = await t.engine.areaCall('outbox.holdFile', { name: 'draft.txt', type: 'text/plain', data: new Uint8Array(1) }) as { blobId: string };
const stale = await t.engine.areaCall('outbox.holdFile', { name: 'old.txt', type: 'text/plain', data: new Uint8Array(1) }) as { blobId: string };
addHeldFileSource(() => [held.blobId]);
expect(await t.engine.areaCall('outbox.dropOrphans', { olderThan: NOW + 1 })).toBe(1);
expect(await t.engine.areaCall('outbox.heldFile', { blobId: held.blobId })).not.toBeNull();
expect(await t.engine.areaCall('outbox.heldFile', { blobId: stale.blobId })).toBeNull();
});
it('drops old files nothing needs', async () => {
const data = new Uint8Array([1]);
const orphan = await t.engine.areaCall('outbox.holdFile', { name: 'a', type: '', data }) as { blobId: string };
lib/store/__tests__/page-side.test.ts
+36
View File
@@ -331,6 +331,42 @@ describe('the page\'s store', () => {
expect(lost).toHaveBeenCalledWith('lost@x');
});
it('still tells the page when a new owner takes over the open that found the key gone', async () => {
await open();
const before = await getAccountRecord('lost@x');
await forgetAccountKey('lost@x');
configureStore({
inputs: () => ({ session: FAKE_SESSION, auth: 'Basic x', invertedKeywordSort: true }),
refreshAuth: async () => null,
onEvent: () => {},
onStoreLost: lost,
});
const store = getStore();
await store.ensureOpen('lost@x');
const worker = FakeWorker.all[FakeWorker.all.length - 1];
const post = worker.postMessage.bind(worker);
let dropped = false;
worker.postMessage = (message) => {
if (message.kind === 'open' && !dropped) {
// The first open never gets its answer: ownership moved under it.
dropped = true;
worker.posted.push(message);
return;
}
post(message);
};
worker.becomeOwner();
await vi.waitFor(() => expect(dropped).toBe(true));
worker.becomeOwner();
await vi.waitFor(() => expect(lost).toHaveBeenCalledWith('lost@x'));
expect(lost).toHaveBeenCalledTimes(1);
const after = await getAccountRecord('lost@x');
const prunes = worker.posted.filter((m) => m.kind === 'prune-files') as unknown as Array<{ keep: string[] }>;
expect(prunes.at(-1)?.keep).toContain(after!.file);
expect(prunes.at(-1)?.keep).not.toContain(before!.file);
store.stop();
});
it('stays quiet on a normal reopen', async () => {
await open();
const worker = await open();
lib/store/__tests__/transport-real.test.ts
+116
View File
@@ -0,0 +1,116 @@
// @vitest-environment node
/** The real FetchTransport against a real local HTTP server: what a fake can't tell about bodies. */
import { createServer, type Server } from 'node:http';
import type { AddressInfo } from 'node:net';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { DEFAULT_SCOPE } from '../area';
import { AccountEngine } from '../engine/account-engine';
import { FetchTransport, TransportError, type StoreSession } from '../engine/transport';
import { mailArea } from '../mail';
import { FakeJmapMail } from './helpers/fake-jmap-mail';
import { FAKE_SESSION, JMAP_ACCOUNT } from './helpers/fake-jmap';
import { openMemoryStore } from './helpers/node-adapter';
const BIG = 3 * 1024 * 1024 + 7;
let server: Server;
let base: string;
let auth: string[] = [];
beforeAll(async () => {
server = createServer((req, res) => {
auth.push(String(req.headers.authorization));
if (req.url?.includes('/whole/')) {
res.writeHead(200, { 'Content-Length': String(BIG) });
res.end(Buffer.alloc(BIG, 5));
return;
}
if (req.url?.includes('/broken/')) {
// Promises more than it sends, then the connection goes.
res.writeHead(200, { 'Content-Length': '500000' });
res.write(Buffer.alloc(1000, 1), () => res.destroy());
return;
}
if (req.url?.includes('/refused/')) {
res.writeHead(404);
res.end('nope');
return;
}
res.writeHead(500);
res.end();
});
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
afterAll(() => {
server.closeAllConnections();
server.close();
});
function transport(): FetchTransport {
const session: StoreSession = { ...FAKE_SESSION, downloadUrl: `${base}/d/{accountId}/{blobId}/{name}?type={type}` };
return new FetchTransport({ session: () => session, auth: () => 'Basic x', refreshAuth: async () => 'Basic x' });
}
describe('FetchTransport against a real server', () => {
it('hands a file over in pieces, none of them the whole file', async () => {
const pieces: number[] = [];
let bytes = 0;
await transport().downloadStream('j', 'whole', 'application/pdf', 'a.pdf', (part) => {
pieces.push(part.length);
bytes += part.length;
expect(part.every((value) => value === 5)).toBe(true);
});
expect(bytes).toBe(BIG);
expect(pieces.length).toBeGreaterThan(1);
expect(Math.max(...pieces)).toBeLessThan(BIG);
expect(auth.at(-1)).toBe('Basic x');
});
it('still gives a whole file from download', async () => {
expect((await transport().download('j', 'whole', 'application/pdf', 'a.pdf')).length).toBe(BIG);
});
it('calls a connection that drops halfway an interrupted transfer, in both', async () => {
const stream = await transport().downloadStream('j', 'broken', 'a/b', 'a', () => {}).catch((error) => error);
expect(stream).toBeInstanceOf(TransportError);
expect(stream).toMatchObject({ kind: 'network', interrupted: true });
const whole = await transport().download('j', 'broken', 'a/b', 'a').catch((error) => error);
expect(whole).toMatchObject({ kind: 'network', interrupted: true });
});
it('says what the server said for a refusal', async () => {
expect(await transport().downloadStream('j', 'refused', 'a/b', 'a', () => {}).catch((error) => error)).toMatchObject({ kind: 'http', status: 404 });
});
it('lets an error from the sink out as it is, and ends the download', async () => {
class Stop extends Error {}
await expect(transport().downloadStream('j', 'whole', 'a/b', 'a', () => { throw new Stop(); })).rejects.toBeInstanceOf(Stop);
});
});
describe('the attachment pass with it', () => {
it('keeps a file that comes in pieces, and carries on past one that breaks', async () => {
const real = transport();
const fake = new FakeJmapMail({ mailboxes: [{ id: 'inbox', role: 'inbox' }] });
fake.addEmail({ id: 'first', blobId: 'refused', size: 500_000_000, receivedAt: new Date(Date.UTC(2026, 9, 3)).toISOString(), attachments: [{ partId: '2', blobId: 'broken', size: 500000, type: 'a/b', name: 'broken' }] as never });
fake.addEmail({ id: 'second', blobId: 'refused', size: 500_000_000, receivedAt: new Date(Date.UTC(2026, 9, 2)).toISOString(), attachments: [{ partId: '2', blobId: 'whole', size: BIG, type: 'a/b', name: 'whole' }] as never });
const db = openMemoryStore();
const engine = new AccountEngine({
account: 'a',
db,
session: FAKE_SESSION,
transport: { request: (calls) => fake.request(calls), download: real.download.bind(real), downloadStream: real.downloadStream.bind(real), upload: real.upload.bind(real) },
areas: [mailArea],
scope: { ...DEFAULT_SCOPE, days: null },
sleep: async () => {},
});
await engine.requestSync();
expect(await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'broken' })).toBeNull();
const kept = await engine.areaCall('mail.getBlob', { jmap: JMAP_ACCOUNT, blobId: 'whole' }) as { blob: Blob };
expect(kept.blob.size).toBe(BIG);
expect(Number(db.get<{ n: number }>("SELECT COUNT(*) AS n FROM blob_chunk WHERE blob_id = 'whole'")?.n)).toBe(4);
// Not given up on: a broken transfer says nothing about the file.
expect(db.get<{ refused: number }>("SELECT refused FROM email_part WHERE blob_id = 'broken'")?.refused).toBe(0);
});
});
lib/store/area.ts
+10
View File
@@ -71,6 +71,10 @@ export interface SyncContext extends StoreContext {
transport: JmapTransport;
/** True once the engine is closing or lost the network: stop at the next batch. */
stopped(): boolean;
/** A newer pass was asked for. Work that can wait stops, and the next pass carries on with it. */
syncWanted(): boolean;
/** Waits, to stay polite to the server. */
pause(ms: number): Promise<void>;
syncState(jmap: string, type: string): SyncStateRow;
/** Call inside the transaction that writes the data, so the two can't disagree. */
setSyncState(jmap: string, type: string, row: Partial<SyncStateRow>): void;
@@ -81,6 +85,8 @@ export interface Syncer {
name: string;
/** The JMAP capability an account must advertise for this to run for it. */
capability: string;
/** Runs after every other syncer of the pass, for work that can wait. */
low?: boolean;
/** Runs one pass for one JMAP account. Must be safe to stop between batches and start again. */
run(ctx: SyncContext, jmap: string): Promise<void>;
}
@@ -102,6 +108,8 @@ export interface MethodContext extends StoreContext {
invertedKeywordSort: boolean;
/** Earlier responses in this request, for creation-id references. */
createdIds: Map<string, string>;
/** The whole request this call came in. */
calls: readonly MethodCall[];
}
export type MethodHandler = (ctx: MethodContext, args: Record<string, unknown>, call: MethodCall) => MethodResult | Promise<MethodResult>;
@@ -121,4 +129,6 @@ export interface StoreArea {
syncers: readonly Syncer[];
calls?: Readonly<Record<string, AreaCall>>;
overlay?: RecordOverlay;
/** At start and when the scope changes: a tighter policy applies at once, not at the next sync. */
scopeChanged?(ctx: StoreContext): void;
}
lib/store/client/api.ts
+9 -2
View File
@@ -81,6 +81,11 @@ class PageStore implements StoreBackend {
private readonly opening = new Map<string, Promise<void>>();
/** The engine can't start in this browser: don't keep every caller waiting for an owner. */
private broken = false;
/**
* Accounts whose key was just made, until their store has opened. A new owner
* can start the open again, and the second try finds the key the first one made.
*/
private readonly keyJustMade = new Set<string>();
/** What the owner's engine was last told to keep. */
private readonly sentScope = new Map<string, string>();
@@ -131,8 +136,9 @@ class PageStore implements StoreBackend {
const run = (async () => {
const client = this.started();
const { keyHex, record, fresh } = await openAccountKey(account, this.hooks()?.keyLayers?.() ?? []);
if (fresh) this.keyJustMade.add(account);
// The old file, if the key alone went, can never be read again.
if (fresh) await client.pruneFiles(await knownStoreFiles()).catch(() => {});
if (this.keyJustMade.has(account)) await client.pruneFiles(await knownStoreFiles()).catch(() => {});
const inputs = this.hooks()?.inputs(account) ?? { session: null, auth: null, invertedKeywordSort: null };
if (inputs.scope) this.sentScope.set(account, JSON.stringify(inputs.scope));
const opened = await client.openAccount({
@@ -144,7 +150,8 @@ class PageStore implements StoreBackend {
invertedKeywordSort: inputs.invertedKeywordSort,
auth: inputs.auth,
});
if (fresh || opened?.created) {
// Whichever try of them gets here first says so.
if (this.keyJustMade.delete(account) || opened?.created) {
if (readOpenedAccounts().includes(account)) this.hooks()?.onStoreLost?.(account);
}
rememberOpened(account, true);
lib/store/drafts/calls.ts
+245
View File
@@ -0,0 +1,245 @@
/** What the page's flush asks the store. Each call is one transaction, so two tabs can't interleave. */
import type { AreaCall, StoreContext } from '../area';
import { upsertEmail, type EmailObject } from '../mail/records';
import { savePatch, EMAIL_TYPE } from '../outbox/email-set';
import type { ConflictSummary } from '../outbox/items';
import { applyPatch, destroyConflicts, stateOf, type RecordState } from '../outbox/patches';
import type { DraftCreate } from './local-email';
import { destroyForDraft, ghostsOf, isPlainDraft, listedDrafts, pendingForFlush, replaceLocalRow, replacesOf, resolveStandIns, versionOf, type DraftError, type DraftRow, type DraftStatus, type DraftVersion } from './pending';
export interface PendingDraft {
localId: string;
jmap: string;
seq: number;
subject: string;
create: DraftCreate;
messageId: string;
replaces: ReturnType<typeof replacesOf>;
status: DraftStatus;
firstAttemptAt: number | null;
/** What this version should look like on the server. */
self: DraftVersion;
/** Earlier versions that may have reached the server. */
ghosts: DraftVersion[];
}
export interface DraftGhost extends DraftVersion {
jmap: string;
}
export interface DraftElsewhere {
id: number;
login: string;
jmap: string | null;
recordId: string;
}
/** What the outbox view shows of a draft that's only on this device. */
export interface LocalDraftSummary {
localId: string;
jmap: string;
subject: string;
status: DraftStatus;
error?: DraftError;
createdAt: number;
}
/** A stand-in the page may still hold is resolved this long after it reached the server. */
const MAP_KEPT_MS = 30 * 24 * 60 * 60 * 1000;
function toPending(row: DraftRow): PendingDraft {
return {
localId: row.local_id,
jmap: row.jmap,
seq: Number(row.seq),
subject: row.subject ?? '',
create: JSON.parse(row.create_json) as DraftCreate,
messageId: row.message_id,
replaces: replacesOf(row),
status: row.status,
firstAttemptAt: row.first_attempt_at === null ? null : Number(row.first_attempt_at),
self: versionOf(row),
ghosts: ghostsOf(row),
};
}
function read(ctx: StoreContext, params: { before: number }): { rows: PendingDraft[]; ghosts: DraftGhost[] } {
return ctx.db.transaction(() => {
// What a closed tab left on the way may have arrived.
ctx.db.run("UPDATE draft_pending SET status = 'uncertain', sending_since = NULL WHERE status = 'sending' AND COALESCE(sending_since, 0) < ?", [params.before]);
ctx.db.run('DELETE FROM draft_id_map WHERE at < ?', [ctx.now() - MAP_KEPT_MS]);
return {
rows: pendingForFlush(ctx.db).map(toPending),
ghosts: ctx.db.all<{ jmap: string; message_id: string; first_at: number; last_at: number; subject: string; size: number }>('SELECT * FROM draft_ghost').map((row) => ({
jmap: row.jmap,
messageId: row.message_id,
from: Number(row.first_at),
to: Number(row.last_at),
subject: row.subject,
size: Number(row.size),
})),
};
});
}
function list(ctx: StoreContext): LocalDraftSummary[] {
return listedDrafts(ctx.db).map((row) => ({
localId: row.local_id,
jmap: row.jmap,
subject: row.subject ?? '',
status: row.status,
createdAt: Number(row.created_at),
...(row.error ? { error: JSON.parse(row.error) as DraftError } : {}),
}));
}
/** False when it isn't there any more, or it's no longer waiting: replaced, discarded, another tab took it. */
function claim(ctx: StoreContext, params: { localId: string }): boolean {
const now = ctx.now();
return ctx.db.run(
`UPDATE draft_pending SET status = 'sending', sending_since = ?, first_attempt_at = COALESCE(first_attempt_at, ?), last_attempt_at = ?
WHERE local_id = ? AND status IN ('waiting', 'uncertain')`,
[now, now, now, params.localId],
).changes === 1;
}
/** Nothing arrived (waiting), it may have (uncertain), or the server said no (refused). */
function failed(ctx: StoreContext, params: { localId: string; status: 'waiting' | 'uncertain' | 'refused'; error?: DraftError; neverSent?: boolean }): void {
ctx.db.run(
`UPDATE draft_pending SET status = ?, error = ?, sending_since = NULL,
first_attempt_at = CASE WHEN ? THEN NULL ELSE first_attempt_at END,
last_attempt_at = CASE WHEN ? THEN NULL ELSE last_attempt_at END
WHERE local_id = ?`,
[params.status, params.error ? JSON.stringify(params.error) : null, params.neverSent ? 1 : 0, params.neverSent ? 1 : 0, params.localId],
);
}
/** Try a refused draft again, once there may be room for it. */
function retry(ctx: StoreContext, params: { localId: string }): void {
ctx.db.run("UPDATE draft_pending SET status = 'waiting', error = NULL WHERE local_id = ? AND status = 'refused'", [params.localId]);
}
interface CreatedParams {
localId: string;
/** The row's place when it was taken: a later one means it was flagged or moved on the way. */
seq: number;
jmap: string;
messageId: string;
serverId: string;
/** The new message as the server has it. Null when reading it back failed. */
email: EmailObject | null;
/** What the server has of the versions this replaces, with null for one it no longer has. */
server: Record<string, EmailObject | null>;
/** Copies of earlier versions of this draft that still look untouched. */
ghostCopies: Array<{ id: string; base: RecordState }>;
}
function addConflict(ctx: StoreContext, jmap: string, recordId: string, summary: ConflictSummary): void {
ctx.db.run(
'INSERT INTO conflict (jmap, type, record_id, summary, local, created_at) VALUES (?, ?, ?, ?, ?, ?)',
[jmap, EMAIL_TYPE, recordId, JSON.stringify(summary), JSON.stringify({ op: 'draft' }), ctx.now()],
);
}
/**
* The new version is on the server. Only now do the versions it replaces start
* to go, by the usual queue: a failure there leaves an old copy, never no copy.
* One changed elsewhere stays, with the user told: both are kept.
*/
function created(ctx: StoreContext, params: CreatedParams): { conflicts: string[] } {
const conflicts: string[] = [];
ctx.db.transaction(() => {
const now = ctx.now();
const row = ctx.db.get<DraftRow>('SELECT * FROM draft_pending WHERE local_id = ?', [params.localId]);
// Replaced or discarded while it was on its way: it left a ghost, and the
// next flush looks at this copy as it does at any other.
if (!row) return;
ctx.db.run('INSERT OR REPLACE INTO draft_id_map (local_id, jmap, server_id, at) VALUES (?, ?, ?, ?)', [row.local_id, row.jmap, params.serverId, now]);
replaceLocalRow(ctx.db, row, params.serverId, params.email);
if (Number(row.seq) !== params.seq) {
const after = JSON.parse(row.create_json) as RecordState;
const patch = JSON.stringify({ keywords: after.keywords, mailboxIds: after.mailboxIds });
savePatch(ctx.db, { jmap: row.jmap, record_id: params.serverId, op: 'update', patch, base: null, server: null, subject: row.subject }, now);
const kept = ctx.db.get<{ json: string; gen: number }>('SELECT json, gen FROM email WHERE jmap = ? AND id = ?', [row.jmap, params.serverId]);
if (kept) upsertEmail(ctx.db, row.jmap, applyPatch(JSON.parse(kept.json) as EmailObject, JSON.parse(patch)), Number(kept.gen));
}
ctx.db.run('DELETE FROM draft_pending WHERE local_id = ?', [row.local_id]);
ctx.db.run('UPDATE draft_elsewhere SET local_id = NULL WHERE local_id = ?', [row.local_id]);
for (const copy of params.ghostCopies) {
const state = JSON.stringify(copy.base);
savePatch(ctx.db, { jmap: row.jmap, record_id: copy.id, op: 'destroy', patch: '{}', base: state, server: state, subject: row.subject }, now);
}
for (const target of replacesOf(row)) {
const server = params.server[target.id];
// Not read on this run (added on the way). With what it looked like known,
// the queue checks it again. Without that, it stays: a spare copy.
if (server === undefined && !target.base) continue;
// A message this device didn't keep: what the server says now is all there is to go by.
const unchanged = server === undefined || (server !== null && (target.base ? !destroyConflicts(target.base, server) : isPlainDraft(ctx.db, target.jmap, server)));
if (unchanged) {
const state = JSON.stringify(target.base ?? stateOf(server as EmailObject));
savePatch(ctx.db, { jmap: target.jmap, record_id: target.id, op: 'destroy', patch: '{}', base: state, server: state, subject: target.subject }, now);
continue;
}
if (server) {
const gen = Number(ctx.db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM email WHERE jmap = ?', [target.jmap])?.gen ?? 0);
upsertEmail(ctx.db, target.jmap, server, gen);
}
const subject = (typeof server?.subject === 'string' ? server.subject : target.subject) ?? row.subject ?? '';
addConflict(ctx, row.jmap, params.serverId, { subject, op: 'draft', reason: server ? 'changed' : 'gone' });
conflicts.push(subject);
}
});
ctx.emit({ type: 'changed', account: ctx.account, collections: ['Email', 'Mailbox', 'Thread'] });
return { conflicts };
}
/** The ghosts of a discarded draft looked up: copies that still look untouched go through the usual queue. */
function ghostsLooked(ctx: StoreContext, params: { jmap: string; messageIds: string[]; found: Array<{ id: string; base: RecordState }> }): void {
ctx.db.transaction(() => {
for (const messageId of params.messageIds) ctx.db.run('DELETE FROM draft_ghost WHERE jmap = ? AND message_id = ?', [params.jmap, messageId]);
for (const copy of params.found) {
const state = JSON.stringify(copy.base);
savePatch(ctx.db, { jmap: params.jmap, record_id: copy.id, op: 'destroy', patch: '{}', base: state, server: state, subject: null }, ctx.now());
}
});
}
/** An old version in another login, to be removed through it once the new one here is up. */
function noteElsewhere(ctx: StoreContext, params: { localId: string; login: string; jmap?: string; id: string }): void {
const waiting = ctx.db.get('SELECT 1 AS x FROM draft_pending WHERE local_id = ?', [params.localId]);
ctx.db.run('INSERT INTO draft_elsewhere (local_id, login, jmap, record_id) VALUES (?, ?, ?, ?)', [waiting ? params.localId : null, params.login, params.jmap ?? null, params.id]);
}
function elsewhereReady(ctx: StoreContext): DraftElsewhere[] {
return ctx.db.all<{ id: number; login: string; jmap: string | null; record_id: string }>('SELECT id, login, jmap, record_id FROM draft_elsewhere WHERE local_id IS NULL')
.map((row) => ({ id: Number(row.id), login: row.login, jmap: row.jmap, recordId: row.record_id }));
}
function elsewhereDone(ctx: StoreContext, params: { id: number }): void {
ctx.db.run('DELETE FROM draft_elsewhere WHERE id = ?', [params.id]);
}
/** A send took care of this draft without the page holding a client's word for it. */
function discard(ctx: StoreContext, params: { jmap: string; id: string }): void {
ctx.db.transaction(() => {
destroyForDraft(ctx, params.jmap, params.id, undefined);
});
ctx.emit({ type: 'changed', account: ctx.account, collections: ['Email', 'Mailbox', 'Thread'] });
}
export const DRAFT_CALLS: Readonly<Record<string, AreaCall>> = {
'drafts.read': read,
'drafts.list': list,
'drafts.claim': claim,
'drafts.failed': failed,
'drafts.retry': retry,
'drafts.created': created,
'drafts.ghostsLooked': ghostsLooked,
'drafts.noteElsewhere': noteElsewhere,
'drafts.elsewhereReady': elsewhereReady,
'drafts.elsewhereDone': elsewhereDone,
'drafts.discard': discard,
'drafts.resolveStandIns': (ctx, params: { ids: string[] }) => resolveStandIns(ctx.db, params.ids),
};
lib/store/drafts/flush.ts
+320
View File
@@ -0,0 +1,320 @@
/**
* Sends the drafts saved on this device, written against a small `io` so the
* page can run it through the JMAP client while the logic is tested next to the
* store (decision 15).
*
* For each draft, in order: look up whether it arrived already when an answer got
* lost, read the versions it replaces, upload its files, create it, and report.
* The store only then queues the old versions for removal, so a failure anywhere
* leaves a copy on the server or on the device, never neither. Anything that
* looks like an outage ends the run and the draft waits. A refusal stays with
* the draft, on the device, for the user to see.
*
* Message-ID lookups match on the client: `header` and `text` filters find no
* Message-ID on either Stalwart (decision 4). A Message-ID alone says little:
* another client can save again under it. So a copy only counts as a version
* made here when it still looks like one. Anything else is left alone: a spare
* copy on the server is always better than no copy.
*/
import { JmapMethodError, responseFor, type JmapResponse, type MethodCall } from '../engine/transport';
import { EMAIL_BODY_PROPERTIES, EMAIL_LIST_PROPERTIES, MAIL_USING, type EmailObject } from '../mail/records';
import { CLOCK_SKEW_MARGIN_MS } from '../outbox/dedupe';
import { isLocalBlobId } from '../outbox/items';
import type { RecordState } from '../outbox/patches';
import type { DraftGhost, PendingDraft } from './calls';
import { blobPartsOf, heldBlobsOf, withUploadedBlobs, type CreatePart } from './local-email';
import type { DraftError, DraftVersion } from './pending';
export interface FoundCopy {
id: string;
base: RecordState;
}
export interface DraftFlushStore {
read(): Promise<{ rows: PendingDraft[]; ghosts: DraftGhost[] }>;
claim(localId: string): Promise<boolean>;
failed(localId: string, status: 'waiting' | 'uncertain' | 'refused', error?: DraftError, neverSent?: boolean): Promise<void>;
created(result: {
localId: string;
seq: number;
jmap: string;
messageId: string;
serverId: string;
email: EmailObject | null;
server: Record<string, EmailObject | null>;
ghostCopies: FoundCopy[];
}): Promise<{ conflicts: string[] }>;
ghostsLooked(jmap: string, messageIds: string[], found: FoundCopy[]): Promise<void>;
}
export interface DraftFlushIo {
store: DraftFlushStore;
request(calls: MethodCall[], using: string[]): Promise<JmapResponse>;
/** Puts a held file on the server. Null when it's no longer on this device. */
upload(jmap: string, blobId: string): Promise<string | null>;
/** The server refused for good: no use retrying. Anything else counts as an outage. */
isRefusal(error: unknown): boolean;
onRefused(subject: string, error: DraftError): void;
onConflict(subject: string): void;
/** One more draft is on the server. */
onSent(): void;
}
const PAGE = 500;
const MAX_PAGES = 5;
/** Set errors that mean "not now" rather than "no". */
const RETRY_LATER = new Set(['serverUnavailable', 'serverFail', 'serverPartialFail', 'rateLimit']);
/** How far a copy's receivedAt may be from the attempts, in either direction, for clocks that differ. */
const CLOSE_TO_ATTEMPT_MS = 15 * 60 * 1000;
/** A server's size counts headers and encoding, which only add to what the store counted: base64 about a third. */
const SIZE_SLACK_BYTES = 64 * 1024;
function bare(id: string): string {
return id.trim().replace(/^<|>$/g, '');
}
function errorText(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function isOutage(error: unknown, io: DraftFlushIo): boolean {
if (error instanceof JmapMethodError) return RETRY_LATER.has(error.type);
return !io.isRefusal(error);
}
interface CopyFound {
id: string;
messageId?: string[] | null;
mailboxIds?: Record<string, boolean>;
keywords?: Record<string, boolean>;
receivedAt?: string;
subject?: string | null;
size?: number;
}
interface Copies {
drafts: string | null;
byMessageId: Map<string, CopyFound[]>;
}
const NO_COPIES: Copies = { drafts: null, byMessageId: new Map() };
/** Copies in Drafts by Message-ID. More than the pages hold can only cost a spare copy. */
async function copiesInDrafts(io: DraftFlushIo, jmap: string, since: number): Promise<Copies> {
const boxes = await io.request([['Mailbox/get', { accountId: jmap, properties: ['id', 'role'] }, 'm']], MAIL_USING);
const drafts = (responseFor(boxes, 'm')[1].list as Array<{ id: string; role?: string | null }>).find((box) => box.role === 'drafts');
if (!drafts) return NO_COPIES;
const byMessageId = new Map<string, CopyFound[]>();
const filter = { operator: 'AND', conditions: [{ inMailbox: drafts.id }, { after: new Date(since - CLOCK_SKEW_MARGIN_MS).toISOString() }] };
for (let page = 0; page < MAX_PAGES; page += 1) {
const response = await io.request([
['Email/query', { accountId: jmap, filter, sort: [{ property: 'receivedAt', isAscending: false }], position: page * PAGE, limit: PAGE }, 'q'],
['Email/get', {
accountId: jmap,
'#ids': { resultOf: 'q', name: 'Email/query', path: '/ids' },
properties: ['id', 'messageId', 'mailboxIds', 'keywords', 'receivedAt', 'subject', 'size'],
}, 'g'],
], MAIL_USING);
const ids = responseFor(response, 'q')[1].ids as string[];
for (const email of responseFor(response, 'g')[1].list as CopyFound[]) {
for (const messageId of email.messageId ?? []) {
const key = bare(messageId);
byMessageId.set(key, [...(byMessageId.get(key) ?? []), email]);
}
}
if (ids.length < PAGE) break;
}
return { drafts: drafts.id, byMessageId };
}
/** Whether a copy is that version, untouched: only in Drafts, still a draft, from the time it was sent, the same subject and size. */
function isThatVersion(copy: CopyFound, drafts: string | null, version: DraftVersion): boolean {
const boxes = Object.keys(copy.mailboxIds ?? {}).filter((box) => copy.mailboxIds?.[box]);
if (boxes.length !== 1 || boxes[0] !== drafts || !copy.keywords?.$draft) return false;
const at = Date.parse(String(copy.receivedAt));
if (!Number.isFinite(at) || at < version.from - CLOSE_TO_ATTEMPT_MS || at > version.to + CLOSE_TO_ATTEMPT_MS) return false;
if ((copy.subject ?? '') !== version.subject) return false;
const size = copy.size ?? 0;
return size >= version.size * 0.9 && size <= version.size * 1.5 + SIZE_SLACK_BYTES;
}
function versionCopies(copies: Copies, version: DraftVersion): FoundCopy[] {
return (copies.byMessageId.get(bare(version.messageId)) ?? [])
.filter((copy) => isThatVersion(copy, copies.drafts, version))
.map((copy) => ({ id: copy.id, base: { keywords: copy.keywords ?? {}, mailboxIds: copy.mailboxIds ?? {} } }));
}
async function readReplaced(io: DraftFlushIo, row: PendingDraft): Promise<Record<string, EmailObject | null>> {
const out: Record<string, EmailObject | null> = {};
const byAccount = new Map<string, string[]>();
for (const target of row.replaces) byAccount.set(target.jmap, [...(byAccount.get(target.jmap) ?? []), target.id]);
for (const [jmap, ids] of byAccount) {
const response = await io.request([['Email/get', { accountId: jmap, ids, properties: EMAIL_LIST_PROPERTIES }, 'r']], MAIL_USING);
const list = responseFor(response, 'r')[1].list as EmailObject[];
for (const id of ids) out[id] = list.find((email) => email.id === id) ?? null;
}
return out;
}
/** Reading the new message back can't fail the flush: it's on the server. */
async function readCreated(io: DraftFlushIo, jmap: string, id: string): Promise<EmailObject | null> {
try {
const response = await io.request([['Email/get', {
accountId: jmap,
ids: [id],
properties: [...EMAIL_LIST_PROPERTIES, ...EMAIL_BODY_PROPERTIES],
fetchAllBodyValues: true,
maxBodyValueBytes: 1_000_000,
}, 'n']], MAIL_USING);
return (responseFor(response, 'n')[1].list as EmailObject[])[0] ?? null;
} catch {
return null;
}
}
function leaves(part: CreatePart): CreatePart[] {
return part.subParts ? part.subParts.flatMap(leaves) : [part];
}
/**
* The part ids a create names that have died: every new version of a message
* gets new ones, and the old ones go with the old version. The same files are in
* the versions this draft replaces, under the ids they have now.
*/
async function livingParts(io: DraftFlushIo, row: PendingDraft, named: CreatePart[]): Promise<Map<string, string>> {
const fresh = new Map<string, string>();
const ids = row.replaces.filter((target) => target.jmap === row.jmap).map((target) => target.id);
if (ids.length === 0) return fresh;
const response = await io.request([['Email/get', { accountId: row.jmap, ids, properties: ['id', 'attachments', 'textBody', 'htmlBody', 'bodyStructure'] }, 'p']], MAIL_USING);
const found = (responseFor(response, 'p')[1].list as Array<{ attachments?: CreatePart[]; textBody?: CreatePart[]; htmlBody?: CreatePart[]; bodyStructure?: CreatePart }>)
.flatMap((email) => [
...(email.attachments ?? []),
// Inline parts Stalwart 1.0 doesn't list under attachments. The text itself isn't a file.
...[...(email.textBody ?? []), ...(email.htmlBody ?? []), ...(email.bodyStructure ? leaves(email.bodyStructure) : [])].filter((part) => !part.type?.startsWith('text/')),
])
.filter((part) => part.blobId);
const free = [...new Map(found.map((part) => [part.blobId, part])).values()];
// What's still alive keeps its part, so a file of the same name can't take it.
const dead = named.filter((part) => {
const at = free.findIndex((candidate) => candidate.blobId === part.blobId);
if (at < 0) return true;
free.splice(at, 1);
return false;
});
for (const part of dead) {
const at = free.findIndex((candidate) => candidate.name === part.name && (candidate.type ?? '') === (part.type ?? ''));
if (at < 0) continue;
fresh.set(part.blobId as string, free[at].blobId as string);
free.splice(at, 1);
}
return fresh;
}
type RowOutcome = 'next' | 'stop';
async function flushRow(io: DraftFlushIo, row: PendingDraft, copies: Copies): Promise<RowOutcome> {
if (!(await io.store.claim(row.localId))) return 'next';
// Each step knows what a failure means: before the create nothing has left.
let serverId = versionCopies(copies, row.self)[0]?.id;
let server: Record<string, EmailObject | null>;
const stay = async (status: 'waiting' | 'uncertain') => {
await io.store.failed(row.localId, status, undefined, status === 'waiting' && row.firstAttemptAt === null);
return 'stop' as const;
};
const refuse = async (error: DraftError) => {
await io.store.failed(row.localId, 'refused', error);
io.onRefused(row.subject, error);
return 'next' as const;
};
try {
server = await readReplaced(io, row);
} catch (error) {
if (isOutage(error, io)) return stay(row.firstAttemptAt === null ? 'waiting' : 'uncertain');
return refuse({ text: errorText(error) });
}
if (!serverId) {
const ids = new Map<string, string>();
try {
for (const blobId of new Set(heldBlobsOf(row.create))) {
const real = await io.upload(row.jmap, blobId);
if (real === null) return refuse({ code: 'file_missing' });
ids.set(blobId, real);
}
} catch (error) {
if (isOutage(error, io)) return stay(row.firstAttemptAt === null ? 'waiting' : 'uncertain');
return refuse({ text: errorText(error) });
}
const creationId = 'draft';
type SetResult = Record<string, Record<string, { id?: string; type?: string; description?: string }>>;
const create = async (): Promise<SetResult> => {
const response = await io.request([['Email/set', {
accountId: row.jmap,
create: { [creationId]: withUploadedBlobs(row.create, ids) },
}, 'c']], MAIL_USING);
return responseFor(response, 'c')[1] as SetResult;
};
let result: SetResult;
try {
result = await create();
if (result.notCreated?.[creationId]?.type === 'blobNotFound') {
const named = blobPartsOf(row.create).filter((part) => !isLocalBlobId(part.blobId));
const fresh = await livingParts(io, row, named);
if (fresh.size > 0) {
for (const [dead, alive] of fresh) ids.set(dead, alive);
result = await create();
}
}
} catch (error) {
// It went out: from here on it may have arrived.
if (isOutage(error, io)) return stay('uncertain');
return refuse({ text: errorText(error) });
}
const refusal = result.notCreated?.[creationId];
if (refusal) {
if (RETRY_LATER.has(refusal.type ?? '')) return stay('uncertain');
return refuse({ type: refusal.type, text: refusal.description ?? refusal.type });
}
serverId = result.created?.[creationId]?.id as string;
if (!serverId) return stay('uncertain');
}
const email = await readCreated(io, row.jmap, serverId);
const ghostCopies = row.ghosts.flatMap((ghost) => versionCopies(copies, ghost)).filter((copy) => copy.id !== serverId);
const { conflicts } = await io.store.created({ localId: row.localId, seq: row.seq, jmap: row.jmap, messageId: row.messageId, serverId, email, server, ghostCopies });
for (const subject of conflicts) io.onConflict(subject);
io.onSent();
return 'next';
}
/** True when all went through to an answer, false when the run stopped on an outage. */
export async function flushDrafts(io: DraftFlushIo): Promise<boolean> {
const { rows, ghosts } = await io.store.read();
if (rows.length === 0 && ghosts.length === 0) return true;
const accounts = new Set([...rows.map((row) => row.jmap), ...ghosts.map((entry) => entry.jmap)]);
for (const jmap of accounts) {
const ours = rows.filter((row) => row.jmap === jmap);
const theirGhosts = ghosts.filter((entry) => entry.jmap === jmap);
const attempts = [
...ours.filter((row) => row.firstAttemptAt !== null).map((row) => row.firstAttemptAt as number),
...ours.flatMap((row) => row.ghosts.map((ghost) => ghost.from)),
...theirGhosts.map((entry) => entry.from),
];
let copies = NO_COPIES;
if (attempts.length > 0) {
try {
copies = await copiesInDrafts(io, jmap, Math.min(...attempts));
} catch (error) {
// Can't look now: nothing may go that might go twice. Refused for good:
// there's nothing to wait for, and a spare copy is the worst that can come of it.
if (isOutage(error, io)) return false;
}
if (theirGhosts.length > 0) {
await io.store.ghostsLooked(jmap, theirGhosts.map((entry) => entry.messageId), theirGhosts.flatMap((entry) => versionCopies(copies, entry)));
}
}
for (const row of ours) {
if ((await flushRow(io, row, copies)) === 'stop') return false;
}
}
return true;
}
lib/store/drafts/local-email.ts
+135
View File
@@ -0,0 +1,135 @@
/**
* What a draft saved on this device looks like to everyone who reads it: the
* Email/set create the client wrote, turned into the list properties and the
* body a server would answer with. The composer reads it back straight after a
* save and matches attachments by name and size.
*/
import { isLocalBlobId } from '../outbox/items';
import type { StorageAdapter } from '../storage';
export type DraftCreate = Record<string, unknown>;
export interface CreatePart {
partId?: string;
blobId?: string;
type?: string;
name?: string | null;
disposition?: string | null;
cid?: string | null;
subParts?: CreatePart[];
}
interface BodyPart extends CreatePart {
size: number;
}
/** A held file's size is known. A part of a kept message's is too. Else the server knows, and the list shows nothing. */
function blobSize(db: StorageAdapter, jmap: string, blobId: string): number {
const row = isLocalBlobId(blobId)
? db.get<{ size: number }>('SELECT size FROM upload WHERE local_id = ?', [blobId])
: db.get<{ size: number }>('SELECT size FROM email_part WHERE jmap = ? AND blob_id = ? ORDER BY size DESC LIMIT 1', [jmap, blobId]);
return Number(row?.size ?? 0);
}
function leaves(part: CreatePart): CreatePart[] {
return part.subParts ? part.subParts.flatMap(leaves) : [part];
}
/** Every blob part a create names, wherever it put them. */
export function blobPartsOf(create: DraftCreate): CreatePart[] {
const listed = (create.attachments ?? []) as CreatePart[];
const tree = create.bodyStructure ? leaves(create.bodyStructure as CreatePart) : [];
return [...listed, ...tree].filter((part) => typeof part.blobId === 'string');
}
export function heldBlobsOf(create: DraftCreate): string[] {
return blobPartsOf(create).map((part) => part.blobId as string).filter(isLocalBlobId);
}
/** The create with the blob ids a flush got for its held files. */
export function withUploadedBlobs(create: DraftCreate, uploaded: ReadonlyMap<string, string>): DraftCreate {
const swap = (part: CreatePart): CreatePart => ({
...part,
...(part.blobId && uploaded.has(part.blobId) ? { blobId: uploaded.get(part.blobId) } : {}),
...(part.subParts ? { subParts: part.subParts.map(swap) } : {}),
});
const out: DraftCreate = { ...create };
if (Array.isArray(create.attachments)) out.attachments = (create.attachments as CreatePart[]).map(swap);
if (create.bodyStructure) out.bodyStructure = swap(create.bodyStructure as CreatePart);
return out;
}
function previewOf(text: string): string {
return text.replace(/\s+/g, ' ').trim().slice(0, 256);
}
function plainFromHtml(html: string): string {
return html.replace(/<(style|script)\b[^<]*(?:<(?!\/\1>)[^<]*)*<\/\1>/gi, ' ').replace(/<[^<>]*>/g, ' ');
}
function headersOf(create: DraftCreate): Array<{ name: string; value: string }> {
const out: Array<{ name: string; value: string }> = [];
for (const [key, value] of Object.entries(create)) {
const match = /^header:([^:]+):asText$/.exec(key);
if (match && typeof value === 'string') out.push({ name: match[1], value: ` ${value}` });
}
return out;
}
/**
* The message as it's answered from the store. `id` also serves as thread id and
* blob id: nothing else has one yet. The raw message doesn't exist until the
* server made it, so an export finds no raw copy to read.
*/
export function localEmailFrom(db: StorageAdapter, jmap: string, id: string, create: DraftCreate, messageId: string, now: number): { list: Record<string, unknown> & { id: string }; body: Record<string, unknown> } {
const values = (create.bodyValues ?? {}) as Record<string, { value?: string }>;
const valueOf = (part: CreatePart) => values[part.partId ?? '']?.value ?? '';
const asBodyPart = (part: CreatePart, type: string): BodyPart => ({ partId: part.partId, type: part.type ?? type, size: valueOf(part).length });
const tree = create.bodyStructure ? leaves(create.bodyStructure as CreatePart) : [];
const textParts = (tree.length ? tree.filter((part) => part.partId && part.type === 'text/plain') : (create.textBody ?? []) as CreatePart[]).map((part) => asBodyPart(part, 'text/plain'));
const htmlParts = (tree.length ? tree.filter((part) => part.partId && part.type === 'text/html') : (create.htmlBody ?? []) as CreatePart[]).map((part) => asBodyPart(part, 'text/html'));
const attachments: BodyPart[] = blobPartsOf(create).map((part) => ({
partId: part.blobId,
blobId: part.blobId,
type: part.type ?? 'application/octet-stream',
name: part.name ?? null,
size: blobSize(db, jmap, part.blobId as string),
disposition: part.disposition ?? 'attachment',
cid: part.cid ?? null,
}));
const text = textParts.map(valueOf).join('\n') || plainFromHtml(htmlParts.map(valueOf).join('\n'));
const size = [...textParts, ...htmlParts, ...attachments].reduce((sum, part) => sum + part.size, 0);
const at = new Date(now).toISOString();
const list = {
id,
threadId: id,
blobId: id,
mailboxIds: create.mailboxIds,
keywords: create.keywords,
size,
receivedAt: at,
sentAt: at,
from: create.from ?? [],
to: create.to ?? [],
cc: create.cc ?? [],
subject: typeof create.subject === 'string' ? create.subject : '',
preview: previewOf(text),
hasAttachment: attachments.some((part) => part.disposition !== 'inline'),
} as Record<string, unknown> & { id: string };
const body: Record<string, unknown> = {
bcc: create.bcc ?? [],
replyTo: create.replyTo ?? null,
messageId: [messageId],
inReplyTo: create.inReplyTo ?? null,
references: create.references ?? null,
headers: headersOf(create),
textBody: textParts.length ? textParts : htmlParts,
// A plain text draft lists its text part as the HTML body too, as a server does (RFC 8621 4.1.4).
htmlBody: htmlParts.length ? htmlParts : textParts,
attachments,
bodyStructure: create.bodyStructure ?? { type: 'multipart/mixed', subParts: [...textParts, ...htmlParts, ...attachments] },
bodyValues: Object.fromEntries(Object.entries(values).map(([partId, value]) => [partId, { value: value.value ?? '', isEncodingProblem: false, isTruncated: false }])),
};
return { list, body };
}
lib/store/drafts/pending.ts
+312
View File
@@ -0,0 +1,312 @@
/**
* Drafts saved on this device. createDraft on the client saves by creating a new
* version and then destroying the one before, and that's how it reaches this
* file: a create hands out a stand-in id, and the destroy that follows names the
* new version in `replacedBy`. Each draft therefore has one pending row however
* often it's saved. A version on the server stays there, hidden here, until the
* new one has arrived (flush.ts).
*
* A version that may already have reached the server (its Message-ID was handed
* over, and the answer got lost) leaves a ghost behind when it's replaced.
*/
import { isStandInId, newStandInId } from '@/lib/jmap/stand-in-ids';
import type { MethodContext, RecordOverlay, StoreContext } from '../area';
import { listObjects } from '../engine/core-schema';
import { deleteEmail, storeBody, upsertEmail, type EmailObject } from '../mail/records';
import { localEmail, nextSeq, pendingRow, queueDestroy, queueUpdate, savePatch, EMAIL_TYPE } from '../outbox/email-set';
import { applyPatch, normalizePatch, patchProblem, stateOf, type Patch, type RecordState } from '../outbox/patches';
import type { StorageAdapter } from '../storage';
import { localEmailFrom, type DraftCreate } from './local-email';
export interface ReplacedDraft {
jmap: string;
id: string;
base: RecordState | null;
subject: string | null;
}
/** What a version sent to the server should look like there, to tell it from anything else under its Message-ID. */
export interface DraftVersion {
messageId: string;
/** Its first and last attempt. */
from: number;
to: number;
subject: string;
size: number;
}
export interface DraftRow {
local_id: string;
jmap: string;
seq: number;
subject: string | null;
create_json: string;
message_id: string;
replaces: string;
status: DraftStatus;
sending_since: number | null;
first_attempt_at: number | null;
last_attempt_at: number | null;
size: number;
ghosts: string;
error: string | null;
created_at: number;
}
/**
* - waiting: goes up by itself when the server answers;
* - sending: on its way (a tab took it);
* - uncertain: the connection broke after it may have arrived. Looked up first;
* - refused: the server said no. Stays here until it's edited or tried again.
*/
export type DraftStatus = 'waiting' | 'sending' | 'uncertain' | 'refused';
export interface DraftError {
code?: 'file_missing';
type?: string;
text?: string;
}
interface SetError { type: string; description?: string }
const notADraft = (): SetError => ({ type: 'serverUnavailable', description: 'This needs the server' });
function rowOf(db: StorageAdapter, localId: string): DraftRow | undefined {
return db.get<DraftRow>('SELECT * FROM draft_pending WHERE local_id = ?', [localId]);
}
export function replacesOf(row: Pick<DraftRow, 'replaces'>): ReplacedDraft[] {
return JSON.parse(row.replaces) as ReplacedDraft[];
}
/** A message of the Drafts folder with the draft keyword, in a request that doesn't also send it. */
function isDraftCreate(ctx: MethodContext, jmap: string, create: DraftCreate): boolean {
if (ctx.calls.some(([name]) => name === 'EmailSubmission/set')) return false;
if (!(create.keywords as Record<string, boolean> | undefined)?.$draft) return false;
const boxes = Object.keys((create.mailboxIds ?? {}) as object);
if (boxes.length === 0) return false;
const drafts = listObjects<{ id: string; role?: string | null }>(ctx.db, jmap, 'Mailbox').filter((box) => box.role === 'drafts').map((box) => box.id);
return boxes.every((box) => drafts.includes(box));
}
export function ghostsOf(row: Pick<DraftRow, 'ghosts'>): DraftVersion[] {
return JSON.parse(row.ghosts) as DraftVersion[];
}
/** This version as a later one sees it, if it may have reached the server. */
export function versionOf(row: DraftRow): DraftVersion {
const from = Number(row.first_attempt_at ?? 0);
return { messageId: row.message_id, from, to: Number(row.last_attempt_at ?? from), subject: row.subject ?? '', size: Number(row.size) };
}
/** Attempts in a long outage are few, but nothing should grow without end. */
const MOST_GHOSTS = 50;
function currentGen(db: StorageAdapter, jmap: string): number {
return Number(db.get<{ gen: number | null }>('SELECT MAX(gen) AS gen FROM email WHERE jmap = ?', [jmap])?.gen ?? 0);
}
/** Writes the row lists and search read. A held file isn't something sync could fetch, and there's no raw message yet. */
function writeLocalRow(db: StorageAdapter, jmap: string, id: string, create: DraftCreate, messageId: string, now: number): { size: number } {
const { list, body } = localEmailFrom(db, jmap, id, create, messageId, now);
const rowid = upsertEmail(db, jmap, list, currentGen(db, jmap));
storeBody(db, rowid, JSON.stringify(list), body);
db.run("DELETE FROM email_part WHERE email_rowid = ? AND (raw = 1 OR blob_id LIKE 'bulwark-local-%')", [rowid]);
return { size: Number(list.size) };
}
export function createLocalDraft(ctx: MethodContext, jmap: string, create: DraftCreate, networkTried: boolean): Record<string, unknown> | SetError & { refused: true } {
if (!isDraftCreate(ctx, jmap, create)) return { ...notADraft(), refused: true };
const id = newStandInId();
const now = ctx.now();
const messageId = (create.messageId as string[] | undefined)?.[0] ?? `${id}@bulwark.invalid`;
const stored: DraftCreate = { ...create, messageId: [messageId] };
const { size } = writeLocalRow(ctx.db, jmap, id, stored, messageId, now);
ctx.db.run(
`INSERT INTO draft_pending (local_id, jmap, seq, subject, create_json, message_id, replaces, status, first_attempt_at, last_attempt_at, size, created_at)
VALUES (?, ?, ?, ?, ?, ?, '[]', ?, ?, ?, ?, ?)`,
[id, jmap, nextSeq(ctx.db), typeof create.subject === 'string' ? create.subject : '', JSON.stringify(stored), messageId, networkTried ? 'uncertain' : 'waiting', networkTried ? now : null, networkTried ? now : null, size, now],
);
return { id, blobId: id, threadId: id, size };
}
/** A flag or a move on a draft that only exists here: it goes into the create. */
export function updateLocalDraft(ctx: Pick<StoreContext, 'db' | 'now'>, jmap: string, id: string, rawPatch: Patch): SetError | null {
const row = rowOf(ctx.db, id);
if (!row) {
const real = mappedId(ctx.db, id);
return real ? queueUpdate(ctx.db, jmap, real, rawPatch, ctx.now()) : { type: 'notFound' };
}
const problem = patchProblem(rawPatch);
if (problem) return problem;
const local = localEmail(ctx.db, row.jmap, id);
const create = JSON.parse(row.create_json) as DraftCreate;
const after = applyPatch(create, normalizePatch(rawPatch));
if (Object.keys(after.mailboxIds ?? {}).length === 0) return { type: 'invalidProperties', description: 'A message has to stay in a folder' };
ctx.db.run('UPDATE draft_pending SET create_json = ?, seq = ? WHERE local_id = ?', [JSON.stringify(after), nextSeq(ctx.db), id]);
if (local) upsertEmail(ctx.db, row.jmap, { ...local.email, keywords: after.keywords, mailboxIds: after.mailboxIds }, local.gen);
return null;
}
/** What `id` is now on the server, for a stand-in that made it there. */
function mappedId(db: StorageAdapter, id: string): string | null {
return db.get<{ server_id: string }>('SELECT server_id FROM draft_id_map WHERE local_id = ?', [id])?.server_id ?? null;
}
export function resolveStandIns(db: StorageAdapter, ids: readonly string[]): Record<string, string | null> {
return Object.fromEntries(ids.map((id) => [id, mappedId(db, id)]));
}
function queueRemoval(db: StorageAdapter, target: ReplacedDraft, now: number): void {
const state = target.base ? JSON.stringify(target.base) : null;
savePatch(db, { jmap: target.jmap, record_id: target.id, op: 'destroy', patch: '{}', base: state, server: state, subject: target.subject }, now);
}
/**
* Takes a pending row away: `into` carries on what it replaced, else the user
* discarded or sent the draft and what it replaced goes too.
*/
function dropPending(ctx: Pick<StoreContext, 'db' | 'now'>, row: DraftRow, into: DraftRow | null): void {
const { db } = ctx;
const replaced = replacesOf(row);
const ghosts = [...ghostsOf(row), ...(row.first_attempt_at !== null ? [versionOf(row)] : [])];
if (into) {
// Only once the new version is up: until then these may be the only copy on the server.
db.run('UPDATE draft_pending SET replaces = ?, ghosts = ? WHERE local_id = ?', [
JSON.stringify([...replacesOf(into), ...replaced]),
JSON.stringify([...ghostsOf(into), ...ghosts].slice(-MOST_GHOSTS)),
into.local_id,
]);
db.run('UPDATE draft_elsewhere SET local_id = ? WHERE local_id = ?', [into.local_id, row.local_id]);
} else {
for (const target of replaced) queueRemoval(db, target, ctx.now());
for (const version of ghosts) {
db.run(
'INSERT OR REPLACE INTO draft_ghost (jmap, message_id, first_at, last_at, subject, size) VALUES (?, ?, ?, ?, ?, ?)',
[row.jmap, version.messageId, version.from, version.to, version.subject, version.size],
);
}
db.run('UPDATE draft_elsewhere SET local_id = NULL WHERE local_id = ?', [row.local_id]);
}
db.run('DELETE FROM draft_pending WHERE local_id = ?', [row.local_id]);
deleteEmail(db, row.jmap, row.local_id);
}
/**
* Email/set destroy of a stand-in, or of any message when a new version on the
* device replaces it. False when it's an ordinary delete for the outbox to queue.
*/
export function destroyForDraft(ctx: Pick<StoreContext, 'db' | 'now'>, jmap: string, id: string, replacedBy: string | undefined): boolean {
const { db } = ctx;
const into = replacedBy ? rowOf(db, replacedBy) ?? null : null;
if (isStandInId(id)) {
const row = rowOf(db, id);
if (row) {
dropPending(ctx, row, into);
return true;
}
const real = mappedId(db, id);
if (!real) return true; // Gone already: that's the same result.
if (into) return destroyForDraft(ctx, jmap, real, replacedBy);
queueDestroy(db, jmap, real, ctx.now());
return true;
}
if (!into) return false;
// The version on the server stays there until the new one has arrived.
const local = localEmail(db, jmap, id);
const waiting = pendingRow(db, jmap, id);
const base = local ? stateOf(local.email) : waiting?.base ? JSON.parse(waiting.base) as RecordState : null;
const entry: ReplacedDraft = { jmap, id, base, subject: typeof local?.email.subject === 'string' ? local.email.subject : waiting?.subject ?? null };
db.run('UPDATE draft_pending SET replaces = ? WHERE local_id = ?', [JSON.stringify([...replacesOf(into), entry]), into.local_id]);
db.run('DELETE FROM outbox_patch WHERE jmap = ? AND type = ? AND record_id = ?', [jmap, EMAIL_TYPE, id]);
deleteEmail(db, jmap, id);
return true;
}
/** A plain draft: only in the Drafts folder, with the keyword. Anything else someone did something with. */
export function isPlainDraft(db: StorageAdapter, jmap: string, email: EmailObject): boolean {
const boxes = Object.entries((email.mailboxIds ?? {}) as Record<string, boolean>).filter(([, on]) => on).map(([box]) => box);
if (boxes.length !== 1 || !(email.keywords as Record<string, boolean> | undefined)?.$draft) return false;
return listObjects<{ id: string; role?: string | null }>(db, jmap, 'Mailbox').some((box) => box.id === boxes[0] && box.role === 'drafts');
}
/** A send naming a draft that's still on the device: the send takes care of it. */
function draftsWithQueuedSend(db: StorageAdapter): Set<string> {
const rows = db.all<{ payload: string }>("SELECT payload FROM outbox WHERE status IN ('waiting', 'sending', 'uncertain', 'held') AND kind = 'email'");
const out = new Set<string>();
for (const row of rows) {
const draft = (JSON.parse(row.payload) as { item: { args: unknown[] } }).item.args[7];
if (typeof draft === 'string' && isStandInId(draft)) out.add(draft);
}
return out;
}
export function allPending(db: StorageAdapter): DraftRow[] {
return db.all<DraftRow>('SELECT * FROM draft_pending ORDER BY seq ASC');
}
/** The drafts that are only here and nothing else speaks for. One a send names is that message, in the outbox. */
export function listedDrafts(db: StorageAdapter): DraftRow[] {
const held = draftsWithQueuedSend(db);
return allPending(db).filter((row) => !held.has(row.local_id));
}
export function pendingForFlush(db: StorageAdapter): DraftRow[] {
const held = draftsWithQueuedSend(db);
return allPending(db).filter((row) => row.status !== 'refused' && !held.has(row.local_id));
}
/**
* Sync must not bring back a version that a draft here replaces, and the folder
* counts stay the server's with what waits here added when they're read.
*/
export const draftsOverlay: RecordOverlay = (ctx, jmap, type, records) => {
if (type !== 'Email' && type !== 'Thread' && type !== 'Mailbox') return records;
const rows = ctx.db.all<Pick<DraftRow, 'replaces'>>('SELECT replaces FROM draft_pending');
if (rows.length === 0) return records;
const replaced = rows.flatMap(replacesOf).filter((target) => target.jmap === jmap);
if (type === 'Email') {
const hidden = new Set(replaced.map((target) => target.id));
return records.filter((record) => !hidden.has(record.id));
}
if (type === 'Thread') {
const hidden = new Set(replaced.map((target) => target.id));
return (records as Array<{ id: string; emailIds: string[] }>).map((thread) => ({ ...thread, emailIds: thread.emailIds.filter((id) => !hidden.has(id)) }));
}
const deltas = new Map<string, { total: number; unread: number }>();
const bump = (box: string, total: number, unread: number) => {
const delta = deltas.get(box) ?? { total: 0, unread: 0 };
deltas.set(box, { total: delta.total + total, unread: delta.unread + unread });
};
for (const row of allPending(ctx.db)) {
if (row.jmap !== jmap) continue;
const create = JSON.parse(row.create_json) as RecordState;
for (const box of Object.keys(create.mailboxIds ?? {})) bump(box, 1, create.keywords?.$seen ? 0 : 1);
}
for (const target of replaced) {
for (const box of Object.keys(target.base?.mailboxIds ?? {})) bump(box, -1, target.base?.keywords?.$seen ? 0 : -1);
}
if (deltas.size === 0) return records;
return (records as Array<{ id: string; totalEmails?: unknown; unreadEmails?: unknown }>).map((box) => {
const delta = deltas.get(box.id);
if (!delta) return box;
const count = (value: unknown, by: number) => (typeof value === 'number' ? Math.max(0, value + by) : value);
return { ...box, totalEmails: count(box.totalEmails, delta.total), unreadEmails: count(box.unreadEmails, delta.unread) };
});
};
/** After a create went through: the real message takes the stand-in's place. */
export function replaceLocalRow(db: StorageAdapter, row: DraftRow, serverId: string, email: EmailObject | null): void {
const local = localEmail(db, row.jmap, row.local_id);
deleteEmail(db, row.jmap, row.local_id);
const gen = currentGen(db, row.jmap);
if (email) {
const { bodyValues, textBody, htmlBody, attachments, bodyStructure, headers, bcc, replyTo, messageId, inReplyTo, references, sender, ...list } = email;
const rowid = upsertEmail(db, row.jmap, list as EmailObject, gen);
storeBody(db, rowid, JSON.stringify(list), { bodyValues, textBody, htmlBody, attachments, bodyStructure, headers, bcc, replyTo, messageId, inReplyTo, references, sender });
return;
}
// It's there, but reading it back failed. Sync brings the text when it can.
if (local) upsertEmail(db, row.jmap, { ...local.email, id: serverId, threadId: serverId, blobId: serverId }, gen);
}
lib/store/drafts/schema.ts
+77
View File
@@ -0,0 +1,77 @@
import type { Migration } from '../storage';
export const DRAFT_MIGRATIONS: readonly Migration[] = [
{
id: 400,
name: 'drafts',
sql: `
-- A draft saved on this device and not on the server yet. One row per
-- draft: a later save takes over from the row before it.
CREATE TABLE draft_pending (
local_id TEXT PRIMARY KEY,
jmap TEXT NOT NULL,
seq INTEGER NOT NULL,
subject TEXT,
-- The create as the client wrote it, with its fixed Message-ID.
create_json TEXT NOT NULL,
message_id TEXT NOT NULL,
-- Versions on the server this one replaces: destroyed after it's there.
replaces TEXT NOT NULL DEFAULT '[]',
status TEXT NOT NULL DEFAULT 'waiting',
sending_since INTEGER,
-- Set once it may have reached the server: then it's looked up first.
first_attempt_at INTEGER,
last_attempt_at INTEGER,
-- What this version should look like on the server.
size INTEGER NOT NULL DEFAULT 0,
-- Earlier versions that may have reached the server. Only removed once
-- this one is there, and only if they still look like those versions.
ghosts TEXT NOT NULL DEFAULT '[]',
error TEXT,
created_at INTEGER NOT NULL
) WITHOUT ROWID;
-- What a stand-in id became. The page may hold one for a long time.
CREATE TABLE draft_id_map (
local_id TEXT PRIMARY KEY,
jmap TEXT NOT NULL,
server_id TEXT NOT NULL,
at INTEGER NOT NULL
) WITHOUT ROWID;
-- A version that may have reached the server, of a draft discarded here
-- since. Looked up and removed when the server is back.
CREATE TABLE draft_ghost (
jmap TEXT NOT NULL,
message_id TEXT NOT NULL,
first_at INTEGER NOT NULL,
last_at INTEGER NOT NULL,
subject TEXT NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY (jmap, message_id)
) WITHOUT ROWID;
-- An old version in another login: this store can't queue it. Removed
-- through that login once local_id is NULL: the new version is up, or the
-- draft was discarded or sent.
CREATE TABLE draft_elsewhere (
id INTEGER PRIMARY KEY AUTOINCREMENT,
local_id TEXT,
login TEXT NOT NULL,
jmap TEXT,
record_id TEXT NOT NULL
);
`,
},
{
id: 401,
name: 'held files in chunks',
sql: `
ALTER TABLE upload ADD COLUMN complete INTEGER NOT NULL DEFAULT 1;
-- Binding a whole file grows the WASM heap by its size, and that never shrinks.
CREATE TABLE upload_chunk (
local_id TEXT NOT NULL,
seq INTEGER NOT NULL,
data BLOB NOT NULL,
PRIMARY KEY (local_id, seq)
);
`,
},
];
lib/store/engine/account-engine.ts
+20 -2
View File
@@ -37,6 +37,7 @@ export class AccountEngine {
private scope: StoreScope;
private readonly transport: JmapTransport;
private readonly syncers: readonly Syncer[];
private readonly areas: readonly StoreArea[];
private readonly methods: ReadonlyMap<string, MethodHandler>;
private readonly calls = new Map<string, AreaCall>();
private readonly overlays: readonly RecordOverlay[];
@@ -58,7 +59,9 @@ export class AccountEngine {
this.session = options.session;
this.scope = options.scope;
this.transport = options.transport;
this.syncers = options.areas.flatMap((area) => area.syncers);
this.areas = options.areas;
const syncers = options.areas.flatMap((area) => area.syncers);
this.syncers = [...syncers.filter((syncer) => !syncer.low), ...syncers.filter((syncer) => syncer.low)];
this.methods = buildMethodTable(options.areas);
this.overlays = options.areas.flatMap((area) => (area.overlay ? [area.overlay] : []));
for (const area of options.areas) {
@@ -75,6 +78,12 @@ export class AccountEngine {
// An offline start has no way to probe the server, so the last answer stays.
this.invertedKeywordSort = options.invertedKeywordSort ?? metaGet(this.db, 'inverted_keyword_sort') === '1';
if (options.invertedKeywordSort !== undefined) metaSet(this.db, 'inverted_keyword_sort', options.invertedKeywordSort ? '1' : '0');
this.applyScope();
}
private applyScope(): void {
const ctx = this.storeContext();
for (const area of this.areas) area.scopeChanged?.(ctx);
}
/** The session kept from the last time, for a start without a network. */
@@ -94,6 +103,7 @@ export class AccountEngine {
// The user may just have made room by keeping less.
this.storageBlockedUntil = 0;
metaSet(this.db, 'scope', JSON.stringify(scope));
this.applyScope();
}
setInvertedKeywordSort(inverted: boolean): void {
@@ -225,15 +235,21 @@ export class AccountEngine {
}
private syncContext(): SyncContext {
const scope = () => this.scope;
return {
account: this.account,
db: this.db,
session: this.session,
scope: this.scope,
// A pass can take long, and a tighter policy applies to the rest of it.
get scope() {
return scope();
},
transport: this.transport,
now: this.now,
emit: this.emitter,
stopped: () => this.closed || !this.online,
syncWanted: () => this.again,
pause: this.sleep,
syncState: (jmap, type) => this.syncState(jmap, type),
setSyncState: (jmap, type, row) => this.setSyncState(jmap, type, row),
overlay: (jmap, type, records) => this.overlay(jmap, type, records),
@@ -287,6 +303,8 @@ export class AccountEngine {
/** Stops syncing and waits for the current batch. The database stays open for the caller to close. */
async close(): Promise<void> {
this.closed = true;
// A large download could otherwise hold up removing the account.
this.transport.abort?.();
await this.running?.catch(() => {});
}
}
lib/store/engine/responder.ts
+2 -2
View File
@@ -56,12 +56,12 @@ export function buildMethodTable(areas: readonly StoreArea[]): Map<string, Metho
/** Answers are copied by structured clone on the way to the page, so callers can't mutate stored objects. */
export async function answerRequest(
ctx: Omit<MethodContext, 'createdIds'>,
ctx: Omit<MethodContext, 'createdIds' | 'calls'>,
methods: ReadonlyMap<string, MethodHandler>,
calls: readonly MethodCall[],
): Promise<JmapResponse | null> {
const done: MethodResponse[] = [];
const full: MethodContext = { ...ctx, createdIds: new Map() };
const full: MethodContext = { ...ctx, createdIds: new Map(), calls };
let answered = 0;
for (const call of calls) {
const [name, rawArgs, callId] = call;
lib/store/engine/transport.ts
+142 -27
View File
@@ -34,6 +34,8 @@ export class TransportError extends Error {
/** The `type` of an RFC 7807 problem body, when there was one. */
readonly problem?: string,
readonly retryAfterMs?: number,
/** The connection worked and this transfer broke or stalled: it says little about the next one. */
readonly interrupted?: boolean,
) {
super(message);
this.name = 'TransportError';
@@ -43,7 +45,14 @@ export class TransportError extends Error {
export interface JmapTransport {
request(calls: MethodCall[], using: string[]): Promise<JmapResponse>;
download(jmap: string, blobId: string, type: string, name: string): Promise<Uint8Array>;
/**
* The same, piece by piece, so a large file isn't held in memory. An error
* thrown by `sink` ends the download and comes out as it is.
*/
downloadStream?(jmap: string, blobId: string, type: string, name: string, sink: (part: Uint8Array) => void): Promise<void>;
upload(jmap: string, data: Uint8Array, type: string): Promise<{ blobId: string; type: string; size: number }>;
/** Ends what's running, now and for good. The engine calls this when it closes. */
abort?(): void;
}
export interface FetchTransportOptions {
@@ -56,23 +65,72 @@ export interface FetchTransportOptions {
}
const REQUEST_TIMEOUT_MS = 120_000;
/** A download may take as long as it takes, as long as bytes keep coming. */
const DOWNLOAD_IDLE_MS = 60_000;
function fillTemplate(template: string, values: Record<string, string>): string {
return template.replace(/\{(\w+)\}/g, (whole, key: string) => (key in values ? encodeURIComponent(values[key]) : whole));
}
/** One transfer's way to be stopped: by its time limit, or by the transport closing. */
interface Transfer {
signal: AbortSignal;
/** Bytes came: for a download, the time limit starts over. */
alive(): void;
/** Whether the time limit ended it. */
timedOut(): boolean;
done(): void;
}
function messageOf(error: unknown, fallback: string): string {
return error instanceof Error && error.message ? error.message : fallback;
}
export class FetchTransport implements JmapTransport {
private readonly running = new Set<AbortController>();
private aborted = false;
constructor(private readonly options: FetchTransportOptions) {}
private async fetchWithAuth(url: string, init: RequestInit): Promise<Response> {
abort(): void {
this.aborted = true;
for (const controller of this.running) controller.abort();
}
private transfer(ms: number, idle: boolean): Transfer {
const controller = new AbortController();
this.running.add(controller);
let timedOut = false;
const arm = () => setTimeout(() => {
timedOut = true;
controller.abort();
}, ms);
let timer = arm();
if (this.aborted) controller.abort();
return {
signal: controller.signal,
alive: () => {
if (!idle) return;
clearTimeout(timer);
timer = arm();
},
timedOut: () => timedOut,
done: () => {
clearTimeout(timer);
this.running.delete(controller);
},
};
}
private async fetchWithAuth(url: string, init: RequestInit, transfer: Transfer): Promise<Response> {
const doFetch = this.options.fetch ?? fetch;
const send = async (header: string | null): Promise<Response> => {
if (!header) throw new TransportError('auth', 'No credentials yet');
const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
if (this.aborted) throw new TransportError('network', 'The store is closing');
try {
return await doFetch(url, { ...init, signal, headers: { ...init.headers, Authorization: header } });
return await doFetch(url, { ...init, signal: transfer.signal, headers: { ...init.headers, Authorization: header } });
} catch (error) {
throw new TransportError('network', error instanceof Error ? error.message : 'Network failure');
throw new TransportError('network', messageOf(error, 'Network failure'), undefined, undefined, undefined, transfer.timedOut());
}
};
let response = await send(this.options.auth());
@@ -87,40 +145,97 @@ export class FetchTransport implements JmapTransport {
return response;
}
/** A JSON body. A connection dropping halfway is a network failure, not a bad answer. */
private async readJson<T>(response: Response): Promise<T> {
try {
return await response.json() as T;
} catch (error) {
if (error instanceof SyntaxError) throw new TransportError('http', 'The server sent something that is not JSON', response.status);
throw new TransportError('network', messageOf(error, 'The response broke off'), undefined, undefined, undefined, true);
}
}
async request(calls: MethodCall[], using: string[]): Promise<JmapResponse> {
const response = await this.fetchWithAuth(this.options.session().apiUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ using, methodCalls: calls }),
});
if (!response.ok) {
let problem: string | undefined;
const transfer = this.transfer(REQUEST_TIMEOUT_MS, false);
try {
const response = await this.fetchWithAuth(this.options.session().apiUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ using, methodCalls: calls }),
}, transfer);
if (!response.ok) {
let problem: string | undefined;
try {
problem = (await response.json() as { type?: string }).type;
} catch {
// Not a problem document.
}
throw new TransportError('http', `JMAP request failed: ${response.status}`, response.status, problem);
}
return await this.readJson<JmapResponse>(response);
} finally {
transfer.done();
}
}
async downloadStream(jmap: string, blobId: string, type: string, name: string, sink: (part: Uint8Array) => void): Promise<void> {
const url = fillTemplate(this.options.session().downloadUrl, { accountId: jmap, blobId, type, name });
const transfer = this.transfer(DOWNLOAD_IDLE_MS, true);
try {
const response = await this.fetchWithAuth(url, { method: 'GET' }, transfer);
if (!response.ok) throw new TransportError('http', `Download failed: ${response.status}`, response.status);
if (!response.body) return;
const reader = response.body.getReader();
try {
problem = (await response.json() as { type?: string }).type;
} catch {
// Not a problem document.
for (;;) {
let read: Awaited<ReturnType<typeof reader.read>>;
try {
read = await reader.read();
} catch (error) {
throw new TransportError('network', messageOf(error, 'The download broke off'), undefined, undefined, undefined, !this.aborted);
}
if (read.done) return;
transfer.alive();
sink(read.value);
}
} finally {
void reader.cancel().catch(() => {});
}
throw new TransportError('http', `JMAP request failed: ${response.status}`, response.status, problem);
} finally {
transfer.done();
}
return await response.json() as JmapResponse;
}
async download(jmap: string, blobId: string, type: string, name: string): Promise<Uint8Array> {
const url = fillTemplate(this.options.session().downloadUrl, { accountId: jmap, blobId, type, name });
const response = await this.fetchWithAuth(url, { method: 'GET' });
if (!response.ok) throw new TransportError('http', `Download failed: ${response.status}`, response.status);
return new Uint8Array(await response.arrayBuffer());
const parts: Uint8Array[] = [];
let size = 0;
await this.downloadStream(jmap, blobId, type, name, (part) => {
parts.push(part);
size += part.length;
});
const data = new Uint8Array(size);
let at = 0;
for (const part of parts) {
data.set(part, at);
at += part.length;
}
return data;
}
async upload(jmap: string, data: Uint8Array, type: string): Promise<{ blobId: string; type: string; size: number }> {
const url = fillTemplate(this.options.session().uploadUrl, { accountId: jmap });
const response = await this.fetchWithAuth(url, {
method: 'POST',
headers: { 'Content-Type': type || 'application/octet-stream' },
body: data as BodyInit,
});
if (!response.ok) throw new TransportError('http', `Upload failed: ${response.status}`, response.status);
return await response.json() as { blobId: string; type: string; size: number };
const transfer = this.transfer(REQUEST_TIMEOUT_MS, false);
try {
const response = await this.fetchWithAuth(url, {
method: 'POST',
headers: { 'Content-Type': type || 'application/octet-stream' },
body: data as BodyInit,
}, transfer);
if (!response.ok) throw new TransportError('http', `Upload failed: ${response.status}`, response.status);
return await this.readJson<{ blobId: string; type: string; size: number }>(response);
} finally {
transfer.done();
}
}
}
lib/store/mail/attachments.ts
+326
View File
@@ -0,0 +1,326 @@
/**
* The files of kept mail: attachments, inline images (so a message offline shows
* them) and the raw message an export reads (decision in
* local-data/desktop-offline-decisions.md). Fetched last in a pass, newest mail
* first, one at a time and paced, and a new pass that's asked for goes first.
*
* What's kept is bounded by the scope: nothing with `attachments` off, nothing
* over `maxAttachmentBytes`, and `maxTotalBytes` in all. At the total a newer
* message's file pushes out raw messages first, then files of older messages,
* oldest first. Never the files of a pinned message. Raw messages come after
* every file and only take room that's free. Files held for a draft or a send
* are in `upload`, pinned files in the files area, and neither is touched here.
*
* A file comes in as a stream where the transport can, written in 1 MB chunks
* that each commit on their own, and is held nowhere whole. Its row in `blob`
* comes last, so a broken download leaves nothing readable.
*/
import type { StoreContext, SyncContext, Syncer } from '../area';
import { TransportError } from '../engine/transport';
import type { StorageAdapter } from '../storage';
import { BLOB_CHUNK_BYTES, dropBlob, putBlob } from './records';
/** Keeps well under the server's 1000 requests a minute, which the user's own clicks count against too. */
const PACE_MS = 200;
/** There's one raw message per message, and nothing waits for them. */
const RAW_PACE_MS = 1000;
const PAGE = 20;
/**
* Parts in a row that don't fit before a pass stops looking. Each check reads the
* kept list, and older mail only has less to push out.
*/
const MOST_PASSED_IN_A_ROW = 50;
const PINNED = '$pinned';
/** What says a file won't come. Anything else, like a 500, is tried at the next sync. */
const PERMANENT_DOWNLOAD_STATUSES = new Set([403, 404, 410]);
const RAW_TYPE = 'message/rfc822';
class DownloadStopped extends Error {}
class DownloadTooLarge extends Error {}
/** Someone else kept the same file while this one was coming in. */
class AlreadyKept extends Error {}
interface Part {
blob_id: string;
size: number;
type: string | null;
name: string | null;
received_at: number;
raw: number;
}
function usedBytes(db: StorageAdapter): number {
return Number(db.get<{ n: number | null }>('SELECT SUM(size) AS n FROM blob')?.n ?? 0);
}
const NOT_PINNED = `NOT EXISTS (
SELECT 1 FROM email_part p JOIN email_keyword k ON k.email_rowid = p.email_rowid
WHERE p.jmap = b.jmap AND p.blob_id = b.blob_id AND k.keyword = '${PINNED}'
)`;
const IS_RAW = `EXISTS (SELECT 1 FROM email_part r WHERE r.jmap = b.jmap AND r.blob_id = b.blob_id AND r.raw = 1)`;
/**
* Whether `size` more can fit, counting what could be pushed out for mail
* received at `at`. A raw message pushes out nothing: it only takes what's free.
*/
function couldFit(db: StorageAdapter, cap: number, size: number, at: number, raw: boolean): boolean {
const used = usedBytes(db);
if (used + size <= cap) return true;
if (raw) return false;
const yielding = Number(db.get<{ n: number | null }>(`SELECT SUM(b.size) AS n FROM blob b WHERE (${IS_RAW} OR b.at < ?) AND ${NOT_PINNED}`, [at])?.n ?? 0);
return used - yielding + size <= cap;
}
/** Pushes out raw messages, then files of mail older than `at`, oldest first, until `size` more fits or nothing is left to push out. */
function pushOut(db: StorageAdapter, cap: number, size: number, at: number): void {
let used = usedBytes(db);
while (used + size > cap) {
const oldest = db.all<{ jmap: string; blob_id: string; size: number }>(`SELECT b.jmap, b.blob_id, b.size FROM blob b WHERE (${IS_RAW} OR b.at < ?) AND ${NOT_PINNED} ORDER BY ${IS_RAW} DESC, b.at ASC LIMIT 50`, [at]);
if (oldest.length === 0) return;
for (const row of oldest) {
if (used + size <= cap) break;
dropBlob(db, row.jmap, row.blob_id);
used -= Number(row.size);
}
}
}
/** Makes room for `size` more. False, with nothing pushed out, when it can't. */
function makeRoom(db: StorageAdapter, cap: number, size: number, at: number, raw: boolean): boolean {
if (!couldFit(db, cap, size, at, raw)) return false;
pushOut(db, cap, size, at);
return true;
}
/** A tighter scope applies at once, not at the next sync. */
export function applyAttachmentScope(ctx: Pick<StoreContext, 'db' | 'scope'>): void {
const { db, scope } = ctx;
db.transaction(() => {
if (!scope.attachments) {
db.run('DELETE FROM blob_chunk');
db.run('DELETE FROM blob');
return;
}
for (const row of db.all<{ jmap: string; blob_id: string }>('SELECT jmap, blob_id FROM blob WHERE size > ?', [scope.maxAttachmentBytes])) {
dropBlob(db, row.jmap, row.blob_id);
}
pushOut(db, scope.maxTotalBytes, 0, Number.MAX_SAFE_INTEGER);
});
}
/** A kept file, piece by piece into a Blob, which the browser can keep out of this worker's memory. */
export function keptBlob(ctx: Pick<StoreContext, 'db' | 'scope'>, jmap: string, blobId: string): { type: string | null; size: number; blob: Blob } | null {
if (!ctx.scope.attachments) return null;
const row = ctx.db.get<{ type: string | null; size: number; data: Uint8Array }>('SELECT type, size, data FROM blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
if (!row) return null;
const type = row.type ?? '';
// A file from before chunks has its bytes in the row; a new one has none there.
let blob = new Blob(row.data?.byteLength ? [row.data as BlobPart] : [], { type });
for (let seq = 0; ; seq += 1) {
const chunk = ctx.db.get<{ data: Uint8Array }>('SELECT data FROM blob_chunk WHERE jmap = ? AND blob_id = ? AND seq = ?', [jmap, blobId, seq]);
if (!chunk) break;
// Each step only refers to the last Blob, so one chunk at a time is in memory here.
blob = new Blob([blob, chunk.data as BlobPart], { type });
}
// Better not there, and fetched again, than a file that reads as empty or cut short.
if (blob.size !== Number(row.size)) {
ctx.db.transaction(() => dropBlob(ctx.db, jmap, blobId));
return null;
}
return { type: row.type, size: Number(row.size), blob };
}
/** A file the user opened online: kept if it belongs to a kept message and the scope allows it. */
export function recordKeptBlob(ctx: Pick<StoreContext, 'db' | 'scope'>, jmap: string, blobId: string, type: string | null, data: Uint8Array): boolean {
const { db, scope } = ctx;
const owner = db.get<{ received_at: number; raw: number }>('SELECT received_at, raw FROM email_part WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
if (!owner || !scope.attachments || data.byteLength > scope.maxAttachmentBytes) return false;
const at = Number(owner.received_at);
return db.transaction(() => {
if (!makeRoom(db, scope.maxTotalBytes, data.byteLength, at, Boolean(owner.raw))) return false;
putBlob(db, jmap, blobId, type, data, at);
return true;
});
}
export type RawRefusal = 'attachments_off' | 'not_kept';
/**
* A whole message as the server has it, by `Email.blobId`, for an offline
* export. It stays a refusal rather than something rebuilt: a rebuilt message
* would lose its signatures and exact bytes.
*/
export function rawBlob(ctx: Pick<StoreContext, 'db' | 'scope'>, jmap: string, blobId: string): { ok: true; type: string; size: number; blob: Blob } | { ok: false; reason: RawRefusal } {
if (!ctx.scope.attachments) return { ok: false, reason: 'attachments_off' };
const isRaw = ctx.db.get('SELECT 1 AS x FROM email_part WHERE jmap = ? AND blob_id = ? AND raw = 1', [jmap, blobId]);
const kept = isRaw ? keptBlob(ctx, jmap, blobId) : null;
if (!kept) return { ok: false, reason: 'not_kept' };
return { ok: true, type: RAW_TYPE, size: kept.size, blob: new Blob([kept.blob], { type: RAW_TYPE }) };
}
/** How many messages (of one folder, or all) an export could write right now. */
export function rawStatus(ctx: Pick<StoreContext, 'db' | 'scope'>, jmap: string, mailboxId?: string): { total: number; kept: number } {
const inFolder = mailboxId ? 'JOIN email_mailbox m ON m.email_rowid = e.rowid AND m.mailbox_id = ?' : '';
const params = mailboxId ? [mailboxId, jmap] : [jmap];
const total = Number(ctx.db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM email e ${inFolder} WHERE e.jmap = ?`, params)?.n ?? 0);
if (!ctx.scope.attachments) return { total, kept: 0 };
const kept = Number(ctx.db.get<{ n: number }>(
`SELECT COUNT(DISTINCT e.rowid) AS n FROM email e ${inFolder}
JOIN email_part p ON p.email_rowid = e.rowid AND p.raw = 1
JOIN blob b ON b.jmap = p.jmap AND b.blob_id = p.blob_id
WHERE e.jmap = ?`,
params,
)?.n ?? 0);
return { total, kept };
}
function isKept(db: StorageAdapter, jmap: string, blobId: string): boolean {
return Boolean(db.get('SELECT 1 AS x FROM blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]));
}
/** What a download left. Never the pieces of a file that's kept: its row says they're whole. */
function clearUnfinished(db: StorageAdapter, jmap: string, blobId: string): void {
if (!isKept(db, jmap, blobId)) db.run('DELETE FROM blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
}
/** Takes a body piece by piece and writes it in chunks, each committed on its own. */
function chunkWriter(ctx: SyncContext, jmap: string, blobId: string) {
const buffer = new Uint8Array(BLOB_CHUNK_BYTES);
let filled = 0;
let seq = 0;
let total = 0;
const write = () => {
if (filled === 0) return;
if (isKept(ctx.db, jmap, blobId)) throw new AlreadyKept();
ctx.db.run('INSERT INTO blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, buffer.subarray(0, filled)]);
seq += 1;
filled = 0;
};
return {
push(part: Uint8Array): void {
total += part.length;
if (total > ctx.scope.maxAttachmentBytes) throw new DownloadTooLarge();
for (let at = 0; at < part.length;) {
const take = Math.min(part.length - at, BLOB_CHUNK_BYTES - filled);
buffer.set(part.subarray(at, at + take), filled);
filled += take;
at += take;
if (filled === BLOB_CHUNK_BYTES) {
if (ctx.stopped() || !ctx.scope.attachments) throw new DownloadStopped();
write();
}
}
},
/** The last piece goes in the transaction that files it, so a small file costs one commit. */
writeLast: write,
get total(): number {
return total;
},
};
}
function refuse(db: StorageAdapter, jmap: string, blobId: string): void {
db.run('UPDATE email_part SET refused = 1 WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
}
type Fetched = 'kept' | 'skipped' | 'stopped';
async function fetchOne(ctx: SyncContext, jmap: string, part: Part): Promise<Fetched> {
const { db, transport } = ctx;
const blobId = String(part.blob_id);
const type = part.type ?? 'application/octet-stream';
const name = part.name ?? 'attachment';
if (isKept(db, jmap, blobId)) return 'kept';
clearUnfinished(db, jmap, blobId);
const writer = chunkWriter(ctx, jmap, blobId);
try {
if (transport.downloadStream) await transport.downloadStream(jmap, blobId, type, name, (piece) => writer.push(piece));
else writer.push(await transport.download(jmap, blobId, type, name));
} catch (error) {
clearUnfinished(db, jmap, blobId);
if (error instanceof AlreadyKept) return 'kept';
if (error instanceof DownloadStopped) return 'stopped';
if (error instanceof DownloadTooLarge) {
refuse(db, jmap, blobId);
return 'skipped';
}
if (error instanceof TransportError && error.kind === 'http') {
if (!PERMANENT_DOWNLOAD_STATUSES.has(error.status ?? 0)) return 'stopped';
refuse(db, jmap, blobId);
return 'skipped';
}
// The connection works and this transfer broke: the next one may be fine.
if (error instanceof TransportError && error.interrupted) return 'skipped';
throw error;
}
const at = Number(part.received_at);
const size = writer.total;
return db.transaction((): Fetched => {
// The user opened it meanwhile, and that copy is whole.
if (isKept(db, jmap, blobId)) return 'kept';
// Its message may have gone while this was on its way, or the rules changed.
const wanted = ctx.scope.attachments && db.get('SELECT 1 AS x FROM email_part WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
if (!wanted || !makeRoom(db, ctx.scope.maxTotalBytes, size, at, Boolean(part.raw))) {
clearUnfinished(db, jmap, blobId);
return 'skipped';
}
writer.writeLast();
db.run(
`INSERT INTO blob (jmap, blob_id, type, size, at, data) VALUES (?, ?, ?, ?, ?, x'')
ON CONFLICT (jmap, blob_id) DO UPDATE SET type = excluded.type, size = excluded.size, at = excluded.at, data = excluded.data`,
[jmap, blobId, part.type, size, at],
);
return 'kept';
});
}
async function fetchKept(ctx: SyncContext, jmap: string): Promise<void> {
const { db } = ctx;
// What a closed app or a stopped pass left unfinished.
db.run('DELETE FROM blob_chunk WHERE jmap = ? AND NOT EXISTS (SELECT 1 FROM blob b WHERE b.jmap = blob_chunk.jmap AND b.blob_id = blob_chunk.blob_id)', [jmap]);
applyAttachmentScope(ctx);
if (!ctx.scope.attachments) return;
// Rows passed over in this pass stay in the list, so the next page starts past them.
let passed = 0;
let passedInARow = 0;
let done = 0;
for (;;) {
const parts = db.all<Part>(
`SELECT p.blob_id, p.size, p.type, p.name, p.received_at, p.raw FROM email_part p
WHERE p.jmap = ? AND p.refused = 0 AND p.size <= ?
AND NOT EXISTS (SELECT 1 FROM blob b WHERE b.jmap = p.jmap AND b.blob_id = p.blob_id)
ORDER BY p.raw ASC, p.received_at DESC, p.blob_id ASC LIMIT ? OFFSET ?`,
[jmap, ctx.scope.maxAttachmentBytes, PAGE, passed],
);
if (parts.length === 0) return;
for (const part of parts) {
if (ctx.stopped() || ctx.syncWanted()) return;
// The same file a second time in this page: kept by now.
if (db.get('SELECT 1 AS x FROM blob WHERE jmap = ? AND blob_id = ?', [jmap, String(part.blob_id)])) continue;
if (!couldFit(db, ctx.scope.maxTotalBytes, Number(part.size), Number(part.received_at), Boolean(part.raw))) {
passed += 1;
passedInARow += 1;
if (passedInARow >= MOST_PASSED_IN_A_ROW) return;
continue;
}
passedInARow = 0;
const outcome = await fetchOne(ctx, jmap, part);
if (outcome === 'stopped') return;
if (outcome === 'skipped') passed += 1;
done += 1;
ctx.emit({ type: 'progress', account: ctx.account, area: 'mail', phase: 'attachments', done, total: null });
await ctx.pause(part.raw ? RAW_PACE_MS : PACE_MS);
}
}
}
export const attachmentSyncer: Syncer = {
name: 'attachments',
capability: 'urn:ietf:params:jmap:mail',
low: true,
run: fetchKept,
};
lib/store/mail/calls.ts
+9 -9
View File
@@ -1,7 +1,8 @@
/** What the page asks the mail part of the store outside JMAP. */
import type { AreaCall, StoreContext } from '../area';
import { putBlob, storeBody, upsertEmail, type EmailObject } from './records';
import { keptBlob, rawBlob, rawStatus, recordKeptBlob } from './attachments';
import { storeBody, upsertEmail, type EmailObject } from './records';
/**
* A message the user opened online: kept with its text if the store doesn't
@@ -34,16 +35,11 @@ function recordOpened(ctx: StoreContext, params: { jmap: string; emails: EmailOb
/** A file of a kept message the user opened. Nothing else is kept: not an export, not a stray file. */
function recordBlob(ctx: StoreContext, params: { jmap: string; blobId: string; type: string | null; data: Uint8Array }): boolean {
if (!ctx.scope.attachments || params.data.byteLength > ctx.scope.maxAttachmentBytes) return false;
const owner = ctx.db.get<{ received_at: number }>('SELECT received_at FROM email_part WHERE jmap = ? AND blob_id = ?', [params.jmap, params.blobId]);
if (!owner) return false;
putBlob(ctx.db, params.jmap, params.blobId, params.type, params.data, Number(owner.received_at));
return true;
return recordKeptBlob(ctx, params.jmap, params.blobId, params.type, params.data);
}
function getBlob(ctx: StoreContext, params: { jmap: string; blobId: string }): { type: string | null; data: Uint8Array } | null {
const row = ctx.db.get<{ type: string | null; data: Uint8Array }>('SELECT type, data FROM blob WHERE jmap = ? AND blob_id = ?', [params.jmap, params.blobId]);
return row ? { type: row.type, data: row.data } : null;
function getBlob(ctx: StoreContext, params: { jmap: string; blobId: string }): { type: string | null; size: number; blob: Blob } | null {
return keptBlob(ctx, params.jmap, params.blobId);
}
/** How much of the disk this account's file takes. */
@@ -57,6 +53,10 @@ export const MAIL_CALLS: Readonly<Record<string, AreaCall>> = {
'mail.recordOpened': recordOpened,
'mail.recordBlob': recordBlob,
'mail.getBlob': getBlob,
/** A whole message for an offline export, or why not. */
'mail.rawBlob': (ctx, params: { jmap: string; blobId: string }) => rawBlob(ctx, params.jmap, params.blobId),
/** How many messages an export could write now, so it can say how many it can't. */
'mail.rawStatus': (ctx, params: { jmap: string; mailboxId?: string }) => rawStatus(ctx, params.jmap, params.mailboxId),
'mail.usage': usage,
/** The session kept with the account, for a start from the local copy. */
'mail.session': (ctx) => ctx.session,
lib/store/mail/index.ts
+3 -1
View File
@@ -1,4 +1,5 @@
import type { StoreArea } from '../area';
import { applyAttachmentScope, attachmentSyncer } from './attachments';
import { MAIL_CALLS } from './calls';
import { MAIL_METHODS } from './methods';
import { MAIL_MIGRATIONS } from './schema';
@@ -9,6 +10,7 @@ export const mailArea: StoreArea = {
name: 'mail',
migrations: [...MAIL_MIGRATIONS, ...SEARCH_MIGRATIONS],
methods: MAIL_METHODS,
syncers: [mailSyncer, identitySyncer, quotaSyncer],
syncers: [mailSyncer, identitySyncer, quotaSyncer, attachmentSyncer],
calls: MAIL_CALLS,
scopeChanged: applyAttachmentScope,
};
lib/store/mail/records.ts
+52 -19
View File
@@ -97,12 +97,21 @@ export function upsertEmail(db: StorageAdapter, jmap: string, email: EmailObject
return rowid;
}
/** Kept files are stored in pieces of this size. */
export const BLOB_CHUNK_BYTES = 1024 * 1024;
export function dropBlob(db: StorageAdapter, jmap: string, blobId: string): void {
db.run('DELETE FROM blob_chunk WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
db.run('DELETE FROM blob WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
}
/** A file that's here as a whole already. */
export function putBlob(db: StorageAdapter, jmap: string, blobId: string, type: string | null, data: Uint8Array, at: number): void {
db.run(
`INSERT INTO blob (jmap, blob_id, type, size, at, data) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (jmap, blob_id) DO UPDATE SET type = excluded.type, size = excluded.size, at = excluded.at, data = excluded.data`,
[jmap, blobId, type, data.byteLength, at, data],
);
dropBlob(db, jmap, blobId);
for (let from = 0, seq = 0; from < data.byteLength; from += BLOB_CHUNK_BYTES, seq += 1) {
db.run('INSERT INTO blob_chunk (jmap, blob_id, seq, data) VALUES (?, ?, ?, ?)', [jmap, blobId, seq, data.subarray(from, from + BLOB_CHUNK_BYTES)]);
}
db.run("INSERT INTO blob (jmap, blob_id, type, size, at, data) VALUES (?, ?, ?, ?, ?, x'')", [jmap, blobId, type, data.byteLength, at]);
}
export function deleteEmail(db: StorageAdapter, jmap: string, id: string): void {
@@ -237,24 +246,48 @@ function forgetParts(db: StorageAdapter, rowid: number): void {
const parts = db.all<{ jmap: string; blob_id: string }>('SELECT jmap, blob_id FROM email_part WHERE email_rowid = ?', [rowid]);
db.run('DELETE FROM email_part WHERE email_rowid = ?', [rowid]);
for (const part of parts) {
db.run(
'DELETE FROM blob WHERE jmap = ? AND blob_id = ? AND NOT EXISTS (SELECT 1 FROM email_part p WHERE p.jmap = ? AND p.blob_id = ?)',
[part.jmap, part.blob_id, part.jmap, part.blob_id],
);
if (db.get('SELECT 1 AS x FROM email_part WHERE jmap = ? AND blob_id = ?', [part.jmap, part.blob_id])) continue;
dropBlob(db, part.jmap, part.blob_id);
}
}
function rememberParts(db: StorageAdapter, rowid: number, attachments: unknown): void {
interface MimePart {
blobId?: unknown;
size?: unknown;
type?: unknown;
name?: unknown;
disposition?: unknown;
subParts?: MimePart[];
}
function leafParts(part: MimePart): MimePart[] {
return part.subParts ? part.subParts.flatMap(leafParts) : [part];
}
/**
* The parts a message needs as files: its attachments, and whatever else the
* body shows. Stalwart 1.0 doesn't list inline images under `attachments`, so
* those come from the body lists and the MIME tree. Text parts are in bodyValues.
*/
function fileParts(body: Record<string, unknown>): MimePart[] {
const listed = (key: string) => (Array.isArray(body[key]) ? body[key] as MimePart[] : []);
const shown = [...listed('textBody'), ...listed('htmlBody'), ...(body.bodyStructure ? leafParts(body.bodyStructure as MimePart) : [])]
.filter((part) => !(typeof part.type === 'string' && part.type.startsWith('text/')) || part.disposition === 'attachment');
return [...listed('attachments'), ...shown];
}
function rememberParts(db: StorageAdapter, rowid: number, list: Record<string, unknown>, body: Record<string, unknown>): void {
const row = db.get<{ jmap: string; received_at: number }>('SELECT jmap, received_at FROM email WHERE rowid = ?', [rowid]);
if (!row || !Array.isArray(attachments)) return;
for (const part of attachments as Array<{ blobId?: unknown; size?: unknown; type?: unknown; name?: unknown }>) {
if (typeof part.blobId !== 'string') continue;
db.run(
`INSERT INTO email_part (jmap, blob_id, email_rowid, received_at, size, type, name) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (jmap, blob_id, email_rowid) DO NOTHING`,
[row.jmap, part.blobId, rowid, row.received_at, typeof part.size === 'number' ? part.size : 0, typeof part.type === 'string' ? part.type : null, typeof part.name === 'string' ? part.name : null],
);
if (!row) return;
const remember = (blobId: string, size: unknown, type: unknown, name: unknown, raw: number) => db.run(
`INSERT INTO email_part (jmap, blob_id, email_rowid, received_at, size, type, name, raw) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (jmap, blob_id, email_rowid) DO NOTHING`,
[row.jmap, blobId, rowid, row.received_at, typeof size === 'number' ? size : 0, typeof type === 'string' ? type : null, typeof name === 'string' ? name : null, raw],
);
for (const part of fileParts(body)) {
if (typeof part.blobId === 'string') remember(part.blobId, part.size, part.type, part.name, 0);
}
if (typeof list.blobId === 'string') remember(list.blobId, list.size, 'message/rfc822', 'message.eml', 1);
}
export function storeBody(db: StorageAdapter, rowid: number, listJson: string, full: Record<string, unknown>): void {
@@ -262,7 +295,7 @@ export function storeBody(db: StorageAdapter, rowid: number, listJson: string, f
const body: Record<string, unknown> = {};
for (const [key, value] of Object.entries(full)) if (!(key in list)) body[key] = value;
const hadBody = Number(db.get<{ body_state: number }>('SELECT body_state FROM email WHERE rowid = ?', [rowid])?.body_state) === 1;
rememberParts(db, rowid, body.attachments);
rememberParts(db, rowid, list, body);
db.run('INSERT INTO email_body (email_rowid, json) VALUES (?, ?) ON CONFLICT (email_rowid) DO UPDATE SET json = excluded.json', [rowid, JSON.stringify(body)]);
db.run('UPDATE email SET body_state = 1 WHERE rowid = ?', [rowid]);
// A first body is one insert. Only a body stored again has an old row to replace.
lib/store/mail/schema.ts
+18
View File
@@ -92,4 +92,22 @@ export const MAIL_MIGRATIONS: readonly Migration[] = [
) WITHOUT ROWID;
`,
},
// In the drafts range: this came with drafts and attachments.
{
id: 410,
name: 'kept files in chunks, raw messages',
sql: `
-- Binding a whole file grows the WASM heap by its size, and that never shrinks.
-- A file counts as kept once its row in blob exists.
CREATE TABLE blob_chunk (
jmap TEXT NOT NULL,
blob_id TEXT NOT NULL,
seq INTEGER NOT NULL,
data BLOB NOT NULL,
PRIMARY KEY (jmap, blob_id, seq)
);
-- 1 for the whole message (Email.blobId), for an export.
ALTER TABLE email_part ADD COLUMN raw INTEGER NOT NULL DEFAULT 0;
`,
},
];
lib/store/mail/sync.ts
+3 -48
View File
@@ -10,14 +10,13 @@
import type { SyncContext, Syncer } from '../area';
import { deleteObject, putObject } from '../engine/core-schema';
import { runChangesLoop } from '../engine/changes';
import { JmapMethodError, responseFor, TransportError, type MethodCall } from '../engine/transport';
import { JmapMethodError, responseFor, type MethodCall } from '../engine/transport';
import {
deleteEmail,
deleteEmailRow,
EMAIL_BODY_PROPERTIES,
EMAIL_LIST_PROPERTIES,
MAIL_USING,
putBlob,
storeBody,
upsertEmail,
type EmailObject,
@@ -113,7 +112,8 @@ async function startFullList(ctx: SyncContext, jmap: string): Promise<ListCursor
async function sweepUnseen(ctx: SyncContext, jmap: string, gen: number): Promise<boolean> {
for (;;) {
if (ctx.stopped()) return false;
const unseen = ctx.db.all<{ rowid: number; id: string }>('SELECT rowid, id FROM email WHERE jmap = ? AND gen < ? LIMIT ?', [jmap, gen, PAGE]);
// A draft saved on this device isn't on the server yet: it has no say in this.
const unseen = ctx.db.all<{ rowid: number; id: string }>("SELECT rowid, id FROM email WHERE jmap = ? AND gen < ? AND id NOT LIKE 'local-%' LIMIT ?", [jmap, gen, PAGE]);
if (unseen.length === 0) return true;
const response = await ctx.transport.request([['Email/get', {
accountId: jmap,
@@ -349,49 +349,6 @@ async function backfillBodies(ctx: SyncContext, jmap: string): Promise<void> {
}
}
// ---- Attachments ----
/**
* The files of kept messages, newest first, within what the user chose: one
* at a time, each in its own transaction, so a reload loses at most one.
*/
async function fetchAttachments(ctx: SyncContext, jmap: string): Promise<void> {
if (!ctx.scope.attachments) return;
let used = Number(ctx.db.get<{ n: number | null }>('SELECT SUM(size) AS n FROM blob')?.n ?? 0);
for (;;) {
if (ctx.stopped()) return;
const parts = ctx.db.all<{ blob_id: string; size: number; type: string | null; name: string | null; received_at: number }>(
`SELECT p.blob_id, p.size, p.type, p.name, p.received_at FROM email_part p
WHERE p.jmap = ? AND p.refused = 0 AND p.size <= ?
AND NOT EXISTS (SELECT 1 FROM blob b WHERE b.jmap = p.jmap AND b.blob_id = p.blob_id)
ORDER BY p.received_at DESC LIMIT 20`,
[jmap, ctx.scope.maxAttachmentBytes],
);
if (parts.length === 0) return;
for (const part of parts) {
if (ctx.stopped()) return;
const blobId = String(part.blob_id);
if (used + Number(part.size) > ctx.scope.maxTotalBytes) return;
let data: Uint8Array;
try {
data = await ctx.transport.download(jmap, blobId, part.type ?? 'application/octet-stream', part.name ?? 'attachment');
} catch (error) {
// Only what says this file won't come. Anything else, like a 500 or a 429, is tried at the next sync.
if (error instanceof TransportError && error.kind === 'http' && PERMANENT_DOWNLOAD_STATUSES.has(error.status ?? 0)) {
ctx.db.run('UPDATE email_part SET refused = 1 WHERE jmap = ? AND blob_id = ?', [jmap, blobId]);
continue;
}
if (error instanceof TransportError && error.kind === 'http') return;
throw error;
}
putBlob(ctx.db, jmap, blobId, part.type, data, Number(part.received_at));
used += data.byteLength;
}
}
}
const PERMANENT_DOWNLOAD_STATUSES = new Set([403, 404, 410]);
// ---- Identities and quotas ----
const IDENTITY_USING = ['urn:ietf:params:jmap:core', 'urn:ietf:params:jmap:mail', 'urn:ietf:params:jmap:submission'];
@@ -481,7 +438,5 @@ export const mailSyncer: Syncer = {
await syncThreads(ctx, jmap);
if (ctx.stopped()) return;
await backfillBodies(ctx, jmap);
if (ctx.stopped()) return;
await fetchAttachments(ctx, jmap);
},
};
lib/store/outbox/calls.ts
+80 -19
View File
@@ -1,6 +1,7 @@
/** What the page asks the outbox. Each call is one SQLite transaction, so two tabs can't interleave. */
import type { AreaCall, StoreContext } from '../area';
import { heldBlobsOf, type DraftCreate } from '../drafts/local-email';
import { metaGet, metaSet } from '../engine/core-schema';
import { deleteEmail, upsertEmail, type EmailObject } from '../mail/records';
import { nextSeq, queueUpdate, type PatchRow } from './email-set';
@@ -136,48 +137,104 @@ function settleStale(ctx: StoreContext, params: { before: number }): number {
).changes;
}
/** Removes an entry and the files it held. */
/** Every held file a waiting message or a draft on this device still needs. */
function neededFiles(ctx: StoreContext): Set<string> {
const forSends = ctx.db.all<OutboxRow>('SELECT * FROM outbox').flatMap((row) => heldBlobIds(toEntry(row).item));
const forDrafts = ctx.db.all<{ create_json: string }>('SELECT create_json FROM draft_pending').flatMap((row) => heldBlobsOf(JSON.parse(row.create_json) as DraftCreate));
return new Set([...forSends, ...forDrafts]);
}
function dropHeld(ctx: StoreContext, blobId: string): void {
ctx.db.run('DELETE FROM upload_chunk WHERE local_id = ?', [blobId]);
ctx.db.run('DELETE FROM upload WHERE local_id = ?', [blobId]);
}
/** Removes an entry and the files only it held. A draft may share one with the message it became. */
function remove(ctx: StoreContext, params: { id: number }): void {
ctx.db.transaction(() => {
const row = ctx.db.get<OutboxRow>('SELECT * FROM outbox WHERE id = ?', [params.id]);
if (!row) return;
for (const blobId of heldBlobIds(toEntry(row).item)) ctx.db.run('DELETE FROM upload WHERE local_id = ?', [blobId]);
ctx.db.run('DELETE FROM outbox WHERE id = ?', [params.id]);
const needed = neededFiles(ctx);
for (const blobId of heldBlobIds(toEntry(row).item)) if (!needed.has(blobId)) dropHeld(ctx, blobId);
});
}
// ---- Files attached offline ----
function holdFile(ctx: StoreContext, params: { jmap?: string; name: string; type: string; data: Uint8Array; localId?: string }): { blobId: string; size: number; type: string } {
const blobId = params.localId ?? `${LOCAL_BLOB_PREFIX}${crypto.randomUUID()}`;
/** The same size the page cuts a file into on its way here. */
const HOLD_CHUNK_BYTES = 1024 * 1024;
/** A file that comes in pieces: nothing can read it until the last one is in. */
function holdStart(ctx: StoreContext, params: { jmap?: string; name: string; type: string; size: number }): { blobId: string } {
const blobId = `${LOCAL_BLOB_PREFIX}${crypto.randomUUID()}`;
ctx.db.run(
'INSERT OR REPLACE INTO upload (local_id, jmap, type, name, size, data, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)',
[blobId, params.jmap ?? null, params.type, params.name, params.data.byteLength, params.data, ctx.now()],
"INSERT INTO upload (local_id, jmap, type, name, size, data, created_at, complete) VALUES (?, ?, ?, ?, ?, x'', ?, 0)",
[blobId, params.jmap ?? null, params.type, params.name, params.size, ctx.now()],
);
return { blobId, size: params.data.byteLength, type: params.type || 'application/octet-stream' };
return { blobId };
}
function heldFile(ctx: StoreContext, params: { blobId: string }): { name: string; type: string; data: Uint8Array } | null {
const row = ctx.db.get<{ name: string | null; type: string | null; data: Uint8Array }>('SELECT name, type, data FROM upload WHERE local_id = ?', [params.blobId]);
return row ? { name: row.name ?? 'attachment', type: row.type ?? 'application/octet-stream', data: row.data } : null;
function holdChunk(ctx: StoreContext, params: { blobId: string; seq: number; data: Uint8Array }): void {
ctx.db.run('INSERT OR REPLACE INTO upload_chunk (local_id, seq, data) VALUES (?, ?, ?)', [params.blobId, params.seq, params.data]);
}
const heldFileSources: Array<(ctx: StoreContext) => Iterable<string>> = [];
/** Another area's records that hold files by their local id, which dropOrphans must leave alone. */
export function addHeldFileSource(source: (ctx: StoreContext) => Iterable<string>): void {
heldFileSources.push(source);
function holdEnd(ctx: StoreContext, params: { blobId: string }): { blobId: string; size: number; type: string } {
ctx.db.run('UPDATE upload SET complete = 1 WHERE local_id = ?', [params.blobId]);
const row = ctx.db.get<{ size: number; type: string | null }>('SELECT size, type FROM upload WHERE local_id = ?', [params.blobId]);
return { blobId: params.blobId, size: Number(row?.size ?? 0), type: row?.type || 'application/octet-stream' };
}
/** Files held for no entry (attached offline to a message that was then discarded). SEAM: the drafts agent keeps its drafts' files here too. */
function holdAbort(ctx: StoreContext, params: { blobId: string }): void {
ctx.db.transaction(() => dropHeld(ctx, params.blobId));
}
/** A file that's here as a whole already: the old database's, on its move here. */
function holdFile(ctx: StoreContext, params: { jmap?: string; name: string; type: string; data: Uint8Array; localId?: string }): { blobId: string; size: number; type: string } {
return ctx.db.transaction(() => {
const blobId = params.localId ?? `${LOCAL_BLOB_PREFIX}${crypto.randomUUID()}`;
dropHeld(ctx, blobId);
ctx.db.run(
"INSERT INTO upload (local_id, jmap, type, name, size, data, created_at) VALUES (?, ?, ?, ?, ?, x'', ?)",
[blobId, params.jmap ?? null, params.type, params.name, params.data.byteLength, ctx.now()],
);
for (let at = 0, seq = 0; at < params.data.byteLength; at += HOLD_CHUNK_BYTES, seq += 1) {
ctx.db.run('INSERT INTO upload_chunk (local_id, seq, data) VALUES (?, ?, ?)', [blobId, seq, params.data.subarray(at, at + HOLD_CHUNK_BYTES)]);
}
return { blobId, size: params.data.byteLength, type: params.type || 'application/octet-stream' };
});
}
/** Piece by piece into a Blob, which the browser can keep out of this worker's memory. */
function heldFile(ctx: StoreContext, params: { blobId: string }): { name: string; type: string; size: number; blob: Blob } | null {
const row = ctx.db.get<{ name: string | null; type: string | null; size: number; data: Uint8Array }>('SELECT name, type, size, data FROM upload WHERE local_id = ? AND complete = 1', [params.blobId]);
if (!row) return null;
const type = row.type || 'application/octet-stream';
// A file from before chunks has its bytes in the row; a new one has none there.
let blob = new Blob(row.data?.byteLength ? [row.data as BlobPart] : [], { type });
for (let seq = 0; ; seq += 1) {
const chunk = ctx.db.get<{ data: Uint8Array }>('SELECT data FROM upload_chunk WHERE local_id = ? AND seq = ?', [params.blobId, seq]);
if (!chunk) break;
// Each step only refers to the last Blob, so one chunk at a time is in memory here.
blob = new Blob([blob, chunk.data as BlobPart], { type });
}
// Pieces missing: as gone, so it's said so, rather than a file that uploads as empty.
if (blob.size !== Number(row.size)) return null;
return { name: row.name ?? 'attachment', type, size: Number(row.size), blob };
}
/**
* Files nothing needs any more: attached to a message that was then discarded,
* or sent or saved long enough ago. Not at once, because an open composer may
* still show one.
*/
function dropOrphans(ctx: StoreContext, params: { olderThan: number }): number {
return ctx.db.transaction(() => {
const needed = new Set(ctx.db.all<OutboxRow>('SELECT * FROM outbox').flatMap((row) => heldBlobIds(toEntry(row).item)));
for (const source of heldFileSources) for (const id of source(ctx)) needed.add(id);
const needed = neededFiles(ctx);
let dropped = 0;
for (const row of ctx.db.all<{ local_id: string }>('SELECT local_id FROM upload WHERE created_at < ?', [params.olderThan])) {
if (needed.has(String(row.local_id))) continue;
ctx.db.run('DELETE FROM upload WHERE local_id = ?', [row.local_id]);
dropHeld(ctx, String(row.local_id));
dropped += 1;
}
return dropped;
@@ -288,6 +345,10 @@ export const OUTBOX_CALLS: Readonly<Record<string, AreaCall>> = {
'outbox.settleStale': settleStale,
'outbox.remove': remove,
'outbox.holdFile': holdFile,
'outbox.holdStart': holdStart,
'outbox.holdChunk': holdChunk,
'outbox.holdEnd': holdEnd,
'outbox.holdAbort': holdAbort,
'outbox.heldFile': heldFile,
'outbox.dropOrphans': dropOrphans,
'outbox.pendingPatches': pendingPatches,
lib/store/outbox/email-set.ts
+25 -14
View File
@@ -4,7 +4,10 @@
* same way. Answers carry no state: a made-up one would reach the server later.
*/
import { isStandInId } from '@/lib/jmap/stand-in-ids';
import { MethodError, type MethodContext, type MethodHandler, type RecordOverlay } from '../area';
import { createLocalDraft, destroyForDraft, updateLocalDraft } from '../drafts/pending';
import type { DraftCreate } from '../drafts/local-email';
import { metaGet, metaSet } from '../engine/core-schema';
import { deleteEmail, upsertEmail, type EmailObject } from '../mail/records';
import type { StorageAdapter } from '../storage';
@@ -24,6 +27,7 @@ export interface PatchRow {
}
const EMAIL = 'Email';
export const EMAIL_TYPE = EMAIL;
/** The next place in the order changes and sends were made. */
export function nextSeq(db: StorageAdapter): number {
@@ -32,11 +36,11 @@ export function nextSeq(db: StorageAdapter): number {
return next;
}
function pendingRow(db: StorageAdapter, jmap: string, id: string): PatchRow | undefined {
export function pendingRow(db: StorageAdapter, jmap: string, id: string): PatchRow | undefined {
return db.get<PatchRow>('SELECT * FROM outbox_patch WHERE jmap = ? AND type = ? AND record_id = ?', [jmap, EMAIL, id]);
}
function savePatch(db: StorageAdapter, row: Omit<PatchRow, 'seq' | 'queued_at' | 'type'>, now: number): void {
export function savePatch(db: StorageAdapter, row: Omit<PatchRow, 'seq' | 'queued_at' | 'type'>, now: number): void {
db.run(
`INSERT INTO outbox_patch (jmap, type, record_id, op, patch, base, server, subject, seq, queued_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@@ -45,7 +49,7 @@ function savePatch(db: StorageAdapter, row: Omit<PatchRow, 'seq' | 'queued_at' |
);
}
function localEmail(db: StorageAdapter, jmap: string, id: string): { email: EmailObject; gen: number } | null {
export function localEmail(db: StorageAdapter, jmap: string, id: string): { email: EmailObject; gen: number } | null {
const row = db.get<{ json: string; gen: number }>('SELECT json, gen FROM email WHERE jmap = ? AND id = ?', [jmap, id]);
return row ? { email: JSON.parse(row.json) as EmailObject, gen: Number(row.gen) } : null;
}
@@ -90,13 +94,16 @@ export function queueDestroy(db: StorageAdapter, jmap: string, id: string, now:
if (local) deleteEmail(db, jmap, id);
}
/**
* SEAM for the drafts agent: Email/set create is theirs. Until it lands every
* create is refused per message, so createDraft throws and the composer shows
* its "draft not saved" state, as it does today.
*/
function createEmails(_ctx: MethodContext, _jmap: string, create: Record<string, unknown>): Record<string, { type: string; description: string }> {
return Object.fromEntries(Object.keys(create).map((id) => [id, { type: 'serverUnavailable', description: 'Saving a draft needs the server' }]));
/** A draft stays on this device until it can go up (lib/store/drafts). Any other new message needs the server. */
function createEmails(ctx: MethodContext, jmap: string, create: Record<string, unknown>, networkTried: boolean): { created: Record<string, unknown>; notCreated: Record<string, unknown> } {
const created: Record<string, unknown> = {};
const notCreated: Record<string, unknown> = {};
for (const [creationId, email] of Object.entries(create)) {
const { refused, ...answer } = createLocalDraft(ctx, jmap, email as DraftCreate, networkTried);
if (refused) notCreated[creationId] = answer;
else created[creationId] = answer;
}
return { created, notCreated };
}
export const emailSet: MethodHandler = (ctx, args) => {
@@ -107,14 +114,15 @@ export const emailSet: MethodHandler = (ctx, args) => {
const result: Record<string, unknown> = { accountId: jmap };
ctx.db.transaction(() => {
if (args.create && Object.keys(args.create as object).length > 0) {
result.created = {};
result.notCreated = createEmails(ctx, jmap, args.create as Record<string, unknown>);
const { created, notCreated } = createEmails(ctx, jmap, args.create as Record<string, unknown>, args.networkTried === true);
result.created = created;
if (Object.keys(notCreated).length > 0) result.notCreated = notCreated;
}
if (args.update) {
const updated: Record<string, null> = {};
const notUpdated: Record<string, unknown> = {};
for (const [id, patch] of Object.entries(args.update as Record<string, Patch>)) {
const problem = queueUpdate(ctx.db, jmap, id, patch, ctx.now());
const problem = isStandInId(id) ? updateLocalDraft(ctx, jmap, id, patch) : queueUpdate(ctx.db, jmap, id, patch, ctx.now());
if (problem) notUpdated[id] = problem;
else updated[id] = null;
}
@@ -122,7 +130,10 @@ export const emailSet: MethodHandler = (ctx, args) => {
if (Object.keys(notUpdated).length > 0) result.notUpdated = notUpdated;
}
if (Array.isArray(args.destroy)) {
for (const id of args.destroy as string[]) queueDestroy(ctx.db, jmap, id, ctx.now());
const replacedBy = typeof args.replacedBy === 'string' ? args.replacedBy : undefined;
for (const id of args.destroy as string[]) {
if (!destroyForDraft(ctx, jmap, id, replacedBy)) queueDestroy(ctx.db, jmap, id, ctx.now());
}
result.destroyed = args.destroy;
}
});
lib/store/outbox/index.ts
+10 -4
View File
@@ -1,13 +1,19 @@
import type { StoreArea } from '../area';
import type { RecordOverlay, StoreArea } from '../area';
import { DRAFT_CALLS } from '../drafts/calls';
import { draftsOverlay } from '../drafts/pending';
import { DRAFT_MIGRATIONS } from '../drafts/schema';
import { OUTBOX_CALLS } from './calls';
import { emailSet, outboxOverlay } from './email-set';
import { OUTBOX_MIGRATIONS } from './schema';
const overlay: RecordOverlay = (ctx, jmap, type, records) => draftsOverlay(ctx, jmap, type, outboxOverlay(ctx, jmap, type, records));
/** Drafts answer the same Email/set and share the queue's order, so they plug in here. */
export const outboxArea: StoreArea = {
name: 'outbox',
migrations: OUTBOX_MIGRATIONS,
migrations: [...OUTBOX_MIGRATIONS, ...DRAFT_MIGRATIONS],
methods: { 'Email/set': emailSet },
syncers: [],
calls: OUTBOX_CALLS,
overlay: outboxOverlay,
calls: { ...OUTBOX_CALLS, ...DRAFT_CALLS },
overlay,
};
lib/store/outbox/items.ts
+2 -1
View File
@@ -91,7 +91,8 @@ export type ConflictReason = 'changed' | 'gone' | 'refused';
export interface ConflictSummary {
subject: string;
op: 'update' | 'destroy';
/** `draft`: a draft saved here met a version changed elsewhere. Both are kept. */
op: 'update' | 'destroy' | 'draft';
reason: ConflictReason;
/** The server's word when it refused. */
detail?: string;