Refactored local delivery to avoid mpsc channel

Cargo.lock

@@ -2058,6 +2058,37 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "email"
+version = "0.11.2"
+dependencies = [
+ "aes",
+ "aes-gcm",
+ "aes-gcm-siv",
+ "bincode",
+ "cbc",
+ "common",
+ "directory",
+ "jmap_proto",
+ "mail-builder",
+ "mail-parser",
+ "nlp",
+ "rasn",
+ "rasn-cms",
+ "rasn-pkix",
+ "rsa",
+ "sequoia-openpgp",
+ "serde",
+ "serde_json",
+ "sieve-rs",
+ "smtp-proto",
+ "spam-filter",
+ "store",
+ "tokio",
+ "trc",
+ "utils",
+]
+
 [[package]]
 name = "ena"
 version = "0.14.3"
@@ -3225,6 +3256,7 @@ dependencies = [
  "common",
  "dashmap",
  "directory",
+ "email",
  "imap_proto",
  "jmap",
  "jmap_proto",
@@ -3471,18 +3503,16 @@ dependencies = [
 name = "jmap"
 version = "0.11.2"
 dependencies = [
- "aes",
  "aes-gcm",
  "aes-gcm-siv",
  "async-stream",
- "async-trait",
  "base64 0.22.1",
  "bincode",
- "cbc",
  "chrono",
  "common",
  "dashmap",
  "directory",
+ "email",
  "form-data",
  "form_urlencoded",
  "futures-util",
@@ -3501,15 +3531,11 @@ dependencies = [
  "nlp",
  "p256",
  "pkcs8",
- "quick-xml 0.36.2",
+ "quick-xml 0.37.1",
  "rand 0.8.5",
- "rasn",
- "rasn-cms",
- "rasn-pkix",
  "reqwest 0.12.9",
  "rev_lines",
  "rsa",
- "sequoia-openpgp",
  "serde",
  "serde_json",
  "sha1",
@@ -3520,9 +3546,9 @@ dependencies = [
  "spam-filter",
  "store",
  "tokio",
- "tokio-tungstenite 0.24.0",
+ "tokio-tungstenite 0.26.1",
  "trc",
- "tungstenite 0.24.0",
+ "tungstenite 0.26.1",
  "utils",
  "x509-parser 0.16.0",
 ]
@@ -4786,6 +4812,7 @@ version = "0.11.2"
 dependencies = [
  "common",
  "directory",
+ "email",
  "imap",
  "jmap",
  "jmap_proto",
@@ -5070,15 +5097,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.36.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quick-xml"
 version = "0.37.1"
@@ -6443,6 +6461,7 @@ dependencies = [
  "common",
  "dashmap",
  "directory",
+ "email",
  "form_urlencoded",
  "http-body-util",
  "hyper 1.5.2",
@@ -6830,6 +6849,7 @@ dependencies = [
  "dashmap",
  "directory",
  "ece",
+ "email",
  "flate2",
  "form_urlencoded",
  "futures",
@@ -7091,14 +7111,14 @@ dependencies = [
 
 [[package]]
 name = "tokio-tungstenite"
-version = "0.24.0"
+version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
+checksum = "be4bf6fecd69fcdede0ec680aaf474cdab988f9de6bc73d3758f0160e3b7025a"
 dependencies = [
  "futures-util",
  "log",
  "tokio",
- "tungstenite 0.24.0",
+ "tungstenite 0.26.1",
 ]
 
 [[package]]
@@ -7312,9 +7332,9 @@ dependencies = [
 
 [[package]]
 name = "tungstenite"
-version = "0.24.0"
+version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
+checksum = "413083a99c579593656008130e29255e54dcaae495be556cc26888f211648c24"
 dependencies = [
  "byteorder",
  "bytes",
@@ -7324,7 +7344,7 @@ dependencies = [
  "log",
  "rand 0.8.5",
  "sha1",
- "thiserror 1.0.69",
+ "thiserror 2.0.9",
  "utf-8",
 ]
 

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "crates/main",
     "crates/jmap",
     "crates/jmap-proto",
+    "crates/email",
     "crates/imap",
     "crates/imap-proto",
     "crates/smtp",

crates/common/src/config/smtp/queue.rs

@@ -85,7 +85,6 @@ pub struct QueueOutboundTimeout {
 #[derive(Debug, Clone)]
 pub struct QueueThrottle {
     pub outbound_concurrency: usize,
-    pub local_concurrency: usize,
     pub sender: Vec<Throttle>,
     pub rcpt: Vec<Throttle>,
     pub host: Vec<Throttle>,
@@ -204,7 +203,6 @@ impl Default for QueueConfig {
             },
             throttle: QueueThrottle {
                 outbound_concurrency: 25,
-                local_concurrency: 10,
                 sender: Default::default(),
                 rcpt: Default::default(),
                 host: Default::default(),
@@ -392,10 +390,6 @@ fn parse_queue_throttle(config: &mut Config) -> QueueThrottle {
             .property_or_default::<usize>("queue.threads.remote", "25")
             .unwrap_or(25)
             .max(1),
-        local_concurrency: config
-            .property_or_default::<usize>("queue.threads.local", "10")
-            .unwrap_or(10)
-            .max(1),
     };
 
     let all_throttles = parse_throttle(

crates/common/src/core.rs

@@ -4,21 +4,33 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 
-use directory::{backend::internal::manage::ManageDirectory, Directory, Type};
+use directory::{backend::internal::manage::ManageDirectory, Directory, QueryBy, Type};
+use jmap_proto::types::{
+    blob::BlobId, collection::Collection, property::Property, state::StateChange,
+};
 use sieve::Sieve;
 use store::{
-    write::{QueueClass, ValueClass},
-    BlobStore, FtsStore, InMemoryStore, IterateParams, Store, ValueKey,
+    dispatch::DocumentSet,
+    roaring::RoaringBitmap,
+    write::{
+        key::DeserializeBigEndian, log::ChangeLogBuilder, now, BatchBuilder, BitmapClass, BlobOp,
+        DirectoryClass, QueueClass, TagValue, ValueClass,
+    },
+    BitmapKey, BlobClass, BlobStore, Deserialize, FtsStore, InMemoryStore, IterateParams, LogKey,
+    Serialize, Store, ValueKey, U32_LEN,
 };
 use trc::AddContext;
+use utils::BlobHash;
 
 use crate::{
+    auth::{AccessToken, ResourceToken, TenantInfo},
     config::smtp::{
         auth::{ArcSealer, DkimSigner},
         queue::RelayHost,
     },
+    ipc::StateEvent,
     ImapId, Inner, MailboxState, Server,
 };
 
@@ -170,6 +182,249 @@ impl Server {
         })
     }
 
+    pub async fn get_used_quota(&self, account_id: u32) -> trc::Result<i64> {
+        self.core
+            .storage
+            .data
+            .get_counter(DirectoryClass::UsedQuota(account_id))
+            .await
+            .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))
+    }
+
+    pub async fn has_available_quota(
+        &self,
+        quotas: &ResourceToken,
+        item_size: u64,
+    ) -> trc::Result<()> {
+        if quotas.quota != 0 {
+            let used_quota = self.get_used_quota(quotas.account_id).await? as u64;
+
+            if used_quota + item_size > quotas.quota {
+                return Err(trc::LimitEvent::Quota
+                    .into_err()
+                    .ctx(trc::Key::Limit, quotas.quota)
+                    .ctx(trc::Key::Size, used_quota));
+            }
+        }
+
+        // SPDX-SnippetBegin
+        // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+        // SPDX-License-Identifier: LicenseRef-SEL
+
+        #[cfg(feature = "enterprise")]
+        if self.core.is_enterprise_edition() {
+            if let Some(tenant) = quotas.tenant.filter(|tenant| tenant.quota != 0) {
+                let used_quota = self.get_used_quota(tenant.id).await? as u64;
+
+                if used_quota + item_size > tenant.quota {
+                    return Err(trc::LimitEvent::TenantQuota
+                        .into_err()
+                        .ctx(trc::Key::Limit, tenant.quota)
+                        .ctx(trc::Key::Size, used_quota));
+                }
+            }
+        }
+
+        // SPDX-SnippetEnd
+
+        Ok(())
+    }
+
+    pub async fn get_resource_token(
+        &self,
+        access_token: &AccessToken,
+        account_id: u32,
+    ) -> trc::Result<ResourceToken> {
+        Ok(if access_token.primary_id == account_id {
+            ResourceToken {
+                account_id,
+                quota: access_token.quota,
+                tenant: access_token.tenant,
+            }
+        } else {
+            let mut quotas = ResourceToken {
+                account_id,
+                ..Default::default()
+            };
+
+            if let Some(principal) = self
+                .core
+                .storage
+                .directory
+                .query(QueryBy::Id(account_id), false)
+                .await
+                .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))?
+            {
+                quotas.quota = principal.quota();
+
+                // SPDX-SnippetBegin
+                // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+                // SPDX-License-Identifier: LicenseRef-SEL
+
+                #[cfg(feature = "enterprise")]
+                if self.core.is_enterprise_edition() {
+                    if let Some(tenant_id) = principal.tenant() {
+                        quotas.tenant = TenantInfo {
+                            id: tenant_id,
+                            quota: self
+                                .core
+                                .storage
+                                .directory
+                                .query(QueryBy::Id(tenant_id), false)
+                                .await
+                                .add_context(|err| {
+                                    err.caused_by(trc::location!()).account_id(tenant_id)
+                                })?
+                                .map(|tenant| tenant.quota())
+                                .unwrap_or_default(),
+                        }
+                        .into();
+                    }
+                }
+
+                // SPDX-SnippetEnd
+            }
+
+            quotas
+        })
+    }
+
+    pub async fn get_property<U>(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        document_id: u32,
+        property: impl AsRef<Property> + Sync + Send,
+    ) -> trc::Result<Option<U>>
+    where
+        U: Deserialize + 'static,
+    {
+        let property = property.as_ref();
+
+        self.core
+            .storage
+            .data
+            .get_value::<U>(ValueKey {
+                account_id,
+                collection: collection.into(),
+                document_id,
+                class: ValueClass::Property(property.into()),
+            })
+            .await
+            .add_context(|err| {
+                err.caused_by(trc::location!())
+                    .account_id(account_id)
+                    .collection(collection)
+                    .document_id(document_id)
+                    .id(property.to_string())
+            })
+    }
+
+    pub async fn get_properties<U, I, P>(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        iterate: &I,
+        property: P,
+    ) -> trc::Result<Vec<(u32, U)>>
+    where
+        I: DocumentSet + Send + Sync,
+        P: AsRef<Property> + Sync + Send,
+        U: Deserialize + 'static,
+    {
+        let property: u8 = property.as_ref().into();
+        let collection: u8 = collection.into();
+        let expected_results = iterate.len();
+        let mut results = Vec::with_capacity(expected_results);
+
+        self.core
+            .storage
+            .data
+            .iterate(
+                IterateParams::new(
+                    ValueKey {
+                        account_id,
+                        collection,
+                        document_id: iterate.min(),
+                        class: ValueClass::Property(property),
+                    },
+                    ValueKey {
+                        account_id,
+                        collection,
+                        document_id: iterate.max(),
+                        class: ValueClass::Property(property),
+                    },
+                ),
+                |key, value| {
+                    let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?;
+                    if iterate.contains(document_id) {
+                        results.push((document_id, U::deserialize(value)?));
+                        Ok(expected_results == 0 || results.len() < expected_results)
+                    } else {
+                        Ok(true)
+                    }
+                },
+            )
+            .await
+            .add_context(|err| {
+                err.caused_by(trc::location!())
+                    .account_id(account_id)
+                    .collection(collection)
+                    .id(property.to_string())
+            })
+            .map(|_| results)
+    }
+
+    pub async fn get_document_ids(
+        &self,
+        account_id: u32,
+        collection: Collection,
+    ) -> trc::Result<Option<RoaringBitmap>> {
+        self.core
+            .storage
+            .data
+            .get_bitmap(BitmapKey::document_ids(account_id, collection))
+            .await
+            .add_context(|err| {
+                err.caused_by(trc::location!())
+                    .account_id(account_id)
+                    .collection(collection)
+            })
+    }
+
+    pub async fn get_tag(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        property: impl AsRef<Property> + Sync + Send,
+        value: impl Into<TagValue<u32>> + Sync + Send,
+    ) -> trc::Result<Option<RoaringBitmap>> {
+        let property = property.as_ref();
+        self.core
+            .storage
+            .data
+            .get_bitmap(BitmapKey {
+                account_id,
+                collection: collection.into(),
+                class: BitmapClass::Tag {
+                    field: property.into(),
+                    value: value.into(),
+                },
+                document_id: 0,
+            })
+            .await
+            .add_context(|err| {
+                err.caused_by(trc::location!())
+                    .account_id(account_id)
+                    .collection(collection)
+                    .id(property.to_string())
+            })
+    }
+
+    pub fn notify_task_queue(&self) {
+        self.inner.ipc.index_tx.notify_one();
+    }
+
     pub async fn total_queued_messages(&self) -> trc::Result<u64> {
         let mut total = 0;
         self.store()
@@ -190,6 +445,166 @@ impl Server {
             .map(|_| total)
     }
 
+    pub fn begin_changes(&self, account_id: u32) -> trc::Result<ChangeLogBuilder> {
+        self.assign_change_id(account_id)
+            .map(ChangeLogBuilder::with_change_id)
+    }
+
+    #[inline(always)]
+    pub fn assign_change_id(&self, _: u32) -> trc::Result<u64> {
+        self.generate_snowflake_id()
+    }
+
+    pub fn generate_snowflake_id(&self) -> trc::Result<u64> {
+        self.inner.data.jmap_id_gen.generate().ok_or_else(|| {
+            trc::StoreEvent::UnexpectedError
+                .into_err()
+                .caused_by(trc::location!())
+                .ctx(trc::Key::Reason, "Failed to generate snowflake id.")
+        })
+    }
+
+    pub async fn commit_changes(
+        &self,
+        account_id: u32,
+        mut changes: ChangeLogBuilder,
+    ) -> trc::Result<u64> {
+        if changes.change_id == u64::MAX || changes.change_id == 0 {
+            changes.change_id = self.assign_change_id(account_id)?;
+        }
+        let state = changes.change_id;
+
+        let mut builder = BatchBuilder::new();
+        builder.with_account_id(account_id).custom(changes);
+        self.core
+            .storage
+            .data
+            .write(builder.build())
+            .await
+            .caused_by(trc::location!())
+            .map(|_| state)
+    }
+
+    pub async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> {
+        let reference_cid = self.inner.data.jmap_id_gen.past_id(before).ok_or_else(|| {
+            trc::StoreEvent::UnexpectedError
+                .caused_by(trc::location!())
+                .ctx(trc::Key::Reason, "Failed to generate reference change id.")
+        })?;
+
+        for collection in [
+            Collection::Email,
+            Collection::Mailbox,
+            Collection::Thread,
+            Collection::Identity,
+            Collection::EmailSubmission,
+        ] {
+            self.core
+                .storage
+                .data
+                .delete_range(
+                    LogKey {
+                        account_id,
+                        collection: collection.into(),
+                        change_id: 0,
+                    },
+                    LogKey {
+                        account_id,
+                        collection: collection.into(),
+                        change_id: reference_cid,
+                    },
+                )
+                .await?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn broadcast_state_change(&self, state_change: StateChange) -> bool {
+        match self
+            .inner
+            .ipc
+            .state_tx
+            .clone()
+            .send(StateEvent::Publish { state_change })
+            .await
+        {
+            Ok(_) => true,
+            Err(_) => {
+                trc::event!(
+                    Server(trc::ServerEvent::ThreadError),
+                    Details = "Error sending state change.",
+                    CausedBy = trc::location!()
+                );
+
+                false
+            }
+        }
+    }
+
+    #[allow(clippy::blocks_in_conditions)]
+    pub async fn put_blob(
+        &self,
+        account_id: u32,
+        data: &[u8],
+        set_quota: bool,
+    ) -> trc::Result<BlobId> {
+        // First reserve the hash
+        let hash = BlobHash::from(data);
+        let mut batch = BatchBuilder::new();
+        let until = now() + self.core.jmap.upload_tmp_ttl;
+
+        batch.with_account_id(account_id).set(
+            BlobOp::Reserve {
+                hash: hash.clone(),
+                until,
+            },
+            (if set_quota { data.len() as u32 } else { 0u32 }).serialize(),
+        );
+        self.core
+            .storage
+            .data
+            .write(batch.build())
+            .await
+            .caused_by(trc::location!())?;
+
+        if !self
+            .core
+            .storage
+            .data
+            .blob_exists(&hash)
+            .await
+            .caused_by(trc::location!())?
+        {
+            // Upload blob to store
+            self.core
+                .storage
+                .blob
+                .put_blob(hash.as_ref(), data)
+                .await
+                .caused_by(trc::location!())?;
+
+            // Commit blob
+            let mut batch = BatchBuilder::new();
+            batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new());
+            self.core
+                .storage
+                .data
+                .write(batch.build())
+                .await
+                .caused_by(trc::location!())?;
+        }
+
+        Ok(BlobId {
+            hash,
+            class: BlobClass::Reserved {
+                account_id,
+                expires: until,
+            },
+            section: None,
+        })
+    }
+
     pub async fn total_accounts(&self) -> trc::Result<u64> {
         self.store()
             .count_principals(None, Type::Individual.into(), None)

crates/common/src/ipc.rs

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::{borrow::Cow, sync::Arc, time::Instant};
+use std::{sync::Arc, time::Instant};
 
 use ahash::RandomState;
 use jmap_proto::types::{state::StateChange, type_state::DataType};
@@ -14,8 +14,8 @@ use mail_auth::{
     report::{tlsrpt::FailureDetails, Record},
 };
 use store::{BlobStore, InMemoryStore, Store};
-use tokio::sync::{mpsc, oneshot};
-use utils::{map::bitmap::Bitmap, BlobHash};
+use tokio::sync::mpsc;
+use utils::map::bitmap::Bitmap;
 
 use crate::{
     config::smtp::{
@@ -25,36 +25,6 @@ use crate::{
     listener::limiter::ConcurrencyLimiter,
 };
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum DeliveryResult {
-    Success,
-    TemporaryFailure {
-        reason: Cow<'static, str>,
-    },
-    PermanentFailure {
-        code: [u8; 3],
-        reason: Cow<'static, str>,
-    },
-}
-
-#[derive(Debug)]
-pub enum DeliveryEvent {
-    Ingest {
-        message: IngestMessage,
-        result_tx: oneshot::Sender<Vec<DeliveryResult>>,
-    },
-    Stop,
-}
-
-#[derive(Debug)]
-pub struct IngestMessage {
-    pub sender_address: String,
-    pub recipients: Vec<String>,
-    pub message_blob: BlobHash,
-    pub message_size: usize,
-    pub session_id: u64,
-}
-
 pub enum HousekeeperEvent {
     AcmeReschedule {
         provider_id: String,

crates/common/src/lib.rs

@@ -33,7 +33,7 @@ use config::{
 use dashmap::DashMap;
 
 use imap_proto::protocol::list::Attribute;
-use ipc::{DeliveryEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent};
+use ipc::{HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent};
 use listener::{
     asn::AsnGeoLookupData, blocked::Security, limiter::ConcurrencyLimiter, tls::AcmeProviders,
 };
@@ -43,7 +43,7 @@ use manager::webadmin::{Resource, WebAdminManager};
 use nlp::bayes::{TokenHash, Weights};
 use parking_lot::{Mutex, RwLock};
 use rustls::sign::CertifiedKey;
-use tokio::sync::{mpsc, Notify};
+use tokio::sync::{mpsc, Notify, Semaphore};
 use tokio_rustls::TlsConnector;
 use utils::{
     cache::{Cache, CacheItemWeight, CacheWithTtl},
@@ -167,10 +167,10 @@ pub struct HttpAuthCache {
 pub struct Ipc {
     pub state_tx: mpsc::Sender<StateEvent>,
     pub housekeeper_tx: mpsc::Sender<HousekeeperEvent>,
-    pub delivery_tx: mpsc::Sender<DeliveryEvent>,
     pub index_tx: Arc<Notify>,
     pub queue_tx: mpsc::Sender<QueueEvent>,
     pub report_tx: mpsc::Sender<ReportingEvent>,
+    pub local_delivery_sm: Arc<Semaphore>,
 }
 
 pub struct TlsConnectors {
@@ -442,10 +442,10 @@ impl Default for Ipc {
         Self {
             state_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0,
             housekeeper_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0,
-            delivery_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0,
             index_tx: Default::default(),
             queue_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0,
             report_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0,
+            local_delivery_sm: Arc::new(Semaphore::new(10)),
         }
     }
 }

crates/common/src/manager/boot.rs

@@ -16,7 +16,7 @@ use store::{
     rand::{distributions::Alphanumeric, thread_rng, Rng},
     Stores,
 };
-use tokio::sync::{mpsc, Notify};
+use tokio::sync::{mpsc, Notify, Semaphore};
 use utils::{
     config::{Config, ConfigKey},
     failed, Semver, UnwrapFailure,
@@ -25,7 +25,7 @@ use utils::{
 use crate::{
     config::{network::AsnGeoLookupConfig, server::Listeners, telemetry::Telemetry},
     core::BuildServer,
-    ipc::{DeliveryEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent},
+    ipc::{HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent},
     Caches, Core, Data, Inner, Ipc, IPC_CHANNEL_BUFFER,
 };
 
@@ -46,7 +46,6 @@ pub struct BootManager {
 pub struct IpcReceivers {
     pub state_rx: Option<mpsc::Receiver<StateEvent>>,
     pub housekeeper_rx: Option<mpsc::Receiver<HousekeeperEvent>>,
-    pub delivery_rx: Option<mpsc::Receiver<DeliveryEvent>>,
     pub queue_rx: Option<mpsc::Receiver<QueueEvent>>,
     pub report_rx: Option<mpsc::Receiver<ReportingEvent>>,
 }
@@ -427,7 +426,7 @@ impl BootManager {
                     core.network.asn_geo_lookup,
                     AsnGeoLookupConfig::Resource { .. }
                 );
-                let (ipc, ipc_rxs) = build_ipc();
+                let (ipc, ipc_rxs) = build_ipc(&mut config);
                 let inner = Arc::new(Inner {
                     shared_core: ArcSwap::from_pointee(core),
                     data,
@@ -484,9 +483,8 @@ impl BootManager {
     }
 }
 
-pub fn build_ipc() -> (Ipc, IpcReceivers) {
+pub fn build_ipc(config: &mut Config) -> (Ipc, IpcReceivers) {
     // Build ipc receivers
-    let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER);
     let (state_tx, state_rx) = mpsc::channel(IPC_CHANNEL_BUFFER);
     let (housekeeper_tx, housekeeper_rx) = mpsc::channel(IPC_CHANNEL_BUFFER);
     let (queue_tx, queue_rx) = mpsc::channel(IPC_CHANNEL_BUFFER);
@@ -495,15 +493,19 @@ pub fn build_ipc() -> (Ipc, IpcReceivers) {
         Ipc {
             state_tx,
             housekeeper_tx,
-            delivery_tx,
             queue_tx,
             report_tx,
             index_tx: Arc::new(Notify::new()),
+            local_delivery_sm: Arc::new(Semaphore::new(
+                config
+                    .property_or_default::<usize>("queue.threads.local", "10")
+                    .unwrap_or(10)
+                    .max(1),
+            )),
         },
         IpcReceivers {
             state_rx: Some(state_rx),
             housekeeper_rx: Some(housekeeper_rx),
-            delivery_rx: Some(delivery_rx),
             queue_rx: Some(queue_rx),
             report_rx: Some(report_rx),
         },

crates/email/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "email"
+version = "0.11.2"
+edition = "2021"
+resolver = "2"
+
+[dependencies]
+utils = { path = "../utils" }
+nlp = { path = "../nlp" }
+store = { path = "../store" }
+trc = { path = "../trc" }
+jmap_proto = { path = "../jmap-proto" }
+common = { path =  "../common" }
+directory = { path =  "../directory" }
+spam-filter = { path =  "../spam-filter" }
+smtp-proto = { version = "0.1", features = ["serde_support"] }
+mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } 
+mail-builder = { version = "0.3", features = ["ludicrous_mode"] }
+sieve-rs = { version = "0.5" } 
+tokio = { version = "1.23", features = ["net", "macros"] }
+serde = { version = "1.0", features = ["derive"]}
+serde_json = "1.0"
+bincode = "1.3.3"
+aes = "0.8.3"
+aes-gcm = "0.10.1"
+aes-gcm-siv = "0.11.1"
+cbc = { version = "0.1.2", features = ["alloc"] }
+rasn = "0.10"
+rasn-cms = "0.10"
+rasn-pkix = "0.10"
+rsa = "0.9.2"
+sequoia-openpgp = { version = "1.16", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] }
+
+[features]
+test_mode = []
+enterprise = []
+
+[dev-dependencies]
+tokio = { version = "1.23", features = ["full"] }

crates/email/src/cache.rs

@@ -11,8 +11,6 @@ use jmap_proto::types::{collection::Collection, property::Property};
 use std::future::Future;
 use trc::AddContext;
 
-use crate::JmapMethods;
-
 pub trait ThreadCache: Sync + Send {
     fn get_cached_thread_ids(
         &self,

crates/email/src/crypto.rs

@@ -0,0 +1,670 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{borrow::Cow, collections::BTreeSet, fmt::Display, io::Cursor};
+
+use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit};
+
+use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary};
+use mail_parser::{decoders::base64::base64_decode, Message, MimeHeaders, PartType};
+use openpgp::{
+    parse::Parse,
+    serialize::stream,
+    types::{KeyFlags, SymmetricAlgorithm},
+};
+use rasn::types::{ObjectIdentifier, OctetString};
+use rasn_cms::{
+    algorithms::{AES128_CBC, AES256_CBC, RSA},
+    pkcs7_compat::EncapsulatedContentInfo,
+    AlgorithmIdentifier, EncryptedContent, EncryptedContentInfo, EncryptedKey, EnvelopedData,
+    IssuerAndSerialNumber, KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo, CONTENT_DATA,
+    CONTENT_ENVELOPED_DATA,
+};
+use rsa::{pkcs1::DecodeRsaPublicKey, Pkcs1v15Encrypt, RsaPublicKey};
+use sequoia_openpgp as openpgp;
+use store::rand::{rngs::StdRng, RngCore, SeedableRng};
+use store::{
+    write::{Bincode, ToBitmaps},
+    Deserialize, Serialize,
+};
+
+const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new();
+
+#[derive(Debug)]
+pub enum EncryptMessageError {
+    AlreadyEncrypted,
+    Error(String),
+}
+
+#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
+pub enum Algorithm {
+    Aes128,
+    Aes256,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub enum EncryptionMethod {
+    PGP,
+    SMIME,
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct EncryptionParams {
+    pub method: EncryptionMethod,
+    pub algo: Algorithm,
+    pub certs: Vec<Vec<u8>>,
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize, Default)]
+#[serde(tag = "type")]
+#[serde(rename_all = "camelCase")]
+pub enum EncryptionType {
+    PGP {
+        algo: Algorithm,
+        certs: String,
+    },
+    SMIME {
+        algo: Algorithm,
+        certs: String,
+    },
+    #[default]
+    Disabled,
+}
+
+#[allow(async_fn_in_trait)]
+pub trait EncryptMessage {
+    async fn encrypt(&self, params: &EncryptionParams) -> Result<Vec<u8>, EncryptMessageError>;
+    fn is_encrypted(&self) -> bool;
+}
+
+impl EncryptMessage for Message<'_> {
+    async fn encrypt(&self, params: &EncryptionParams) -> Result<Vec<u8>, EncryptMessageError> {
+        let root = self.root_part();
+        let raw_message = self.raw_message();
+        let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize);
+        let mut inner_message = Vec::with_capacity(raw_message.len());
+
+        // Move MIME headers and body to inner message
+        for header in root.headers() {
+            (if header.name.is_mime_header() {
+                &mut inner_message
+            } else {
+                &mut outer_message
+            })
+            .extend_from_slice(&raw_message[header.offset_field()..header.offset_end()]);
+        }
+        inner_message.extend_from_slice(b"\r\n");
+        inner_message.extend_from_slice(&raw_message[root.raw_body_offset()..]);
+
+        // Encrypt inner message
+        match params.method {
+            EncryptionMethod::PGP => {
+                // Prepare encrypted message
+                let boundary = make_boundary("_");
+                outer_message.extend_from_slice(
+                    concat!(
+                        "Content-Type: multipart/encrypted;\r\n\t",
+                        "protocol=\"application/pgp-encrypted\";\r\n\t",
+                        "boundary=\""
+                    )
+                    .as_bytes(),
+                );
+                outer_message.extend_from_slice(boundary.as_bytes());
+                outer_message.extend_from_slice(
+                    concat!(
+                        "\"\r\n\r\n",
+                        "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n",
+                        "--"
+                    )
+                    .as_bytes(),
+                );
+                outer_message.extend_from_slice(boundary.as_bytes());
+                outer_message.extend_from_slice(
+                    concat!(
+                        "\r\nContent-Type: application/pgp-encrypted\r\n\r\n",
+                        "Version: 1\r\n\r\n--"
+                    )
+                    .as_bytes(),
+                );
+                outer_message.extend_from_slice(boundary.as_bytes());
+                outer_message.extend_from_slice(
+                    concat!(
+                        "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n",
+                        "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n"
+                    )
+                    .as_bytes(),
+                );
+
+                let certs = params
+                    .certs
+                    .iter()
+                    .map(openpgp::Cert::from_bytes)
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(|err| {
+                        EncryptMessageError::Error(format!(
+                            "Failed to parse OpenPGP public key: {}",
+                            err
+                        ))
+                    })?;
+
+                // Encrypt contents (TODO: use rayon)
+                let algo = params.algo;
+                let encrypted_contents = tokio::task::spawn_blocking(move || {
+                    // Parse public key
+                    let mut keys = Vec::with_capacity(certs.len());
+                    let policy = openpgp::policy::StandardPolicy::new();
+
+                    for cert in &certs {
+                        for key in cert
+                            .keys()
+                            .with_policy(&policy, None)
+                            .supported()
+                            .alive()
+                            .revoked(false)
+                            .key_flags(KeyFlags::empty().set_transport_encryption())
+                        {
+                            keys.push(key);
+                        }
+                    }
+
+                    // Compose a writer stack corresponding to the output format and
+                    // packet structure we want.
+                    let mut sink = Vec::with_capacity(inner_message.len());
+
+                    // Stream an OpenPGP message.
+                    let message = stream::Armorer::new(stream::Message::new(&mut sink))
+                        .build()
+                        .map_err(|err| {
+                            EncryptMessageError::Error(format!("Failed to create armorer: {}", err))
+                        })?;
+                    let message = stream::Encryptor2::for_recipients(message, keys)
+                        .symmetric_algo(match algo {
+                            Algorithm::Aes128 => SymmetricAlgorithm::AES128,
+                            Algorithm::Aes256 => SymmetricAlgorithm::AES256,
+                        })
+                        .build()
+                        .map_err(|err| {
+                            EncryptMessageError::Error(format!(
+                                "Failed to build encryptor: {}",
+                                err
+                            ))
+                        })?;
+                    let mut message =
+                        stream::LiteralWriter::new(message).build().map_err(|err| {
+                            EncryptMessageError::Error(format!(
+                                "Failed to create literal writer: {}",
+                                err
+                            ))
+                        })?;
+                    std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err(
+                        |err| {
+                            EncryptMessageError::Error(format!(
+                                "Failed to encrypt message: {}",
+                                err
+                            ))
+                        },
+                    )?;
+                    message.finalize().map_err(|err| {
+                        EncryptMessageError::Error(format!("Failed to finalize message: {}", err))
+                    })?;
+
+                    String::from_utf8(sink).map_err(|err| {
+                        EncryptMessageError::Error(format!(
+                            "Failed to convert encrypted message to UTF-8: {}",
+                            err
+                        ))
+                    })
+                })
+                .await
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
+                })??;
+                outer_message.extend_from_slice(encrypted_contents.as_bytes());
+                outer_message.extend_from_slice(b"\r\n--");
+                outer_message.extend_from_slice(boundary.as_bytes());
+                outer_message.extend_from_slice(b"--\r\n");
+            }
+            EncryptionMethod::SMIME => {
+                // Generate random IV
+                let mut rng = StdRng::from_entropy();
+                let mut iv = vec![0u8; 16];
+                rng.fill_bytes(&mut iv);
+
+                // Generate random key
+                let mut key = vec![0u8; params.algo.key_size()];
+                rng.fill_bytes(&mut key);
+
+                // Encrypt contents (TODO: use rayon)
+                let algo = params.algo;
+                let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || {
+                    (algo.encrypt(&key, &iv, &inner_message), key, iv)
+                })
+                .await
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
+                })?;
+
+                // Encrypt key using public keys
+                #[allow(clippy::mutable_key_type)]
+                let mut recipient_infos = BTreeSet::new();
+                for cert in &params.certs {
+                    let cert =
+                        rasn::der::decode::<rasn_pkix::Certificate>(cert).map_err(|err| {
+                            EncryptMessageError::Error(format!(
+                                "Failed to parse certificate: {}",
+                                err
+                            ))
+                        })?;
+
+                    let public_key = RsaPublicKey::from_pkcs1_der(
+                        cert.tbs_certificate
+                            .subject_public_key_info
+                            .subject_public_key
+                            .as_raw_slice(),
+                    )
+                    .map_err(|err| {
+                        EncryptMessageError::Error(format!("Failed to parse public key: {}", err))
+                    })?;
+                    let encrypted_key = public_key
+                        .encrypt(&mut rng, Pkcs1v15Encrypt, &key[..])
+                        .map_err(|err| {
+                            EncryptMessageError::Error(format!("Failed to encrypt key: {}", err))
+                        })
+                        .unwrap();
+
+                    recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo(
+                        KeyTransRecipientInfo {
+                            version: 0.into(),
+                            rid: RecipientIdentifier::IssuerAndSerialNumber(
+                                IssuerAndSerialNumber {
+                                    issuer: cert.tbs_certificate.issuer,
+                                    serial_number: cert.tbs_certificate.serial_number,
+                                },
+                            ),
+                            key_encryption_algorithm: AlgorithmIdentifier {
+                                algorithm: RSA.into(),
+                                parameters: Some(
+                                    rasn::der::encode(&())
+                                        .map_err(|err| {
+                                            EncryptMessageError::Error(format!(
+                                                "Failed to encode RSA algorithm identifier: {}",
+                                                err
+                                            ))
+                                        })?
+                                        .into(),
+                                ),
+                            },
+                            encrypted_key: EncryptedKey::from(encrypted_key),
+                        },
+                    ));
+                }
+
+                let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo {
+                    content_type: CONTENT_ENVELOPED_DATA.into(),
+                    content: Some(
+                        rasn::der::encode(&EnvelopedData {
+                            version: 0.into(),
+                            originator_info: None,
+                            recipient_infos,
+                            encrypted_content_info: EncryptedContentInfo {
+                                content_type: CONTENT_DATA.into(),
+                                content_encryption_algorithm: AlgorithmIdentifier {
+                                    algorithm: params.algo.to_algorithm_identifier(),
+                                    parameters: Some(
+                                        rasn::der::encode(&OctetString::from(iv))
+                                            .map_err(|err| {
+                                                EncryptMessageError::Error(format!(
+                                                    "Failed to encode IV: {}",
+                                                    err
+                                                ))
+                                            })?
+                                            .into(),
+                                    ),
+                                },
+                                encrypted_content: Some(EncryptedContent::from(encrypted_contents)),
+                            },
+                            unprotected_attrs: None,
+                        })
+                        .map_err(|err| {
+                            EncryptMessageError::Error(format!(
+                                "Failed to encode EnvelopedData: {}",
+                                err
+                            ))
+                        })?
+                        .into(),
+                    ),
+                })
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err))
+                })?;
+
+                // Generate message
+                outer_message.extend_from_slice(
+                    concat!(
+                        "Content-Type: application/pkcs7-mime;\r\n",
+                        "\tname=\"smime.p7m\";\r\n",
+                        "\tsmime-type=enveloped-data\r\n",
+                        "Content-Disposition: attachment;\r\n",
+                        "\tfilename=\"smime.p7m\"\r\n",
+                        "Content-Transfer-Encoding: base64\r\n\r\n"
+                    )
+                    .as_bytes(),
+                );
+                base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err))
+                })?;
+            }
+        }
+
+        Ok(outer_message)
+    }
+
+    fn is_encrypted(&self) -> bool {
+        if self.content_type().is_some_and(|ct| {
+            let main_type = ct.c_type.as_ref();
+            let sub_type = ct
+                .c_subtype
+                .as_ref()
+                .map(|s| s.as_ref())
+                .unwrap_or_default();
+
+            (main_type.eq_ignore_ascii_case("application")
+                && (sub_type.eq_ignore_ascii_case("pkcs7-mime")
+                    || sub_type.eq_ignore_ascii_case("pkcs7-signature")
+                    || (sub_type.eq_ignore_ascii_case("octet-stream")
+                        && self.attachment_name().is_some_and(|name| {
+                            name.rsplit_once('.')
+                                .is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext))
+                        }))))
+                || (main_type.eq_ignore_ascii_case("multipart")
+                    && sub_type.eq_ignore_ascii_case("encrypted"))
+        }) {
+            return true;
+        }
+
+        if self.parts.len() <= 2 {
+            let mut text_part = None;
+            let mut is_multipart = false;
+
+            for part in &self.parts {
+                match &part.body {
+                    PartType::Text(text) => {
+                        text_part = Some(text.as_ref());
+                    }
+                    PartType::Multipart(_) => {
+                        is_multipart = true;
+                    }
+                    _ => (),
+                }
+            }
+
+            match text_part {
+                Some(text) if self.parts.len() == 1 || is_multipart => {
+                    if text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") {
+                        return true;
+                    }
+                }
+                _ => (),
+            }
+        }
+
+        false
+    }
+}
+
+impl Algorithm {
+    fn key_size(&self) -> usize {
+        match self {
+            Algorithm::Aes128 => 16,
+            Algorithm::Aes256 => 32,
+        }
+    }
+
+    fn to_algorithm_identifier(self) -> ObjectIdentifier {
+        match self {
+            Algorithm::Aes128 => AES128_CBC.into(),
+            Algorithm::Aes256 => AES256_CBC.into(),
+        }
+    }
+
+    fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec<u8> {
+        match self {
+            Algorithm::Aes128 => cbc::Encryptor::<aes::Aes128>::new(key.into(), iv.into())
+                .encrypt_padded_vec_mut::<Pkcs7>(contents),
+            Algorithm::Aes256 => cbc::Encryptor::<aes::Aes256>::new(key.into(), iv.into())
+                .encrypt_padded_vec_mut::<Pkcs7>(contents),
+        }
+    }
+}
+
+pub fn try_parse_certs(
+    expected_method: EncryptionMethod,
+    cert: Vec<u8>,
+) -> Result<Vec<Vec<u8>>, Cow<'static, str>> {
+    // Check if it's a PEM file
+    let (method, certs) = if let Some(result) = try_parse_pem(&cert)? {
+        result
+    } else if rasn::der::decode::<rasn_pkix::Certificate>(&cert[..]).is_ok() {
+        (EncryptionMethod::SMIME, vec![cert])
+    } else if let Ok(cert_) = openpgp::Cert::from_bytes(&cert[..]) {
+        if !has_pgp_keys(cert_) {
+            (EncryptionMethod::PGP, vec![cert])
+        } else {
+            return Err("Could not find any suitable keys in certificate".into());
+        }
+    } else {
+        return Err("Could not find any valid certificates".into());
+    };
+
+    if method == expected_method {
+        Ok(certs)
+    } else {
+        Err("No valid certificates found for the selected encryption".into())
+    }
+}
+
+fn has_pgp_keys(cert: openpgp::Cert) -> bool {
+    cert.keys()
+        .with_policy(&P, None)
+        .supported()
+        .alive()
+        .revoked(false)
+        .key_flags(KeyFlags::empty().set_transport_encryption())
+        .next()
+        .is_some()
+}
+
+#[allow(clippy::type_complexity)]
+fn try_parse_pem(
+    bytes_: &[u8],
+) -> Result<Option<(EncryptionMethod, Vec<Vec<u8>>)>, Cow<'static, str>> {
+    if let Some(internal) = std::str::from_utf8(bytes_)
+        .ok()
+        .and_then(|cert| cert.strip_prefix("-----STALWART CERTIFICATE-----"))
+    {
+        return base64_decode(internal.as_bytes())
+            .ok_or(Cow::from("Failed to decode base64"))
+            .and_then(|bytes| {
+                Bincode::<EncryptionParams>::deserialize(&bytes)
+                    .map_err(|_| Cow::from("Failed to deserialize internal certificate"))
+            })
+            .map(|params| Some((params.inner.method, params.inner.certs)));
+    }
+
+    let mut bytes = bytes_.iter().enumerate();
+    let mut buf = vec![];
+    let mut method = None;
+    let mut certs = vec![];
+
+    loop {
+        // Find start of PEM block
+        let mut start_pos = 0;
+        for (pos, &ch) in bytes.by_ref() {
+            if ch.is_ascii_whitespace() {
+                continue;
+            } else if ch == b'-' {
+                start_pos = pos;
+                break;
+            } else {
+                return Ok(None);
+            }
+        }
+
+        // Find block type
+        for (_, &ch) in bytes.by_ref() {
+            match ch {
+                b'-' => (),
+                b'\n' => break,
+                _ => {
+                    if ch.is_ascii() {
+                        buf.push(ch.to_ascii_uppercase());
+                    } else {
+                        return Ok(None);
+                    }
+                }
+            }
+        }
+        if buf.is_empty() {
+            break;
+        }
+
+        // Find type
+        let tag = std::str::from_utf8(&buf).unwrap();
+        if tag.contains("CERTIFICATE") {
+            if method.is_some_and(|m| m == EncryptionMethod::PGP) {
+                return Err("Cannot mix OpenPGP and S/MIME certificates".into());
+            } else {
+                method = Some(EncryptionMethod::SMIME);
+            }
+        } else if tag.contains("PGP") {
+            if method.is_some_and(|m| m == EncryptionMethod::SMIME) {
+                return Err("Cannot mix OpenPGP and S/MIME certificates".into());
+            } else {
+                method = Some(EncryptionMethod::PGP);
+            }
+        } else {
+            // Ignore block
+            let mut found_end = false;
+            for (_, &ch) in bytes.by_ref() {
+                if ch == b'-' {
+                    found_end = true;
+                } else if ch == b'\n' && found_end {
+                    break;
+                }
+            }
+            buf.clear();
+            continue;
+        }
+
+        // Collect base64
+        buf.clear();
+        let mut found_end = false;
+        let mut end_pos = 0;
+        for (pos, &ch) in bytes.by_ref() {
+            match ch {
+                b'-' => {
+                    found_end = true;
+                }
+                b'\n' => {
+                    if found_end {
+                        end_pos = pos;
+                        break;
+                    }
+                }
+                _ => {
+                    if !ch.is_ascii_whitespace() {
+                        buf.push(ch);
+                    }
+                }
+            }
+        }
+
+        // Decode base64
+        let cert =
+            base64_decode(&buf).ok_or_else(|| Cow::from("Failed to decode base64 certificate."))?;
+        match method.unwrap() {
+            EncryptionMethod::PGP => match openpgp::Cert::from_bytes(bytes_) {
+                Ok(cert) => {
+                    if !has_pgp_keys(cert) {
+                        return Err("Could not find any suitable keys in OpenPGP public key".into());
+                    }
+                    certs.push(
+                        bytes_
+                            .get(start_pos..end_pos + 1)
+                            .unwrap_or_default()
+                            .to_vec(),
+                    );
+                }
+                Err(err) => {
+                    return Err(format!("Failed to decode OpenPGP public key: {err}").into())
+                }
+            },
+            EncryptionMethod::SMIME => {
+                if let Err(err) = rasn::der::decode::<rasn_pkix::Certificate>(&cert) {
+                    return Err(format!("Failed to decode X509 certificate: {err}").into());
+                }
+                certs.push(cert);
+            }
+        }
+        buf.clear();
+    }
+
+    Ok(method.map(|method| (method, certs)))
+}
+
+impl Serialize for &EncryptionParams {
+    fn serialize(self) -> Vec<u8> {
+        let len = bincode::serialized_size(&self).unwrap_or_default();
+        let mut buf = Vec::with_capacity(len as usize + 1);
+        buf.push(1);
+        let _ = bincode::serialize_into(&mut buf, &self);
+        buf
+    }
+}
+
+impl Deserialize for EncryptionParams {
+    fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
+        let version = *bytes
+            .first()
+            .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?;
+        match version {
+            1 if bytes.len() > 1 => bincode::deserialize(&bytes[1..]).map_err(|err| {
+                trc::EventType::Store(trc::StoreEvent::DeserializeError)
+                    .from_bincode_error(err)
+                    .caused_by(trc::location!())
+            }),
+
+            _ => Err(trc::StoreEvent::DeserializeError
+                .into_err()
+                .caused_by(trc::location!())
+                .ctx(trc::Key::Value, version as u64)),
+        }
+    }
+}
+
+impl ToBitmaps for &EncryptionParams {
+    fn to_bitmaps(&self, _: &mut Vec<store::write::Operation>, _: u8, _: bool) {
+        unreachable!()
+    }
+}
+
+impl Display for EncryptionMethod {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            EncryptionMethod::PGP => write!(f, "OpenPGP"),
+            EncryptionMethod::SMIME => write!(f, "S/MIME"),
+        }
+    }
+}
+
+impl Display for Algorithm {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Algorithm::Aes128 => write!(f, "AES-128"),
+            Algorithm::Aes256 => write!(f, "AES-256"),
+        }
+    }
+}

crates/email/src/delivery.rs

@@ -4,36 +4,128 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use common::{
-    ipc::{DeliveryResult, IngestMessage},
-    Server,
-};
+use common::Server;
 use directory::Permission;
 use jmap_proto::types::{state::StateChange, type_state::DataType};
 use mail_parser::MessageParser;
-use std::future::Future;
+use std::{borrow::Cow, future::Future};
 use store::ahash::AHashMap;
+use utils::BlobHash;
 
 use crate::{
-    email::{
-        bayes::EmailBayesTrain,
-        ingest::{EmailIngest, IngestEmail, IngestSource},
-    },
+    ingest::{EmailIngest, IngestEmail, IngestSource},
     mailbox::INBOX_ID,
-    sieve::{get::SieveScriptGet, ingest::SieveScriptIngest},
+    sieve::SieveScriptIngest,
 };
 
-use super::state::StateManager;
+#[derive(Debug)]
+pub struct IngestMessage {
+    pub sender_address: String,
+    pub recipients: Vec<String>,
+    pub message_blob: BlobHash,
+    pub message_size: usize,
+    pub session_id: u64,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum LocalDeliveryStatus {
+    Success,
+    TemporaryFailure {
+        reason: Cow<'static, str>,
+    },
+    PermanentFailure {
+        code: [u8; 3],
+        reason: Cow<'static, str>,
+    },
+}
+
+pub struct LocalDeliveryResult {
+    pub status: Vec<LocalDeliveryStatus>,
+    pub autogenerated: Vec<AutogeneratedMessage>,
+}
+
+pub struct AutogeneratedMessage {
+    pub sender_address: String,
+    pub recipients: Vec<String>,
+    pub message: Vec<u8>,
+}
 
 pub trait MailDelivery: Sync + Send {
     fn deliver_message(
         &self,
         message: IngestMessage,
-    ) -> impl Future<Output = Vec<DeliveryResult>> + Send;
+    ) -> impl Future<Output = LocalDeliveryResult> + Send;
 }
 
+/*
+
+let semaphore = Arc::new(Semaphore::new(
+            inner
+                .shared_core
+                .load()
+                .smtp
+                .queue
+                .throttle
+                .local_concurrency,
+        ));
+
+        loop {
+            let permit = match semaphore.clone().acquire_owned().await {
+                Ok(permit) => permit,
+                Err(_) => {
+                    trc::error!(trc::StoreEvent::UnexpectedError
+                        .into_err()
+                        .details("Semaphore error")
+                        .caused_by(trc::location!()));
+                    break;
+                }
+            };
+
+            match delivery_rx.recv().await {
+                Some(event) => match event {
+                    DeliveryEvent::Ingest { message, result_tx } => {
+                        let server = inner.build_server();
+
+                        tokio::spawn(async move {
+                            result_tx.send(server.deliver_message(message).await).ok();
+
+                            drop(permit);
+                        });
+                    }
+                    DeliveryEvent::Stop => break,
+                },
+                None => {
+                    break;
+                }
+            }
+        }
+
+*/
+
 impl MailDelivery for Server {
-    async fn deliver_message(&self, message: IngestMessage) -> Vec<DeliveryResult> {
+    async fn deliver_message(&self, message: IngestMessage) -> LocalDeliveryResult {
+        // Obtain permit
+        let _permit = match self.inner.ipc.local_delivery_sm.acquire().await {
+            Ok(permit) => permit,
+            Err(_) => {
+                trc::error!(
+                    trc::Error::new(trc::EventType::Server(trc::ServerEvent::ThreadError))
+                        .details("Failed to obtain semaphore permit.")
+                        .span_id(message.session_id)
+                        .caused_by(trc::location!())
+                );
+
+                return LocalDeliveryResult {
+                    status: (0..message.recipients.len())
+                        .map(|_| LocalDeliveryStatus::TemporaryFailure {
+                            reason: "Temporary I/O error.".into(),
+                        })
+                        .collect::<Vec<_>>(),
+                    autogenerated: vec![],
+                };
+            }
+        };
+
         // Read message
         let raw_message = match self
             .core
@@ -51,11 +143,14 @@ impl MailDelivery for Server {
                     CausedBy = trc::location!()
                 );
 
-                return (0..message.recipients.len())
-                    .map(|_| DeliveryResult::TemporaryFailure {
-                        reason: "Blob not found.".into(),
-                    })
-                    .collect::<Vec<_>>();
+                return LocalDeliveryResult {
+                    status: (0..message.recipients.len())
+                        .map(|_| LocalDeliveryStatus::TemporaryFailure {
+                            reason: "Blob not found.".into(),
+                        })
+                        .collect::<Vec<_>>(),
+                    autogenerated: vec![],
+                };
             }
             Err(err) => {
                 trc::error!(err
@@ -63,17 +158,24 @@ impl MailDelivery for Server {
                     .span_id(message.session_id)
                     .caused_by(trc::location!()));
 
-                return (0..message.recipients.len())
-                    .map(|_| DeliveryResult::TemporaryFailure {
-                        reason: "Temporary I/O error.".into(),
-                    })
-                    .collect::<Vec<_>>();
+                return LocalDeliveryResult {
+                    status: (0..message.recipients.len())
+                        .map(|_| LocalDeliveryStatus::TemporaryFailure {
+                            reason: "Temporary I/O error.".into(),
+                        })
+                        .collect::<Vec<_>>(),
+                    autogenerated: vec![],
+                };
             }
         };
 
         // Obtain the UIDs for each recipient
         let mut uids: AHashMap<u32, usize> = AHashMap::with_capacity(message.recipients.len());
-        let mut results = Vec::with_capacity(message.recipients.len());
+        let mut result = LocalDeliveryResult {
+            status: Vec::with_capacity(message.recipients.len()),
+            autogenerated: Vec::new(),
+        };
+
         for rcpt in message.recipients {
             let uid = match self
                 .email_to_id(&self.core.storage.directory, &rcpt, message.session_id)
@@ -82,7 +184,7 @@ impl MailDelivery for Server {
                 Ok(Some(uid)) => uid,
                 Ok(None) => {
                     // Something went wrong
-                    results.push(DeliveryResult::PermanentFailure {
+                    result.status.push(LocalDeliveryStatus::PermanentFailure {
                         code: [5, 5, 0],
                         reason: "Mailbox not found.".into(),
                     });
@@ -94,19 +196,19 @@ impl MailDelivery for Server {
                         .ctx(trc::Key::To, rcpt)
                         .span_id(message.session_id)
                         .caused_by(trc::location!()));
-                    results.push(DeliveryResult::TemporaryFailure {
+                    result.status.push(LocalDeliveryStatus::TemporaryFailure {
                         reason: "Address lookup failed.".into(),
                     });
                     continue;
                 }
             };
-            if let Some(result) = uids.get(&uid).and_then(|pos| results.get(*pos)) {
-                results.push(result.clone());
+            if let Some(status) = uids.get(&uid).and_then(|pos| result.status.get(*pos)) {
+                result.status.push(status.clone());
                 continue;
             }
 
             // Obtain access token
-            let result = match self.get_access_token(uid).await.and_then(|token| {
+            let status = match self.get_access_token(uid).await.and_then(|token| {
                 token
                     .assert_has_permission(Permission::EmailReceive)
                     .map(|_| token)
@@ -114,17 +216,6 @@ impl MailDelivery for Server {
                 Ok(access_token) => {
                     // Check if there is an active sieve script
                     match self.sieve_script_get_active(uid).await {
-                        Ok(Some(active_script)) => {
-                            self.sieve_script_ingest(
-                                &access_token,
-                                &raw_message,
-                                &message.sender_address,
-                                &rcpt,
-                                message.session_id,
-                                active_script,
-                            )
-                            .await
-                        }
                         Ok(None) => {
                             // Ingest message
                             self.email_ingest(IngestEmail {
@@ -142,6 +233,18 @@ impl MailDelivery for Server {
                             })
                             .await
                         }
+                        Ok(Some(active_script)) => {
+                            self.sieve_script_ingest(
+                                &access_token,
+                                &raw_message,
+                                &message.sender_address,
+                                &rcpt,
+                                message.session_id,
+                                active_script,
+                                &mut result.autogenerated,
+                            )
+                            .await
+                        }
                         Err(err) => Err(err),
                     }
                 }
@@ -149,7 +252,7 @@ impl MailDelivery for Server {
                 Err(err) => Err(err),
             };
 
-            let result = match result {
+            let status = match status {
                 Ok(ingested_message) => {
                     // Notify state change
                     if ingested_message.change_id != u64::MAX {
@@ -163,28 +266,28 @@ impl MailDelivery for Server {
                         .await;
                     }
 
-                    DeliveryResult::Success
+                    LocalDeliveryStatus::Success
                 }
                 Err(err) => {
-                    let result = match err.as_ref() {
+                    let status = match err.as_ref() {
                         trc::EventType::Limit(trc::LimitEvent::Quota) => {
-                            DeliveryResult::TemporaryFailure {
+                            LocalDeliveryStatus::TemporaryFailure {
                                 reason: "Mailbox over quota.".into(),
                             }
                         }
                         trc::EventType::Limit(trc::LimitEvent::TenantQuota) => {
-                            DeliveryResult::TemporaryFailure {
+                            LocalDeliveryStatus::TemporaryFailure {
                                 reason: "Organization over quota.".into(),
                             }
                         }
                         trc::EventType::Security(trc::SecurityEvent::Unauthorized) => {
-                            DeliveryResult::PermanentFailure {
+                            LocalDeliveryStatus::PermanentFailure {
                                 code: [5, 5, 0],
                                 reason: "This account is not authorized to receive email.".into(),
                             }
                         }
                         trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => {
-                            DeliveryResult::PermanentFailure {
+                            LocalDeliveryStatus::PermanentFailure {
                                 code: err
                                     .value(trc::Key::Code)
                                     .and_then(|v| v.to_uint())
@@ -199,7 +302,7 @@ impl MailDelivery for Server {
                                     .into(),
                             }
                         }
-                        _ => DeliveryResult::TemporaryFailure {
+                        _ => LocalDeliveryStatus::TemporaryFailure {
                             reason: "Transient server failure.".into(),
                         },
                     };
@@ -208,16 +311,16 @@ impl MailDelivery for Server {
                         .ctx(trc::Key::To, rcpt.to_string())
                         .span_id(message.session_id));
 
-                    result
+                    status
                 }
             };
 
             // Cache response for UID to avoid duplicate deliveries
-            uids.insert(uid, results.len());
+            uids.insert(uid, result.status.len());
 
-            results.push(result);
+            result.status.push(status);
         }
 
-        results
+        result
     }
 }

crates/email/src/ingest.rs

@@ -10,7 +10,11 @@ use std::{
     time::{Duration, Instant},
 };
 
-use common::{auth::ResourceToken, Server};
+use common::{
+    auth::{AccessToken, ResourceToken},
+    Server,
+};
+use directory::Permission;
 use jmap_proto::{
     object::Object,
     types::{
@@ -23,11 +27,11 @@ use mail_parser::{
     PartType,
 };
 
-use rand::Rng;
 use spam_filter::{
     analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, SpamFilterInput,
 };
 use std::future::Future;
+use store::rand::Rng;
 use store::{
     ahash::AHashSet,
     query::Filter,
@@ -42,12 +46,8 @@ use trc::{AddContext, MessageIngestEvent};
 use utils::map::vec_map::VecMap;
 
 use crate::{
-    blob::upload::BlobUpload,
-    changes::write::ChangeLog,
-    email::index::{IndexMessage, VisitValues, MAX_ID_LENGTH},
+    index::{IndexMessage, VisitValues, MAX_ID_LENGTH},
     mailbox::{UidMailbox, INBOX_ID, JUNK_ID},
-    services::index::Indexer,
-    JmapMethods,
 };
 
 use super::{
@@ -104,6 +104,7 @@ pub trait EmailIngest: Sync + Send {
         account_id: u32,
         mailbox_id: u32,
     ) -> impl Future<Output = trc::Result<u32>> + Send;
+    fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool;
 }
 
 impl EmailIngest for Server {
@@ -437,7 +438,6 @@ impl EmailIngest for Server {
         // Obtain a documentId and changeId
         let change_id = self
             .assign_change_id(account_id)
-            .await
             .caused_by(trc::location!())?;
 
         // Store blob
@@ -646,7 +646,6 @@ impl EmailIngest for Server {
             let mut batch = BatchBuilder::new();
             let change_id = self
                 .assign_change_id(account_id)
-                .await
                 .caused_by(trc::location!())?;
             let mut changes = ChangeLogBuilder::with_change_id(change_id);
             batch
@@ -702,7 +701,7 @@ impl EmailIngest for Server {
             match self.core.storage.data.write(batch.build()).await {
                 Ok(_) => return Ok(Some(thread_id)),
                 Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => {
-                    let backoff = rand::thread_rng().gen_range(50..=300);
+                    let backoff = store::rand::thread_rng().gen_range(50..=300);
                     tokio::time::sleep(Duration::from_millis(backoff)).await;
                     try_count += 1;
                 }
@@ -728,6 +727,12 @@ impl EmailIngest for Server {
             .await
             .and_then(|v| v.last_counter_id().map(|id| id as u32))
     }
+
+    fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool {
+        self.core.spam.bayes.as_ref().is_some_and(|bayes| {
+            bayes.account_classify && access_token.has_permission(Permission::SpamFilterTrain)
+        })
+    }
 }
 
 pub struct LogEmailInsert(Option<u32>);

crates/email/src/lib.rs

@@ -0,0 +1,14 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod cache;
+pub mod crypto;
+pub mod delivery;
+pub mod index;
+pub mod ingest;
+pub mod mailbox;
+pub mod metadata;
+pub mod sieve;

crates/email/src/mailbox.rs

@@ -0,0 +1,495 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{future::Future, slice::Iter};
+
+use common::{config::jmap::settings::SpecialUse, Server};
+use jmap_proto::{
+    object::{
+        index::{IndexAs, IndexProperty, ObjectIndexBuilder},
+        Object,
+    },
+    types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value},
+};
+use store::{
+    ahash::AHashSet,
+    query::Filter,
+    rand,
+    roaring::RoaringBitmap,
+    write::{
+        BatchBuilder, BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto,
+        TagValue, ToBitmaps,
+    },
+    Serialize, U32_LEN,
+};
+use trc::AddContext;
+use utils::codec::leb128::{Leb128Iterator, Leb128Vec};
+
+use crate::cache::ThreadCache;
+
+pub const INBOX_ID: u32 = 0;
+pub const TRASH_ID: u32 = 1;
+pub const JUNK_ID: u32 = 2;
+pub const DRAFTS_ID: u32 = 3;
+pub const SENT_ID: u32 = 4;
+pub const ARCHIVE_ID: u32 = 5;
+pub const TOMBSTONE_ID: u32 = u32::MAX - 1;
+
+#[derive(Debug)]
+pub struct ExpandPath<'x> {
+    pub path: Vec<&'x str>,
+    pub found_names: Vec<(String, u32, u32)>,
+}
+
+pub static SCHEMA: &[IndexProperty] = &[
+    IndexProperty::new(Property::Name)
+        .index_as(IndexAs::Text {
+            tokenize: true,
+            index: true,
+        })
+        .required(),
+    IndexProperty::new(Property::Role).index_as(IndexAs::Text {
+        tokenize: false,
+        index: true,
+    }),
+    IndexProperty::new(Property::Role).index_as(IndexAs::HasProperty),
+    IndexProperty::new(Property::ParentId).index_as(IndexAs::Integer),
+    IndexProperty::new(Property::SortOrder).index_as(IndexAs::Integer),
+    IndexProperty::new(Property::IsSubscribed).index_as(IndexAs::IntegerList),
+    IndexProperty::new(Property::Acl).index_as(IndexAs::Acl),
+];
+
+#[derive(Debug, Clone, Copy)]
+pub struct UidMailbox {
+    pub mailbox_id: u32,
+    pub uid: u32,
+}
+
+pub trait MailboxFnc: Sync + Send {
+    fn mailbox_get_or_create(
+        &self,
+        account_id: u32,
+    ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send;
+
+    fn mailbox_create_path(
+        &self,
+        account_id: u32,
+        path: &str,
+    ) -> impl Future<Output = trc::Result<Option<(u32, Option<u64>)>>> + Send;
+
+    fn mailbox_count_threads(
+        &self,
+        account_id: u32,
+        document_ids: Option<RoaringBitmap>,
+    ) -> impl Future<Output = trc::Result<usize>> + Send;
+
+    fn mailbox_unread_tags(
+        &self,
+        account_id: u32,
+        document_id: u32,
+        message_ids: &Option<RoaringBitmap>,
+    ) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send;
+
+    fn mailbox_expand_path<'x>(
+        &self,
+        account_id: u32,
+        path: &'x str,
+        exact_match: bool,
+    ) -> impl Future<Output = trc::Result<Option<ExpandPath<'x>>>> + Send;
+
+    fn mailbox_get_by_name(
+        &self,
+        account_id: u32,
+        path: &str,
+    ) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
+
+    fn mailbox_get_by_role(
+        &self,
+        account_id: u32,
+        role: &str,
+    ) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
+}
+
+impl MailboxFnc for Server {
+    async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result<RoaringBitmap> {
+        let mut mailbox_ids = self
+            .get_document_ids(account_id, Collection::Mailbox)
+            .await?
+            .unwrap_or_default();
+        if !mailbox_ids.is_empty() {
+            return Ok(mailbox_ids);
+        }
+
+        #[cfg(feature = "test_mode")]
+        if mailbox_ids.is_empty() && account_id == 0 {
+            return Ok(mailbox_ids);
+        }
+
+        let mut batch = BatchBuilder::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::Mailbox);
+
+        // Create mailboxes
+        let mut last_document_id = ARCHIVE_ID;
+        for folder in &self.core.jmap.default_folders {
+            let (role, document_id) = match folder.special_use {
+                SpecialUse::Inbox => ("inbox", INBOX_ID),
+                SpecialUse::Trash => ("trash", TRASH_ID),
+                SpecialUse::Junk => ("junk", JUNK_ID),
+                SpecialUse::Drafts => ("drafts", DRAFTS_ID),
+                SpecialUse::Sent => ("sent", SENT_ID),
+                SpecialUse::Archive => ("archive", ARCHIVE_ID),
+                SpecialUse::None => {
+                    last_document_id += 1;
+                    ("", last_document_id)
+                }
+                SpecialUse::Shared => unreachable!(),
+            };
+
+            let mut object = Object::with_capacity(4)
+                .with_property(Property::Name, folder.name.clone())
+                .with_property(Property::ParentId, Value::Id(0u64.into()))
+                .with_property(
+                    Property::Cid,
+                    Value::UnsignedInt(rand::random::<u32>() as u64),
+                );
+            if !role.is_empty() {
+                object.set(Property::Role, role);
+            }
+            if folder.subscribe {
+                object.set(
+                    Property::IsSubscribed,
+                    Value::List(vec![Value::Id(account_id.into())]),
+                );
+            }
+            batch
+                .create_document_with_id(document_id)
+                .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(object));
+            mailbox_ids.insert(document_id);
+        }
+
+        self.core
+            .storage
+            .data
+            .write(batch.build())
+            .await
+            .caused_by(trc::location!())
+            .map(|_| mailbox_ids)
+    }
+
+    async fn mailbox_create_path(
+        &self,
+        account_id: u32,
+        path: &str,
+    ) -> trc::Result<Option<(u32, Option<u64>)>> {
+        let expanded_path =
+            if let Some(expand_path) = self.mailbox_expand_path(account_id, path, false).await? {
+                expand_path
+            } else {
+                return Ok(None);
+            };
+
+        let mut next_parent_id = 0;
+        let mut path = expanded_path.path.into_iter().enumerate().peekable();
+        'outer: while let Some((pos, name)) = path.peek() {
+            let is_inbox = *pos == 0 && name.eq_ignore_ascii_case("inbox");
+
+            for (part, parent_id, document_id) in &expanded_path.found_names {
+                if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox")))
+                    && *parent_id == next_parent_id
+                {
+                    next_parent_id = *document_id;
+                    path.next();
+                    continue 'outer;
+                }
+            }
+            break;
+        }
+
+        // Create missing folders
+        if path.peek().is_some() {
+            let mut changes = self.begin_changes(account_id)?;
+
+            for (_, name) in path {
+                if name.len() > self.core.jmap.mailbox_name_max_len {
+                    return Ok(None);
+                }
+                let mut batch = BatchBuilder::new();
+                batch
+                    .with_account_id(account_id)
+                    .with_collection(Collection::Mailbox)
+                    .create_document()
+                    .custom(
+                        ObjectIndexBuilder::new(SCHEMA).with_changes(
+                            Object::with_capacity(3)
+                                .with_property(Property::Name, name)
+                                .with_property(
+                                    Property::ParentId,
+                                    Value::Id(Id::from(next_parent_id)),
+                                )
+                                .with_property(
+                                    Property::Cid,
+                                    Value::UnsignedInt(rand::random::<u32>() as u64),
+                                ),
+                        ),
+                    );
+                let document_id = self
+                    .store()
+                    .write_expect_id(batch)
+                    .await
+                    .caused_by(trc::location!())?;
+                changes.log_insert(Collection::Mailbox, document_id);
+                next_parent_id = document_id + 1;
+            }
+            let change_id = changes.change_id;
+            let mut batch = BatchBuilder::new();
+
+            batch
+                .with_account_id(account_id)
+                .with_collection(Collection::Mailbox)
+                .custom(changes);
+            self.store()
+                .write(batch.build())
+                .await
+                .caused_by(trc::location!())?;
+
+            Ok(Some((next_parent_id - 1, Some(change_id))))
+        } else {
+            Ok(Some((next_parent_id - 1, None)))
+        }
+    }
+
+    async fn mailbox_count_threads(
+        &self,
+        account_id: u32,
+        document_ids: Option<RoaringBitmap>,
+    ) -> trc::Result<usize> {
+        if let Some(document_ids) = document_ids {
+            let mut thread_ids = AHashSet::default();
+            self.get_cached_thread_ids(account_id, document_ids.into_iter())
+                .await
+                .caused_by(trc::location!())?
+                .into_iter()
+                .for_each(|(_, thread_id)| {
+                    thread_ids.insert(thread_id);
+                });
+            Ok(thread_ids.len())
+        } else {
+            Ok(0)
+        }
+    }
+
+    async fn mailbox_unread_tags(
+        &self,
+        account_id: u32,
+        document_id: u32,
+        message_ids: &Option<RoaringBitmap>,
+    ) -> trc::Result<Option<RoaringBitmap>> {
+        if let (Some(message_ids), Some(mailbox_message_ids)) = (
+            message_ids,
+            self.get_tag(
+                account_id,
+                Collection::Email,
+                Property::MailboxIds,
+                document_id,
+            )
+            .await?,
+        ) {
+            if let Some(mut seen) = self
+                .get_tag(
+                    account_id,
+                    Collection::Email,
+                    Property::Keywords,
+                    Keyword::Seen,
+                )
+                .await?
+            {
+                seen ^= message_ids;
+                seen &= &mailbox_message_ids;
+                if !seen.is_empty() {
+                    Ok(Some(seen))
+                } else {
+                    Ok(None)
+                }
+            } else {
+                Ok(mailbox_message_ids.into())
+            }
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn mailbox_expand_path<'x>(
+        &self,
+        account_id: u32,
+        path: &'x str,
+        exact_match: bool,
+    ) -> trc::Result<Option<ExpandPath<'x>>> {
+        let path = path
+            .split('/')
+            .filter_map(|p| {
+                let p = p.trim();
+                if !p.is_empty() {
+                    p.into()
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+        if path.is_empty() || path.len() > self.core.jmap.mailbox_max_depth {
+            return Ok(None);
+        }
+
+        let mut filter = Vec::with_capacity(path.len() + 2);
+        let mut has_inbox = false;
+        filter.push(Filter::Or);
+        for (pos, item) in path.iter().enumerate() {
+            if pos == 0 && item.eq_ignore_ascii_case("inbox") {
+                has_inbox = true;
+            } else {
+                filter.push(Filter::eq(Property::Name, *item));
+            }
+        }
+        filter.push(Filter::End);
+
+        let mut document_ids = if filter.len() > 2 {
+            self.store()
+                .filter(account_id, Collection::Mailbox, filter)
+                .await
+                .caused_by(trc::location!())?
+                .results
+        } else {
+            RoaringBitmap::new()
+        };
+        if has_inbox {
+            document_ids.insert(INBOX_ID);
+        }
+        if exact_match && (document_ids.len() as usize) < path.len() {
+            return Ok(None);
+        }
+
+        let mut found_names = Vec::new();
+        for document_id in document_ids {
+            if let Some(mut obj) = self
+                .get_property::<Object<Value>>(
+                    account_id,
+                    Collection::Mailbox,
+                    document_id,
+                    Property::Value,
+                )
+                .await?
+            {
+                if let Some(Value::Text(value)) = obj.properties.remove(&Property::Name) {
+                    found_names.push((
+                        value,
+                        if let Some(Value::Id(value)) = obj.properties.remove(&Property::ParentId) {
+                            value.document_id()
+                        } else {
+                            0
+                        },
+                        document_id + 1,
+                    ));
+                } else {
+                    return Ok(None);
+                }
+            } else {
+                return Ok(None);
+            }
+        }
+
+        Ok(Some(ExpandPath { path, found_names }))
+    }
+
+    async fn mailbox_get_by_name(&self, account_id: u32, path: &str) -> trc::Result<Option<u32>> {
+        Ok(self
+            .mailbox_expand_path(account_id, path, true)
+            .await?
+            .and_then(|ep| {
+                let mut next_parent_id = 0;
+                'outer: for (pos, name) in ep.path.iter().enumerate() {
+                    let is_inbox = pos == 0 && name.eq_ignore_ascii_case("inbox");
+
+                    for (part, parent_id, document_id) in &ep.found_names {
+                        if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox")))
+                            && *parent_id == next_parent_id
+                        {
+                            next_parent_id = *document_id;
+                            continue 'outer;
+                        }
+                    }
+                    return None;
+                }
+                Some(next_parent_id - 1)
+            }))
+    }
+
+    async fn mailbox_get_by_role(&self, account_id: u32, role: &str) -> trc::Result<Option<u32>> {
+        self.store()
+            .filter(
+                account_id,
+                Collection::Mailbox,
+                vec![Filter::eq(Property::Role, role)],
+            )
+            .await
+            .caused_by(trc::location!())
+            .map(|r| r.results.min())
+    }
+}
+
+impl PartialEq for UidMailbox {
+    fn eq(&self, other: &Self) -> bool {
+        self.mailbox_id == other.mailbox_id
+    }
+}
+
+impl Eq for UidMailbox {}
+
+impl ToBitmaps for UidMailbox {
+    fn to_bitmaps(&self, ops: &mut Vec<Operation>, field: u8, set: bool) {
+        ops.push(Operation::Bitmap {
+            class: BitmapClass::Tag {
+                field,
+                value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)),
+            },
+            set,
+        });
+    }
+}
+
+impl SerializeInto for UidMailbox {
+    fn serialize_into(&self, buf: &mut Vec<u8>) {
+        buf.push_leb128(self.mailbox_id);
+        buf.push_leb128(self.uid);
+    }
+}
+
+impl DeserializeFrom for UidMailbox {
+    fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option<Self> {
+        Some(UidMailbox {
+            mailbox_id: bytes.next_leb128()?,
+            uid: bytes.next_leb128()?,
+        })
+    }
+}
+
+impl Serialize for UidMailbox {
+    fn serialize(self) -> Vec<u8> {
+        let mut buf = Vec::with_capacity(U32_LEN * 2);
+        self.serialize_into(&mut buf);
+        buf
+    }
+}
+
+impl UidMailbox {
+    pub fn new(mailbox_id: u32, uid: u32) -> Self {
+        UidMailbox { mailbox_id, uid }
+    }
+
+    pub fn new_unassigned(mailbox_id: u32) -> Self {
+        UidMailbox { mailbox_id, uid: 0 }
+    }
+}

crates/email/src/sieve.rs

@@ -4,33 +4,31 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::borrow::Cow;
+use std::{borrow::Cow, sync::Arc};
 
-use common::{
-    auth::AccessToken, listener::stream::NullIo, scripts::plugins::PluginContext, Server,
+use crate::{
+    delivery::AutogeneratedMessage,
+    ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail},
+    mailbox::{MailboxFnc, INBOX_ID, TRASH_ID},
 };
+use common::{auth::AccessToken, scripts::plugins::PluginContext, Server};
 use directory::{backend::internal::PrincipalField, Permission, QueryBy};
-use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property};
+use jmap_proto::{
+    object::Object,
+    types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value},
+};
 use mail_parser::MessageParser;
-use sieve::{Envelope, Event, Input, Mailbox, Recipient};
-use smtp::core::{Session, SessionAddress};
+use serde::ser::SerializeSeq;
+use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve};
 use store::{
     ahash::AHashSet,
-    write::{now, BatchBuilder, Bincode, F_VALUE},
+    blake3,
+    query::Filter,
+    write::{assert::HashedValue, now, BatchBuilder, Bincode, BlobOp, F_VALUE},
+    Deserialize, Serialize,
 };
 use trc::{AddContext, SieveEvent};
 
-use crate::{
-    email::{
-        bayes::EmailBayesTrain,
-        ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail},
-    },
-    mailbox::{get::MailboxGet, set::MailboxSet, INBOX_ID, TRASH_ID},
-    sieve::SeenIdHash,
-    JmapMethods,
-};
-
-use super::{get::SieveScriptGet, ActiveScript};
 use std::future::Future;
 
 struct SieveMessage<'x> {
@@ -39,7 +37,27 @@ struct SieveMessage<'x> {
     pub flags: Vec<Keyword>,
 }
 
+pub struct ActiveScript {
+    pub document_id: u32,
+    pub script_name: String,
+    pub script: Arc<Sieve>,
+    pub seen_ids: SeenIds,
+}
+
+#[derive(Debug, Clone)]
+pub struct SeenIdHash {
+    hash: [u8; 32],
+    expiry: u64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct SeenIds {
+    pub ids: AHashSet<SeenIdHash>,
+    pub has_changes: bool,
+}
+
 pub trait SieveScriptIngest: Sync + Send {
+    #[allow(clippy::too_many_arguments)]
     fn sieve_script_ingest(
         &self,
         access_token: &AccessToken,
@@ -48,7 +66,25 @@ pub trait SieveScriptIngest: Sync + Send {
         envelope_to: &str,
         session_id: u64,
         active_script: ActiveScript,
+        autogenerated: &mut Vec<AutogeneratedMessage>,
     ) -> impl Future<Output = trc::Result<IngestedEmail>> + Send;
+
+    fn sieve_script_get_active(
+        &self,
+        account_id: u32,
+    ) -> impl Future<Output = trc::Result<Option<ActiveScript>>> + Send;
+
+    fn sieve_script_get_by_name(
+        &self,
+        account_id: u32,
+        name: &str,
+    ) -> impl Future<Output = trc::Result<Option<Sieve>>> + Send;
+
+    fn sieve_script_compile(
+        &self,
+        account_id: u32,
+        document_id: u32,
+    ) -> impl Future<Output = trc::Result<(Sieve, Object<Value>)>> + Send;
 }
 
 impl SieveScriptIngest for Server {
@@ -61,6 +97,7 @@ impl SieveScriptIngest for Server {
         envelope_to: &str,
         session_id: u64,
         mut active_script: ActiveScript,
+        autogenerated: &mut Vec<AutogeneratedMessage>,
     ) -> trc::Result<IngestedEmail> {
         // Parse message
         let message = if let Some(message) = MessageParser::new().parse(raw_message) {
@@ -347,10 +384,8 @@ impl SieveScriptIngest for Server {
                         input = true.into();
                         if let Some(message) = messages.get(message_id) {
                             let recipients = match recipient {
-                                Recipient::Address(rcpt) => vec![SessionAddress::new(rcpt)],
-                                Recipient::Group(rcpts) => {
-                                    rcpts.into_iter().map(SessionAddress::new).collect()
-                                }
+                                Recipient::Address(rcpt) => vec![rcpt],
+                                Recipient::Group(rcpts) => rcpts,
                                 Recipient::List(_) => {
                                     // Not yet implemented
                                     continue;
@@ -363,28 +398,24 @@ impl SieveScriptIngest for Server {
                                     From = mail_from.clone(),
                                     To = recipients
                                         .iter()
-                                        .map(|r| trc::Value::String(r.address_lcase.clone()))
+                                        .map(|r| trc::Value::String(r.clone()))
                                         .collect::<Vec<_>>(),
                                     Size = message.raw_message.len(),
                                     SpanId = session_id
                                 );
 
-                                Session::<NullIo>::sieve(
-                                    self.clone(),
-                                    SessionAddress::new(mail_from.clone()),
+                                autogenerated.push(AutogeneratedMessage {
+                                    sender_address: mail_from.clone(),
                                     recipients,
-                                    message.raw_message.to_vec(),
-                                    0,
-                                )
-                                .queue_message()
-                                .await;
+                                    message: message.raw_message.to_vec(),
+                                });
                             } else {
                                 trc::event!(
                                     Sieve(SieveEvent::MessageTooLarge),
                                     From = mail_from.clone(),
                                     To = recipients
                                         .iter()
-                                        .map(|r| trc::Value::String(r.address_lcase.clone()))
+                                        .map(|r| trc::Value::String(r.clone()))
                                         .collect::<Vec<_>>(),
                                     Size = message.raw_message.len(),
                                     Limit = self.core.jmap.mail_max_size,
@@ -521,7 +552,9 @@ impl SieveScriptIngest for Server {
                     Bincode::new(active_script.seen_ids),
                     F_VALUE,
                 );
-            let _ = self.write_batch(batch).await;
+            if let Err(err) = self.store().write(batch).await.caused_by(trc::location!()) {
+                trc::error!(err.details("Failed to save Sieve seen ids changes."));
+            }
         }
 
         if let Some(reject_reason) = reject_reason {
@@ -538,6 +571,189 @@ impl SieveScriptIngest for Server {
             Err(last_temp_error.unwrap())
         }
     }
+
+    async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result<Option<ActiveScript>> {
+        // Find the currently active script
+        if let Some(document_id) = self
+            .store()
+            .filter(
+                account_id,
+                Collection::SieveScript,
+                vec![Filter::eq(Property::IsActive, 1u32)],
+            )
+            .await
+            .caused_by(trc::location!())?
+            .results
+            .min()
+        {
+            let (script, mut script_object) =
+                self.sieve_script_compile(account_id, document_id).await?;
+            Ok(Some(ActiveScript {
+                document_id,
+                script: Arc::new(script),
+                script_name: script_object
+                    .properties
+                    .remove(&Property::Name)
+                    .and_then(|name| name.try_unwrap_string())
+                    .unwrap_or_else(|| account_id.to_string()),
+                seen_ids: self
+                    .get_property::<Bincode<SeenIds>>(
+                        account_id,
+                        Collection::SieveScript,
+                        document_id,
+                        Property::EmailIds,
+                    )
+                    .await?
+                    .map(|seen_ids| seen_ids.inner)
+                    .unwrap_or_default(),
+            }))
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn sieve_script_get_by_name(
+        &self,
+        account_id: u32,
+        name: &str,
+    ) -> trc::Result<Option<Sieve>> {
+        // Find the script by name
+        if let Some(document_id) = self
+            .store()
+            .filter(
+                account_id,
+                Collection::SieveScript,
+                vec![Filter::eq(Property::Name, name)],
+            )
+            .await
+            .caused_by(trc::location!())?
+            .results
+            .min()
+        {
+            self.sieve_script_compile(account_id, document_id)
+                .await
+                .map(|(sieve, _)| Some(sieve))
+        } else {
+            Ok(None)
+        }
+    }
+
+    #[allow(clippy::blocks_in_conditions)]
+    async fn sieve_script_compile(
+        &self,
+        account_id: u32,
+        document_id: u32,
+    ) -> trc::Result<(Sieve, Object<Value>)> {
+        // Obtain script object
+        let script_object = self
+            .get_property::<HashedValue<Object<Value>>>(
+                account_id,
+                Collection::SieveScript,
+                document_id,
+                Property::Value,
+            )
+            .await?
+            .ok_or_else(|| {
+                trc::StoreEvent::NotFound
+                    .into_err()
+                    .caused_by(trc::location!())
+                    .document_id(document_id)
+            })?;
+
+        // Obtain the sieve script length
+        let (script_offset, blob_id) = script_object
+            .inner
+            .properties
+            .get(&Property::BlobId)
+            .and_then(|v| v.as_blob_id())
+            .and_then(|v| (v.section.as_ref()?.size, v).into())
+            .ok_or_else(|| {
+                trc::StoreEvent::NotFound
+                    .into_err()
+                    .caused_by(trc::location!())
+                    .document_id(document_id)
+            })?;
+
+        // Obtain the sieve script blob
+        let script_bytes = self
+            .core
+            .storage
+            .blob
+            .get_blob(blob_id.hash.as_ref(), 0..usize::MAX)
+            .await
+            .caused_by(trc::location!())?
+            .ok_or_else(|| {
+                trc::StoreEvent::NotFound
+                    .into_err()
+                    .caused_by(trc::location!())
+                    .document_id(document_id)
+            })?;
+
+        // Obtain the precompiled script
+        if let Some(sieve) = script_bytes
+            .get(script_offset..)
+            .and_then(|bytes| Bincode::<Sieve>::deserialize(bytes).ok())
+        {
+            Ok((sieve.inner, script_object.inner))
+        } else {
+            // Deserialization failed, probably because the script compiler version changed
+            match self.core.sieve.untrusted_compiler.compile(
+                script_bytes.get(0..script_offset).ok_or_else(|| {
+                    trc::StoreEvent::NotFound
+                        .into_err()
+                        .caused_by(trc::location!())
+                        .document_id(document_id)
+                })?,
+            ) {
+                Ok(sieve) => {
+                    // Store updated compiled sieve script
+                    let sieve = Bincode::new(sieve);
+                    let compiled_bytes = (&sieve).serialize();
+                    let mut updated_sieve_bytes =
+                        Vec::with_capacity(script_offset + compiled_bytes.len());
+                    updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]);
+                    updated_sieve_bytes.extend_from_slice(&compiled_bytes);
+
+                    // Store updated blob
+                    let mut new_blob_id = blob_id.clone();
+                    new_blob_id.hash = self
+                        .put_blob(account_id, &updated_sieve_bytes, false)
+                        .await?
+                        .hash;
+                    let mut new_script_object = script_object.inner.clone();
+                    new_script_object.set(Property::BlobId, new_blob_id.clone());
+
+                    // Update script object
+                    let mut batch = BatchBuilder::new();
+                    batch
+                        .with_account_id(account_id)
+                        .with_collection(Collection::SieveScript)
+                        .update_document(document_id)
+                        .assert_value(Property::Value, &script_object)
+                        .set(Property::Value, (&new_script_object).serialize())
+                        .clear(BlobOp::Link {
+                            hash: blob_id.hash.clone(),
+                        })
+                        .set(
+                            BlobOp::Link {
+                                hash: new_blob_id.hash,
+                            },
+                            Vec::new(),
+                        );
+                    self.store()
+                        .write(batch.build())
+                        .await
+                        .caused_by(trc::location!())?;
+
+                    Ok((sieve.inner, new_script_object))
+                }
+                Err(error) => Err(trc::StoreEvent::UnexpectedError
+                    .caused_by(trc::location!())
+                    .reason(error)
+                    .details("Failed to compile Sieve script")),
+            }
+        }
+    }
 }
 
 #[inline(always)]
@@ -554,3 +770,107 @@ pub fn is_valid_role(role: &str) -> bool {
     ]
     .contains(&role)
 }
+
+impl SeenIdHash {
+    pub fn new(id: &str, expiry: u64) -> Self {
+        let mut hasher = blake3::Hasher::new();
+        hasher.update(id.as_bytes());
+        SeenIdHash {
+            hash: hasher.finalize().into(),
+            expiry,
+        }
+    }
+}
+
+impl PartialOrd for SeenIdHash {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for SeenIdHash {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.expiry.cmp(&other.expiry)
+    }
+}
+
+impl std::hash::Hash for SeenIdHash {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.hash.hash(state);
+    }
+}
+
+impl PartialEq for SeenIdHash {
+    fn eq(&self, other: &Self) -> bool {
+        self.hash == other.hash
+    }
+}
+
+impl Eq for SeenIdHash {}
+
+// SeenIds serializer
+impl serde::Serialize for SeenIds {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let mut seq = serializer.serialize_seq((self.ids.len() * 2).into())?;
+        for id in &self.ids {
+            seq.serialize_element(&id.expiry)?;
+            seq.serialize_element(&id.hash)?;
+        }
+
+        seq.end()
+    }
+}
+
+impl<'de> serde::Deserialize<'de> for SeenIds {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        deserializer.deserialize_seq(SeenIdsVisitor)
+    }
+}
+
+struct SeenIdsVisitor;
+
+impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor {
+    type Value = SeenIds;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("invalid SeenIds")
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'de>,
+    {
+        let num_entries = seq.size_hint().unwrap_or(0) / 2;
+        let mut seen_ids = SeenIds {
+            ids: AHashSet::with_capacity(num_entries),
+            has_changes: false,
+        };
+        let now = now();
+
+        for _ in 0..num_entries {
+            let expiry = seq
+                .next_element::<u64>()?
+                .ok_or_else(|| serde::de::Error::custom("Expected expiry."))?;
+            if expiry > now {
+                seen_ids.ids.insert(SeenIdHash {
+                    hash: seq
+                        .next_element()?
+                        .ok_or_else(|| serde::de::Error::custom("Expected hash."))?,
+                    expiry,
+                });
+            } else {
+                seq.next_element::<[u8; 32]>()?
+                    .ok_or_else(|| serde::de::Error::custom("Expected hash."))?;
+                seen_ids.has_changes = true;
+            }
+        }
+
+        Ok(seen_ids)
+    }
+}

crates/imap/Cargo.toml

@@ -12,6 +12,7 @@ directory = { path = "../directory" }
 trc = { path = "../trc" }
 store = { path = "../store" }
 common = { path = "../common" }
+email = { path = "../email" }
 nlp = { path = "../nlp" }
 utils = { path = "../utils" }
 mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } 

crates/imap/src/core/mailbox.rs

@@ -11,12 +11,11 @@ use common::{
     AccountId, Mailbox,
 };
 use directory::{backend::internal::PrincipalField, QueryBy};
+use email::mailbox::{MailboxFnc, INBOX_ID};
 use imap_proto::protocol::list::Attribute;
 use jmap::{
     auth::acl::{AclMethods, EffectiveAcl},
     changes::get::ChangesLookup,
-    mailbox::{get::MailboxGet, set::MailboxSet, INBOX_ID},
-    JmapMethods,
 };
 use jmap_proto::{
     object::Object,

crates/imap/src/core/message.rs

@@ -8,8 +8,8 @@ use std::{collections::BTreeMap, sync::Arc};
 
 use ahash::AHashMap;
 use common::{listener::SessionStream, NextMailboxState};
+use email::mailbox::UidMailbox;
 use imap_proto::protocol::{expunge, select::Exists, Sequence};
-use jmap::{mailbox::UidMailbox, JmapMethods};
 use jmap_proto::{
     object::Object,
     types::{collection::Collection, property::Property, value::Value},

crates/imap/src/op/acl.rs

@@ -11,6 +11,7 @@ use directory::{
     backend::internal::{manage::ChangedPrincipals, PrincipalField},
     Permission, QueryBy, Type,
 };
+use email::mailbox::SCHEMA;
 use imap_proto::{
     protocol::acl::{
         Arguments, GetAclResponse, ListRightsResponse, ModRightsOp, MyRightsResponse, Rights,
@@ -19,10 +20,7 @@ use imap_proto::{
     Command, ResponseCode, StatusResponse,
 };
 
-use jmap::{
-    auth::acl::EffectiveAcl, changes::write::ChangeLog, mailbox::set::SCHEMA,
-    services::state::StateManager, JmapMethods,
-};
+use jmap::auth::acl::EffectiveAcl;
 use jmap_proto::{
     object::{index::ObjectIndexBuilder, Object},
     types::{
@@ -352,7 +350,8 @@ impl<T: SessionStream> Session<T> {
                 );
             if !batch.is_empty() {
                 data.server
-                    .write_batch(batch)
+                    .store()
+                    .write(batch)
                     .await
                     .imap_ctx(&arguments.tag, trc::location!())?;
                 let mut changes = ChangeLogBuilder::new();

crates/imap/src/op/append.rs

@@ -7,6 +7,7 @@
 use std::{sync::Arc, time::Instant};
 
 use directory::Permission;
+use email::ingest::{EmailIngest, IngestEmail, IngestSource};
 use imap_proto::{
     protocol::{append::Arguments, select::HighestModSeq},
     receiver::Request,
@@ -18,13 +19,6 @@ use crate::{
     spawn_op,
 };
 use common::{listener::SessionStream, MailboxId};
-use jmap::{
-    email::{
-        bayes::EmailBayesTrain,
-        ingest::{EmailIngest, IngestEmail, IngestSource},
-    },
-    services::state::StateManager,
-};
 use jmap_proto::types::{acl::Acl, keyword::Keyword, state::StateChange, type_state::DataType};
 use mail_parser::MessageParser;
 

crates/imap/src/op/copy_move.rs

@@ -7,6 +7,10 @@
 use std::{sync::Arc, time::Instant};
 
 use directory::Permission;
+use email::{
+    ingest::EmailIngest,
+    mailbox::{UidMailbox, JUNK_ID},
+};
 use imap_proto::{
     protocol::copy_move::Arguments, receiver::Request, Command, ResponseCode, ResponseType,
     StatusResponse,
@@ -17,13 +21,7 @@ use crate::{
     spawn_op,
 };
 use common::{listener::SessionStream, MailboxId};
-use jmap::{
-    changes::write::ChangeLog,
-    email::{bayes::EmailBayesTrain, copy::EmailCopy, ingest::EmailIngest, set::TagManager},
-    mailbox::{UidMailbox, JUNK_ID},
-    services::{index::Indexer, state::StateManager},
-    JmapMethods,
-};
+use jmap::email::{bayes::EmailBayesTrain, copy::EmailCopy, set::TagManager};
 use jmap_proto::{
     error::set::SetErrorType,
     types::{
@@ -236,7 +234,6 @@ impl<T: SessionStream> SessionData<T> {
                     changelog.change_id = self
                         .server
                         .assign_change_id(account_id)
-                        .await
                         .imap_ctx(&arguments.tag, trc::location!())?;
                 }
                 batch.value(Property::Cid, changelog.change_id, F_VALUE);
@@ -270,7 +267,8 @@ impl<T: SessionStream> SessionData<T> {
 
                 // Write changes
                 self.server
-                    .write_batch(batch)
+                    .store()
+                    .write(batch)
                     .await
                     .imap_ctx(&arguments.tag, trc::location!())?;
 

crates/imap/src/op/create.rs

@@ -13,14 +13,13 @@ use crate::{
 };
 use common::{listener::SessionStream, Account, Mailbox};
 use directory::Permission;
+use email::mailbox::SCHEMA;
 use imap_proto::{
     protocol::{create::Arguments, list::Attribute},
     receiver::Request,
     Command, ResponseCode, StatusResponse,
 };
-use jmap::{
-    changes::write::ChangeLog, mailbox::set::SCHEMA, services::state::StateManager, JmapMethods,
-};
+use jmap::JmapMethods;
 use jmap_proto::{
     object::{index::ObjectIndexBuilder, Object},
     types::{
@@ -79,7 +78,6 @@ impl<T: SessionStream> SessionData<T> {
         let mut changes = self
             .server
             .begin_changes(params.account_id)
-            .await
             .imap_ctx(&arguments.tag, trc::location!())?;
 
         let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0);
@@ -105,7 +103,8 @@ impl<T: SessionStream> SessionData<T> {
                 .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(mailbox));
             let mailbox_id = self
                 .server
-                .write_batch_expect_id(batch)
+                .store()
+                .write_expect_id(batch)
                 .await
                 .imap_ctx(&arguments.tag, trc::location!())?;
             changes.log_insert(Collection::Mailbox, mailbox_id);
@@ -121,7 +120,8 @@ impl<T: SessionStream> SessionData<T> {
             .with_collection(Collection::Mailbox)
             .custom(changes);
         self.server
-            .write_batch(batch)
+            .store()
+            .write(batch)
             .await
             .imap_ctx(&arguments.tag, trc::location!())?;
 

crates/imap/src/op/delete.rs

@@ -15,7 +15,7 @@ use directory::Permission;
 use imap_proto::{
     protocol::delete::Arguments, receiver::Request, Command, ResponseCode, StatusResponse,
 };
-use jmap::{changes::write::ChangeLog, mailbox::set::MailboxSet, services::state::StateManager};
+use jmap::mailbox::set::MailboxSet;
 use jmap_proto::types::{state::StateChange, type_state::DataType};
 use store::write::log::ChangeLogBuilder;
 

crates/imap/src/op/expunge.rs

@@ -8,6 +8,7 @@ use std::{sync::Arc, time::Instant};
 
 use ahash::AHashMap;
 use directory::Permission;
+use email::mailbox::UidMailbox;
 use imap_proto::{
     parser::parse_sequence_set,
     receiver::{Request, Token},
@@ -17,13 +18,7 @@ use trc::AddContext;
 
 use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData};
 use common::{listener::SessionStream, ImapId};
-use jmap::{
-    changes::write::ChangeLog,
-    email::{delete::EmailDeletion, set::TagManager},
-    mailbox::UidMailbox,
-    services::state::StateManager,
-    JmapMethods,
-};
+use jmap::email::{delete::EmailDeletion, set::TagManager};
 use jmap_proto::types::{
     acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property,
     state::StateChange, type_state::DataType,
@@ -251,10 +246,16 @@ impl<T: SessionStream> SessionData<T> {
                     mailboxes.update_batch(&mut batch, Property::MailboxIds);
                     keywords.update_batch(&mut batch, Property::Keywords);
                     if changelog.change_id == u64::MAX {
-                        changelog.change_id = self.server.assign_change_id(account_id).await?
+                        changelog.change_id = self.server.assign_change_id(account_id)?
                     }
                     batch.value(Property::Cid, changelog.change_id, F_VALUE);
-                    match self.server.write_batch(batch).await {
+                    match self
+                        .server
+                        .store()
+                        .write(batch)
+                        .await
+                        .caused_by(trc::location!())
+                    {
                         Ok(_) => {
                             changelog.log_update(Collection::Email, Id::from_parts(thread_id, id));
                             changelog.log_child_update(Collection::Mailbox, mailbox_id.mailbox_id);

crates/imap/src/op/fetch.rs

@@ -13,6 +13,7 @@ use crate::{
 use ahash::AHashMap;
 use common::listener::SessionStream;
 use directory::Permission;
+use email::metadata::MessageMetadata;
 use imap_proto::{
     parser::PushUnique,
     protocol::{
@@ -26,13 +27,7 @@ use imap_proto::{
     receiver::Request,
     Command, ResponseCode, ResponseType, StatusResponse,
 };
-use jmap::{
-    blob::download::BlobDownload,
-    changes::{get::ChangesLookup, write::ChangeLog},
-    email::metadata::MessageMetadata,
-    services::state::StateManager,
-    JmapMethods,
-};
+use jmap::{blob::download::BlobDownload, changes::get::ChangesLookup};
 use jmap_proto::types::{
     acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property,
     state::StateChange, type_state::DataType,
@@ -42,6 +37,7 @@ use store::{
     query::log::{Change, Query},
     write::{assert::HashedValue, BatchBuilder, Bincode, F_BITMAP, F_VALUE},
 };
+use trc::AddContext;
 
 use super::{FromModSeq, ImapContext};
 
@@ -532,7 +528,6 @@ impl<T: SessionStream> SessionData<T> {
             let mut changelog = self
                 .server
                 .begin_changes(account_id)
-                .await
                 .imap_ctx(&arguments.tag, trc::location!())?;
             for (id, mut keywords) in set_seen_ids {
                 keywords.inner.push(Keyword::Seen);
@@ -545,7 +540,13 @@ impl<T: SessionStream> SessionData<T> {
                     .value(Property::Keywords, keywords.inner, F_VALUE)
                     .value(Property::Keywords, Keyword::Seen, F_BITMAP)
                     .value(Property::Cid, changelog.change_id, F_VALUE);
-                match self.server.write_batch(batch).await {
+                match self
+                    .server
+                    .store()
+                    .write(batch)
+                    .await
+                    .caused_by(trc::location!())
+                {
                     Ok(_) => {
                         changelog.log_update(Collection::Email, id);
                     }

crates/imap/src/op/rename.rs

@@ -12,13 +12,11 @@ use crate::{
 };
 use common::listener::SessionStream;
 use directory::Permission;
+use email::mailbox::SCHEMA;
 use imap_proto::{
     protocol::rename::Arguments, receiver::Request, Command, ResponseCode, StatusResponse,
 };
-use jmap::{
-    auth::acl::EffectiveAcl, changes::write::ChangeLog, mailbox::set::SCHEMA,
-    services::state::StateManager, JmapMethods,
-};
+use jmap::auth::acl::EffectiveAcl;
 use jmap_proto::{
     object::{index::ObjectIndexBuilder, Object},
     types::{
@@ -138,7 +136,6 @@ impl<T: SessionStream> SessionData<T> {
         let mut changes = self
             .server
             .begin_changes(params.account_id)
-            .await
             .imap_ctx(&arguments.tag, trc::location!())?;
 
         let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0);
@@ -163,7 +160,8 @@ impl<T: SessionStream> SessionData<T> {
 
             let mailbox_id = self
                 .server
-                .write_batch_expect_id(batch)
+                .store()
+                .write_expect_id(batch)
                 .await
                 .imap_ctx(&arguments.tag, trc::location!())?;
 
@@ -195,7 +193,8 @@ impl<T: SessionStream> SessionData<T> {
         let change_id = changes.change_id;
         batch.custom(changes);
         self.server
-            .write_batch(batch)
+            .store()
+            .write(batch)
             .await
             .imap_ctx(&arguments.tag, trc::location!())?;
 

crates/imap/src/op/status.rs

@@ -19,7 +19,6 @@ use imap_proto::{
     receiver::Request,
     Command, ResponseCode, StatusResponse,
 };
-use jmap::JmapMethods;
 use jmap_proto::{
     object::Object,
     types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value},

crates/imap/src/op/store.rs

@@ -13,6 +13,7 @@ use crate::{
 use ahash::AHashSet;
 use common::listener::SessionStream;
 use directory::Permission;
+use email::{ingest::EmailIngest, mailbox::UidMailbox};
 use imap_proto::{
     protocol::{
         fetch::{DataItem, FetchItem},
@@ -23,11 +24,8 @@ use imap_proto::{
     Command, ResponseCode, ResponseType, StatusResponse,
 };
 use jmap::{
-    changes::{get::ChangesLookup, write::ChangeLog},
+    changes::get::ChangesLookup,
     email::{bayes::EmailBayesTrain, set::TagManager},
-    mailbox::UidMailbox,
-    services::{index::Indexer, state::StateManager},
-    JmapMethods,
 };
 use jmap_proto::types::{
     acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property,
@@ -37,6 +35,7 @@ use store::{
     query::log::{Change, Query},
     write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, ValueClass, F_VALUE},
 };
+use trc::AddContext;
 
 use super::{FromModSeq, ImapContext};
 
@@ -291,7 +290,6 @@ impl<T: SessionStream> SessionData<T> {
                         changelog.change_id = self
                             .server
                             .assign_change_id(account_id)
-                            .await
                             .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?
                     }
                     batch.value(Property::Cid, changelog.change_id, F_VALUE);
@@ -310,7 +308,13 @@ impl<T: SessionStream> SessionData<T> {
                         has_spam_train_tasks = true;
                     }
 
-                    match self.server.write_batch(batch).await {
+                    match self
+                        .server
+                        .store()
+                        .write(batch)
+                        .await
+                        .caused_by(trc::location!())
+                    {
                         Ok(_) => {
                             // Set all current mailboxes as changed if the Seen tag changed
                             if seen_changed {

crates/imap/src/op/subscribe.rs

@@ -12,13 +12,9 @@ use crate::{
 };
 use common::listener::SessionStream;
 use directory::Permission;
+use email::mailbox::SCHEMA;
 use imap_proto::{receiver::Request, Command, ResponseCode, StatusResponse};
-use jmap::{
-    changes::write::ChangeLog,
-    mailbox::set::{MailboxSubscribe, SCHEMA},
-    services::state::StateManager,
-    JmapMethods,
-};
+use jmap::mailbox::set::MailboxSubscribe;
 use jmap_proto::{
     object::{index::ObjectIndexBuilder, Object},
     types::{
@@ -129,7 +125,6 @@ impl<T: SessionStream> SessionData<T> {
             let mut changes = self
                 .server
                 .begin_changes(account_id)
-                .await
                 .imap_ctx(&tag, trc::location!())?;
             let mut batch = BatchBuilder::new();
             batch
@@ -148,7 +143,8 @@ impl<T: SessionStream> SessionData<T> {
             let change_id = changes.change_id;
             batch.custom(changes);
             self.server
-                .write_batch(batch)
+                .store()
+                .write(batch)
                 .await
                 .imap_ctx(&tag, trc::location!())?;
 

crates/imap/src/op/thread.rs

@@ -13,6 +13,7 @@ use crate::{
 use ahash::AHashMap;
 use common::listener::SessionStream;
 use directory::Permission;
+use email::cache::ThreadCache;
 use imap_proto::{
     protocol::{
         thread::{Arguments, Response},
@@ -21,7 +22,6 @@ use imap_proto::{
     receiver::Request,
     Command, StatusResponse,
 };
-use jmap::email::cache::ThreadCache;
 use trc::AddContext;
 
 impl<T: SessionStream> Session<T> {

crates/jmap/Cargo.toml

@@ -14,6 +14,7 @@ common = { path =  "../common" }
 directory = { path =  "../directory" }
 trc = { path = "../trc" }
 spam-filter = { path = "../spam-filter" }
+email = { path = "../email" }
 smtp-proto = { version = "0.1" }
 mail-parser = { version = "0.9", features = ["full_encoding", "serde_support", "ludicrous_mode"] } 
 mail-builder = { version = "0.3", features = ["ludicrous_mode"] }
@@ -27,8 +28,6 @@ hyper-util = { version = "0.1.1", features = ["tokio"] }
 http-body-util = "0.1.0"
 form_urlencoded = "1.1.0"
 tokio = { version = "1.23", features = ["rt"] }
-aes-gcm = "0.10.1"
-aes-gcm-siv = "0.11.1"
 bincode = "1.3.3"
 form-data = { version = "0.5.0", features = ["sync"], default-features = false }
 mime = "0.3.17"
@@ -40,25 +39,20 @@ hkdf = "0.12.3"
 sha1 = "0.10"
 sha2 = "0.10"
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]}
-tokio-tungstenite = "0.24"
-tungstenite = "0.24"
+tokio-tungstenite = "0.26"
+tungstenite = "0.26"
 chrono = "0.4"
 dashmap = "6.0"
-aes = "0.8.3"
-cbc = { version = "0.1.2", features = ["alloc"] }
-sequoia-openpgp = { version = "1.16", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] }
 rand = "0.8.5"
 pkcs8 = { version = "0.10.2", features = ["alloc", "std"] }
-rasn = "0.10"
-rasn-cms = "0.10"
-rasn-pkix = "0.10"
-rsa = "0.9.2"
-async-trait = "0.1.68"
 lz4_flex = { version = "0.11", default-features = false }
 rev_lines = "0.3.0"
 x509-parser = "0.16.0"
-quick-xml = "0.36"
+quick-xml = "0.37"
 memory-stats = "1.2.0"
+aes-gcm = "0.10.1"
+aes-gcm-siv = "0.11.1"
+rsa = "0.9.2"
 
 [features]
 test_mode = []

crates/jmap/src/api/form.rs

@@ -9,10 +9,9 @@ use std::{borrow::Cow, fmt::Write, future::Future};
 use chrono::Utc;
 use common::{
     config::network::{ContactForm, FieldOrDefault},
-    ip_to_bytes,
-    ipc::{DeliveryResult, IngestMessage},
-    psl, Server, KV_RATE_LIMIT_CONTACT,
+    ip_to_bytes, psl, Server, KV_RATE_LIMIT_CONTACT,
 };
+use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery};
 use hyper::StatusCode;
 use mail_auth::common::cache::NoCache;
 use mail_builder::{
@@ -32,7 +31,7 @@ use trc::AddContext;
 use utils::BlobHash;
 use x509_parser::nom::AsBytes;
 
-use crate::{auth::oauth::FormData, services::ingest::MailDelivery};
+use crate::auth::oauth::FormData;
 
 use super::{
     http::{HttpSessionData, ToHttpResponse},
@@ -210,13 +209,16 @@ impl FormHandler for Server {
                     session_id: session.session_id,
                 })
                 .await
+                .status
             {
                 match result {
-                    DeliveryResult::Success => {
+                    LocalDeliveryStatus::Success => {
                         has_success = true;
                     }
-                    DeliveryResult::TemporaryFailure { reason }
-                    | DeliveryResult::PermanentFailure { reason, .. } => failure = Some(reason),
+                    LocalDeliveryStatus::TemporaryFailure { reason }
+                    | LocalDeliveryStatus::PermanentFailure { reason, .. } => {
+                        failure = Some(reason)
+                    }
                 }
             }
 

crates/jmap/src/api/management/enterprise/undelete.rs

@@ -13,6 +13,10 @@ use std::str::FromStr;
 use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
 use common::{auth::AccessToken, enterprise::undelete::DeletedBlob, Server};
 use directory::backend::internal::manage::ManageDirectory;
+use email::{
+    ingest::{EmailIngest, IngestEmail, IngestSource},
+    mailbox::INBOX_ID,
+};
 use hyper::Method;
 use jmap_proto::types::collection::Collection;
 use mail_parser::{DateTime, MessageParser};
@@ -29,9 +33,6 @@ use crate::{
         HttpRequest, HttpResponse, JsonResponse,
     },
     blob::download::BlobDownload,
-    email::ingest::{EmailIngest, IngestEmail, IngestSource},
-    mailbox::INBOX_ID,
-    JmapMethods,
 };
 
 #[derive(serde::Deserialize, serde::Serialize)]

crates/jmap/src/api/management/stores.rs

@@ -15,6 +15,10 @@ use directory::{
     backend::internal::manage::{self, ManageDirectory},
     Permission,
 };
+use email::{
+    ingest::EmailIngest,
+    mailbox::{UidMailbox, SCHEMA},
+};
 use hyper::Method;
 use jmap_proto::{
     object::{index::ObjectIndexBuilder, Object},
@@ -30,10 +34,7 @@ use crate::{
         http::{HttpSessionData, ToHttpResponse},
         HttpRequest, HttpResponse, JsonResponse,
     },
-    email::ingest::EmailIngest,
-    mailbox::{set::SCHEMA, UidMailbox},
     services::index::Indexer,
-    JmapMethods,
 };
 
 use super::decode_path_element;
@@ -344,7 +345,8 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u
             )
             .clear(Property::EmailIds);
         server
-            .write_batch(batch)
+            .store()
+            .write(batch)
             .await
             .caused_by(trc::location!())?;
         mailbox_count += 1;
@@ -388,7 +390,8 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u
             .assert_value(ValueClass::Property(Property::MailboxIds.into()), &uids)
             .value(Property::MailboxIds, uids.inner, F_VALUE);
         server
-            .write_batch(batch)
+            .store()
+            .write(batch)
             .await
             .caused_by(trc::location!())?;
         email_count += 1;

crates/jmap/src/api/request.rs

@@ -30,7 +30,6 @@ use crate::{
     principal::{get::PrincipalGet, query::PrincipalQuery},
     push::{get::PushSubscriptionFetch, set::PushSubscriptionSet},
     quota::{get::QuotaGet, query::QuotaQuery},
-    services::state::StateManager,
     sieve::{
         get::SieveScriptGet, query::SieveScriptQuery, set::SieveScriptSet,
         validate::SieveScriptValidate,

crates/jmap/src/auth/acl.rs

@@ -30,8 +30,6 @@ use store::{
 use trc::AddContext;
 use utils::map::bitmap::Bitmap;
 
-use crate::JmapMethods;
-
 pub trait AclMethods: Sync + Send {
     fn shared_documents(
         &self,

crates/jmap/src/blob/copy.rs

@@ -10,6 +10,7 @@ use jmap_proto::{
     method::copy::{CopyBlobRequest, CopyBlobResponse},
     types::blob::BlobId,
 };
+use trc::AddContext;
 
 use std::future::Future;
 use store::{
@@ -18,8 +19,6 @@ use store::{
 };
 use utils::map::vec_map::VecMap;
 
-use crate::JmapMethods;
-
 use super::download::BlobDownload;
 
 pub trait BlobCopy: Sync + Send {
@@ -55,7 +54,10 @@ impl BlobCopy for Server {
                     },
                     0u32.serialize(),
                 );
-                self.write_batch(batch).await?;
+                self.store()
+                    .write(batch)
+                    .await
+                    .caused_by(trc::location!())?;
                 let dest_blob_id = BlobId {
                     hash: blob_id.hash.clone(),
                     class: BlobClass::Reserved {

crates/jmap/src/blob/download.rs

@@ -129,6 +129,7 @@ impl BlobDownload for Server {
             }))
     }
 
+    #[inline(always)]
     async fn get_blob(&self, hash: &BlobHash, range: Range<usize>) -> trc::Result<Option<Vec<u8>>> {
         self.core
             .storage

crates/jmap/src/blob/get.rs

@@ -5,6 +5,7 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::mailbox::UidMailbox;
 use jmap_proto::{
     method::{
         get::{GetRequest, GetResponse},
@@ -26,7 +27,6 @@ use sha2::{Sha256, Sha512};
 use store::BlobClass;
 use utils::map::vec_map::VecMap;
 
-use crate::{mailbox::UidMailbox, JmapMethods};
 use std::future::Future;
 
 use super::download::BlobDownload;

crates/jmap/src/blob/upload.rs

@@ -14,16 +14,12 @@ use jmap_proto::{
         BlobUploadRequest, BlobUploadResponse, BlobUploadResponseObject, DataSourceObject,
     },
     request::reference::MaybeReference,
-    types::{blob::BlobId, id::Id},
+    types::id::Id,
 };
-use store::{
-    write::{now, BatchBuilder, BlobOp},
-    BlobClass, Serialize,
-};
-use trc::AddContext;
-use utils::BlobHash;
 
-use crate::{auth::rate_limit::RateLimiter, JmapMethods};
+use trc::AddContext;
+
+use crate::auth::rate_limit::RateLimiter;
 
 use super::{download::BlobDownload, UploadResponse};
 use std::future::Future;
@@ -46,13 +42,6 @@ pub trait BlobUpload: Sync + Send {
         data: &[u8],
         access_token: Arc<AccessToken>,
     ) -> impl Future<Output = trc::Result<UploadResponse>> + Send;
-
-    fn put_blob(
-        &self,
-        account_id: u32,
-        data: &[u8],
-        set_quota: bool,
-    ) -> impl Future<Output = trc::Result<BlobId>> + Send;
 }
 
 impl BlobUpload for Server {
@@ -261,52 +250,4 @@ impl BlobUpload for Server {
             size: data.len(),
         })
     }
-
-    #[allow(clippy::blocks_in_conditions)]
-    async fn put_blob(&self, account_id: u32, data: &[u8], set_quota: bool) -> trc::Result<BlobId> {
-        // First reserve the hash
-        let hash = BlobHash::from(data);
-        let mut batch = BatchBuilder::new();
-        let until = now() + self.core.jmap.upload_tmp_ttl;
-
-        batch.with_account_id(account_id).set(
-            BlobOp::Reserve {
-                hash: hash.clone(),
-                until,
-            },
-            (if set_quota { data.len() as u32 } else { 0u32 }).serialize(),
-        );
-        self.write_batch(batch).await?;
-
-        if !self
-            .core
-            .storage
-            .data
-            .blob_exists(&hash)
-            .await
-            .caused_by(trc::location!())?
-        {
-            // Upload blob to store
-            self.core
-                .storage
-                .blob
-                .put_blob(hash.as_ref(), data)
-                .await
-                .caused_by(trc::location!())?;
-
-            // Commit blob
-            let mut batch = BatchBuilder::new();
-            batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new());
-            self.write_batch(batch).await?;
-        }
-
-        Ok(BlobId {
-            hash,
-            class: BlobClass::Reserved {
-                account_id,
-                expires: until,
-            },
-            section: None,
-        })
-    }
 }

crates/jmap/src/changes/mod.rs

@@ -7,4 +7,3 @@
 pub mod get;
 pub mod query;
 pub mod state;
-pub mod write;

crates/jmap/src/changes/write.rs

@@ -1,112 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
- *
- * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
- */
-
-use std::time::Duration;
-
-use common::Server;
-use jmap_proto::types::collection::Collection;
-use std::future::Future;
-use store::{
-    write::{log::ChangeLogBuilder, BatchBuilder},
-    LogKey,
-};
-use trc::AddContext;
-
-pub trait ChangeLog: Sync + Send {
-    fn begin_changes(
-        &self,
-        account_id: u32,
-    ) -> impl Future<Output = trc::Result<ChangeLogBuilder>> + Send;
-    fn assign_change_id(&self, account_id: u32) -> impl Future<Output = trc::Result<u64>> + Send;
-    fn generate_snowflake_id(&self) -> trc::Result<u64>;
-    fn commit_changes(
-        &self,
-        account_id: u32,
-        changes: ChangeLogBuilder,
-    ) -> impl Future<Output = trc::Result<u64>> + Send;
-    fn delete_changes(
-        &self,
-        account_id: u32,
-        before: Duration,
-    ) -> impl Future<Output = trc::Result<()>> + Send;
-}
-
-impl ChangeLog for Server {
-    async fn begin_changes(&self, account_id: u32) -> trc::Result<ChangeLogBuilder> {
-        self.assign_change_id(account_id)
-            .await
-            .map(ChangeLogBuilder::with_change_id)
-    }
-
-    async fn assign_change_id(&self, _: u32) -> trc::Result<u64> {
-        self.generate_snowflake_id()
-    }
-
-    fn generate_snowflake_id(&self) -> trc::Result<u64> {
-        self.inner.data.jmap_id_gen.generate().ok_or_else(|| {
-            trc::StoreEvent::UnexpectedError
-                .into_err()
-                .caused_by(trc::location!())
-                .ctx(trc::Key::Reason, "Failed to generate snowflake id.")
-        })
-    }
-
-    async fn commit_changes(
-        &self,
-        account_id: u32,
-        mut changes: ChangeLogBuilder,
-    ) -> trc::Result<u64> {
-        if changes.change_id == u64::MAX || changes.change_id == 0 {
-            changes.change_id = self.assign_change_id(account_id).await?;
-        }
-        let state = changes.change_id;
-
-        let mut builder = BatchBuilder::new();
-        builder.with_account_id(account_id).custom(changes);
-        self.core
-            .storage
-            .data
-            .write(builder.build())
-            .await
-            .caused_by(trc::location!())
-            .map(|_| state)
-    }
-
-    async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> {
-        let reference_cid = self.inner.data.jmap_id_gen.past_id(before).ok_or_else(|| {
-            trc::StoreEvent::UnexpectedError
-                .caused_by(trc::location!())
-                .ctx(trc::Key::Reason, "Failed to generate reference change id.")
-        })?;
-
-        for collection in [
-            Collection::Email,
-            Collection::Mailbox,
-            Collection::Thread,
-            Collection::Identity,
-            Collection::EmailSubmission,
-        ] {
-            self.core
-                .storage
-                .data
-                .delete_range(
-                    LogKey {
-                        account_id,
-                        collection: collection.into(),
-                        change_id: 0,
-                    },
-                    LogKey {
-                        account_id,
-                        collection: collection.into(),
-                        change_id: reference_cid,
-                    },
-                )
-                .await?;
-        }
-
-        Ok(())
-    }
-}

crates/jmap/src/email/bayes.rs

@@ -6,8 +6,8 @@
 
 use std::future::Future;
 
-use common::{auth::AccessToken, Server};
-use directory::Permission;
+use common::Server;
+use email::metadata::MessageMetadata;
 use jmap_proto::types::{collection::Collection, property::Property};
 use mail_parser::Message;
 use spam_filter::{
@@ -16,10 +16,6 @@ use spam_filter::{
 use store::write::{Bincode, TaskQueueClass};
 use trc::StoreEvent;
 
-use crate::{changes::write::ChangeLog, JmapMethods};
-
-use super::metadata::MessageMetadata;
-
 pub trait EmailBayesTrain: Sync + Send {
     fn email_bayes_train(
         &self,
@@ -35,8 +31,6 @@ pub trait EmailBayesTrain: Sync + Send {
         document_id: u32,
         learn_spam: bool,
     ) -> impl Future<Output = trc::Result<TaskQueueClass>> + Send;
-
-    fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool;
 }
 
 impl EmailBayesTrain for Server {
@@ -83,10 +77,4 @@ impl EmailBayesTrain for Server {
             learn_spam,
         })
     }
-
-    fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool {
-        self.core.spam.bayes.as_ref().is_some_and(|bayes| {
-            bayes.account_classify && access_token.has_permission(Permission::SpamFilterTrain)
-        })
-    }
 }

crates/jmap/src/email/body.rs

@@ -4,16 +4,14 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
+use email::metadata::{MessageMetadataContents, MetadataPartType};
 use jmap_proto::{
     object::Object,
     types::{blob::BlobId, property::Property, value::Value},
 };
 use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType};
 
-use super::{
-    headers::HeaderToValue,
-    metadata::{MessageMetadataContents, MetadataPartType},
-};
+use super::headers::HeaderToValue;
 
 pub trait ToBodyPart {
     fn to_body_part(

crates/jmap/src/email/copy.rs

@@ -8,6 +8,12 @@ use common::{
     auth::{AccessToken, ResourceToken},
     Server,
 };
+use email::{
+    index::{EmailIndexBuilder, TrimTextValue, VisitValues, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH},
+    ingest::{EmailIngest, IngestedEmail, LogEmailInsert},
+    mailbox::{MailboxFnc, UidMailbox},
+    metadata::MessageMetadata,
+};
 use jmap_proto::{
     error::set::SetError,
     method::{
@@ -45,22 +51,9 @@ use store::{
 use trc::AddContext;
 use utils::map::vec_map::VecMap;
 
-use crate::{
-    api::http::HttpSessionData,
-    auth::acl::AclMethods,
-    changes::{state::StateManager, write::ChangeLog},
-    mailbox::{set::MailboxSet, UidMailbox},
-    services::index::Indexer,
-    JmapMethods,
-};
+use crate::{api::http::HttpSessionData, auth::acl::AclMethods, changes::state::StateManager};
 use std::future::Future;
 
-use super::{
-    index::{EmailIndexBuilder, TrimTextValue, VisitValues, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH},
-    ingest::{EmailIngest, IngestedEmail, LogEmailInsert},
-    metadata::MessageMetadata,
-};
-
 pub trait EmailCopy: Sync + Send {
     fn email_copy(
         &self,
@@ -413,7 +406,7 @@ impl EmailCopy for Server {
         }
 
         // Prepare batch
-        let change_id = self.assign_change_id(account_id).await?;
+        let change_id = self.assign_change_id(account_id)?;
         let mut batch = BatchBuilder::new();
         batch
             .with_account_id(account_id)

crates/jmap/src/email/crypto.rs

@@ -4,661 +4,20 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::{
-    borrow::Cow, collections::BTreeSet, fmt::Display, future::Future, io::Cursor, sync::Arc,
-};
+use std::{future::Future, sync::Arc};
 
-use crate::{
-    api::{http::ToHttpResponse, HttpResponse, JsonResponse},
-    JmapMethods,
-};
-use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit};
+use crate::api::{http::ToHttpResponse, HttpResponse, JsonResponse};
 use common::{auth::AccessToken, Server};
 use directory::backend::internal::manage;
+use email::crypto::{
+    try_parse_certs, EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams,
+    EncryptionType,
+};
 use jmap_proto::types::{collection::Collection, property::Property};
-use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary};
-use mail_parser::{decoders::base64::base64_decode, Message, MessageParser, MimeHeaders, PartType};
-use openpgp::{
-    parse::Parse,
-    serialize::stream,
-    types::{KeyFlags, SymmetricAlgorithm},
-};
-use rand::{rngs::StdRng, RngCore, SeedableRng};
-use rasn::types::{ObjectIdentifier, OctetString};
-use rasn_cms::{
-    algorithms::{AES128_CBC, AES256_CBC, RSA},
-    pkcs7_compat::EncapsulatedContentInfo,
-    AlgorithmIdentifier, EncryptedContent, EncryptedContentInfo, EncryptedKey, EnvelopedData,
-    IssuerAndSerialNumber, KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo, CONTENT_DATA,
-    CONTENT_ENVELOPED_DATA,
-};
-use rsa::{pkcs1::DecodeRsaPublicKey, Pkcs1v15Encrypt, RsaPublicKey};
-use sequoia_openpgp as openpgp;
+use mail_builder::encoders::base64::base64_encode_mime;
+use mail_parser::MessageParser;
 use serde_json::json;
-use store::{
-    write::{BatchBuilder, Bincode, ToBitmaps, F_CLEAR, F_VALUE},
-    Deserialize, Serialize,
-};
-
-const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new();
-
-#[derive(Debug)]
-pub enum EncryptMessageError {
-    AlreadyEncrypted,
-    Error(String),
-}
-
-#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
-pub enum Algorithm {
-    Aes128,
-    Aes256,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-pub enum EncryptionMethod {
-    PGP,
-    SMIME,
-}
-
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
-pub struct EncryptionParams {
-    pub method: EncryptionMethod,
-    pub algo: Algorithm,
-    pub certs: Vec<Vec<u8>>,
-}
-
-#[derive(Debug, serde::Serialize, serde::Deserialize, Default)]
-#[serde(tag = "type")]
-#[serde(rename_all = "camelCase")]
-pub enum EncryptionType {
-    PGP {
-        algo: Algorithm,
-        certs: String,
-    },
-    SMIME {
-        algo: Algorithm,
-        certs: String,
-    },
-    #[default]
-    Disabled,
-}
-
-#[allow(async_fn_in_trait)]
-pub trait EncryptMessage {
-    async fn encrypt(&self, params: &EncryptionParams) -> Result<Vec<u8>, EncryptMessageError>;
-    fn is_encrypted(&self) -> bool;
-}
-
-impl EncryptMessage for Message<'_> {
-    async fn encrypt(&self, params: &EncryptionParams) -> Result<Vec<u8>, EncryptMessageError> {
-        let root = self.root_part();
-        let raw_message = self.raw_message();
-        let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize);
-        let mut inner_message = Vec::with_capacity(raw_message.len());
-
-        // Move MIME headers and body to inner message
-        for header in root.headers() {
-            (if header.name.is_mime_header() {
-                &mut inner_message
-            } else {
-                &mut outer_message
-            })
-            .extend_from_slice(&raw_message[header.offset_field()..header.offset_end()]);
-        }
-        inner_message.extend_from_slice(b"\r\n");
-        inner_message.extend_from_slice(&raw_message[root.raw_body_offset()..]);
-
-        // Encrypt inner message
-        match params.method {
-            EncryptionMethod::PGP => {
-                // Prepare encrypted message
-                let boundary = make_boundary("_");
-                outer_message.extend_from_slice(
-                    concat!(
-                        "Content-Type: multipart/encrypted;\r\n\t",
-                        "protocol=\"application/pgp-encrypted\";\r\n\t",
-                        "boundary=\""
-                    )
-                    .as_bytes(),
-                );
-                outer_message.extend_from_slice(boundary.as_bytes());
-                outer_message.extend_from_slice(
-                    concat!(
-                        "\"\r\n\r\n",
-                        "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n",
-                        "--"
-                    )
-                    .as_bytes(),
-                );
-                outer_message.extend_from_slice(boundary.as_bytes());
-                outer_message.extend_from_slice(
-                    concat!(
-                        "\r\nContent-Type: application/pgp-encrypted\r\n\r\n",
-                        "Version: 1\r\n\r\n--"
-                    )
-                    .as_bytes(),
-                );
-                outer_message.extend_from_slice(boundary.as_bytes());
-                outer_message.extend_from_slice(
-                    concat!(
-                        "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n",
-                        "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n"
-                    )
-                    .as_bytes(),
-                );
-
-                let certs = params
-                    .certs
-                    .iter()
-                    .map(openpgp::Cert::from_bytes)
-                    .collect::<Result<Vec<_>, _>>()
-                    .map_err(|err| {
-                        EncryptMessageError::Error(format!(
-                            "Failed to parse OpenPGP public key: {}",
-                            err
-                        ))
-                    })?;
-
-                // Encrypt contents (TODO: use rayon)
-                let algo = params.algo;
-                let encrypted_contents = tokio::task::spawn_blocking(move || {
-                    // Parse public key
-                    let mut keys = Vec::with_capacity(certs.len());
-                    let policy = openpgp::policy::StandardPolicy::new();
-
-                    for cert in &certs {
-                        for key in cert
-                            .keys()
-                            .with_policy(&policy, None)
-                            .supported()
-                            .alive()
-                            .revoked(false)
-                            .key_flags(KeyFlags::empty().set_transport_encryption())
-                        {
-                            keys.push(key);
-                        }
-                    }
-
-                    // Compose a writer stack corresponding to the output format and
-                    // packet structure we want.
-                    let mut sink = Vec::with_capacity(inner_message.len());
-
-                    // Stream an OpenPGP message.
-                    let message = stream::Armorer::new(stream::Message::new(&mut sink))
-                        .build()
-                        .map_err(|err| {
-                            EncryptMessageError::Error(format!("Failed to create armorer: {}", err))
-                        })?;
-                    let message = stream::Encryptor2::for_recipients(message, keys)
-                        .symmetric_algo(match algo {
-                            Algorithm::Aes128 => SymmetricAlgorithm::AES128,
-                            Algorithm::Aes256 => SymmetricAlgorithm::AES256,
-                        })
-                        .build()
-                        .map_err(|err| {
-                            EncryptMessageError::Error(format!(
-                                "Failed to build encryptor: {}",
-                                err
-                            ))
-                        })?;
-                    let mut message =
-                        stream::LiteralWriter::new(message).build().map_err(|err| {
-                            EncryptMessageError::Error(format!(
-                                "Failed to create literal writer: {}",
-                                err
-                            ))
-                        })?;
-                    std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err(
-                        |err| {
-                            EncryptMessageError::Error(format!(
-                                "Failed to encrypt message: {}",
-                                err
-                            ))
-                        },
-                    )?;
-                    message.finalize().map_err(|err| {
-                        EncryptMessageError::Error(format!("Failed to finalize message: {}", err))
-                    })?;
-
-                    String::from_utf8(sink).map_err(|err| {
-                        EncryptMessageError::Error(format!(
-                            "Failed to convert encrypted message to UTF-8: {}",
-                            err
-                        ))
-                    })
-                })
-                .await
-                .map_err(|err| {
-                    EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
-                })??;
-                outer_message.extend_from_slice(encrypted_contents.as_bytes());
-                outer_message.extend_from_slice(b"\r\n--");
-                outer_message.extend_from_slice(boundary.as_bytes());
-                outer_message.extend_from_slice(b"--\r\n");
-            }
-            EncryptionMethod::SMIME => {
-                // Generate random IV
-                let mut rng = StdRng::from_entropy();
-                let mut iv = vec![0u8; 16];
-                rng.fill_bytes(&mut iv);
-
-                // Generate random key
-                let mut key = vec![0u8; params.algo.key_size()];
-                rng.fill_bytes(&mut key);
-
-                // Encrypt contents (TODO: use rayon)
-                let algo = params.algo;
-                let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || {
-                    (algo.encrypt(&key, &iv, &inner_message), key, iv)
-                })
-                .await
-                .map_err(|err| {
-                    EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
-                })?;
-
-                // Encrypt key using public keys
-                #[allow(clippy::mutable_key_type)]
-                let mut recipient_infos = BTreeSet::new();
-                for cert in &params.certs {
-                    let cert =
-                        rasn::der::decode::<rasn_pkix::Certificate>(cert).map_err(|err| {
-                            EncryptMessageError::Error(format!(
-                                "Failed to parse certificate: {}",
-                                err
-                            ))
-                        })?;
-
-                    let public_key = RsaPublicKey::from_pkcs1_der(
-                        cert.tbs_certificate
-                            .subject_public_key_info
-                            .subject_public_key
-                            .as_raw_slice(),
-                    )
-                    .map_err(|err| {
-                        EncryptMessageError::Error(format!("Failed to parse public key: {}", err))
-                    })?;
-                    let encrypted_key = public_key
-                        .encrypt(&mut rng, Pkcs1v15Encrypt, &key[..])
-                        .map_err(|err| {
-                            EncryptMessageError::Error(format!("Failed to encrypt key: {}", err))
-                        })
-                        .unwrap();
-
-                    recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo(
-                        KeyTransRecipientInfo {
-                            version: 0.into(),
-                            rid: RecipientIdentifier::IssuerAndSerialNumber(
-                                IssuerAndSerialNumber {
-                                    issuer: cert.tbs_certificate.issuer,
-                                    serial_number: cert.tbs_certificate.serial_number,
-                                },
-                            ),
-                            key_encryption_algorithm: AlgorithmIdentifier {
-                                algorithm: RSA.into(),
-                                parameters: Some(
-                                    rasn::der::encode(&())
-                                        .map_err(|err| {
-                                            EncryptMessageError::Error(format!(
-                                                "Failed to encode RSA algorithm identifier: {}",
-                                                err
-                                            ))
-                                        })?
-                                        .into(),
-                                ),
-                            },
-                            encrypted_key: EncryptedKey::from(encrypted_key),
-                        },
-                    ));
-                }
-
-                let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo {
-                    content_type: CONTENT_ENVELOPED_DATA.into(),
-                    content: Some(
-                        rasn::der::encode(&EnvelopedData {
-                            version: 0.into(),
-                            originator_info: None,
-                            recipient_infos,
-                            encrypted_content_info: EncryptedContentInfo {
-                                content_type: CONTENT_DATA.into(),
-                                content_encryption_algorithm: AlgorithmIdentifier {
-                                    algorithm: params.algo.to_algorithm_identifier(),
-                                    parameters: Some(
-                                        rasn::der::encode(&OctetString::from(iv))
-                                            .map_err(|err| {
-                                                EncryptMessageError::Error(format!(
-                                                    "Failed to encode IV: {}",
-                                                    err
-                                                ))
-                                            })?
-                                            .into(),
-                                    ),
-                                },
-                                encrypted_content: Some(EncryptedContent::from(encrypted_contents)),
-                            },
-                            unprotected_attrs: None,
-                        })
-                        .map_err(|err| {
-                            EncryptMessageError::Error(format!(
-                                "Failed to encode EnvelopedData: {}",
-                                err
-                            ))
-                        })?
-                        .into(),
-                    ),
-                })
-                .map_err(|err| {
-                    EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err))
-                })?;
-
-                // Generate message
-                outer_message.extend_from_slice(
-                    concat!(
-                        "Content-Type: application/pkcs7-mime;\r\n",
-                        "\tname=\"smime.p7m\";\r\n",
-                        "\tsmime-type=enveloped-data\r\n",
-                        "Content-Disposition: attachment;\r\n",
-                        "\tfilename=\"smime.p7m\"\r\n",
-                        "Content-Transfer-Encoding: base64\r\n\r\n"
-                    )
-                    .as_bytes(),
-                );
-                base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| {
-                    EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err))
-                })?;
-            }
-        }
-
-        Ok(outer_message)
-    }
-
-    fn is_encrypted(&self) -> bool {
-        if self.content_type().is_some_and(|ct| {
-            let main_type = ct.c_type.as_ref();
-            let sub_type = ct
-                .c_subtype
-                .as_ref()
-                .map(|s| s.as_ref())
-                .unwrap_or_default();
-
-            (main_type.eq_ignore_ascii_case("application")
-                && (sub_type.eq_ignore_ascii_case("pkcs7-mime")
-                    || sub_type.eq_ignore_ascii_case("pkcs7-signature")
-                    || (sub_type.eq_ignore_ascii_case("octet-stream")
-                        && self.attachment_name().is_some_and(|name| {
-                            name.rsplit_once('.')
-                                .is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext))
-                        }))))
-                || (main_type.eq_ignore_ascii_case("multipart")
-                    && sub_type.eq_ignore_ascii_case("encrypted"))
-        }) {
-            return true;
-        }
-
-        if self.parts.len() <= 2 {
-            let mut text_part = None;
-            let mut is_multipart = false;
-
-            for part in &self.parts {
-                match &part.body {
-                    PartType::Text(text) => {
-                        text_part = Some(text.as_ref());
-                    }
-                    PartType::Multipart(_) => {
-                        is_multipart = true;
-                    }
-                    _ => (),
-                }
-            }
-
-            match text_part {
-                Some(text) if self.parts.len() == 1 || is_multipart => {
-                    if text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") {
-                        return true;
-                    }
-                }
-                _ => (),
-            }
-        }
-
-        false
-    }
-}
-
-impl Algorithm {
-    fn key_size(&self) -> usize {
-        match self {
-            Algorithm::Aes128 => 16,
-            Algorithm::Aes256 => 32,
-        }
-    }
-
-    fn to_algorithm_identifier(self) -> ObjectIdentifier {
-        match self {
-            Algorithm::Aes128 => AES128_CBC.into(),
-            Algorithm::Aes256 => AES256_CBC.into(),
-        }
-    }
-
-    fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec<u8> {
-        match self {
-            Algorithm::Aes128 => cbc::Encryptor::<aes::Aes128>::new(key.into(), iv.into())
-                .encrypt_padded_vec_mut::<Pkcs7>(contents),
-            Algorithm::Aes256 => cbc::Encryptor::<aes::Aes256>::new(key.into(), iv.into())
-                .encrypt_padded_vec_mut::<Pkcs7>(contents),
-        }
-    }
-}
-
-pub fn try_parse_certs(
-    expected_method: EncryptionMethod,
-    cert: Vec<u8>,
-) -> Result<Vec<Vec<u8>>, Cow<'static, str>> {
-    // Check if it's a PEM file
-    let (method, certs) = if let Some(result) = try_parse_pem(&cert)? {
-        result
-    } else if rasn::der::decode::<rasn_pkix::Certificate>(&cert[..]).is_ok() {
-        (EncryptionMethod::SMIME, vec![cert])
-    } else if let Ok(cert_) = openpgp::Cert::from_bytes(&cert[..]) {
-        if !has_pgp_keys(cert_) {
-            (EncryptionMethod::PGP, vec![cert])
-        } else {
-            return Err("Could not find any suitable keys in certificate".into());
-        }
-    } else {
-        return Err("Could not find any valid certificates".into());
-    };
-
-    if method == expected_method {
-        Ok(certs)
-    } else {
-        Err("No valid certificates found for the selected encryption".into())
-    }
-}
-
-fn has_pgp_keys(cert: openpgp::Cert) -> bool {
-    cert.keys()
-        .with_policy(&P, None)
-        .supported()
-        .alive()
-        .revoked(false)
-        .key_flags(KeyFlags::empty().set_transport_encryption())
-        .next()
-        .is_some()
-}
-
-#[allow(clippy::type_complexity)]
-fn try_parse_pem(
-    bytes_: &[u8],
-) -> Result<Option<(EncryptionMethod, Vec<Vec<u8>>)>, Cow<'static, str>> {
-    if let Some(internal) = std::str::from_utf8(bytes_)
-        .ok()
-        .and_then(|cert| cert.strip_prefix("-----STALWART CERTIFICATE-----"))
-    {
-        return base64_decode(internal.as_bytes())
-            .ok_or(Cow::from("Failed to decode base64"))
-            .and_then(|bytes| {
-                Bincode::<EncryptionParams>::deserialize(&bytes)
-                    .map_err(|_| Cow::from("Failed to deserialize internal certificate"))
-            })
-            .map(|params| Some((params.inner.method, params.inner.certs)));
-    }
-
-    let mut bytes = bytes_.iter().enumerate();
-    let mut buf = vec![];
-    let mut method = None;
-    let mut certs = vec![];
-
-    loop {
-        // Find start of PEM block
-        let mut start_pos = 0;
-        for (pos, &ch) in bytes.by_ref() {
-            if ch.is_ascii_whitespace() {
-                continue;
-            } else if ch == b'-' {
-                start_pos = pos;
-                break;
-            } else {
-                return Ok(None);
-            }
-        }
-
-        // Find block type
-        for (_, &ch) in bytes.by_ref() {
-            match ch {
-                b'-' => (),
-                b'\n' => break,
-                _ => {
-                    if ch.is_ascii() {
-                        buf.push(ch.to_ascii_uppercase());
-                    } else {
-                        return Ok(None);
-                    }
-                }
-            }
-        }
-        if buf.is_empty() {
-            break;
-        }
-
-        // Find type
-        let tag = std::str::from_utf8(&buf).unwrap();
-        if tag.contains("CERTIFICATE") {
-            if method.is_some_and(|m| m == EncryptionMethod::PGP) {
-                return Err("Cannot mix OpenPGP and S/MIME certificates".into());
-            } else {
-                method = Some(EncryptionMethod::SMIME);
-            }
-        } else if tag.contains("PGP") {
-            if method.is_some_and(|m| m == EncryptionMethod::SMIME) {
-                return Err("Cannot mix OpenPGP and S/MIME certificates".into());
-            } else {
-                method = Some(EncryptionMethod::PGP);
-            }
-        } else {
-            // Ignore block
-            let mut found_end = false;
-            for (_, &ch) in bytes.by_ref() {
-                if ch == b'-' {
-                    found_end = true;
-                } else if ch == b'\n' && found_end {
-                    break;
-                }
-            }
-            buf.clear();
-            continue;
-        }
-
-        // Collect base64
-        buf.clear();
-        let mut found_end = false;
-        let mut end_pos = 0;
-        for (pos, &ch) in bytes.by_ref() {
-            match ch {
-                b'-' => {
-                    found_end = true;
-                }
-                b'\n' => {
-                    if found_end {
-                        end_pos = pos;
-                        break;
-                    }
-                }
-                _ => {
-                    if !ch.is_ascii_whitespace() {
-                        buf.push(ch);
-                    }
-                }
-            }
-        }
-
-        // Decode base64
-        let cert =
-            base64_decode(&buf).ok_or_else(|| Cow::from("Failed to decode base64 certificate."))?;
-        match method.unwrap() {
-            EncryptionMethod::PGP => match openpgp::Cert::from_bytes(bytes_) {
-                Ok(cert) => {
-                    if !has_pgp_keys(cert) {
-                        return Err("Could not find any suitable keys in OpenPGP public key".into());
-                    }
-                    certs.push(
-                        bytes_
-                            .get(start_pos..end_pos + 1)
-                            .unwrap_or_default()
-                            .to_vec(),
-                    );
-                }
-                Err(err) => {
-                    return Err(format!("Failed to decode OpenPGP public key: {err}").into())
-                }
-            },
-            EncryptionMethod::SMIME => {
-                if let Err(err) = rasn::der::decode::<rasn_pkix::Certificate>(&cert) {
-                    return Err(format!("Failed to decode X509 certificate: {err}").into());
-                }
-                certs.push(cert);
-            }
-        }
-        buf.clear();
-    }
-
-    Ok(method.map(|method| (method, certs)))
-}
-
-impl Serialize for &EncryptionParams {
-    fn serialize(self) -> Vec<u8> {
-        let len = bincode::serialized_size(&self).unwrap_or_default();
-        let mut buf = Vec::with_capacity(len as usize + 1);
-        buf.push(1);
-        let _ = bincode::serialize_into(&mut buf, &self);
-        buf
-    }
-}
-
-impl Deserialize for EncryptionParams {
-    fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
-        let version = *bytes
-            .first()
-            .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?;
-        match version {
-            1 if bytes.len() > 1 => bincode::deserialize(&bytes[1..]).map_err(|err| {
-                trc::EventType::Store(trc::StoreEvent::DeserializeError)
-                    .from_bincode_error(err)
-                    .caused_by(trc::location!())
-            }),
-
-            _ => Err(trc::StoreEvent::DeserializeError
-                .into_err()
-                .caused_by(trc::location!())
-                .ctx(trc::Key::Value, version as u64)),
-        }
-    }
-}
-
-impl ToBitmaps for &EncryptionParams {
-    fn to_bitmaps(&self, _: &mut Vec<store::write::Operation>, _: u8, _: bool) {
-        unreachable!()
-    }
-}
+use store::{write::{BatchBuilder, Bincode, F_CLEAR, F_VALUE}, Serialize};
 
 pub trait CryptoHandler: Sync + Send {
     fn handle_crypto_get(
@@ -777,21 +136,3 @@ impl CryptoHandler for Server {
         .into_http_response())
     }
 }
-
-impl Display for EncryptionMethod {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            EncryptionMethod::PGP => write!(f, "OpenPGP"),
-            EncryptionMethod::SMIME => write!(f, "S/MIME"),
-        }
-    }
-}
-
-impl Display for Algorithm {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Algorithm::Aes128 => write!(f, "AES-128"),
-            Algorithm::Aes256 => write!(f, "AES-256"),
-        }
-    }
-}

crates/jmap/src/email/delete.rs

@@ -7,6 +7,11 @@
 use std::time::Duration;
 
 use common::{Server, KV_LOCK_PURGE_ACCOUNT};
+use email::{
+    index::EmailIndexBuilder,
+    mailbox::{UidMailbox, JUNK_ID, TOMBSTONE_ID, TRASH_ID},
+    metadata::MessageMetadata,
+};
 use jmap_proto::types::{
     collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange,
     type_state::DataType,
@@ -23,14 +28,6 @@ use store::{
 use trc::{AddContext, StoreEvent};
 use utils::codec::leb128::Leb128Reader;
 
-use crate::{
-    changes::write::ChangeLog,
-    mailbox::{UidMailbox, JUNK_ID, TOMBSTONE_ID, TRASH_ID},
-    services::state::StateManager,
-    JmapMethods,
-};
-
-use super::{index::EmailIndexBuilder, metadata::MessageMetadata};
 use rand::prelude::SliceRandom;
 use std::future::Future;
 

crates/jmap/src/email/get.rs

@@ -5,6 +5,11 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::{
+    cache::ThreadCache,
+    mailbox::UidMailbox,
+    metadata::{MessageMetadata, MetadataPartType},
+};
 use jmap_proto::{
     method::get::{GetRequest, GetResponse},
     object::{email::GetArguments, Object},
@@ -25,15 +30,13 @@ use trc::{AddContext, StoreEvent};
 
 use crate::{
     auth::acl::AclMethods, blob::download::BlobDownload, changes::state::StateManager,
-    email::headers::HeaderToValue, mailbox::UidMailbox, JmapMethods,
+    email::headers::HeaderToValue,
 };
 use std::future::Future;
 
 use super::{
     body::{ToBodyPart, TruncateBody},
-    cache::ThreadCache,
     headers::IntoForm,
-    metadata::{MessageMetadata, MetadataPartType},
 };
 
 pub trait EmailGet: Sync + Send {

crates/jmap/src/email/import.rs

@@ -5,6 +5,10 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::{
+    ingest::{EmailIngest, IngestEmail, IngestSource},
+    mailbox::MailboxFnc,
+};
 use jmap_proto::{
     error::set::{SetError, SetErrorType},
     method::import::{ImportEmailRequest, ImportEmailResponse},
@@ -22,13 +26,9 @@ use utils::map::vec_map::VecMap;
 
 use crate::{
     api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload,
-    changes::state::StateManager, mailbox::set::MailboxSet, JmapMethods,
+    changes::state::StateManager,
 };
 
-use super::{
-    bayes::EmailBayesTrain,
-    ingest::{EmailIngest, IngestEmail, IngestSource},
-};
 use std::future::Future;
 
 pub trait EmailImport: Sync + Send {

crates/jmap/src/email/mod.rs

@@ -6,16 +6,12 @@
 
 pub mod bayes;
 pub mod body;
-pub mod cache;
 pub mod copy;
 pub mod crypto;
 pub mod delete;
 pub mod get;
 pub mod headers;
 pub mod import;
-pub mod index;
-pub mod ingest;
-pub mod metadata;
 pub mod parse;
 pub mod query;
 pub mod set;

crates/jmap/src/email/parse.rs

@@ -5,6 +5,7 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::index::PREVIEW_LENGTH;
 use jmap_proto::{
     method::parse::{ParseEmailRequest, ParseEmailResponse},
     object::Object,
@@ -21,7 +22,6 @@ use crate::blob::download::BlobDownload;
 use super::{
     body::{ToBodyPart, TruncateBody},
     headers::HeaderToValue,
-    index::PREVIEW_LENGTH,
 };
 
 pub trait EmailParse: Sync + Send {

crates/jmap/src/email/query.rs

@@ -5,6 +5,7 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::cache::ThreadCache;
 use jmap_proto::{
     method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty},
     object::email::QueryArguments,
@@ -23,8 +24,6 @@ use store::{
 
 use crate::{auth::acl::AclMethods, JmapMethods};
 
-use super::cache::ThreadCache;
-
 pub trait EmailQuery: Sync + Send {
     fn email_query(
         &self,

crates/jmap/src/email/set.rs

@@ -7,6 +7,10 @@
 use std::{borrow::Cow, collections::HashMap, slice::IterMut};
 
 use common::{auth::AccessToken, Server};
+use email::{
+    ingest::{EmailIngest, IngestEmail, IngestSource},
+    mailbox::{MailboxFnc, UidMailbox},
+};
 use jmap_proto::{
     error::set::{SetError, SetErrorType},
     method::set::{RequestArguments, SetRequest, SetResponse},
@@ -42,20 +46,14 @@ use store::{
 use trc::AddContext;
 
 use crate::{
-    api::http::HttpSessionData,
-    auth::acl::AclMethods,
-    blob::download::BlobDownload,
-    changes::{state::StateManager, write::ChangeLog},
-    mailbox::{set::MailboxSet, UidMailbox},
-    JmapMethods,
+    api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload,
+    changes::state::StateManager, JmapMethods,
 };
 use std::future::Future;
 
 use super::{
-    bayes::EmailBayesTrain,
     delete::EmailDeletion,
     headers::{BuildHeader, ValueToHeader},
-    ingest::{EmailIngest, IngestEmail, IngestSource},
 };
 
 pub trait EmailSet: Sync + Send {
@@ -892,7 +890,7 @@ impl EmailSet for Server {
 
                 // Update last change id
                 if changes.change_id == u64::MAX {
-                    changes.change_id = self.assign_change_id(account_id).await?;
+                    changes.change_id = self.assign_change_id(account_id)?;
                 }
                 batch.value(Property::Cid, changes.change_id, F_VALUE);
             }

crates/jmap/src/email/snippet.rs

@@ -5,6 +5,7 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::metadata::{MessageMetadata, MetadataPartType};
 use jmap_proto::{
     method::{
         query::Filter,
@@ -16,9 +17,8 @@ use mail_parser::{decoders::html::html_to_text, GetHeader, HeaderName, PartType}
 use nlp::language::{search_snippet::generate_snippet, stemmer::Stemmer, Language};
 use store::{backend::MAX_TOKEN_LENGTH, write::Bincode};
 
-use crate::{auth::acl::AclMethods, blob::download::BlobDownload, JmapMethods};
+use crate::{auth::acl::AclMethods, blob::download::BlobDownload};
 
-use super::metadata::{MessageMetadata, MetadataPartType};
 use std::future::Future;
 
 pub trait EmailSearchSnippet: Sync + Send {

crates/jmap/src/identity/get.rs

@@ -18,7 +18,7 @@ use store::{
 use trc::AddContext;
 use utils::sanitize_email;
 
-use crate::{changes::state::StateManager, JmapMethods};
+use crate::changes::state::StateManager;
 
 use std::future::Future;
 

crates/jmap/src/identity/set.rs

@@ -19,10 +19,9 @@ use jmap_proto::{
 };
 use std::future::Future;
 use store::write::{log::ChangeLogBuilder, BatchBuilder, F_CLEAR, F_VALUE};
+use trc::AddContext;
 use utils::sanitize_email;
 
-use crate::{changes::write::ChangeLog, JmapMethods};
-
 pub trait IdentitySet: Sync + Send {
     fn identity_set(
         &self,
@@ -102,7 +101,11 @@ impl IdentitySet for Server {
                 .with_collection(Collection::Identity)
                 .create_document()
                 .value(Property::Value, identity, F_VALUE);
-            let document_id = self.write_batch_expect_id(batch).await?;
+            let document_id = self
+                .store()
+                .write_expect_id(batch)
+                .await
+                .caused_by(trc::location!())?;
             identity_ids.insert(document_id);
             changes.log_insert(Collection::Identity, document_id);
             response.created(id, document_id);
@@ -158,7 +161,10 @@ impl IdentitySet for Server {
                 .with_collection(Collection::Identity)
                 .update_document(document_id)
                 .value(Property::Value, identity, F_VALUE);
-            self.write_batch(batch).await?;
+            self.store()
+                .write(batch)
+                .await
+                .caused_by(trc::location!())?;
             changes.log_update(Collection::Identity, document_id);
             response.updated.append(id, None);
         }
@@ -174,7 +180,10 @@ impl IdentitySet for Server {
                     .with_collection(Collection::Identity)
                     .delete_document(document_id)
                     .value(Property::Value, (), F_VALUE | F_CLEAR);
-                self.write_batch(batch).await?;
+                self.store()
+                    .write(batch)
+                    .await
+                    .caused_by(trc::location!())?;
                 changes.log_delete(Collection::Identity, document_id);
                 response.destroyed.push(id);
             } else {

crates/jmap/src/lib.rs

@@ -8,33 +8,24 @@ use std::{fmt::Display, future::Future, sync::Arc, time::Duration};
 
 use changes::state::StateManager;
 use common::{
-    auth::{AccessToken, ResourceToken, TenantInfo},
     manager::boot::{BootManager, IpcReceivers},
     Inner, Server,
 };
-use directory::QueryBy;
 use jmap_proto::{
     method::{
         query::{QueryRequest, QueryResponse},
         set::{SetRequest, SetResponse},
     },
-    types::{collection::Collection, property::Property},
+    types::collection::Collection,
 };
 use services::{
-    delivery::spawn_delivery_manager, housekeeper::spawn_housekeeper,
-    index::spawn_email_queue_task, state::spawn_state_manager,
+    housekeeper::spawn_housekeeper, index::spawn_email_queue_task, state::spawn_state_manager,
 };
 
 use store::{
-    dispatch::DocumentSet,
     fts::FtsFilter,
     query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet},
     roaring::RoaringBitmap,
-    write::{
-        key::DeserializeBigEndian, AssignedIds, BatchBuilder, BitmapClass, DirectoryClass,
-        TagValue, ValueClass,
-    },
-    BitmapKey, Deserialize, IterateParams, ValueKey, U32_LEN,
 };
 use trc::AddContext;
 
@@ -88,9 +79,6 @@ impl StartServices for BootManager {
 
 impl SpawnServices for IpcReceivers {
     fn spawn_services(&mut self, inner: Arc<Inner>) {
-        // Spawn delivery manager
-        spawn_delivery_manager(inner.clone(), self.delivery_rx.take().unwrap());
-
         // Spawn state manager
         spawn_state_manager(inner.clone(), self.state_rx.take().unwrap());
 
@@ -103,138 +91,6 @@ impl SpawnServices for IpcReceivers {
 }
 
 impl JmapMethods for Server {
-    async fn get_property<U>(
-        &self,
-        account_id: u32,
-        collection: Collection,
-        document_id: u32,
-        property: impl AsRef<Property> + Sync + Send,
-    ) -> trc::Result<Option<U>>
-    where
-        U: Deserialize + 'static,
-    {
-        let property = property.as_ref();
-
-        self.core
-            .storage
-            .data
-            .get_value::<U>(ValueKey {
-                account_id,
-                collection: collection.into(),
-                document_id,
-                class: ValueClass::Property(property.into()),
-            })
-            .await
-            .add_context(|err| {
-                err.caused_by(trc::location!())
-                    .account_id(account_id)
-                    .collection(collection)
-                    .document_id(document_id)
-                    .id(property.to_string())
-            })
-    }
-
-    async fn get_properties<U, I, P>(
-        &self,
-        account_id: u32,
-        collection: Collection,
-        iterate: &I,
-        property: P,
-    ) -> trc::Result<Vec<(u32, U)>>
-    where
-        I: DocumentSet + Send + Sync,
-        P: AsRef<Property> + Sync + Send,
-        U: Deserialize + 'static,
-    {
-        let property: u8 = property.as_ref().into();
-        let collection: u8 = collection.into();
-        let expected_results = iterate.len();
-        let mut results = Vec::with_capacity(expected_results);
-
-        self.core
-            .storage
-            .data
-            .iterate(
-                IterateParams::new(
-                    ValueKey {
-                        account_id,
-                        collection,
-                        document_id: iterate.min(),
-                        class: ValueClass::Property(property),
-                    },
-                    ValueKey {
-                        account_id,
-                        collection,
-                        document_id: iterate.max(),
-                        class: ValueClass::Property(property),
-                    },
-                ),
-                |key, value| {
-                    let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?;
-                    if iterate.contains(document_id) {
-                        results.push((document_id, U::deserialize(value)?));
-                        Ok(expected_results == 0 || results.len() < expected_results)
-                    } else {
-                        Ok(true)
-                    }
-                },
-            )
-            .await
-            .add_context(|err| {
-                err.caused_by(trc::location!())
-                    .account_id(account_id)
-                    .collection(collection)
-                    .id(property.to_string())
-            })
-            .map(|_| results)
-    }
-
-    async fn get_document_ids(
-        &self,
-        account_id: u32,
-        collection: Collection,
-    ) -> trc::Result<Option<RoaringBitmap>> {
-        self.core
-            .storage
-            .data
-            .get_bitmap(BitmapKey::document_ids(account_id, collection))
-            .await
-            .add_context(|err| {
-                err.caused_by(trc::location!())
-                    .account_id(account_id)
-                    .collection(collection)
-            })
-    }
-
-    async fn get_tag(
-        &self,
-        account_id: u32,
-        collection: Collection,
-        property: impl AsRef<Property> + Sync + Send,
-        value: impl Into<TagValue<u32>> + Sync + Send,
-    ) -> trc::Result<Option<RoaringBitmap>> {
-        let property = property.as_ref();
-        self.core
-            .storage
-            .data
-            .get_bitmap(BitmapKey {
-                account_id,
-                collection: collection.into(),
-                class: BitmapClass::Tag {
-                    field: property.into(),
-                    value: value.into(),
-                },
-                document_id: 0,
-            })
-            .await
-            .add_context(|err| {
-                err.caused_by(trc::location!())
-                    .account_id(account_id)
-                    .collection(collection)
-                    .id(property.to_string())
-            })
-    }
-
     async fn prepare_set_response<T: Sync + Send>(
         &self,
         request: &SetRequest<T>,
@@ -252,109 +108,6 @@ impl JmapMethods for Server {
         )
     }
 
-    async fn get_resource_token(
-        &self,
-        access_token: &AccessToken,
-        account_id: u32,
-    ) -> trc::Result<ResourceToken> {
-        Ok(if access_token.primary_id == account_id {
-            ResourceToken {
-                account_id,
-                quota: access_token.quota,
-                tenant: access_token.tenant,
-            }
-        } else {
-            let mut quotas = ResourceToken {
-                account_id,
-                ..Default::default()
-            };
-
-            if let Some(principal) = self
-                .core
-                .storage
-                .directory
-                .query(QueryBy::Id(account_id), false)
-                .await
-                .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))?
-            {
-                quotas.quota = principal.quota();
-
-                // SPDX-SnippetBegin
-                // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
-                // SPDX-License-Identifier: LicenseRef-SEL
-
-                #[cfg(feature = "enterprise")]
-                if self.core.is_enterprise_edition() {
-                    if let Some(tenant_id) = principal.tenant() {
-                        quotas.tenant = TenantInfo {
-                            id: tenant_id,
-                            quota: self
-                                .core
-                                .storage
-                                .directory
-                                .query(QueryBy::Id(tenant_id), false)
-                                .await
-                                .add_context(|err| {
-                                    err.caused_by(trc::location!()).account_id(tenant_id)
-                                })?
-                                .map(|tenant| tenant.quota())
-                                .unwrap_or_default(),
-                        }
-                        .into();
-                    }
-                }
-
-                // SPDX-SnippetEnd
-            }
-
-            quotas
-        })
-    }
-
-    async fn get_used_quota(&self, account_id: u32) -> trc::Result<i64> {
-        self.core
-            .storage
-            .data
-            .get_counter(DirectoryClass::UsedQuota(account_id))
-            .await
-            .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))
-    }
-
-    async fn has_available_quota(&self, quotas: &ResourceToken, item_size: u64) -> trc::Result<()> {
-        if quotas.quota != 0 {
-            let used_quota = self.get_used_quota(quotas.account_id).await? as u64;
-
-            if used_quota + item_size > quotas.quota {
-                return Err(trc::LimitEvent::Quota
-                    .into_err()
-                    .ctx(trc::Key::Limit, quotas.quota)
-                    .ctx(trc::Key::Size, used_quota));
-            }
-        }
-
-        // SPDX-SnippetBegin
-        // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
-        // SPDX-License-Identifier: LicenseRef-SEL
-
-        #[cfg(feature = "enterprise")]
-        if self.core.is_enterprise_edition() {
-            if let Some(tenant) = quotas.tenant.filter(|tenant| tenant.quota != 0) {
-                let used_quota = self.get_used_quota(tenant.id).await? as u64;
-
-                if used_quota + item_size > tenant.quota {
-                    return Err(trc::LimitEvent::TenantQuota
-                        .into_err()
-                        .ctx(trc::Key::Limit, tenant.quota)
-                        .ctx(trc::Key::Size, used_quota));
-                }
-            }
-        }
-
-        // SPDX-SnippetEnd
-
-        Ok(())
-    }
-
     async fn filter(
         &self,
         account_id: u32,
@@ -466,21 +219,6 @@ impl JmapMethods for Server {
         Ok(response)
     }
 
-    async fn write_batch(&self, batch: BatchBuilder) -> trc::Result<AssignedIds> {
-        self.core
-            .storage
-            .data
-            .write(batch.build())
-            .await
-            .caused_by(trc::location!())
-    }
-
-    async fn write_batch_expect_id(&self, batch: BatchBuilder) -> trc::Result<u32> {
-        self.write_batch(batch)
-            .await
-            .and_then(|ids| ids.last_document_id().caused_by(trc::location!()))
-    }
-
     fn increment_config_version(&self) {
         self.inner
             .data
@@ -490,62 +228,12 @@ impl JmapMethods for Server {
 }
 
 pub trait JmapMethods: Sync + Send {
-    fn get_property<U>(
-        &self,
-        account_id: u32,
-        collection: Collection,
-        document_id: u32,
-        property: impl AsRef<Property> + Sync + Send,
-    ) -> impl Future<Output = trc::Result<Option<U>>> + Send
-    where
-        U: Deserialize + 'static;
-
-    fn get_properties<U, I, P>(
-        &self,
-        account_id: u32,
-        collection: Collection,
-        iterate: &I,
-        property: P,
-    ) -> impl Future<Output = trc::Result<Vec<(u32, U)>>> + Send
-    where
-        I: DocumentSet + Send + Sync,
-        P: AsRef<Property> + Sync + Send,
-        U: Deserialize + 'static;
-
-    fn get_document_ids(
-        &self,
-        account_id: u32,
-        collection: Collection,
-    ) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send;
-
-    fn get_tag(
-        &self,
-        account_id: u32,
-        collection: Collection,
-        property: impl AsRef<Property> + Sync + Send,
-        value: impl Into<TagValue<u32>> + Sync + Send,
-    ) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send;
-
     fn prepare_set_response<T: Sync + Send>(
         &self,
         request: &SetRequest<T>,
         collection: Collection,
     ) -> impl Future<Output = trc::Result<SetResponse>> + Send;
 
-    fn get_resource_token(
-        &self,
-        access_token: &AccessToken,
-        account_id: u32,
-    ) -> impl Future<Output = trc::Result<ResourceToken>> + Send;
-
-    fn get_used_quota(&self, account_id: u32) -> impl Future<Output = trc::Result<i64>> + Send;
-
-    fn has_available_quota(
-        &self,
-        quotas: &ResourceToken,
-        item_size: u64,
-    ) -> impl Future<Output = trc::Result<()>> + Send;
-
     fn filter(
         &self,
         account_id: u32,
@@ -574,16 +262,6 @@ pub trait JmapMethods: Sync + Send {
         response: QueryResponse,
     ) -> impl Future<Output = trc::Result<QueryResponse>> + Send;
 
-    fn write_batch(
-        &self,
-        batch: BatchBuilder,
-    ) -> impl Future<Output = trc::Result<AssignedIds>> + Send;
-
-    fn write_batch_expect_id(
-        &self,
-        batch: BatchBuilder,
-    ) -> impl Future<Output = trc::Result<u32>> + Send;
-
     fn increment_config_version(&self);
 }
 

crates/jmap/src/mailbox/get.rs

@@ -5,22 +5,18 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::mailbox::MailboxFnc;
 use jmap_proto::{
     method::get::{GetRequest, GetResponse, RequestArguments},
     object::Object,
-    types::{acl::Acl, collection::Collection, keyword::Keyword, property::Property, value::Value},
+    types::{acl::Acl, collection::Collection, property::Property, value::Value},
 };
-use store::{ahash::AHashSet, query::Filter, roaring::RoaringBitmap};
-use trc::AddContext;
 
 use crate::{
     auth::acl::{AclMethods, EffectiveAcl},
     changes::state::StateManager,
-    email::cache::ThreadCache,
-    JmapMethods,
 };
 
-use super::{set::MailboxSet, INBOX_ID};
 use std::future::Future;
 
 pub trait MailboxGet: Sync + Send {
@@ -29,38 +25,6 @@ pub trait MailboxGet: Sync + Send {
         request: GetRequest<RequestArguments>,
         access_token: &AccessToken,
     ) -> impl Future<Output = trc::Result<GetResponse>> + Send;
-
-    fn mailbox_count_threads(
-        &self,
-        account_id: u32,
-        document_ids: Option<RoaringBitmap>,
-    ) -> impl Future<Output = trc::Result<usize>> + Send;
-
-    fn mailbox_unread_tags(
-        &self,
-        account_id: u32,
-        document_id: u32,
-        message_ids: &Option<RoaringBitmap>,
-    ) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send;
-
-    fn mailbox_expand_path<'x>(
-        &self,
-        account_id: u32,
-        path: &'x str,
-        exact_match: bool,
-    ) -> impl Future<Output = trc::Result<Option<ExpandPath<'x>>>> + Send;
-
-    fn mailbox_get_by_name(
-        &self,
-        account_id: u32,
-        path: &str,
-    ) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
-
-    fn mailbox_get_by_role(
-        &self,
-        account_id: u32,
-        role: &str,
-    ) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
 }
 
 impl MailboxGet for Server {
@@ -282,182 +246,4 @@ impl MailboxGet for Server {
         }
         Ok(response)
     }
-
-    async fn mailbox_count_threads(
-        &self,
-        account_id: u32,
-        document_ids: Option<RoaringBitmap>,
-    ) -> trc::Result<usize> {
-        if let Some(document_ids) = document_ids {
-            let mut thread_ids = AHashSet::default();
-            self.get_cached_thread_ids(account_id, document_ids.into_iter())
-                .await
-                .caused_by(trc::location!())?
-                .into_iter()
-                .for_each(|(_, thread_id)| {
-                    thread_ids.insert(thread_id);
-                });
-            Ok(thread_ids.len())
-        } else {
-            Ok(0)
-        }
-    }
-
-    async fn mailbox_unread_tags(
-        &self,
-        account_id: u32,
-        document_id: u32,
-        message_ids: &Option<RoaringBitmap>,
-    ) -> trc::Result<Option<RoaringBitmap>> {
-        if let (Some(message_ids), Some(mailbox_message_ids)) = (
-            message_ids,
-            self.get_tag(
-                account_id,
-                Collection::Email,
-                Property::MailboxIds,
-                document_id,
-            )
-            .await?,
-        ) {
-            if let Some(mut seen) = self
-                .get_tag(
-                    account_id,
-                    Collection::Email,
-                    Property::Keywords,
-                    Keyword::Seen,
-                )
-                .await?
-            {
-                seen ^= message_ids;
-                seen &= &mailbox_message_ids;
-                if !seen.is_empty() {
-                    Ok(Some(seen))
-                } else {
-                    Ok(None)
-                }
-            } else {
-                Ok(mailbox_message_ids.into())
-            }
-        } else {
-            Ok(None)
-        }
-    }
-
-    async fn mailbox_expand_path<'x>(
-        &self,
-        account_id: u32,
-        path: &'x str,
-        exact_match: bool,
-    ) -> trc::Result<Option<ExpandPath<'x>>> {
-        let path = path
-            .split('/')
-            .filter_map(|p| {
-                let p = p.trim();
-                if !p.is_empty() {
-                    p.into()
-                } else {
-                    None
-                }
-            })
-            .collect::<Vec<_>>();
-        if path.is_empty() || path.len() > self.core.jmap.mailbox_max_depth {
-            return Ok(None);
-        }
-
-        let mut filter = Vec::with_capacity(path.len() + 2);
-        let mut has_inbox = false;
-        filter.push(Filter::Or);
-        for (pos, item) in path.iter().enumerate() {
-            if pos == 0 && item.eq_ignore_ascii_case("inbox") {
-                has_inbox = true;
-            } else {
-                filter.push(Filter::eq(Property::Name, *item));
-            }
-        }
-        filter.push(Filter::End);
-
-        let mut document_ids = if filter.len() > 2 {
-            self.filter(account_id, Collection::Mailbox, filter)
-                .await?
-                .results
-        } else {
-            RoaringBitmap::new()
-        };
-        if has_inbox {
-            document_ids.insert(INBOX_ID);
-        }
-        if exact_match && (document_ids.len() as usize) < path.len() {
-            return Ok(None);
-        }
-
-        let mut found_names = Vec::new();
-        for document_id in document_ids {
-            if let Some(mut obj) = self
-                .get_property::<Object<Value>>(
-                    account_id,
-                    Collection::Mailbox,
-                    document_id,
-                    Property::Value,
-                )
-                .await?
-            {
-                if let Some(Value::Text(value)) = obj.properties.remove(&Property::Name) {
-                    found_names.push((
-                        value,
-                        if let Some(Value::Id(value)) = obj.properties.remove(&Property::ParentId) {
-                            value.document_id()
-                        } else {
-                            0
-                        },
-                        document_id + 1,
-                    ));
-                } else {
-                    return Ok(None);
-                }
-            } else {
-                return Ok(None);
-            }
-        }
-
-        Ok(Some(ExpandPath { path, found_names }))
-    }
-
-    async fn mailbox_get_by_name(&self, account_id: u32, path: &str) -> trc::Result<Option<u32>> {
-        Ok(self
-            .mailbox_expand_path(account_id, path, true)
-            .await?
-            .and_then(|ep| {
-                let mut next_parent_id = 0;
-                'outer: for (pos, name) in ep.path.iter().enumerate() {
-                    let is_inbox = pos == 0 && name.eq_ignore_ascii_case("inbox");
-
-                    for (part, parent_id, document_id) in &ep.found_names {
-                        if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox")))
-                            && *parent_id == next_parent_id
-                        {
-                            next_parent_id = *document_id;
-                            continue 'outer;
-                        }
-                    }
-                    return None;
-                }
-                Some(next_parent_id - 1)
-            }))
-    }
-
-    async fn mailbox_get_by_role(&self, account_id: u32, role: &str) -> trc::Result<Option<u32>> {
-        self.filter(
-            account_id,
-            Collection::Mailbox,
-            vec![Filter::eq(Property::Role, role)],
-        )
-        .await
-        .map(|r| r.results.min())
-    }
-}
-
-#[derive(Debug)]
-pub struct ExpandPath<'x> {
-    pub path: Vec<&'x str>,
-    pub found_names: Vec<(String, u32, u32)>,
 }

crates/jmap/src/mailbox/mod.rs

@@ -4,84 +4,6 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::slice::Iter;
-
-use store::{
-    write::{
-        BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto, TagValue, ToBitmaps,
-    },
-    Serialize, U32_LEN,
-};
-use utils::codec::leb128::{Leb128Iterator, Leb128Vec};
-
 pub mod get;
 pub mod query;
 pub mod set;
-
-pub const INBOX_ID: u32 = 0;
-pub const TRASH_ID: u32 = 1;
-pub const JUNK_ID: u32 = 2;
-pub const DRAFTS_ID: u32 = 3;
-pub const SENT_ID: u32 = 4;
-pub const ARCHIVE_ID: u32 = 5;
-pub const TOMBSTONE_ID: u32 = u32::MAX - 1;
-
-#[derive(Debug, Clone, Copy)]
-pub struct UidMailbox {
-    pub mailbox_id: u32,
-    pub uid: u32,
-}
-
-impl PartialEq for UidMailbox {
-    fn eq(&self, other: &Self) -> bool {
-        self.mailbox_id == other.mailbox_id
-    }
-}
-
-impl Eq for UidMailbox {}
-
-impl ToBitmaps for UidMailbox {
-    fn to_bitmaps(&self, ops: &mut Vec<Operation>, field: u8, set: bool) {
-        ops.push(Operation::Bitmap {
-            class: BitmapClass::Tag {
-                field,
-                value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)),
-            },
-            set,
-        });
-    }
-}
-
-impl SerializeInto for UidMailbox {
-    fn serialize_into(&self, buf: &mut Vec<u8>) {
-        buf.push_leb128(self.mailbox_id);
-        buf.push_leb128(self.uid);
-    }
-}
-
-impl DeserializeFrom for UidMailbox {
-    fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option<Self> {
-        Some(UidMailbox {
-            mailbox_id: bytes.next_leb128()?,
-            uid: bytes.next_leb128()?,
-        })
-    }
-}
-
-impl Serialize for UidMailbox {
-    fn serialize(self) -> Vec<u8> {
-        let mut buf = Vec::with_capacity(U32_LEN * 2);
-        self.serialize_into(&mut buf);
-        buf
-    }
-}
-
-impl UidMailbox {
-    pub fn new(mailbox_id: u32, uid: u32) -> Self {
-        UidMailbox { mailbox_id, uid }
-    }
-
-    pub fn new_unassigned(mailbox_id: u32) -> Self {
-        UidMailbox { mailbox_id, uid: 0 }
-    }
-}

crates/jmap/src/mailbox/query.rs

@@ -5,6 +5,7 @@
  */
 
 use common::{auth::AccessToken, Server};
+use email::mailbox::MailboxFnc;
 use jmap_proto::{
     method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty},
     object::{mailbox::QueryArguments, Object},
@@ -19,8 +20,6 @@ use store::{
 use crate::{auth::acl::AclMethods, JmapMethods, UpdateResults};
 use std::future::Future;
 
-use super::set::MailboxSet;
-
 pub trait MailboxQuery: Sync + Send {
     fn mailbox_query(
         &self,

crates/jmap/src/mailbox/set.rs

@@ -4,16 +4,13 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use common::{auth::AccessToken, config::jmap::settings::SpecialUse, Server};
+use common::{auth::AccessToken, Server};
 use directory::Permission;
+use email::mailbox::{MailboxFnc, SCHEMA};
 use jmap_proto::{
     error::set::{SetError, SetErrorType},
     method::set::{SetRequest, SetResponse},
-    object::{
-        index::{IndexAs, IndexProperty, ObjectIndexBuilder},
-        mailbox::SetArguments,
-        Object,
-    },
+    object::{index::ObjectIndexBuilder, mailbox::SetArguments, Object},
     response::references::EvalObjectReferences,
     types::{
         acl::Acl,
@@ -34,18 +31,15 @@ use store::{
         BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE,
     },
 };
-use trc::AddContext;
 
 use crate::{
     auth::acl::{AclMethods, EffectiveAcl},
-    changes::write::ChangeLog,
     email::delete::EmailDeletion,
     JmapMethods,
 };
 
-use super::{get::MailboxGet, ARCHIVE_ID, DRAFTS_ID, SENT_ID};
 #[allow(unused_imports)]
-use super::{UidMailbox, INBOX_ID, JUNK_ID, TRASH_ID};
+use email::mailbox::{UidMailbox, INBOX_ID, JUNK_ID, TRASH_ID};
 use std::future::Future;
 
 pub struct SetContext<'x> {
@@ -57,24 +51,6 @@ pub struct SetContext<'x> {
     will_destroy: Vec<Id>,
 }
 
-pub static SCHEMA: &[IndexProperty] = &[
-    IndexProperty::new(Property::Name)
-        .index_as(IndexAs::Text {
-            tokenize: true,
-            index: true,
-        })
-        .required(),
-    IndexProperty::new(Property::Role).index_as(IndexAs::Text {
-        tokenize: false,
-        index: true,
-    }),
-    IndexProperty::new(Property::Role).index_as(IndexAs::HasProperty),
-    IndexProperty::new(Property::ParentId).index_as(IndexAs::Integer),
-    IndexProperty::new(Property::SortOrder).index_as(IndexAs::Integer),
-    IndexProperty::new(Property::IsSubscribed).index_as(IndexAs::IntegerList),
-    IndexProperty::new(Property::Acl).index_as(IndexAs::Acl),
-];
-
 pub trait MailboxSet: Sync + Send {
     fn mailbox_set(
         &self,
@@ -97,17 +73,6 @@ pub trait MailboxSet: Sync + Send {
         update: Option<(u32, HashedValue<Object<Value>>)>,
         ctx: &SetContext,
     ) -> impl Future<Output = trc::Result<Result<ObjectIndexBuilder, SetError>>> + Send;
-
-    fn mailbox_get_or_create(
-        &self,
-        account_id: u32,
-    ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send;
-
-    fn mailbox_create_path(
-        &self,
-        account_id: u32,
-        path: &str,
-    ) -> impl Future<Output = trc::Result<Option<(u32, Option<u64>)>>> + Send;
 }
 
 impl MailboxSet for Server {
@@ -801,148 +766,6 @@ impl MailboxSet for Server {
             .with_current_opt(current)
             .validate())
     }
-
-    async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result<RoaringBitmap> {
-        let mut mailbox_ids = self
-            .get_document_ids(account_id, Collection::Mailbox)
-            .await?
-            .unwrap_or_default();
-        if !mailbox_ids.is_empty() {
-            return Ok(mailbox_ids);
-        }
-
-        #[cfg(feature = "test_mode")]
-        if mailbox_ids.is_empty() && account_id == 0 {
-            return Ok(mailbox_ids);
-        }
-
-        let mut batch = BatchBuilder::new();
-        batch
-            .with_account_id(account_id)
-            .with_collection(Collection::Mailbox);
-
-        // Create mailboxes
-        let mut last_document_id = ARCHIVE_ID;
-        for folder in &self.core.jmap.default_folders {
-            let (role, document_id) = match folder.special_use {
-                SpecialUse::Inbox => ("inbox", INBOX_ID),
-                SpecialUse::Trash => ("trash", TRASH_ID),
-                SpecialUse::Junk => ("junk", JUNK_ID),
-                SpecialUse::Drafts => ("drafts", DRAFTS_ID),
-                SpecialUse::Sent => ("sent", SENT_ID),
-                SpecialUse::Archive => ("archive", ARCHIVE_ID),
-                SpecialUse::None => {
-                    last_document_id += 1;
-                    ("", last_document_id)
-                }
-                SpecialUse::Shared => unreachable!(),
-            };
-
-            let mut object = Object::with_capacity(4)
-                .with_property(Property::Name, folder.name.clone())
-                .with_property(Property::ParentId, Value::Id(0u64.into()))
-                .with_property(
-                    Property::Cid,
-                    Value::UnsignedInt(rand::random::<u32>() as u64),
-                );
-            if !role.is_empty() {
-                object.set(Property::Role, role);
-            }
-            if folder.subscribe {
-                object.set(
-                    Property::IsSubscribed,
-                    Value::List(vec![Value::Id(account_id.into())]),
-                );
-            }
-            batch
-                .create_document_with_id(document_id)
-                .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(object));
-            mailbox_ids.insert(document_id);
-        }
-
-        self.core
-            .storage
-            .data
-            .write(batch.build())
-            .await
-            .caused_by(trc::location!())
-            .map(|_| mailbox_ids)
-    }
-
-    async fn mailbox_create_path(
-        &self,
-        account_id: u32,
-        path: &str,
-    ) -> trc::Result<Option<(u32, Option<u64>)>> {
-        let expanded_path =
-            if let Some(expand_path) = self.mailbox_expand_path(account_id, path, false).await? {
-                expand_path
-            } else {
-                return Ok(None);
-            };
-
-        let mut next_parent_id = 0;
-        let mut path = expanded_path.path.into_iter().enumerate().peekable();
-        'outer: while let Some((pos, name)) = path.peek() {
-            let is_inbox = *pos == 0 && name.eq_ignore_ascii_case("inbox");
-
-            for (part, parent_id, document_id) in &expanded_path.found_names {
-                if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox")))
-                    && *parent_id == next_parent_id
-                {
-                    next_parent_id = *document_id;
-                    path.next();
-                    continue 'outer;
-                }
-            }
-            break;
-        }
-
-        // Create missing folders
-        if path.peek().is_some() {
-            let mut changes = self.begin_changes(account_id).await?;
-
-            for (_, name) in path {
-                if name.len() > self.core.jmap.mailbox_name_max_len {
-                    return Ok(None);
-                }
-                let mut batch = BatchBuilder::new();
-                batch
-                    .with_account_id(account_id)
-                    .with_collection(Collection::Mailbox)
-                    .create_document()
-                    .custom(
-                        ObjectIndexBuilder::new(SCHEMA).with_changes(
-                            Object::with_capacity(3)
-                                .with_property(Property::Name, name)
-                                .with_property(
-                                    Property::ParentId,
-                                    Value::Id(Id::from(next_parent_id)),
-                                )
-                                .with_property(
-                                    Property::Cid,
-                                    Value::UnsignedInt(rand::random::<u32>() as u64),
-                                ),
-                        ),
-                    );
-                let document_id = self.write_batch_expect_id(batch).await?;
-                changes.log_insert(Collection::Mailbox, document_id);
-                next_parent_id = document_id + 1;
-            }
-            let change_id = changes.change_id;
-            let mut batch = BatchBuilder::new();
-
-            batch
-                .with_account_id(account_id)
-                .with_collection(Collection::Mailbox)
-                .custom(changes);
-            self.write_batch(batch).await?;
-
-            Ok(Some((next_parent_id - 1, Some(change_id))))
-        } else {
-            Ok(Some((next_parent_id - 1, None)))
-        }
-    }
 }
 
 pub trait MailboxSubscribe {

crates/jmap/src/principal/get.rs

@@ -13,8 +13,6 @@ use jmap_proto::{
 };
 use std::future::Future;
 
-use crate::JmapMethods;
-
 pub trait PrincipalGet: Sync + Send {
     fn principal_get(
         &self,

crates/jmap/src/push/get.rs

@@ -21,8 +21,6 @@ use store::{
 };
 use utils::map::bitmap::Bitmap;
 
-use crate::JmapMethods;
-
 use super::{EncryptionKeys, PushSubscription};
 use std::future::Future;
 

crates/jmap/src/push/set.rs

@@ -24,8 +24,9 @@ use store::{
     rand::{distributions::Alphanumeric, thread_rng, Rng},
     write::{now, BatchBuilder, F_CLEAR, F_VALUE},
 };
+use trc::AddContext;
 
-use crate::{services::state::StateManager, JmapMethods};
+use crate::services::state::StateManager;
 
 const EXPIRES_MAX: i64 = 7 * 24 * 3600; // 7 days
 const VERIFICATION_CODE_LEN: usize = 32;
@@ -119,7 +120,11 @@ impl PushSubscriptionSet for Server {
                 .with_collection(Collection::PushSubscription)
                 .create_document()
                 .value(Property::Value, push, F_VALUE);
-            let document_id = self.write_batch_expect_id(batch).await?;
+            let document_id = self
+                .store()
+                .write_expect_id(batch)
+                .await
+                .caused_by(trc::location!())?;
             push_ids.insert(document_id);
             response.created.insert(
                 id,
@@ -180,7 +185,10 @@ impl PushSubscriptionSet for Server {
                 .with_collection(Collection::PushSubscription)
                 .update_document(document_id)
                 .value(Property::Value, push, F_VALUE);
-            self.write_batch(batch).await?;
+            self.store()
+                .write(batch)
+                .await
+                .caused_by(trc::location!())?;
             response.updated.append(id, None);
         }
 
@@ -195,7 +203,10 @@ impl PushSubscriptionSet for Server {
                     .with_collection(Collection::PushSubscription)
                     .delete_document(document_id)
                     .value(Property::Value, (), F_VALUE | F_CLEAR);
-                self.write_batch(batch).await?;
+                self.store()
+                    .write(batch)
+                    .await
+                    .caused_by(trc::location!())?;
                 response.destroyed.push(id);
             } else {
                 response.not_destroyed.append(id, SetError::not_found());

crates/jmap/src/quota/get.rs

@@ -12,8 +12,6 @@ use jmap_proto::{
 };
 use std::future::Future;
 
-use crate::JmapMethods;
-
 pub trait QuotaGet: Sync + Send {
     fn quota_get(
         &self,

crates/jmap/src/services/delivery.rs

@@ -1,57 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
- *
- * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
- */
-
-use std::sync::Arc;
-
-use common::{core::BuildServer, ipc::DeliveryEvent, Inner};
-use tokio::sync::{mpsc, Semaphore};
-
-use super::ingest::MailDelivery;
-
-pub fn spawn_delivery_manager(inner: Arc<Inner>, mut delivery_rx: mpsc::Receiver<DeliveryEvent>) {
-    tokio::spawn(async move {
-        let semaphore = Arc::new(Semaphore::new(
-            inner
-                .shared_core
-                .load()
-                .smtp
-                .queue
-                .throttle
-                .local_concurrency,
-        ));
-
-        loop {
-            let permit = match semaphore.clone().acquire_owned().await {
-                Ok(permit) => permit,
-                Err(_) => {
-                    trc::error!(trc::StoreEvent::UnexpectedError
-                        .into_err()
-                        .details("Semaphore error")
-                        .caused_by(trc::location!()));
-                    break;
-                }
-            };
-
-            match delivery_rx.recv().await {
-                Some(event) => match event {
-                    DeliveryEvent::Ingest { message, result_tx } => {
-                        let server = inner.build_server();
-
-                        tokio::spawn(async move {
-                            result_tx.send(server.deliver_message(message).await).ok();
-
-                            drop(permit);
-                        });
-                    }
-                    DeliveryEvent::Stop => break,
-                },
-                None => {
-                    break;
-                }
-            }
-        }
-    });
-}

crates/jmap/src/services/gossip/ping.rs

@@ -10,8 +10,6 @@ use common::{
 };
 use trc::ClusterEvent;
 
-use crate::services::index::Indexer;
-
 use super::{request::Request, Gossiper, PeerStatus};
 
 impl Gossiper {

crates/jmap/src/services/index.rs

@@ -11,6 +11,7 @@ use directory::{
     backend::internal::{manage::ManageDirectory, PrincipalField},
     Type,
 };
+use email::{index::IndexMessageText, metadata::MessageMetadata};
 use jmap_proto::types::{collection::Collection, property::Property};
 use store::{
     ahash::AHashMap,
@@ -27,12 +28,7 @@ use std::future::Future;
 use trc::{AddContext, TaskQueueEvent};
 use utils::{BlobHash, BLOB_HASH_LEN};
 
-use crate::{
-    blob::download::BlobDownload,
-    changes::write::ChangeLog,
-    email::{bayes::EmailBayesTrain, index::IndexMessageText, metadata::MessageMetadata},
-    JmapMethods,
-};
+use crate::{blob::download::BlobDownload, email::bayes::EmailBayesTrain};
 
 #[derive(Debug, Clone)]
 pub struct EmailTask {
@@ -81,7 +77,6 @@ pub trait Indexer: Sync + Send {
         account_id: Option<u32>,
         tenant_id: Option<u32>,
     ) -> impl Future<Output = trc::Result<()>> + Send;
-    fn notify_task_queue(&self);
 }
 
 impl Indexer for Server {
@@ -313,10 +308,6 @@ impl Indexer for Server {
         }
     }
 
-    fn notify_task_queue(&self) {
-        self.inner.ipc.index_tx.notify_one();
-    }
-
     async fn reindex(&self, account_id: Option<u32>, tenant_id: Option<u32>) -> trc::Result<()> {
         let accounts = if let Some(account_id) = account_id {
             RoaringBitmap::from_sorted_iter([account_id]).unwrap()

crates/jmap/src/services/mod.rs

@@ -4,9 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-pub mod delivery;
 pub mod gossip;
 pub mod housekeeper;
 pub mod index;
-pub mod ingest;
 pub mod state;

crates/jmap/src/services/state.rs

@@ -372,11 +372,6 @@ pub trait StateManager: Sync + Send {
         types: Bitmap<DataType>,
     ) -> impl Future<Output = trc::Result<mpsc::Receiver<StateChange>>> + Send;
 
-    fn broadcast_state_change(
-        &self,
-        state_change: StateChange,
-    ) -> impl Future<Output = bool> + Send;
-
     fn update_push_subscriptions(&self, account_id: u32) -> impl Future<Output = bool> + Send;
 }
 
@@ -407,28 +402,6 @@ impl StateManager for Server {
         Ok(change_rx)
     }
 
-    async fn broadcast_state_change(&self, state_change: StateChange) -> bool {
-        match self
-            .inner
-            .ipc
-            .state_tx
-            .clone()
-            .send(StateEvent::Publish { state_change })
-            .await
-        {
-            Ok(_) => true,
-            Err(_) => {
-                trc::event!(
-                    Server(ServerEvent::ThreadError),
-                    Details = "Error sending state change.",
-                    CausedBy = trc::location!()
-                );
-
-                false
-            }
-        }
-    }
-
     async fn update_push_subscriptions(&self, account_id: u32) -> bool {
         let push_subs = match self.fetch_push_subscriptions(account_id).await {
             Ok(push_subs) => push_subs,

crates/jmap/src/sieve/get.rs

@@ -4,29 +4,16 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::sync::Arc;
-
 use common::Server;
 use jmap_proto::{
     method::get::{GetRequest, GetResponse, RequestArguments},
     object::Object,
     types::{collection::Collection, property::Property, value::Value},
 };
-use sieve::Sieve;
-use store::{
-    query::Filter,
-    write::{assert::HashedValue, BatchBuilder, Bincode, BlobOp},
-    BlobClass, Deserialize, Serialize,
-};
+use store::BlobClass;
 
-use crate::{
-    blob::{download::BlobDownload, upload::BlobUpload},
-    changes::state::StateManager,
-    sieve::SeenIds,
-    JmapMethods,
-};
+use crate::changes::state::StateManager;
 
-use super::ActiveScript;
 use std::future::Future;
 
 pub trait SieveScriptGet: Sync + Send {
@@ -34,23 +21,6 @@ pub trait SieveScriptGet: Sync + Send {
         &self,
         request: GetRequest<RequestArguments>,
     ) -> impl Future<Output = trc::Result<GetResponse>> + Send;
-
-    fn sieve_script_get_active(
-        &self,
-        account_id: u32,
-    ) -> impl Future<Output = trc::Result<Option<ActiveScript>>> + Send;
-
-    fn sieve_script_get_by_name(
-        &self,
-        account_id: u32,
-        name: &str,
-    ) -> impl Future<Output = trc::Result<Option<Sieve>>> + Send;
-
-    fn sieve_script_compile(
-        &self,
-        account_id: u32,
-        document_id: u32,
-    ) -> impl Future<Output = trc::Result<(Sieve, Object<Value>)>> + Send;
 }
 
 impl SieveScriptGet for Server {
@@ -145,176 +115,4 @@ impl SieveScriptGet for Server {
 
         Ok(response)
     }
-
-    async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result<Option<ActiveScript>> {
-        // Find the currently active script
-        if let Some(document_id) = self
-            .filter(
-                account_id,
-                Collection::SieveScript,
-                vec![Filter::eq(Property::IsActive, 1u32)],
-            )
-            .await?
-            .results
-            .min()
-        {
-            let (script, mut script_object) =
-                self.sieve_script_compile(account_id, document_id).await?;
-            Ok(Some(ActiveScript {
-                document_id,
-                script: Arc::new(script),
-                script_name: script_object
-                    .properties
-                    .remove(&Property::Name)
-                    .and_then(|name| name.try_unwrap_string())
-                    .unwrap_or_else(|| account_id.to_string()),
-                seen_ids: self
-                    .get_property::<Bincode<SeenIds>>(
-                        account_id,
-                        Collection::SieveScript,
-                        document_id,
-                        Property::EmailIds,
-                    )
-                    .await?
-                    .map(|seen_ids| seen_ids.inner)
-                    .unwrap_or_default(),
-            }))
-        } else {
-            Ok(None)
-        }
-    }
-
-    async fn sieve_script_get_by_name(
-        &self,
-        account_id: u32,
-        name: &str,
-    ) -> trc::Result<Option<Sieve>> {
-        // Find the script by name
-        if let Some(document_id) = self
-            .filter(
-                account_id,
-                Collection::SieveScript,
-                vec![Filter::eq(Property::Name, name)],
-            )
-            .await?
-            .results
-            .min()
-        {
-            self.sieve_script_compile(account_id, document_id)
-                .await
-                .map(|(sieve, _)| Some(sieve))
-        } else {
-            Ok(None)
-        }
-    }
-
-    #[allow(clippy::blocks_in_conditions)]
-    async fn sieve_script_compile(
-        &self,
-        account_id: u32,
-        document_id: u32,
-    ) -> trc::Result<(Sieve, Object<Value>)> {
-        // Obtain script object
-        let script_object = self
-            .get_property::<HashedValue<Object<Value>>>(
-                account_id,
-                Collection::SieveScript,
-                document_id,
-                Property::Value,
-            )
-            .await?
-            .ok_or_else(|| {
-                trc::StoreEvent::NotFound
-                    .into_err()
-                    .caused_by(trc::location!())
-                    .document_id(document_id)
-            })?;
-
-        // Obtain the sieve script length
-        let (script_offset, blob_id) = script_object
-            .inner
-            .properties
-            .get(&Property::BlobId)
-            .and_then(|v| v.as_blob_id())
-            .and_then(|v| (v.section.as_ref()?.size, v).into())
-            .ok_or_else(|| {
-                trc::StoreEvent::NotFound
-                    .into_err()
-                    .caused_by(trc::location!())
-                    .document_id(document_id)
-            })?;
-
-        // Obtain the sieve script blob
-        let script_bytes = self
-            .get_blob(&blob_id.hash, 0..usize::MAX)
-            .await?
-            .ok_or_else(|| {
-                trc::StoreEvent::NotFound
-                    .into_err()
-                    .caused_by(trc::location!())
-                    .document_id(document_id)
-            })?;
-
-        // Obtain the precompiled script
-        if let Some(sieve) = script_bytes
-            .get(script_offset..)
-            .and_then(|bytes| Bincode::<Sieve>::deserialize(bytes).ok())
-        {
-            Ok((sieve.inner, script_object.inner))
-        } else {
-            // Deserialization failed, probably because the script compiler version changed
-            match self.core.sieve.untrusted_compiler.compile(
-                script_bytes.get(0..script_offset).ok_or_else(|| {
-                    trc::StoreEvent::NotFound
-                        .into_err()
-                        .caused_by(trc::location!())
-                        .document_id(document_id)
-                })?,
-            ) {
-                Ok(sieve) => {
-                    // Store updated compiled sieve script
-                    let sieve = Bincode::new(sieve);
-                    let compiled_bytes = (&sieve).serialize();
-                    let mut updated_sieve_bytes =
-                        Vec::with_capacity(script_offset + compiled_bytes.len());
-                    updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]);
-                    updated_sieve_bytes.extend_from_slice(&compiled_bytes);
-
-                    // Store updated blob
-                    let mut new_blob_id = blob_id.clone();
-                    new_blob_id.hash = self
-                        .put_blob(account_id, &updated_sieve_bytes, false)
-                        .await?
-                        .hash;
-                    let mut new_script_object = script_object.inner.clone();
-                    new_script_object.set(Property::BlobId, new_blob_id.clone());
-
-                    // Update script object
-                    let mut batch = BatchBuilder::new();
-                    batch
-                        .with_account_id(account_id)
-                        .with_collection(Collection::SieveScript)
-                        .update_document(document_id)
-                        .assert_value(Property::Value, &script_object)
-                        .set(Property::Value, (&new_script_object).serialize())
-                        .clear(BlobOp::Link {
-                            hash: blob_id.hash.clone(),
-                        })
-                        .set(
-                            BlobOp::Link {
-                                hash: new_blob_id.hash,
-                            },
-                            Vec::new(),
-                        );
-                    self.write_batch(batch).await?;
-
-                    Ok((sieve.inner, new_script_object))
-                }
-                Err(error) => Err(trc::StoreEvent::UnexpectedError
-                    .caused_by(trc::location!())
-                    .reason(error)
-                    .details("Failed to compile Sieve script")),
-            }
-        }
-    }
 }

crates/jmap/src/sieve/mod.rs

@@ -4,137 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::sync::Arc;
-
-use serde::ser::SerializeSeq;
-use sieve::Sieve;
-use store::{ahash::AHashSet, blake3, write::now};
-
 pub mod get;
-pub mod ingest;
 pub mod query;
 pub mod set;
 pub mod validate;
-
-pub struct ActiveScript {
-    pub document_id: u32,
-    pub script_name: String,
-    pub script: Arc<Sieve>,
-    pub seen_ids: SeenIds,
-}
-
-#[derive(Debug, Clone)]
-pub struct SeenIdHash {
-    hash: [u8; 32],
-    expiry: u64,
-}
-
-#[derive(Debug, Clone, Default)]
-pub struct SeenIds {
-    pub ids: AHashSet<SeenIdHash>,
-    pub has_changes: bool,
-}
-
-impl SeenIdHash {
-    pub fn new(id: &str, expiry: u64) -> Self {
-        let mut hasher = blake3::Hasher::new();
-        hasher.update(id.as_bytes());
-        SeenIdHash {
-            hash: hasher.finalize().into(),
-            expiry,
-        }
-    }
-}
-
-impl PartialOrd for SeenIdHash {
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for SeenIdHash {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.expiry.cmp(&other.expiry)
-    }
-}
-
-impl std::hash::Hash for SeenIdHash {
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.hash.hash(state);
-    }
-}
-
-impl PartialEq for SeenIdHash {
-    fn eq(&self, other: &Self) -> bool {
-        self.hash == other.hash
-    }
-}
-
-impl Eq for SeenIdHash {}
-
-// SeenIds serializer
-impl serde::Serialize for SeenIds {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        let mut seq = serializer.serialize_seq((self.ids.len() * 2).into())?;
-        for id in &self.ids {
-            seq.serialize_element(&id.expiry)?;
-            seq.serialize_element(&id.hash)?;
-        }
-
-        seq.end()
-    }
-}
-
-impl<'de> serde::Deserialize<'de> for SeenIds {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        deserializer.deserialize_seq(SeenIdsVisitor)
-    }
-}
-
-struct SeenIdsVisitor;
-
-impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor {
-    type Value = SeenIds;
-
-    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-        formatter.write_str("invalid SeenIds")
-    }
-
-    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
-    where
-        A: serde::de::SeqAccess<'de>,
-    {
-        let num_entries = seq.size_hint().unwrap_or(0) / 2;
-        let mut seen_ids = SeenIds {
-            ids: AHashSet::with_capacity(num_entries),
-            has_changes: false,
-        };
-        let now = now();
-
-        for _ in 0..num_entries {
-            let expiry = seq
-                .next_element::<u64>()?
-                .ok_or_else(|| serde::de::Error::custom("Expected expiry."))?;
-            if expiry > now {
-                seen_ids.ids.insert(SeenIdHash {
-                    hash: seq
-                        .next_element()?
-                        .ok_or_else(|| serde::de::Error::custom("Expected hash."))?,
-                    expiry,
-                });
-            } else {
-                seq.next_element::<[u8; 32]>()?
-                    .ok_or_else(|| serde::de::Error::custom("Expected hash."))?;
-                seen_ids.has_changes = true;
-            }
-        }
-
-        Ok(seen_ids)
-    }
-}

crates/jmap/src/sieve/set.rs

@@ -36,13 +36,9 @@ use store::{
     },
     BlobClass,
 };
+use trc::AddContext;
 
-use crate::{
-    api::http::HttpSessionData,
-    blob::{download::BlobDownload, upload::BlobUpload},
-    changes::write::ChangeLog,
-    JmapMethods,
-};
+use crate::{api::http::HttpSessionData, blob::download::BlobDownload, JmapMethods};
 use std::future::Future;
 
 pub struct SetContext<'x> {
@@ -152,7 +148,11 @@ impl SieveScriptSet for Server {
                             }
                         }
 
-                        let document_id = self.write_batch_expect_id(batch).await?;
+                        let document_id = self
+                            .store()
+                            .write_expect_id(batch)
+                            .await
+                            .caused_by(trc::location!())?;
                         sieve_ids.insert(document_id);
                         changes.log_insert(Collection::SieveScript, document_id);
 
@@ -444,7 +444,10 @@ impl SieveScriptSet for Server {
             }
         }
 
-        self.write_batch(batch).await?;
+        self.store()
+            .write(batch)
+            .await
+            .caused_by(trc::location!())?;
         Ok(true)
     }
 

crates/jmap/src/submission/get.rs

@@ -13,7 +13,7 @@ use jmap_proto::{
 use smtp::queue::{self, spool::SmtpSpool};
 use std::future::Future;
 
-use crate::{changes::state::StateManager, JmapMethods};
+use crate::changes::state::StateManager;
 
 pub trait EmailSubmissionGet: Sync + Send {
     fn email_submission_get(

crates/jmap/src/submission/set.rs

@@ -10,6 +10,7 @@ use common::{
     listener::{stream::NullIo, ServerInstance},
     Server,
 };
+use email::metadata::MessageMetadata;
 use jmap_proto::{
     error::set::{SetError, SetErrorType},
     method::set::{self, SetRequest, SetResponse},
@@ -38,12 +39,10 @@ use smtp::{
 };
 use smtp_proto::{request::parser::Rfc5321Parser, MailFrom, RcptTo};
 use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder, Bincode};
+use trc::AddContext;
 use utils::{map::vec_map::VecMap, sanitize_email};
 
-use crate::{
-    blob::download::BlobDownload, changes::write::ChangeLog, email::metadata::MessageMetadata,
-    JmapMethods,
-};
+use crate::blob::download::BlobDownload;
 use std::future::Future;
 
 pub static SCHEMA: &[IndexProperty] = &[
@@ -107,7 +106,11 @@ impl EmailSubmissionSet for Server {
                         .with_collection(Collection::EmailSubmission)
                         .create_document()
                         .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(submission));
-                    let document_id = self.write_batch_expect_id(batch).await?;
+                    let document_id = self
+                        .store()
+                        .write_expect_id(batch)
+                        .await
+                        .caused_by(trc::location!())?;
                     changes.log_insert(Collection::EmailSubmission, document_id);
                     response.created(id, document_id);
                 }
@@ -193,7 +196,10 @@ impl EmailSubmissionSet for Server {
                                             .with_property(Property::UndoStatus, undo_status),
                                     ),
                             );
-                        self.write_batch(batch).await?;
+                        self.store()
+                            .write(batch)
+                            .await
+                            .caused_by(trc::location!())?;
                         changes.log_update(Collection::EmailSubmission, document_id);
                         response.updated.append(id, None);
                     } else {
@@ -242,7 +248,10 @@ impl EmailSubmissionSet for Server {
                     .with_collection(Collection::EmailSubmission)
                     .delete_document(document_id)
                     .custom(ObjectIndexBuilder::new(SCHEMA).with_current(submission));
-                self.write_batch(batch).await?;
+                self.store()
+                    .write(batch)
+                    .await
+                    .caused_by(trc::location!())?;
                 changes.log_delete(Collection::EmailSubmission, document_id);
                 response.destroyed.push(id);
             } else {

crates/jmap/src/thread/get.rs

@@ -14,7 +14,7 @@ use std::future::Future;
 use store::query::{sort::Pagination, Comparator, ResultSet};
 use trc::AddContext;
 
-use crate::{changes::state::StateManager, JmapMethods};
+use crate::changes::state::StateManager;
 
 pub trait ThreadGet: Sync + Send {
     fn thread_get(

crates/jmap/src/vacation/set.rs

@@ -28,10 +28,9 @@ use store::write::{
     log::{Changes, LogInsert},
     BatchBuilder, BlobOp, DirectoryClass, F_CLEAR, F_VALUE,
 };
+use trc::AddContext;
 
 use crate::{
-    blob::upload::BlobUpload,
-    changes::write::ChangeLog,
     sieve::set::{ObjectBlobId, SieveScriptSet, SCHEMA},
     JmapMethods,
 };
@@ -119,7 +118,7 @@ impl VacationResponseSet for Server {
 
         // Prepare write batch
         let mut batch = BatchBuilder::new();
-        let change_id = self.assign_change_id(account_id).await?;
+        let change_id = self.assign_change_id(account_id)?;
         batch
             .with_change_id(change_id)
             .with_account_id(account_id)
@@ -307,7 +306,11 @@ impl VacationResponseSet for Server {
             // Write changes
             batch.custom(obj);
             let document_id = if !batch.is_empty() {
-                let ids = self.write_batch(batch).await?;
+                let ids = self
+                    .store()
+                    .write(batch)
+                    .await
+                    .caused_by(trc::location!())?;
                 response.new_state = Some(change_id.into());
                 match document_id {
                     Some(document_id) => document_id,
@@ -350,7 +353,10 @@ impl VacationResponseSet for Server {
 
             // Write changes
             if !batch.is_empty() {
-                self.write_batch(batch).await?;
+                self.store()
+                    .write(batch)
+                    .await
+                    .caused_by(trc::location!())?;
                 response.new_state = Some(change_id.into());
             }
         }

crates/jmap/src/websocket/stream.rs

@@ -76,7 +76,8 @@ impl WebSocketHandler for Server {
                 let _ = stream
                     .send(Message::Text(
                         WebSocketRequestError::from(RequestError::internal_server_error())
-                            .to_json(),
+                            .to_json()
+                            .into(),
                     ))
                     .await;
                 return;
@@ -128,7 +129,7 @@ impl WebSocketHandler for Server {
                                             response
                                         },
                                     };
-                                    if let Err(err) = stream.send(Message::Text(response)).await {
+                                    if let Err(err) = stream.send(Message::Text(response.into())).await {
                                         trc::event!(Jmap(JmapEvent::WebsocketError),
                                                     Details = "Failed to send text message",
                                                     SpanId = session.session_id,
@@ -208,7 +209,7 @@ impl WebSocketHandler for Server {
                 // Send any queued changes
                 let elapsed = last_changes_sent.elapsed();
                 if elapsed >= throttle {
-                    if let Err(err) = stream.send(Message::Text(changes.to_json())).await {
+                    if let Err(err) = stream.send(Message::Text(changes.to_json().into())).await {
                         trc::event!(
                             Jmap(JmapEvent::WebsocketError),
                             Details = "Failed to send state change message.",
@@ -224,7 +225,7 @@ impl WebSocketHandler for Server {
                     next_event = throttle - elapsed;
                 }
             } else if last_heartbeat.elapsed() > heartbeat {
-                if let Err(err) = stream.send(Message::Ping(vec![])).await {
+                if let Err(err) = stream.send(Message::Ping(Vec::<u8>::new().into())).await {
                     trc::event!(
                         Jmap(JmapEvent::WebsocketError),
                         Details = "Failed to send ping message.",

crates/managesieve/src/op/deletescript.rs

@@ -9,7 +9,7 @@ use std::time::Instant;
 use common::listener::SessionStream;
 use directory::Permission;
 use imap_proto::receiver::Request;
-use jmap::{changes::write::ChangeLog, sieve::set::SieveScriptSet};
+use jmap::sieve::set::SieveScriptSet;
 use jmap_proto::types::collection::Collection;
 use store::write::log::ChangeLogBuilder;
 use trc::AddContext;

crates/managesieve/src/op/getscript.rs

@@ -9,7 +9,7 @@ use std::time::Instant;
 use common::listener::SessionStream;
 use directory::Permission;
 use imap_proto::receiver::Request;
-use jmap::{blob::download::BlobDownload, sieve::set::ObjectBlobId, JmapMethods};
+use jmap::{blob::download::BlobDownload, sieve::set::ObjectBlobId};
 use jmap_proto::{
     object::Object,
     types::{collection::Collection, property::Property, value::Value},

crates/managesieve/src/op/havespace.rs

@@ -9,7 +9,6 @@ use std::time::Instant;
 use common::listener::SessionStream;
 use directory::Permission;
 use imap_proto::receiver::Request;
-use jmap::JmapMethods;
 use trc::AddContext;
 
 use crate::core::{Command, ResponseCode, Session, StatusResponse};

crates/managesieve/src/op/listscripts.rs

@@ -8,7 +8,6 @@ use std::time::Instant;
 
 use common::listener::SessionStream;
 use directory::Permission;
-use jmap::JmapMethods;
 use jmap_proto::{
     object::Object,
     types::{collection::Collection, property::Property, value::Value},

crates/managesieve/src/op/putscript.rs

@@ -10,7 +10,6 @@ use common::listener::SessionStream;
 use directory::Permission;
 use imap_proto::receiver::Request;
 use jmap::{
-    blob::upload::BlobUpload,
     sieve::set::{ObjectBlobId, SCHEMA},
     JmapMethods,
 };
@@ -188,7 +187,8 @@ impl<T: SessionStream> Session<T> {
                     ),
             );
             self.server
-                .write_batch(batch)
+                .store()
+                .write(batch)
                 .await
                 .caused_by(trc::location!())?;
 
@@ -248,7 +248,8 @@ impl<T: SessionStream> Session<T> {
 
             let assigned_ids = self
                 .server
-                .write_batch(batch)
+                .store()
+                .write(batch)
                 .await
                 .caused_by(trc::location!())?;
 

crates/managesieve/src/op/renamescript.rs

@@ -9,7 +9,7 @@ use std::time::Instant;
 use common::listener::SessionStream;
 use directory::Permission;
 use imap_proto::receiver::Request;
-use jmap::{changes::write::ChangeLog, sieve::set::SCHEMA, JmapMethods};
+use jmap::sieve::set::SCHEMA;
 use jmap_proto::{
     object::{index::ObjectIndexBuilder, Object},
     types::{collection::Collection, property::Property, value::Value},
@@ -93,7 +93,8 @@ impl<T: SessionStream> Session<T> {
             );
         if !batch.is_empty() {
             self.server
-                .write_batch(batch)
+                .store()
+                .write(batch)
                 .await
                 .caused_by(trc::location!())?;
             let mut changelog = ChangeLogBuilder::new();

crates/managesieve/src/op/setactive.rs

@@ -9,7 +9,7 @@ use std::time::Instant;
 use common::listener::SessionStream;
 use directory::Permission;
 use imap_proto::receiver::Request;
-use jmap::{changes::write::ChangeLog, sieve::set::SieveScriptSet};
+use jmap::sieve::set::SieveScriptSet;
 use jmap_proto::types::collection::Collection;
 use store::write::log::ChangeLogBuilder;
 use trc::AddContext;

crates/pop3/Cargo.toml

@@ -13,6 +13,7 @@ imap = { path = "../imap" }
 utils = { path = "../utils" }
 trc = { path = "../trc" }
 jmap_proto = { path = "../jmap-proto" }
+email = { path = "../email" }
 mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } 
 mail-send = { version = "0.4", default-features = false, features = ["cram-md5", "ring", "tls12"] }
 rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] }

crates/pop3/src/mailbox.rs

@@ -7,10 +7,7 @@
 use std::collections::BTreeMap;
 
 use common::listener::SessionStream;
-use jmap::{
-    mailbox::{set::MailboxSet, UidMailbox, INBOX_ID},
-    JmapMethods,
-};
+use email::mailbox::{MailboxFnc, UidMailbox, INBOX_ID};
 use jmap_proto::{
     object::Object,
     types::{collection::Collection, property::Property, value::Value},

crates/pop3/src/op/delete.rs

@@ -8,9 +8,7 @@ use std::time::Instant;
 
 use common::listener::SessionStream;
 use directory::Permission;
-use jmap::{
-    changes::write::ChangeLog, email::delete::EmailDeletion, services::state::StateManager,
-};
+use jmap::email::delete::EmailDeletion;
 use jmap_proto::types::{state::StateChange, type_state::DataType};
 use store::roaring::RoaringBitmap;
 use trc::AddContext;

crates/pop3/src/op/fetch.rs

@@ -8,7 +8,8 @@ use std::time::Instant;
 
 use common::listener::SessionStream;
 use directory::Permission;
-use jmap::{blob::download::BlobDownload, email::metadata::MessageMetadata, JmapMethods};
+use email::metadata::MessageMetadata;
+use jmap::blob::download::BlobDownload;
 use jmap_proto::types::{collection::Collection, property::Property};
 use store::write::Bincode;
 use trc::AddContext;

crates/smtp/Cargo.toml

@@ -17,6 +17,7 @@ utils = { path =  "../utils" }
 nlp = { path =  "../nlp" }
 directory = { path =  "../directory" }
 common = { path =  "../common" }
+email = { path =  "../email" }
 spam-filter = { path =  "../spam-filter" }
 trc = { path = "../trc" }
 mail-auth = { version = "0.6" }

crates/smtp/src/core/mod.rs

@@ -7,18 +7,14 @@
 use std::{
     hash::Hash,
     net::IpAddr,
-    sync::{Arc, LazyLock},
+    sync::Arc,
     time::{Duration, Instant},
 };
 
 use common::{
     auth::AccessToken,
     config::smtp::auth::VerifyStrategy,
-    listener::{
-        asn::AsnGeoLookupResult,
-        limiter::{ConcurrencyLimiter, InFlight},
-        ServerInstance,
-    },
+    listener::{asn::AsnGeoLookupResult, limiter::InFlight, ServerInstance},
     Inner, Server,
 };
 use directory::Directory;
@@ -27,7 +23,6 @@ use smtp_proto::request::receiver::{
     BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver,
 };
 use tokio::io::{AsyncRead, AsyncWrite};
-use utils::snowflake::SnowflakeIdGenerator;
 
 use crate::{
     inbound::auth::SaslToken,
@@ -219,18 +214,6 @@ impl PartialOrd for SessionAddress {
     }
 }
 
-static SIEVE: LazyLock<Arc<ServerInstance>> = LazyLock::new(|| {
-    Arc::new(ServerInstance {
-        id: "sieve".to_string(),
-        protocol: common::config::server::ServerProtocol::Lmtp,
-        acceptor: common::listener::TcpAcceptor::Plain,
-        limiter: ConcurrencyLimiter::new(0),
-        shutdown_rx: tokio::sync::watch::channel(false).1,
-        proxy_networks: vec![],
-        span_id_gen: Arc::new(SnowflakeIdGenerator::new()),
-    })
-});
-
 impl Session<common::listener::stream::NullIo> {
     pub fn local(
         server: Server,
@@ -268,20 +251,6 @@ impl Session<common::listener::stream::NullIo> {
         }
     }
 
-    pub fn sieve(
-        server: Server,
-        mail_from: SessionAddress,
-        rcpt_to: Vec<SessionAddress>,
-        message: Vec<u8>,
-        session_id: u64,
-    ) -> Self {
-        Self::local(
-            server,
-            SIEVE.clone(),
-            SessionData::local(mail_from.into(), rcpt_to, message, session_id),
-        )
-    }
-
     pub fn has_failed(&mut self) -> Option<String> {
         if self.stream.tx_buf.first().map_or(true, |&c| c == b'2') {
             self.stream.tx_buf.clear();

crates/smtp/src/inbound/spawn.rs

@@ -68,12 +68,6 @@ impl SessionManager for SmtpSessionManager {
                 .report_tx
                 .send(common::ipc::ReportingEvent::Stop)
                 .await;
-            let _ = self
-                .inner
-                .ipc
-                .delivery_tx
-                .send(common::ipc::DeliveryEvent::Stop)
-                .await;
         }
     }
 }

crates/smtp/src/outbound/delivery.rs

@@ -271,7 +271,7 @@ impl DeliveryAttempt {
                     let delivery_result = message
                         .deliver_local(
                             recipients.iter_mut().filter(|r| r.domain_idx == domain_idx),
-                            &server.inner.ipc.delivery_tx,
+                            &server,
                         )
                         .await;
 

crates/smtp/src/outbound/local.rs

@@ -4,20 +4,24 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use common::ipc::{DeliveryEvent, DeliveryResult, IngestMessage};
+use common::Server;
+use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery};
 use smtp_proto::Response;
-use tokio::sync::{mpsc, oneshot};
-use trc::ServerEvent;
+use trc::SieveEvent;
 
-use crate::queue::{
-    Error, ErrorDetails, HostResponse, Message, Recipient, Status, RCPT_STATUS_CHANGED,
+use crate::{
+    queue::{
+        quota::HasQueueQuota, spool::SmtpSpool, DomainPart, Error, ErrorDetails, HostResponse,
+        Message, MessageSource, Recipient, Status, RCPT_STATUS_CHANGED,
+    },
+    reporting::SmtpReporting,
 };
 
 impl Message {
     pub async fn deliver_local(
         &self,
         recipients: impl Iterator<Item = &mut Recipient>,
-        delivery_tx: &mpsc::Sender<DeliveryEvent>,
+        server: &Server,
     ) -> Status<(), Error> {
         // Prepare recipients list
         let mut total_rcpt = 0;
@@ -37,54 +41,22 @@ impl Message {
             pending_recipients.push(rcpt);
         }
 
-        // Create oneshot channel
-        let (result_tx, result_rx) = oneshot::channel();
-
-        // Deliver message to JMAP server
-        let delivery_result = match delivery_tx
-            .send(DeliveryEvent::Ingest {
-                message: IngestMessage {
-                    sender_address: self.return_path_lcase.clone(),
-                    recipients: recipient_addresses,
-                    message_blob: self.blob_hash.clone(),
-                    message_size: self.size,
-                    session_id: self.span_id,
-                },
-                result_tx,
+        // Deliver message
+        let delivery_result = server
+            .deliver_message(IngestMessage {
+                sender_address: self.return_path_lcase.clone(),
+                recipients: recipient_addresses,
+                message_blob: self.blob_hash.clone(),
+                message_size: self.size,
+                session_id: self.span_id,
             })
-            .await
-        {
-            Ok(_) => {
-                // Wait for result
-                match result_rx.await {
-                    Ok(delivery_result) => delivery_result,
-                    Err(_) => {
-                        trc::event!(
-                            Server(ServerEvent::ThreadError),
-                            CausedBy = trc::location!(),
-                            SpanId = self.span_id,
-                            Reason = "Result channel closed",
-                        );
-                        return Status::local_error();
-                    }
-                }
-            }
-            Err(_) => {
-                trc::event!(
-                    Server(ServerEvent::ThreadError),
-                    CausedBy = trc::location!(),
-                    SpanId = self.span_id,
-                    Reason = "TX channel closed",
-                );
-                return Status::local_error();
-            }
-        };
+            .await;
 
         // Process delivery results
-        for (rcpt, result) in pending_recipients.into_iter().zip(delivery_result) {
+        for (rcpt, result) in pending_recipients.into_iter().zip(delivery_result.status) {
             rcpt.flags |= RCPT_STATUS_CHANGED;
             match result {
-                DeliveryResult::Success => {
+                LocalDeliveryStatus::Success => {
                     rcpt.status = Status::Completed(HostResponse {
                         hostname: "localhost".to_string(),
                         response: Response {
@@ -95,7 +67,7 @@ impl Message {
                     });
                     total_completed += 1;
                 }
-                DeliveryResult::TemporaryFailure { reason } => {
+                LocalDeliveryStatus::TemporaryFailure { reason } => {
                     rcpt.status = Status::TemporaryFailure(HostResponse {
                         hostname: ErrorDetails {
                             entity: "localhost".to_string(),
@@ -108,7 +80,7 @@ impl Message {
                         },
                     });
                 }
-                DeliveryResult::PermanentFailure { code, reason } => {
+                LocalDeliveryStatus::PermanentFailure { code, reason } => {
                     total_completed += 1;
                     rcpt.status = Status::PermanentFailure(HostResponse {
                         hostname: ErrorDetails {
@@ -125,6 +97,56 @@ impl Message {
             }
         }
 
+        // Process autogenerated messages
+        for autogenerated in delivery_result.autogenerated {
+            let from_addr_lcase = autogenerated.sender_address.to_lowercase();
+            let from_addr_domain = from_addr_lcase.domain_part().to_string();
+
+            let mut message = server.new_message(
+                autogenerated.sender_address,
+                from_addr_lcase,
+                from_addr_domain,
+                self.span_id,
+            );
+            for rcpt in autogenerated.recipients {
+                message.add_recipient(rcpt, server).await;
+            }
+
+            // Sign message
+            let signature = server
+                .sign_message(
+                    &mut message,
+                    &server.core.sieve.sign,
+                    &autogenerated.message,
+                )
+                .await;
+
+            // Queue Message
+            message.size = autogenerated.message.len() + signature.as_ref().map_or(0, |s| s.len());
+            if server.has_quota(&mut message).await {
+                message
+                    .queue(
+                        signature.as_deref(),
+                        &autogenerated.message,
+                        self.span_id,
+                        server,
+                        MessageSource::Autogenerated,
+                    )
+                    .await;
+            } else {
+                trc::event!(
+                    Sieve(SieveEvent::QuotaExceeded),
+                    SpanId = self.span_id,
+                    From = message.return_path_lcase,
+                    To = message
+                        .recipients
+                        .into_iter()
+                        .map(|r| trc::Value::from(r.address_lcase))
+                        .collect::<Vec<_>>(),
+                );
+            }
+        }
+
         if total_completed == total_rcpt {
             Status::Completed(())
         } else {

crates/store/src/dispatch/store.rs

@@ -182,7 +182,8 @@ impl Store {
         result.caused_by(trc::location!())
     }
 
-    pub async fn write(&self, batch: Batch) -> trc::Result<AssignedIds> {
+    pub async fn write(&self, batch: impl Into<Batch>) -> trc::Result<AssignedIds> {
+        let batch = batch.into();
         #[cfg(feature = "test_mode")]
         if std::env::var("PARANOID_WRITE").is_ok_and(|v| v == "1") {
             let mut account_id = u32::MAX;
@@ -301,6 +302,13 @@ impl Store {
         result
     }
 
+    #[inline]
+    pub async fn write_expect_id(&self, batch: impl Into<Batch>) -> trc::Result<u32> {
+        self.write(batch)
+            .await
+            .and_then(|ids| ids.last_document_id())
+    }
+
     pub async fn purge_store(&self) -> trc::Result<()> {
         // Delete expired reports
         let now = now();
@@ -925,3 +933,9 @@ impl Store {
         }
     }
 }
+
+impl From<BatchBuilder> for Batch {
+    fn from(builder: BatchBuilder) -> Self {
+        builder.build()
+    }
+}

tests/Cargo.toml

@@ -29,6 +29,7 @@ imap_proto = { path = "../crates/imap-proto" }
 pop3 = { path = "../crates/pop3", features = ["test_mode"] }
 smtp = { path = "../crates/smtp", features = ["test_mode", "enterprise"] }
 common = { path = "../crates/common", features = ["test_mode", "enterprise"] }
+email = { path = "../crates/email", features = ["test_mode"] }
 spam-filter = { path = "../crates/spam-filter", features = ["test_mode", "enterprise"] }
 trc = { path = "../crates/trc" }
 managesieve = { path = "../crates/managesieve", features = ["test_mode", "enterprise"] }

tests/src/imap/mod.rs

@@ -334,7 +334,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest {
     let cache = Caches::parse(&mut config);
 
     let store = core.storage.data.clone();
-    let (ipc, mut ipc_rxs) = build_ipc();
+    let (ipc, mut ipc_rxs) = build_ipc(&mut config);
     let inner = Arc::new(Inner {
         shared_core: core.into_shared(),
         data,

tests/src/jmap/auth_acl.rs

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use jmap::mailbox::{INBOX_ID, TRASH_ID};
+use ::email::mailbox::{INBOX_ID, TRASH_ID};
 use jmap_client::{
     core::{
         error::{MethodError, MethodErrorType},