diff --git a/protos/rustmailer.proto b/protos/rustmailer.proto index 877163c..9ac9811 100644 --- a/protos/rustmailer.proto +++ b/protos/rustmailer.proto @@ -135,6 +135,10 @@ message Account { int64 updated_at = 16; // Method used to access and manage emails. MailerType mailer_type = 17; + // Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). + // - If `None` or not provided, the client will connect directly to the API server. + // - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. + optional uint64 use_proxy = 18; } // PagedAccount represents a paginated list of Account messages. @@ -173,6 +177,10 @@ message AccountCreateRequest { int64 incremental_sync_interval_sec = 9; // Method used to access and manage emails. MailerType mailer_type = 10; + // Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). + // - If `None` or not provided, the client will connect directly to the API server. + // - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. + optional uint64 use_proxy = 11; } // AccountUpdateRequest defines the parameters for updating an existing email account. @@ -195,6 +203,10 @@ message AccountUpdateRequest { optional int64 full_sync_interval_min = 8; // Optional: Update the interval (in seconds) for incremental synchronization updates. optional int64 incremental_sync_interval_sec = 9; + // Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). + // - If `None` or not provided, the client will connect directly to the API server. + // - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. + optional uint64 use_proxy = 10; } // AccountError represents an error encountered during account processing. @@ -338,7 +350,7 @@ service AutoConfigService { // MailBox represents a mail folder on an IMAP server. message MailBox { // The unique hash of the mailbox, serving as a primary key. - uint64 mailbox_hash = 1; + uint64 mailbox_id = 1; // The hash of the account to which this mailbox belongs, serving as a secondary key. uint64 account_hash = 2; // The unique name of the mailbox (e.g., "INBOX", "Sent Items"). @@ -696,6 +708,17 @@ message EmailEnvelope { // The identifier of the thread this email belongs to. // This is computed based on `in_reply_to` / `references` / `message_id`. uint64 thread_id = 26; + // The `mid` field is reserved for potential integration with other backend models. + // For instance, it can be used to store the email index or ID from external services like the Gmail API. + // This ID could be used for reference or identification purposes in scenarios where an external service + // provides an identifier for the email in question. + // This field is optional, meaning that it may be `None` if no external service identifier is available. + optional string mid = 27; + // A list of labels applied to the message. + // Each element is a string representing a Gmail label ID (e.g., "INBOX", "UNREAD"). + // This field reflects the current labels associated with the email. + // Note: This field is populated only for Gmail API accounts. For other account types, it will be empty. + repeated string label_ids = 28; } // FetchMessageContentRequest is used to fetch specific content sections of an email message. diff --git a/src/modules/account/payload.rs b/src/modules/account/payload.rs index eace935..eeb4976 100644 --- a/src/modules/account/payload.rs +++ b/src/modules/account/payload.rs @@ -60,6 +60,10 @@ pub struct AccountCreateRequest { /// Incremental sync interval (seconds), default 60s #[oai(validator(minimum(value = "10"), maximum(value = "3600")))] pub incremental_sync_interval_sec: i64, + /// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). + /// - If `None` or not provided, the client will connect directly to the API server. + /// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. + pub use_proxy: Option, } impl AccountCreateRequest { @@ -139,6 +143,10 @@ pub struct AccountUpdateRequest { /// Incremental sync interval (seconds) #[oai(validator(minimum(value = "10"), maximum(value = "3600")))] pub incremental_sync_interval_sec: Option, + /// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). + /// - If `None` or not provided, the client will connect directly to the API server. + /// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. + pub use_proxy: Option, } impl AccountUpdateRequest { diff --git a/src/modules/account/since.rs b/src/modules/account/since.rs index ef4b4d3..ab39d75 100644 --- a/src/modules/account/since.rs +++ b/src/modules/account/since.rs @@ -6,7 +6,7 @@ use crate::{ modules::error::{code::ErrorCode, RustMailerResult}, raise_error, }; -use chrono::{Datelike, Days, Local, Months, NaiveDate, Utc}; +use chrono::{Datelike, Days, Local, Months, NaiveDate, TimeZone, Utc}; use poem_openapi::{Enum, Object}; use serde::{Deserialize, Serialize}; @@ -101,7 +101,7 @@ impl RelativeDate { Ok(()) } - pub fn calculate_date(&self) -> RustMailerResult { + fn compute_date(&self) -> RustMailerResult> { if self.value == 0 { return Err(raise_error!( "Value must be greater than 0".into(), @@ -125,8 +125,6 @@ impl RelativeDate { })?; let naive_date = date.date_naive(); - - // Check if the date is before 1970 if naive_date.year() < 1970 { return Err(raise_error!( format!( @@ -137,8 +135,18 @@ impl RelativeDate { )); } + Ok(date) + } + + pub fn calculate_date(&self) -> RustMailerResult { + let date = self.compute_date()?; Ok(date.format("%d-%b-%Y").to_string()) } + + pub fn calculate_gmail_date(&self) -> RustMailerResult { + let date = self.compute_date()?; + Ok(date.format("/%Y/%m/%d").to_string()) + } } impl DateSince { @@ -218,6 +226,20 @@ impl DateSince { Ok(date.format("%d-%b-%Y").to_string()) } + pub fn format_for_gmail(&self, fixed: &str) -> RustMailerResult { + let date = NaiveDate::parse_from_str(fixed, "%Y-%m-%d").map_err(|_| { + raise_error!( + format!( + "Invalid date format. Expected 'YYYY-MM-DD'. Example: '2024-11-19'. Provided: '{}'", + fixed + ), + ErrorCode::InvalidParameter + ) + })?; + + Ok(date.format("/%Y/%m/%d").to_string()) + } + pub fn since_date(&self) -> RustMailerResult { // Handle the case where only one of `fixed` or `relative` is provided if let Some(r) = &self.relative { @@ -234,6 +256,19 @@ impl DateSince { )) } } + + pub fn since_gmail_date(&self) -> RustMailerResult { + if let Some(r) = &self.relative { + r.calculate_gmail_date() + } else if let Some(f) = &self.fixed { + self.format_for_gmail(f) + } else { + Err(raise_error!( + "You must provide either a 'fixed' or 'relative' date.".to_string(), + ErrorCode::InvalidParameter + )) + } + } } #[cfg(test)] diff --git a/src/modules/account/v2.rs b/src/modules/account/v2.rs index 743b4c4..117e06a 100644 --- a/src/modules/account/v2.rs +++ b/src/modules/account/v2.rs @@ -18,6 +18,7 @@ use crate::{ since::DateSince, status::AccountRunningState, }, + cache::imap::mailbox::MailBox, database::{insert_impl, list_all_impl}, error::RustMailerResult, }, @@ -28,7 +29,6 @@ use crate::id; use crate::modules::account::payload::AccountCreateRequest; use crate::modules::account::payload::AccountUpdateRequest; use crate::modules::account::payload::MinimalAccount; -use crate::modules::cache::imap::mailbox::MailBox; use crate::modules::cache::imap::manager::EnvelopeFlagsManager; use crate::modules::cache::imap::task::IMAP_TASKS; use crate::modules::context::controller::SYNC_CONTROLLER; @@ -109,6 +109,10 @@ pub struct AccountV2 { pub created_at: i64, /// Last update timestamp (UNIX epoch milliseconds) pub updated_at: i64, + /// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). + /// - If `None` or not provided, the client will connect directly to the API server. + /// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. + pub use_proxy: Option, } impl AccountV2 { @@ -145,6 +149,7 @@ impl AccountV2 { incremental_sync_interval_sec: request.incremental_sync_interval_sec, created_at: utc_now!(), updated_at: utc_now!(), + use_proxy: request.use_proxy, }) } @@ -411,6 +416,11 @@ impl AccountV2 { if let Some(mailboxes) = request.sync_folders { new.sync_folders = mailboxes; } + + if let Some(use_proxy) = request.use_proxy { + new.use_proxy = Some(use_proxy); + } + if let Some(full_sync_interval_min) = &request.full_sync_interval_min { new.full_sync_interval_min = Some(*full_sync_interval_min); } @@ -469,6 +479,7 @@ impl From for AccountV2 { known_folders: value.known_folders, created_at: value.created_at, updated_at: value.updated_at, + use_proxy: None, } } } diff --git a/src/modules/cache/imap/address.rs b/src/modules/cache/imap/address.rs index 09979d9..417ae4a 100644 --- a/src/modules/cache/imap/address.rs +++ b/src/modules/cache/imap/address.rs @@ -14,7 +14,7 @@ use tracing::info; use crate::{ id, modules::{ - cache::imap::v2::EmailEnvelopeV2, + cache::{imap::v2::EmailEnvelopeV3, vendor::gmail::sync::envelope::GmailEnvelope}, database::{batch_delete_impl, filter_by_secondary_key_impl, manager::DB_MANAGER}, error::{code::ErrorCode, RustMailerResult}, utils::envelope_hash, @@ -165,7 +165,7 @@ impl AddressEntity { Ok(()) } - pub fn extract(envelope: &EmailEnvelopeV2) -> Vec { + pub fn extract(envelope: &EmailEnvelopeV3) -> Vec { let from = envelope.from.as_ref().map(|f| f.address.clone()).flatten(); let envelope_hash = envelope.create_envelope_id(); let date = envelope.date.clone(); @@ -180,7 +180,83 @@ impl AddressEntity { account_id, mailbox_id, id: id!(96), - from: from.clone(), + from, + to: None, + cc: None, + envelope_hash, + date, + internal_date, + }); + } + (None, Some(cc)) => { + entities.extend(cc.iter().map(|c| { + let from = from.clone(); + AddressEntity { + account_id, + mailbox_id, + id: id!(96), + from, + to: None, + cc: c.address.clone(), + envelope_hash, + date: date.clone(), + internal_date: internal_date.clone(), + } + })); + } + (Some(to), None) => { + entities.extend(to.iter().map(|t| { + let from = from.clone(); + AddressEntity { + account_id, + mailbox_id, + id: id!(96), + from, + to: t.address.clone(), + cc: None, + envelope_hash, + date: date.clone(), + internal_date: internal_date.clone(), + } + })); + } + (Some(to), Some(cc)) => { + entities.extend(to.iter().flat_map(|t| { + let from = from.clone(); + cc.iter().map(move |c| AddressEntity { + account_id, + mailbox_id, + id: id!(96), + from: from.clone(), + to: t.address.clone(), + cc: c.address.clone(), + envelope_hash, + date: date.clone(), + internal_date: internal_date.clone(), + }) + })); + } + } + + entities + } + + pub fn extract2(envelope: &GmailEnvelope) -> Vec { + let from = envelope.from.as_ref().map(|f| f.address.clone()).flatten(); + let envelope_hash = envelope.create_envelope_id(); + let date = envelope.date.clone(); + let internal_date = Some(envelope.internal_date.clone()); + let account_id = envelope.account_id; + let mailbox_id = envelope.label_id; + let mut entities = Vec::new(); + + match (&envelope.to, &envelope.cc) { + (None, None) => { + entities.push(AddressEntity { + account_id, + mailbox_id, + id: id!(96), + from: from, to: None, cc: None, envelope_hash, diff --git a/src/modules/cache/imap/manager.rs b/src/modules/cache/imap/manager.rs index 058c20e..9a57f1b 100644 --- a/src/modules/cache/imap/manager.rs +++ b/src/modules/cache/imap/manager.rs @@ -15,7 +15,7 @@ use crate::modules::cache::imap::flags_to_hash; use crate::modules::cache::imap::mailbox::EnvelopeFlag; use crate::modules::cache::imap::minimal::MinimalEnvelope; use crate::modules::cache::imap::thread::EmailThread; -use crate::modules::cache::imap::v2::EmailEnvelopeV2; +use crate::modules::cache::imap::v2::EmailEnvelopeV3; use crate::modules::context::Initialize; use crate::modules::error::RustMailerResult; use crate::modules::hook::channel::{Event, EVENT_CHANNEL}; @@ -75,7 +75,7 @@ impl EnvelopeFlagsManager { pub async fn clean_account(account_id: u64) -> RustMailerResult<()> { FLAGS_STATE_MAP.remove(&account_id); - EmailEnvelopeV2::clean_account(account_id).await?; + EmailEnvelopeV3::clean_account(account_id).await?; MinimalEnvelope::clean_account(account_id).await?; AddressEntity::clean_account(account_id).await?; EmailThread::clean_account(account_id).await @@ -99,7 +99,7 @@ impl EnvelopeFlagsManager { FLAGS_STATE_MAP.remove(&account_id); } } - EmailEnvelopeV2::clean_envelopes(account_id, mailbox_id, to_delete_uid).await?; + EmailEnvelopeV3::clean_envelopes(account_id, mailbox_id, to_delete_uid).await?; MinimalEnvelope::clean_envelopes(account_id, mailbox_id, to_delete_uid).await?; AddressEntity::clean_envelopes(account_id, mailbox_id, to_delete_uid).await?; EmailThread::clean_envelopes(account_id, mailbox_id, to_delete_uid).await @@ -110,7 +110,7 @@ impl EnvelopeFlagsManager { if let Some(mailbox_map) = FLAGS_STATE_MAP.get(&account_id) { mailbox_map.remove(&mailbox_id); } - EmailEnvelopeV2::clean_mailbox_envelopes(account_id, mailbox_id).await?; + EmailEnvelopeV3::clean_mailbox_envelopes(account_id, mailbox_id).await?; MinimalEnvelope::clean_mailbox_envelopes(account_id, mailbox_id).await?; AddressEntity::clean_mailbox_envelopes(account_id, mailbox_id).await?; EmailThread::clean_mailbox_envelopes(account_id, mailbox_id).await @@ -141,7 +141,7 @@ impl EnvelopeFlagsManager { if !account.minimal_sync() && EventHookTask::event_watched(account.id, EventType::EmailFlagsChanged).await? { - if let Some(current) = EmailEnvelopeV2::find(account.id, mailbox_id, uid).await? { + if let Some(current) = EmailEnvelopeV3::find(account.id, mailbox_id, uid).await? { let (added, removed) = Self::diff_envelope_flags(¤t.flags, &flags); EVENT_CHANNEL .queue(Event::new( @@ -171,7 +171,7 @@ impl EnvelopeFlagsManager { let flags_hash = flags_to_hash(&flags); if !account.minimal_sync() { - EmailEnvelopeV2::update_flags(account.id, mailbox_id, uid, &flags, flags_hash) + EmailEnvelopeV3::update_flags(account.id, mailbox_id, uid, &flags, flags_hash) .await?; } MinimalEnvelope::update_flags(account.id, mailbox_id, uid, flags_hash).await?; diff --git a/src/modules/cache/imap/minimal.rs b/src/modules/cache/imap/minimal.rs index 68ff8c3..4b2975d 100644 --- a/src/modules/cache/imap/minimal.rs +++ b/src/modules/cache/imap/minimal.rs @@ -11,7 +11,7 @@ use tracing::{error, info}; use crate::{ modules::{ - cache::imap::{v2::EmailEnvelopeV2, manager::EnvelopeFlagsManager}, + cache::imap::{manager::EnvelopeFlagsManager, v2::EmailEnvelopeV3}, database::{ batch_delete_impl, batch_insert_impl, delete_impl, filter_by_secondary_key_impl, manager::DB_MANAGER, update_impl, @@ -94,7 +94,7 @@ impl MinimalEnvelope { } info!( - "Finished deleting envelopes for mailbox_hash={} account_id={} total_deleted={} in {:?}", + "Finished deleting envelopes for mailbox_id={} account_id={} total_deleted={} in {:?}", mailbox_id, account_id, total_deleted, @@ -105,11 +105,11 @@ impl MinimalEnvelope { pub async fn clean_envelopes( account_id: u64, - mailbox_hash: u64, + mailbox_id: u64, to_delete_uid: &[u32], ) -> RustMailerResult<()> { for uid in to_delete_uid { - let key = envelope_hash(account_id, mailbox_hash, *uid); + let key = envelope_hash(account_id, mailbox_id, *uid); delete_impl(DB_MANAGER.envelope_db(), move |rw| { rw.get() .primary::(key) @@ -152,7 +152,7 @@ impl MinimalEnvelope { .await .map_err(|e| { error!( - "Failed to update flags: account_id={}, mailbox_hash={}, uid={}, error={:?}", + "Failed to update flags: account_id={}, mailbox_id={}, uid={}, error={:?}", account_id, mailbox_id, uid, e ); e @@ -196,8 +196,8 @@ impl MinimalEnvelope { } } -impl From<&EmailEnvelopeV2> for MinimalEnvelope { - fn from(value: &EmailEnvelopeV2) -> Self { +impl From<&EmailEnvelopeV3> for MinimalEnvelope { + fn from(value: &EmailEnvelopeV3) -> Self { Self { account_id: value.account_id, mailbox_id: value.mailbox_id, diff --git a/src/modules/cache/imap/mod.rs b/src/modules/cache/imap/mod.rs index ce7ad2f..ad3b91a 100644 --- a/src/modules/cache/imap/mod.rs +++ b/src/modules/cache/imap/mod.rs @@ -7,9 +7,18 @@ use std::sync::LazyLock; use crate::{ calculate_hash, modules::{ - cache::imap::{ - address::AddressEntity, envelope::EmailEnvelope, minimal::MinimalEnvelope, - thread::EmailThread, v2::EmailEnvelopeV2, + cache::{ + imap::{ + address::AddressEntity, + envelope::EmailEnvelope, + minimal::MinimalEnvelope, + thread::EmailThread, + v2::{EmailEnvelopeV2, EmailEnvelopeV3}, + }, + vendor::gmail::sync::{ + envelope::GmailEnvelope, + labels::{GmailCheckPoint, GmailLabels}, + }, }, database::ModelsAdapter, }, @@ -33,10 +42,14 @@ pub static ENVELOPE_MODELS: LazyLock = LazyLock::new(|| { let mut adapter = ModelsAdapter::new(); adapter.register_model::(); adapter.register_model::(); + adapter.register_model::(); adapter.register_model::(); adapter.register_model::(); adapter.register_model::(); adapter.register_model::(); + adapter.register_model::(); + adapter.register_model::(); + adapter.register_model::(); adapter.models }); diff --git a/src/modules/cache/imap/sync/mod.rs b/src/modules/cache/imap/sync/mod.rs index 1d29987..8f19027 100644 --- a/src/modules/cache/imap/sync/mod.rs +++ b/src/modules/cache/imap/sync/mod.rs @@ -3,7 +3,7 @@ // Unauthorized copying, modification, or distribution is prohibited. use crate::modules::{ - account::{status::AccountRunningState, v2::AccountV2}, + account::{entity::MailerType, status::AccountRunningState, v2::AccountV2}, cache::imap::{mailbox::MailBox, manager::EnvelopeFlagsManager}, error::RustMailerResult, hook::{ @@ -30,6 +30,11 @@ pub mod sync_type; static SYNC_COUNTER: AtomicUsize = AtomicUsize::new(0); pub async fn execute_imap_sync(account: &AccountV2) -> RustMailerResult<()> { + assert!( + matches!(account.mailer_type, MailerType::ImapSmtp), + "Bug: Unexpected mailer type, expected ImapSmtp, found: {:?}", + account.mailer_type + ); let start_time = Instant::now(); let account_id = account.id; diff --git a/src/modules/cache/imap/sync/rebuild.rs b/src/modules/cache/imap/sync/rebuild.rs index 7ce7c94..757ebee 100644 --- a/src/modules/cache/imap/sync/rebuild.rs +++ b/src/modules/cache/imap/sync/rebuild.rs @@ -122,11 +122,11 @@ pub async fn should_rebuild_cache( mailbox_count: usize, local_envelope_count: usize, ) -> RustMailerResult { - // If both remote mailboxes and local envelopes exist, no rebuild is needed. + // If both local mailboxes and local envelopes exist, no rebuild is needed. if mailbox_count > 0 && local_envelope_count > 0 { return Ok(false); } - // If there are remote mailboxes but no local envelopes, clear the mailboxes. + // If there are local mailboxes but no local envelopes, clear the mailboxes. if mailbox_count > 0 { let mailboxes = MailBox::list_all(account.id).await?; MailBox::batch_delete(mailboxes).await?; @@ -134,7 +134,7 @@ pub async fn should_rebuild_cache( if local_envelope_count > 0 { EnvelopeFlagsManager::clean_account(account.id).await? } - // If either remote mailboxes or local envelopes were missing, cache rebuild is required. + // If either local mailboxes or local envelopes were missing, cache rebuild is required. Ok(true) } diff --git a/src/modules/cache/imap/sync/sync_type.rs b/src/modules/cache/imap/sync/sync_type.rs index 7b95dc0..f33e6fe 100644 --- a/src/modules/cache/imap/sync/sync_type.rs +++ b/src/modules/cache/imap/sync/sync_type.rs @@ -4,7 +4,7 @@ use crate::{ modules::{ - account::{status::AccountRunningState, v2::AccountV2}, + account::{entity::MailerType, status::AccountRunningState, v2::AccountV2}, error::RustMailerResult, }, utc_now, @@ -27,24 +27,38 @@ pub async fn determine_sync_type(account: &AccountV2) -> RustMailerResult { let now = utc_now!(); - if is_time_for_full_sync( - now, - info.last_full_sync_start, - account - .full_sync_interval_min - .unwrap_or(DEFAULT_FULL_SYNC_INTERVAL_MIN), - ) { - AccountRunningState::set_full_sync_start(account.id).await?; - SyncType::FullSync - } else if is_time_for_incremental_sync( + let incremental_sync = is_time_for_incremental_sync( now, info.last_incremental_sync_start, account.incremental_sync_interval_sec, - ) { - AccountRunningState::set_incremental_sync_start(account.id).await?; - SyncType::IncrementalSync - } else { - SyncType::SkipSync + ); + + match account.mailer_type { + MailerType::ImapSmtp => { + if is_time_for_full_sync( + now, + info.last_full_sync_start, + account + .full_sync_interval_min + .unwrap_or(DEFAULT_FULL_SYNC_INTERVAL_MIN), + ) { + AccountRunningState::set_full_sync_start(account.id).await?; + SyncType::FullSync + } else if incremental_sync { + AccountRunningState::set_incremental_sync_start(account.id).await?; + SyncType::IncrementalSync + } else { + SyncType::SkipSync + } + } + MailerType::GmailApi => { + if incremental_sync { + AccountRunningState::set_incremental_sync_start(account.id).await?; + SyncType::IncrementalSync + } else { + SyncType::SkipSync + } + } } } None => { diff --git a/src/modules/cache/imap/thread.rs b/src/modules/cache/imap/thread.rs index b02491d..19b8cae 100644 --- a/src/modules/cache/imap/thread.rs +++ b/src/modules/cache/imap/thread.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use crate::{ modules::{ - cache::imap::v2::EmailEnvelopeV2, + cache::imap::v2::EmailEnvelopeV3, database::{ batch_delete_impl, delete_impl, manager::DB_MANAGER, paginate_secondary_scan_impl, }, @@ -139,7 +139,7 @@ impl EmailThread { page: u64, page_size: u64, desc: bool, - ) -> RustMailerResult> { + ) -> RustMailerResult> { let threads = paginate_secondary_scan_impl::( DB_MANAGER.envelope_db(), Some(page), @@ -151,7 +151,7 @@ impl EmailThread { .await?; let fetch_tasks = threads.items.into_iter().map(|thread| async move { - EmailEnvelopeV2::get(thread.envelope_id) + EmailEnvelopeV3::get(thread.envelope_id) .await? .ok_or_else(|| { raise_error!( @@ -161,7 +161,7 @@ impl EmailThread { }) }); - let results: RustMailerResult> = + let results: RustMailerResult> = join_all(fetch_tasks).await.into_iter().collect(); let envelopes = results?; diff --git a/src/modules/cache/imap/v2.rs b/src/modules/cache/imap/v2.rs index 2a90f55..65daffd 100644 --- a/src/modules/cache/imap/v2.rs +++ b/src/modules/cache/imap/v2.rs @@ -138,6 +138,125 @@ impl EmailEnvelopeV2 { pub fn create_envelope_id(&self) -> u64 { envelope_hash(self.account_id, self.mailbox_id, self.uid) } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)] +#[native_model(id = 1, version = 3, from = EmailEnvelopeV2)] +#[native_db(primary_key(pk -> String), secondary_key(create_envelope_id -> u64, unique))] +pub struct EmailEnvelopeV3 { + /// The ID of the account owning the email. + #[secondary_key] + pub account_id: u64, + /// The unique identifier of the mailbox where the email is stored (e.g., `MailBox::id`). + /// Used for indexing to avoid updating indexes when mailboxes are renamed. + #[secondary_key] + pub mailbox_id: u64, + /// The decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent"). + pub mailbox_name: String, + /// The unique identifier (IMAP UID) of the email within the mailbox. + pub uid: u32, + /// The date and time the email was received by the server, as a Unix timestamp in milliseconds. + /// If `None`, the internal date is unavailable. + pub internal_date: Option, + /// The size of the email in bytes. + pub size: u32, + /// The flags associated with the email (e.g., `\Seen`, `\Answered`, `\Flagged`). + /// Represented as a list of `EnvelopeFlag` for standard or custom flags. + pub flags: Vec, + /// A hash of the email's flags for efficient comparison or indexing. + pub flags_hash: u64, + /// The blind carbon copy (BCC) recipient(s) of the email, if any. + pub bcc: Option>, + /// The carbon copy (CC) recipient(s) of the email, if any. + pub cc: Option>, + /// The date the email was sent, as a Unix timestamp in milliseconds, if available. + pub date: Option, + /// The sender's address, including name and email, if available. + pub from: Option, + /// The message ID of the email to which this email is a reply, if applicable. + pub in_reply_to: Option, + /// The actual sender's address, if different from the `from` field. + pub sender: Option, + /// The return address for undeliverable emails, if specified. + pub return_address: Option, + /// The unique message ID of the email, typically used for threading. + pub message_id: Option, + /// The subject of the email, if available. + pub subject: Option, + /// The name of the thread this email belongs to, if applicable. + pub thread_name: Option, + /// The identifier of the thread this email belongs to. + /// This is computed based on `in_reply_to` / `references` / `message_id`. + #[secondary_key] + pub thread_id: u64, + /// The MIME version of the email (e.g., "1.0"), if specified. + pub mime_version: Option, + /// A list of message IDs referenced by this email, used for threading. + pub references: Option>, + /// The address(es) to which replies should be sent, if specified. + pub reply_to: Option>, + /// The primary recipient(s) of the email, if any. + pub to: Option>, + /// A list of attachments included in the email, if any. + /// + /// Each `ImapAttachment` item contains metadata including the part ID and MIME type, + /// which indicates the exact location of the attachment in the raw message structure. + /// This allows the backend to directly fetch specific attachments without retrieving + /// the entire message content. + /// + /// This is particularly useful for accounts configured with minimal sync, where full + /// message bodies are not cached locally. By including this data in the API response, + /// the client can request to download only the required attachment via a follow-up + /// API call, improving both efficiency and user experience. + /// + /// Developers do not need to understand the internal IMAP part structure — this + /// metadata provides a clean abstraction for fetching specific attachments. + pub attachments: Option>, + /// Metadata for the email's body parts (e.g., plain text, HTML), if available. + /// + /// Each `EmailBodyPart` contains detailed metadata (such as part ID, content type, + /// and charset) describing a portion of the email body. This enables precise access + /// to body content, such as plain text or HTML sections, without downloading the full + /// raw message from the server. + /// + /// This is especially helpful for lightweight clients or minimized-sync accounts that + /// do not cache full email content. The frontend can pass this metadata back to the + /// server to retrieve only the desired portion of the message (e.g., the HTML body), + /// which significantly reduces bandwidth and latency. + /// + /// By abstracting the complexity of MIME part navigation, developers can efficiently + /// retrieve specific parts of an email without handling the low-level IMAP structure. + pub body_meta: Option>, + /// Details about how the email was received, if available. + pub received: Option, + /// The `mid` field is reserved for potential integration with other backend models. + /// For instance, it can be used to store the email index or ID from external services like the Gmail API. + /// This ID could be used for reference or identification purposes in scenarios where an external service + /// provides an identifier for the email in question. + /// + /// This field is optional, meaning that it may be `None` if no external service identifier is available. + pub mid: Option, + /// A list of labels applied to the message. + /// + /// Each element is a string representing a Gmail label ID (e.g., "INBOX", "UNREAD"). + /// This field reflects the current labels associated with the email. + /// + /// Note: This field is populated only for Gmail API accounts. For other account types, it will be empty. + pub label_ids: Vec, +} + +impl EmailEnvelopeV3 { + pub fn pk(&self) -> String { + format!( + "{}_{}", + self.internal_date.unwrap_or(utc_now!()), + envelope_hash(self.account_id, self.mailbox_id, self.uid) + ) + } + + pub fn create_envelope_id(&self) -> u64 { + envelope_hash(self.account_id, self.mailbox_id, self.uid) + } pub fn compute_thread_id(&self) -> u64 { if self.in_reply_to.is_some() && self.references.as_ref().map_or(false, |r| !r.is_empty()) { @@ -153,10 +272,10 @@ impl EmailEnvelopeV2 { account_id: u64, mailbox_id: u64, uid: u32, - ) -> RustMailerResult> { + ) -> RustMailerResult> { secondary_find_impl( DB_MANAGER.envelope_db(), - EmailEnvelopeV2Key::create_envelope_id, + EmailEnvelopeV3Key::create_envelope_id, envelope_hash(account_id, mailbox_id, uid), ) .await @@ -166,10 +285,10 @@ impl EmailEnvelopeV2 { account_id: u64, mailbox_id: u64, thread_id: u64, - ) -> RustMailerResult> { - let envelopes = filter_by_secondary_key_impl::( + ) -> RustMailerResult> { + let envelopes = filter_by_secondary_key_impl::( DB_MANAGER.envelope_db(), - EmailEnvelopeV2Key::thread_id, + EmailEnvelopeV3Key::thread_id, thread_id, ) .await?; @@ -193,16 +312,16 @@ impl EmailEnvelopeV2 { Ok(result) } - pub async fn get(envelope_id: u64) -> RustMailerResult> { + pub async fn get(envelope_id: u64) -> RustMailerResult> { secondary_find_impl( DB_MANAGER.envelope_db(), - EmailEnvelopeV2Key::create_envelope_id, + EmailEnvelopeV3Key::create_envelope_id, envelope_id, ) .await } - pub async fn save_envelopes(envelopes: Vec) -> RustMailerResult<()> { + pub async fn save_envelopes(envelopes: Vec) -> RustMailerResult<()> { with_transaction(DB_MANAGER.envelope_db(), move |rw| { for mut e in envelopes { // --- Preprocessing --- @@ -226,7 +345,7 @@ impl EmailEnvelopeV2 { ); // --- Store full & minimal envelope --- - rw.insert::(e) + rw.insert::(e) .map_err(|err| raise_error!(format!("{:#?}", err), ErrorCode::InternalError))?; rw.insert::(minimal) .map_err(|err| raise_error!(format!("{:#?}", err), ErrorCode::InternalError))?; @@ -272,13 +391,13 @@ impl EmailEnvelopeV2 { page: u64, page_size: u64, desc: bool, - ) -> RustMailerResult> { + ) -> RustMailerResult> { paginate_secondary_scan_impl( DB_MANAGER.envelope_db(), Some(page), Some(page_size), Some(desc), - EmailEnvelopeV2Key::mailbox_id, + EmailEnvelopeV3Key::mailbox_id, mailbox_id, ) .await @@ -298,8 +417,8 @@ impl EmailEnvelopeV2 { DB_MANAGER.envelope_db(), move |rw| { rw.get() - .secondary::( - EmailEnvelopeV2Key::create_envelope_id, + .secondary::( + EmailEnvelopeV3Key::create_envelope_id, envelope_hash(account_id, mailbox_id, uid), ) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? @@ -320,7 +439,7 @@ impl EmailEnvelopeV2 { .await .map_err(|e| { error!( - "Failed to update flags: account_id={}, mailbox_hash={}, uid={}, error={:?}", + "Failed to update flags: account_id={}, mailbox_id={}, uid={}, error={:?}", account_id, mailbox_id, uid, e ); e @@ -335,15 +454,15 @@ impl EmailEnvelopeV2 { let start_time = Instant::now(); loop { let deleted = batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| { - let to_delete: Vec = rw + let to_delete: Vec = rw .scan() - .secondary(EmailEnvelopeV2Key::mailbox_id) + .secondary(EmailEnvelopeV3Key::mailbox_id) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .start_with(mailbox_id) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .take(BATCH_SIZE) .filter_map(Result::ok) // filter only Ok values - .filter(|e: &EmailEnvelopeV2| e.account_id == account_id) + .filter(|e: &EmailEnvelopeV3| e.account_id == account_id) .collect(); Ok(to_delete) }) @@ -356,7 +475,7 @@ impl EmailEnvelopeV2 { } info!( - "Finished deleting envelopes for mailbox_hash={} account_id={} total_deleted={} in {:?}", + "Finished deleting envelopes for mailbox_id={} account_id={} total_deleted={} in {:?}", mailbox_id, account_id, total_deleted, @@ -374,7 +493,7 @@ impl EmailEnvelopeV2 { let key = envelope_hash(account_id, mailbox_id, *uid); delete_impl(DB_MANAGER.envelope_db(), move |rw| { rw.get() - .secondary::(EmailEnvelopeV2Key::create_envelope_id, key) + .secondary::(EmailEnvelopeV3Key::create_envelope_id, key) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .ok_or_else(|| { raise_error!("envelope missing".into(), ErrorCode::InternalError) @@ -391,9 +510,9 @@ impl EmailEnvelopeV2 { let start_time = Instant::now(); loop { let deleted = batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| { - let to_delete: Vec = rw + let to_delete: Vec = rw .scan() - .secondary(EmailEnvelopeV2Key::account_id) + .secondary(EmailEnvelopeV3Key::account_id) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .start_with(account_id) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? @@ -484,3 +603,71 @@ impl From for EmailEnvelope { } } } + +impl From for EmailEnvelopeV3 { + fn from(value: EmailEnvelopeV2) -> Self { + Self { + account_id: value.account_id, + mailbox_id: value.mailbox_id, + mailbox_name: value.mailbox_name, + uid: value.uid, + internal_date: value.internal_date, + size: value.size, + flags: value.flags, + flags_hash: value.flags_hash, + bcc: value.bcc, + cc: value.cc, + date: value.date, + from: value.from, + in_reply_to: value.in_reply_to, + sender: value.sender, + return_address: value.return_address, + message_id: value.message_id, + subject: value.subject, + thread_name: value.thread_name, + thread_id: value.thread_id, + mime_version: value.mime_version, + references: value.references, + reply_to: value.reply_to, + to: value.to, + attachments: value.attachments, + body_meta: value.body_meta, + received: value.received, + mid: None, + label_ids: vec![], + } + } +} + +impl From for EmailEnvelopeV2 { + fn from(value: EmailEnvelopeV3) -> Self { + Self { + account_id: value.account_id, + mailbox_id: value.mailbox_id, + mailbox_name: value.mailbox_name, + uid: value.uid, + internal_date: value.internal_date, + size: value.size, + flags: value.flags, + flags_hash: value.flags_hash, + bcc: value.bcc, + cc: value.cc, + date: value.date, + from: value.from, + in_reply_to: value.in_reply_to, + sender: value.sender, + return_address: value.return_address, + message_id: value.message_id, + subject: value.subject, + thread_name: value.thread_name, + thread_id: value.thread_id, + mime_version: value.mime_version, + references: value.references, + reply_to: value.reply_to, + to: value.to, + attachments: value.attachments, + body_meta: value.body_meta, + received: value.received, + } + } +} diff --git a/src/modules/cache/mod.rs b/src/modules/cache/mod.rs index 9dfdc70..9c0bd8c 100644 --- a/src/modules/cache/mod.rs +++ b/src/modules/cache/mod.rs @@ -4,3 +4,4 @@ pub mod disk; pub mod imap; +pub mod vendor; diff --git a/src/modules/cache/vendor/gmail/mod.rs b/src/modules/cache/vendor/gmail/mod.rs new file mode 100644 index 0000000..edc3925 --- /dev/null +++ b/src/modules/cache/vendor/gmail/mod.rs @@ -0,0 +1,6 @@ +// Copyright © 2025 rustmailer.com +// Licensed under RustMailer License Agreement v1.0 +// Unauthorized copying, modification, or distribution is prohibited. + +pub mod model; +pub mod sync; diff --git a/src/modules/cache/vendor/gmail/model/history.rs b/src/modules/cache/vendor/gmail/model/history.rs new file mode 100644 index 0000000..9efd39c --- /dev/null +++ b/src/modules/cache/vendor/gmail/model/history.rs @@ -0,0 +1,67 @@ +// Copyright © 2025 rustmailer.com +// Licensed under RustMailer License Agreement v1.0 +// Unauthorized copying, modification, or distribution is prohibited. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct HistoryList { + #[serde(default)] + pub history: Vec, + #[serde(rename = "historyId")] + pub history_id: String, + #[serde(rename = "nextPageToken")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_page_token: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct MessageIndex { + pub id: String, + #[serde(default, rename = "labelIds")] + pub label_ids: Vec, + #[serde(rename = "threadId")] + pub thread_id: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct MessageObject { + pub message: MessageIndex, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct LabelMessageObject { + #[serde(rename = "labelIds")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub label_ids: Option>, + + pub message: MessageIndex, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct History { + pub id: String, + #[serde(rename = "labelsAdded")] + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub labels_added: Vec, + #[serde(rename = "labelsRemoved")] + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub labels_removed: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub messages: Vec, + #[serde(rename = "messagesAdded")] + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub messages_added: Vec, + #[serde(rename = "messagesDeleted")] + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub messages_deleted: Vec, +} + +impl History { + pub fn has_changes(&self) -> bool { + !(self.labels_added.is_empty() + && self.labels_removed.is_empty() + && self.messages_added.is_empty() + && self.messages_deleted.is_empty()) + } +} diff --git a/src/modules/cache/vendor/gmail/model/labels.rs b/src/modules/cache/vendor/gmail/model/labels.rs new file mode 100644 index 0000000..74a4c87 --- /dev/null +++ b/src/modules/cache/vendor/gmail/model/labels.rs @@ -0,0 +1,73 @@ +// Copyright © 2025 rustmailer.com +// Licensed under RustMailer License Agreement v1.0 +// Unauthorized copying, modification, or distribution is prohibited. + +use serde::{Deserialize, Serialize}; +use crate::modules::cache::vendor::gmail::sync::labels::GmailLabels; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct LabelList { + pub labels: Vec