diff --git a/src/modules/account/since.rs b/src/modules/account/since.rs index ab39d75..c8e0189 100644 --- a/src/modules/account/since.rs +++ b/src/modules/account/since.rs @@ -6,7 +6,7 @@ use crate::{ modules::error::{code::ErrorCode, RustMailerResult}, raise_error, }; -use chrono::{Datelike, Days, Local, Months, NaiveDate, TimeZone, Utc}; +use chrono::{Datelike, Days, Local, Months, NaiveDate, Utc}; use poem_openapi::{Enum, Object}; use serde::{Deserialize, Serialize}; diff --git a/src/modules/cache/imap/address.rs b/src/modules/cache/imap/address.rs index 417ae4a..6cdf2b6 100644 --- a/src/modules/cache/imap/address.rs +++ b/src/modules/cache/imap/address.rs @@ -73,7 +73,7 @@ impl AddressEntity { } pub async fn clean_account(account_id: u64) -> RustMailerResult<()> { - const BATCH_SIZE: usize = 500; + const BATCH_SIZE: usize = 200; let mut total_deleted = 0usize; let start_time = Instant::now(); loop { diff --git a/src/modules/cache/imap/minimal.rs b/src/modules/cache/imap/minimal.rs index 4b2975d..d369812 100644 --- a/src/modules/cache/imap/minimal.rs +++ b/src/modules/cache/imap/minimal.rs @@ -68,7 +68,7 @@ impl MinimalEnvelope { } pub async fn clean_mailbox_envelopes(account_id: u64, mailbox_id: u64) -> RustMailerResult<()> { - const BATCH_SIZE: usize = 500; + const BATCH_SIZE: usize = 200; let mut total_deleted = 0usize; let start_time = Instant::now(); loop { @@ -162,7 +162,7 @@ impl MinimalEnvelope { } pub async fn clean_account(account_id: u64) -> RustMailerResult<()> { - const BATCH_SIZE: usize = 500; + const BATCH_SIZE: usize = 200; let mut total_deleted = 0usize; let start_time = Instant::now(); loop { diff --git a/src/modules/cache/imap/task.rs b/src/modules/cache/imap/task.rs index 53f57f6..142b372 100644 --- a/src/modules/cache/imap/task.rs +++ b/src/modules/cache/imap/task.rs @@ -4,6 +4,7 @@ use crate::modules::account::entity::{AuthType, MailerType}; use crate::modules::cache::imap::sync::execute_imap_sync; +use crate::modules::cache::vendor::gmail::sync::execute_gmail_sync; use crate::modules::oauth2::token::OAuth2AccessToken; use crate::modules::scheduler::periodic::TaskHandle; use crate::modules::{ @@ -19,7 +20,7 @@ use tracing::{error, warn}; static _DESCRIPTION: &str = "This task periodically synchronizes mailbox data for a specified account, ensuring that all local data is up-to-date."; const TASK_INTERVAL: Duration = Duration::from_secs(10); -pub static IMAP_TASKS: LazyLock = LazyLock::new(AccountSyncTask::new); +pub static SYNC_TASKS: LazyLock = LazyLock::new(AccountSyncTask::new); static LAST_WARN_TIME: AtomicI64 = AtomicI64::new(0); const WARN_INTERVAL_MS: i64 = 600_000; @@ -34,7 +35,7 @@ impl AccountSyncTask { } } - pub async fn start_account_task(&self, account_id: u64, email: String) { + pub async fn start_account_sync_task(&self, account_id: u64, email: String) { let task_name = format!("account-sync-task-{}-{}", account_id, &email); let periodic_task = PeriodicTask::new(&task_name); let task = move |param: Option| { @@ -54,34 +55,48 @@ impl AccountSyncTask { ); } } else { - if matches!(account.mailer_type, MailerType::ImapSmtp) { - if let AuthType::OAuth2 = account - .imap - .as_ref() - .expect( - "BUG: account.imap is None, but this should never happen here", - ) - .auth - .auth_type - { - if OAuth2AccessToken::get(account.id).await?.is_none() { - if utc_now!() % 300_000 == 0 { - warn!("Account {}: Sync aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id); + match account.mailer_type { + MailerType::ImapSmtp => { + if let AuthType::OAuth2 = account.imap.as_ref().expect("BUG: account.imap is None, but this should never happen here").auth.auth_type { + if OAuth2AccessToken::get(account.id).await?.is_none() { + if utc_now!() % 300_000 == 0 { + warn!("Account {}: Sync aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id); + } + return Ok(()); + } } - return Ok(()); - } - } - if let Err(e) = execute_imap_sync(&account).await { - STATUS_DISPATCHER - .append_error( - account_id, - format!("error in account sync task: {:#?}", e), + if let Err(e) = execute_imap_sync(&account).await { + STATUS_DISPATCHER + .append_error( + account_id, + format!("error in account sync task: {:#?}", e), + ) + .await; + error!( + "Failed to synchronize mailbox data for '{}': {:?}", + account_id, e ) - .await; - error!( - "Failed to synchronize mailbox data for '{}': {:?}", - account_id, e - ) + } + } + MailerType::GmailApi => { + if OAuth2AccessToken::get(account.id).await?.is_none() { + if utc_now!() % 300_000 == 0 { + warn!("Account {}: Sync aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id); + } + return Ok(()); + } + if let Err(e) = execute_gmail_sync(&account).await { + STATUS_DISPATCHER + .append_error( + account_id, + format!("error in account sync task: {:#?}", e), + ) + .await; + error!( + "Failed to synchronize mailbox data for '{}': {:?}", + account_id, e + ) + } } } } diff --git a/src/modules/cache/vendor/gmail/sync/client.rs b/src/modules/cache/vendor/gmail/sync/client.rs index 70f070b..9be672c 100644 --- a/src/modules/cache/vendor/gmail/sync/client.rs +++ b/src/modules/cache/vendor/gmail/sync/client.rs @@ -131,7 +131,7 @@ impl GmailClient { max_results: u32, ) -> RustMailerResult { let mut url = format!( - "https://gmail.googleapis.com/gmail/v1/users/me/history?labelIds={}&maxResults={}&startHistoryId={}", + "https://gmail.googleapis.com/gmail/v1/users/me/history?labelId={}&maxResults={}&startHistoryId={}", label_id, max_results, start_history_id ); diff --git a/src/modules/cache/vendor/gmail/sync/envelope.rs b/src/modules/cache/vendor/gmail/sync/envelope.rs index a9ae041..39f2787 100644 --- a/src/modules/cache/vendor/gmail/sync/envelope.rs +++ b/src/modules/cache/vendor/gmail/sync/envelope.rs @@ -2,8 +2,6 @@ // Licensed under RustMailer License Agreement v1.0 // Unauthorized copying, modification, or distribution is prohibited. -use std::time::Instant; - use crate::{ calculate_hash, id, modules::{ @@ -22,10 +20,12 @@ use crate::{ }, raise_error, }; +use itertools::Itertools; use native_db::*; use native_model::{native_model, Model}; use poem_openapi::Object; use serde::{Deserialize, Serialize}; +use std::time::Instant; use tracing::info; #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)] @@ -174,10 +174,10 @@ impl GmailEnvelope { Some(e.internal_date), e.date, ); - // --- Store envelope --- rw.insert::(e) .map_err(|err| raise_error!(format!("{:#?}", err), ErrorCode::InternalError))?; + // --- Thread upsert --- match rw .get() @@ -261,6 +261,40 @@ impl GmailEnvelope { ); Ok(()) } + + pub async fn clean_account(account_id: u64) -> RustMailerResult<()> { + const BATCH_SIZE: usize = 200; + let mut total_deleted = 0usize; + let start_time = Instant::now(); + loop { + let deleted = batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| { + let to_delete: Vec = rw + .scan() + .secondary(GmailEnvelopeKey::account_id) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .start_with(account_id) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .take(BATCH_SIZE) + .try_collect() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(to_delete) + }) + .await?; + total_deleted += deleted; + // If this batch is empty, break the loop + if deleted == 0 { + break; + } + } + + info!( + "Finished deleting gmail envelopes for account_id={} total_deleted={} in {:?}", + account_id, + total_deleted, + start_time.elapsed() + ); + Ok(()) + } } impl From for EmailEnvelopeV3 { diff --git a/src/modules/cache/vendor/gmail/sync/flow.rs b/src/modules/cache/vendor/gmail/sync/flow.rs index e9300a9..07c13d6 100644 --- a/src/modules/cache/vendor/gmail/sync/flow.rs +++ b/src/modules/cache/vendor/gmail/sync/flow.rs @@ -19,7 +19,7 @@ use crate::{ raise_error, }; -const ENVELOPE_BATCH_SIZE: u32 = 500; +const ENVELOPE_BATCH_SIZE: u32 = 100; pub async fn fetch_and_save_since_date( account: &AccountV2, @@ -37,7 +37,7 @@ pub async fn fetch_and_save_since_date( let mut page_token: Option = None; let mut page = 1; // Used only for tracking sync progress let semaphore = Arc::new(Semaphore::new(10)); - let mut max_history_id = None; + let mut history_ids = Vec::new(); loop { let resp = GmailClient::list_messages( account_id, @@ -109,7 +109,10 @@ pub async fn fetch_and_save_since_date( }) .collect::>>()?; inserted_count += envelopes.len(); - max_history_id = compute_max_history_id(&envelopes); + let hid = compute_max_history_id(&envelopes); + if let Some(hid) = hid { + history_ids.push(hid.to_string()); + } GmailEnvelope::save_envelopes(envelopes).await?; } // Break if API response has no next page @@ -118,7 +121,8 @@ pub async fn fetch_and_save_since_date( } page += 1; } - Ok((inserted_count, max_history_id)) + let hid = max_history_id(&history_ids).map(|s| s.to_string()); + Ok((inserted_count, hid)) } pub async fn fetch_and_save_full_label( @@ -145,8 +149,8 @@ pub async fn fetch_and_save_full_label( // Each page returns message IDs, and we still need to fetch message details individually. let mut page_token: Option = None; let mut page = 1; // Used only for tracking sync progress - let semaphore = Arc::new(Semaphore::new(10)); - let mut max_history_id = None; + let semaphore = Arc::new(Semaphore::new(5)); + let mut history_ids = Vec::new(); loop { let resp = GmailClient::list_messages( account_id, @@ -205,7 +209,10 @@ pub async fn fetch_and_save_full_label( }) .collect::>>()?; inserted_count += envelopes.len(); - max_history_id = compute_max_history_id(&envelopes); + let hid = compute_max_history_id(&envelopes); + if let Some(hid) = hid { + history_ids.push(hid.to_string()); + } GmailEnvelope::save_envelopes(envelopes).await?; } // Break if API response has no next page @@ -214,45 +221,68 @@ pub async fn fetch_and_save_full_label( } page += 1; } - Ok((inserted_count, max_history_id)) + let hid = max_history_id(&history_ids).map(|s| s.to_string()); + Ok((inserted_count, hid)) } -fn max_history_id_fallback(a: &str, b: &str) -> String { - // Try to parse as u64 +fn max_history_id_fallback<'a>(a: &'a str, b: &'a str) -> &'a str { match (a.parse::(), b.parse::()) { (Ok(a_num), Ok(b_num)) => { if a_num >= b_num { - a.to_string() + a } else { - b.to_string() + b } } - // If parsing fails, fall back to length + lexicographical comparison _ => { if a.len() > b.len() { - a.to_string() + a } else if b.len() > a.len() { - b.to_string() + b + } else if a >= b { + a } else { - // Same length, compare lexicographically - if a >= b { - a.to_string() - } else { - b.to_string() - } + b } } } } -fn compute_max_history_id(envelopes: &[GmailEnvelope]) -> Option { +pub fn max_history_id(ids: &[String]) -> Option<&str> { + ids.iter() + .map(|s| s.as_str()) + .reduce(|a, b| max_history_id_fallback(a, b)) +} + +fn compute_max_history_id<'a>(envelopes: &'a [GmailEnvelope]) -> Option<&'a str> { envelopes .iter() .map(|e| e.history_id.as_str()) .fold(None, |max_id, curr| { Some(match max_id { - Some(m) => max_history_id_fallback(m.as_str(), curr), - None => curr.to_string(), + Some(m) => max_history_id_fallback(m, curr), + None => curr, }) }) } + +#[cfg(test)] +mod tests { + use crate::modules::cache::vendor::gmail::sync::flow::max_history_id_fallback; + + #[tokio::test] + async fn test1() { + let ids = vec![ + "2671855", "2671863", "2671871", "2671881", "2671891", "2671898", "100865", "81974", + "81967", "2671905", "531772", "531769", "3296", "1385924", + ]; + + let max_id = ids + .iter() + .cloned() + .reduce(|a, b| max_history_id_fallback(a, b)) + .unwrap(); + + assert_eq!(max_id, "2671905"); + } +} diff --git a/src/modules/cache/vendor/gmail/sync/history.rs b/src/modules/cache/vendor/gmail/sync/history.rs index 5b35030..15a7df6 100644 --- a/src/modules/cache/vendor/gmail/sync/history.rs +++ b/src/modules/cache/vendor/gmail/sync/history.rs @@ -2,7 +2,6 @@ // Licensed under RustMailer License Agreement v1.0 // Unauthorized copying, modification, or distribution is prohibited. - use ahash::{AHashSet, HashSet}; use tokio::task::JoinHandle; use tracing::{info, warn}; @@ -16,6 +15,7 @@ use crate::{ cleanup_single_label, client::GmailClient, envelope::GmailEnvelope, + flow::max_history_id, labels::{GmailCheckPoint, GmailLabels}, rebuild::rebuild_single_label_cache, }, @@ -33,8 +33,9 @@ pub async fn handle_history( let account_id = account.id; let use_proxy = account.use_proxy.clone(); let remote_labels = find_existing_remote_labels(local_labels, remote_labels); + let checkpoint = GmailCheckPoint::get(account_id).await?; + let mut history_ids = Vec::with_capacity(remote_labels.len()); for remote in remote_labels { - let checkpoint = GmailCheckPoint::get(remote.id).await?; let mut page_token = None; loop { let mut list = match GmailClient::list_history( @@ -55,8 +56,11 @@ pub async fn handle_history( code, } => { if code == ErrorCode::GmailApiInvalidHistoryId { - handle_invalid_history_id(account, &remote).await?; - continue; + let history_id = handle_invalid_history_id(account, &remote).await?; + if let Some(history_id) = history_id { + history_ids.push(history_id); + } + break; } else { return Err(raise_error!(message, code)); } @@ -70,17 +74,19 @@ pub async fn handle_history( .into_iter() .filter(|h| h.has_changes()) .collect(); - apply_history(account_id, use_proxy, &remote, history_list).await?; if page_token.is_none() { - GmailCheckPoint::new(account_id, remote.id, list.history_id) - .save() - .await?; + history_ids.push(list.history_id); break; } } GmailLabels::upsert(remote).await?; } + let max = max_history_id(&history_ids); + if let Some(history_id) = max { + let checkpoint = GmailCheckPoint::new(account_id, history_id.to_string()); + checkpoint.save().await?; + } Ok(()) } @@ -214,7 +220,7 @@ pub async fn apply_history( async fn handle_invalid_history_id( account: &AccountV2, label: &GmailLabels, -) -> RustMailerResult<()> { +) -> RustMailerResult> { info!( "Account {}: Invalid history ID detected for label '{}'. Rebuilding local state...", account.id, label.name @@ -229,11 +235,5 @@ async fn handle_invalid_history_id( "Account {}: Upserted label '{}' into local database", account.id, label.name ); - rebuild_single_label_cache(account, label).await?; - info!( - "Account {}: Rebuilt local cache for label '{}'", - account.id, label.name - ); - - Ok(()) + rebuild_single_label_cache(account, label).await } diff --git a/src/modules/cache/vendor/gmail/sync/labels.rs b/src/modules/cache/vendor/gmail/sync/labels.rs index 06cb2c2..b39548c 100644 --- a/src/modules/cache/vendor/gmail/sync/labels.rs +++ b/src/modules/cache/vendor/gmail/sync/labels.rs @@ -13,7 +13,7 @@ use crate::{ cache::imap::mailbox::MailBox, database::{ async_find_impl, batch_delete_impl, batch_insert_impl, delete_impl, - filter_by_secondary_key_impl, insert_impl, manager::DB_MANAGER, upsert_impl, + filter_by_secondary_key_impl, manager::DB_MANAGER, upsert_impl, }, error::{code::ErrorCode, RustMailerResult}, }, @@ -37,10 +37,6 @@ pub struct GmailLabels { } impl GmailLabels { - pub async fn save(&self) -> RustMailerResult<()> { - insert_impl(DB_MANAGER.envelope_db(), self.to_owned()).await - } - pub async fn upsert(label: GmailLabels) -> RustMailerResult<()> { upsert_impl(DB_MANAGER.envelope_db(), label).await } @@ -85,6 +81,22 @@ impl GmailLabels { .await?; Ok(()) } + + pub async fn clean(account_id: u64) -> RustMailerResult<()> { + batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| { + let labels: Vec = rw + .scan() + .secondary::(GmailLabelsKey::account_id) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .start_with(account_id) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .try_collect() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(labels) + }) + .await?; + Ok(()) + } } impl From for MailBox { @@ -110,14 +122,8 @@ impl From for MailBox { #[native_model(id = 8, version = 1)] #[native_db] pub struct GmailCheckPoint { - /// Primary key for the checkpoint. - /// Generated by hashing the combination of `account_id` and `label_id` (as a string). - /// Uniquely identifies the sync state of a specific label for a specific account. - #[primary_key] - pub id: u64, - /// The Gmail account ID this checkpoint belongs to. - #[secondary_key] + #[primary_key] pub account_id: u64, /// The latest Gmail `historyId` for incremental synchronization. @@ -127,30 +133,28 @@ pub struct GmailCheckPoint { /// Creation timestamp in UNIX epoch milliseconds. /// Records when this checkpoint was initially created. pub created_at: i64, - - /// Last update timestamp in UNIX epoch milliseconds. - /// Records the most recent time this checkpoint was updated. - pub updated_at: i64, } impl GmailCheckPoint { - pub async fn get(id: u64) -> RustMailerResult { - let entity = async_find_impl(DB_MANAGER.envelope_db(), id).await?; + pub async fn get(account_id: u64) -> RustMailerResult { + let entity = async_find_impl(DB_MANAGER.envelope_db(), account_id).await?; entity.ok_or_else(|| { raise_error!( - format!("GmailCheckPoint not found for id={}", id), + format!("GmailCheckPoint not found for id={}", account_id), ErrorCode::ResourceNotFound ) }) } - pub fn new(account_id: u64, label_id: u64, max_history_id: String) -> Self { + pub async fn find(account_id: u64) -> RustMailerResult> { + async_find_impl(DB_MANAGER.envelope_db(), account_id).await + } + + pub fn new(account_id: u64, history_id: String) -> Self { Self { - id: label_id, account_id, - history_id: max_history_id, + history_id, created_at: utc_now!(), - updated_at: utc_now!(), } } // Upsert is used here to overwrite the existing record @@ -158,28 +162,21 @@ impl GmailCheckPoint { upsert_impl(DB_MANAGER.envelope_db(), self.to_owned()).await } - pub async fn list_all(account_id: u64) -> RustMailerResult> { - filter_by_secondary_key_impl( - DB_MANAGER.envelope_db(), - GmailLabelsKey::account_id, - account_id, - ) - .await - } - pub async fn clean(account_id: u64) -> RustMailerResult<()> { - batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| { - let to_delete: Vec = rw - .scan() - .secondary(GmailCheckPointKey::account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .start_with(account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(to_delete) - }) - .await?; + if Self::find(account_id).await?.is_some() { + delete_impl(DB_MANAGER.envelope_db(), move |rw| { + rw.get() + .primary::(account_id) + .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))? + .ok_or_else(|| { + raise_error!( + "gmail history id checkpoint missing".into(), + ErrorCode::InternalError + ) + }) + }) + .await?; + } Ok(()) } } diff --git a/src/modules/cache/vendor/gmail/sync/mod.rs b/src/modules/cache/vendor/gmail/sync/mod.rs index be193ae..54ae1fa 100644 --- a/src/modules/cache/vendor/gmail/sync/mod.rs +++ b/src/modules/cache/vendor/gmail/sync/mod.rs @@ -64,12 +64,9 @@ pub async fn execute_gmail_sync(account: &AccountV2) -> RustMailerResult<()> { .collect(); let local_labels = GmailLabels::list_all(account.id).await?; - // How to determine if a rebuild is needed? - // Simplified rule: if the local label does not exist, trigger a rebuild. - // We do not check how many local message metadata entries exist, - // since that would be expensive. - let local_checkpoints = GmailCheckPoint::list_all(account.id).await?; - if should_rebuild_cache(account, local_labels.len(), local_checkpoints.len()).await? { + + let checkpoint = GmailCheckPoint::find(account.id).await?; + if should_rebuild_cache(account, &local_labels, checkpoint).await? { AccountRunningState::set_initial_sync_folders( account.id, remote_labels @@ -78,7 +75,6 @@ pub async fn execute_gmail_sync(account: &AccountV2) -> RustMailerResult<()> { .collect(), ) .await?; - match &account.date_since { Some(date_since) => { rebuild_cache_since_date(account, &remote_labels, date_since).await?; @@ -120,8 +116,14 @@ pub async fn execute_gmail_sync(account: &AccountV2) -> RustMailerResult<()> { } if !missing_labels.is_empty() { + info!( + count = missing_labels.len(), + labels = ?missing_labels, + "Inserting missing Gmail labels into database" + ); GmailLabels::batch_insert(&missing_labels).await?; for label in &missing_labels { + //During incremental synchronization, if any labels are found missing or not fully synchronized, the checkpoint does not need to be updated. rebuild_single_label_cache(account, label).await?; } } @@ -130,24 +132,32 @@ pub async fn execute_gmail_sync(account: &AccountV2) -> RustMailerResult<()> { pub async fn should_rebuild_cache( account: &AccountV2, - local_labels_count: usize, - local_checkpoints_count: usize, + local_labels: &[GmailLabels], + checkpoint: Option, ) -> RustMailerResult { // If both local labels and checkpoint exist, no rebuild is needed. - if local_labels_count > 0 && local_checkpoints_count > 0 { + if !local_labels.is_empty() && checkpoint.is_some() { return Ok(false); } - // If there are local mailboxes but no checkpoints, clear the mailboxes. - if local_labels_count > 0 { - let mailboxes = GmailLabels::list_all(account.id).await?; - GmailLabels::batch_delete(mailboxes).await?; + + info!( + account_id = account.id, + label_count = local_labels.len(), + "Rebuilding cache: cleaning local labels and checkpoints" + ); + + if !local_labels.is_empty() { + GmailLabels::batch_delete(local_labels.to_vec()).await?; } - if local_checkpoints_count > 0 { - //这个要清理,清理掉本地缓存的所有信息,包括关联的索引信息,比如thread, checkpoint也是 - //EnvelopeFlagsManager::clean_account(account.id).await? + if checkpoint.is_some() { GmailCheckPoint::clean(account.id).await?; } - // If either remote mailboxes or local envelopes were missing, cache rebuild is required. + GmailEnvelope::clean_account(account.id).await?; + AddressEntity::clean_account(account.id).await?; + EmailThread::clean_account(account.id).await?; + + info!(account_id = account.id, "Cache cleaning completed"); + Ok(true) } diff --git a/src/modules/cache/vendor/gmail/sync/rebuild.rs b/src/modules/cache/vendor/gmail/sync/rebuild.rs index eebe309..fcb2ccd 100644 --- a/src/modules/cache/vendor/gmail/sync/rebuild.rs +++ b/src/modules/cache/vendor/gmail/sync/rebuild.rs @@ -4,9 +4,9 @@ use crate::modules::{ account::{since::DateSince, v2::AccountV2}, - cache::{ - vendor::gmail::sync::flow::{fetch_and_save_full_label, fetch_and_save_since_date}, - vendor::gmail::sync::labels::{GmailCheckPoint, GmailLabels}, + cache::vendor::gmail::sync::{ + flow::{fetch_and_save_full_label, fetch_and_save_since_date, max_history_id}, + labels::{GmailCheckPoint, GmailLabels}, }, error::RustMailerResult, }; @@ -21,6 +21,8 @@ pub async fn rebuild_cache( let mut total_inserted = 0; GmailLabels::batch_insert(remote_labels).await?; + let mut history_ids = Vec::with_capacity(remote_labels.len()); + for label in remote_labels { if label.exists == 0 { info!( @@ -32,15 +34,14 @@ pub async fn rebuild_cache( match fetch_and_save_full_label(account, label, label.exists, true).await { Ok((inserted, max_history_id)) => { total_inserted += inserted; + if let Some(history_id) = max_history_id { - GmailCheckPoint::new(account.id, label.id, history_id) - .save() - .await?; + history_ids.push(history_id); } } Err(e) => { warn!( - "Account {}: Failed to sync label '{}'. Error: {}. Removing label entry.", + "Account {}: Failed to sync label '{}'. Error: {:#?}. Removing label entry.", account.id, &label.name, e ); if let Err(del_err) = GmailLabels::delete(label.id).await { @@ -52,6 +53,11 @@ pub async fn rebuild_cache( } } } + let max = max_history_id(&history_ids); + if let Some(history_id) = max { + let checkpoint = GmailCheckPoint::new(account.id, history_id.to_string()); + checkpoint.save().await?; + } let elapsed_time = start_time.elapsed().as_secs(); info!( "Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \ @@ -71,6 +77,7 @@ pub async fn rebuild_cache_since_date( let date = date_since.since_gmail_date()?; GmailLabels::batch_insert(remote_labels).await?; + let mut history_ids = Vec::with_capacity(remote_labels.len()); for label in remote_labels { if label.exists == 0 { info!( @@ -83,12 +90,8 @@ pub async fn rebuild_cache_since_date( match fetch_and_save_since_date(account, date.as_str(), label, true).await { Ok((inserted, max_history_id)) => { total_inserted += inserted; - // After each label finishes syncing, record its checkpoint individually. - // This avoids fetching a large amount of unnecessary history records. if let Some(history_id) = max_history_id { - GmailCheckPoint::new(account.id, label.id, history_id) - .save() - .await?; + history_ids.push(history_id); } } Err(e) => { @@ -106,6 +109,11 @@ pub async fn rebuild_cache_since_date( } } + let max = max_history_id(&history_ids); + if let Some(history_id) = max { + let checkpoint = GmailCheckPoint::new(account.id, history_id.to_string()); + checkpoint.save().await?; + } let elapsed_time = start_time.elapsed().as_secs(); info!( "Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \ @@ -118,7 +126,7 @@ pub async fn rebuild_cache_since_date( pub async fn rebuild_single_label_cache( account: &AccountV2, label: &GmailLabels, -) -> RustMailerResult<()> { +) -> RustMailerResult> { if label.exists > 0 { match &account.date_since { Some(date_since) => { @@ -129,15 +137,11 @@ pub async fn rebuild_single_label_cache( "Account {}: Label '{}' synced successfully. {} messages inserted.", account.id, label.name, inserted ); - if let Some(history_id) = max_history_id { - GmailCheckPoint::new(account.id, label.id, history_id) - .save() - .await?; - } + return Ok(max_history_id); } Err(e) => { warn!( - "Account {}: Failed to sync mailbox '{}'. Error: {}. Removing mailbox entry.", + "Account {}: Failed to sync mailbox '{}'. Error: {}. Removing label entry.", account.id, &label.name, e ); if let Err(del_err) = GmailLabels::delete(label.id).await { @@ -155,15 +159,11 @@ pub async fn rebuild_single_label_cache( "Account {}: Label '{}' synced successfully. {} messages inserted.", account.id, label.name, inserted ); - if let Some(history_id) = max_history_id { - GmailCheckPoint::new(account.id, label.id, history_id) - .save() - .await?; - } + return Ok(max_history_id); } Err(e) => { warn!( - "Account {}: Failed to sync label '{}'. Error: {}. Removing label entry.", + "Account {}: Failed to sync label '{}'. Error: {:#?}. Removing label entry.", account.id, &label.name, e ); if let Err(del_err) = GmailLabels::delete(label.id).await { @@ -176,5 +176,5 @@ pub async fn rebuild_single_label_cache( }, } } - Ok(()) + Ok(None) } diff --git a/src/modules/cache/vendor/gmail/sync/sync_labels.rs b/src/modules/cache/vendor/gmail/sync/sync_labels.rs index 79fec25..42057b6 100644 --- a/src/modules/cache/vendor/gmail/sync/sync_labels.rs +++ b/src/modules/cache/vendor/gmail/sync/sync_labels.rs @@ -113,6 +113,5 @@ pub async fn retrieve_label_metadata( Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)), } } - Ok(details) } diff --git a/src/modules/cache/vendor/tests.rs b/src/modules/cache/vendor/tests.rs index 7db8f24..4a84e25 100644 --- a/src/modules/cache/vendor/tests.rs +++ b/src/modules/cache/vendor/tests.rs @@ -38,7 +38,7 @@ async fn access_token() -> String { grpc_client.set_send_compressed(CompressionEncoding::GZIP); let request = GetOAuth2TokensRequest { - account_id: 7397694139904449, + account_id: 1908057970788951, }; let mut request = poem_grpc::Request::new(request); @@ -54,14 +54,13 @@ async fn access_token() -> String { async fn test1() { let access_token = access_token().await; let url = "https://gmail.googleapis.com/gmail/v1/users/me/labels/Label_6886728075529239043"; - let url = "https://gmail.googleapis.com/gmail/v1/users/me/messages?labelIds=INBOX&maxResults=10&pageToken=08792416985640480557"; let url = "https://gmail.googleapis.com/gmail/v1/users/me/messages/198e590baf688394?format=metadata"; let url = "https://gmail.googleapis.com/gmail/v1/users/me/messages?labelIds=INBOX&maxResults=10"; let url = "https://gmail.googleapis.com/gmail/v1/users/me/labels/Label_6886728075529239043"; let url = "https://gmail.googleapis.com/gmail/v1/users/me/labels"; - + let url = "https://gmail.googleapis.com/gmail/v1/users/me/messages?labelIds=SENT&maxResults=20"; let mut builder = reqwest::ClientBuilder::new() .user_agent(rustmailer_version!()) .timeout(Duration::from_secs(10)) diff --git a/src/modules/context/controller.rs b/src/modules/context/controller.rs index 6957f51..d3abae3 100644 --- a/src/modules/context/controller.rs +++ b/src/modules/context/controller.rs @@ -2,7 +2,7 @@ // Licensed under RustMailer License Agreement v1.0 // Unauthorized copying, modification, or distribution is prohibited. -use crate::modules::{cache::imap::task::IMAP_TASKS, error::RustMailerResult}; +use crate::modules::{cache::imap::task::SYNC_TASKS, error::RustMailerResult}; use std::{sync::LazyLock, time::Duration}; use tokio::sync::mpsc; use tracing::{error, info}; @@ -44,10 +44,10 @@ impl SyncController { async fn start_syncer(account_id: u64, email: String) -> RustMailerResult> { info!( - "IMAP syncer starting for account: {}-{}.", + "Account syncer starting for account: {}-{}.", account_id, email ); - IMAP_TASKS.start_account_task(account_id, email).await; + SYNC_TASKS.start_account_sync_task(account_id, email).await; tokio::time::sleep(Duration::from_millis(100)).await; Ok(Some(())) } diff --git a/src/modules/context/executors.rs b/src/modules/context/executors.rs index dcc2347..3719b10 100644 --- a/src/modules/context/executors.rs +++ b/src/modules/context/executors.rs @@ -119,7 +119,7 @@ impl EmailClientExecutors { let active_accounts: Vec = accounts.into_iter().filter(|a| a.enabled).collect(); if active_accounts.is_empty() { - info!("No active accounts found for IMAP initialization."); + info!("No active accounts found for account initialization."); return Ok(()); } info!( diff --git a/src/modules/database/tests.rs b/src/modules/database/tests.rs index 351e73d..63ea023 100644 --- a/src/modules/database/tests.rs +++ b/src/modules/database/tests.rs @@ -8,7 +8,10 @@ use crate::{ id, modules::{ account::{entity::AccountKey, v2::AccountV2}, - cache::imap::mailbox::MailBox, + cache::{ + imap::{mailbox::MailBox, ENVELOPE_MODELS}, + vendor::gmail::sync::{envelope::GmailEnvelope, flow::max_history_id}, + }, database::META_MODELS, hook::{ entity::{EventHooks, HookType, HttpConfig, HttpMethod}, @@ -111,3 +114,34 @@ fn test6() { println!("{}", serde_json::to_string_pretty(&test).unwrap()); } + +#[test] +fn test7() { + let database = Builder::new() + .create( + &ENVELOPE_MODELS, + PathBuf::from("D://rustmailer_data//envelope.db"), + ) + .unwrap(); + //database.compact().unwrap(); + let r_transaction = database.r_transaction().unwrap(); + let entities: Vec = r_transaction + .scan() + .primary() + .unwrap() + .all() + .unwrap() + .try_collect() + .unwrap(); + // println!("{:#?}", entities); + + let history_ids: Vec = entities + .into_iter() + .filter(|e| e.label_name == "INBOX") + .map(|e| e.history_id) + .collect(); + println!("{}", history_ids.len()); + + let max_id = max_history_id(&history_ids); + println!("{:#?}", max_id); +} diff --git a/src/modules/hook/http/mod.rs b/src/modules/hook/http/mod.rs index 8e46f5e..e5fdb88 100644 --- a/src/modules/hook/http/mod.rs +++ b/src/modules/hook/http/mod.rs @@ -4,6 +4,8 @@ use dashmap::DashMap; use http::header::{AUTHORIZATION, CONTENT_TYPE}; +use http::StatusCode; +use tracing::error; use crate::modules::error::code::ErrorCode; use crate::modules::hook::entity::HttpMethod; @@ -119,7 +121,7 @@ impl HttpClient { .await .map_err(|e| { raise_error!( - format!("Request failed for URL {}: {:#?}", url, e), + format!("Request failed: {:#?}", e), ErrorCode::InternalError ) })?; @@ -127,7 +129,7 @@ impl HttpClient { if res.status().is_success() { let json: serde_json::Value = res.json().await.map_err(|e| { raise_error!( - format!("Failed to parse response from URL {}: {:#?}", url, e), + format!("Failed to parse response: {:#?}", e), ErrorCode::InternalError ) })?; @@ -136,14 +138,21 @@ impl HttpClient { let status = res.status(); let text = res.text().await.map_err(|e| { raise_error!( - format!("Failed to read error response from URL {}: {:#?}", url, e), + format!("Failed to read error response: {:#?}", e), ErrorCode::InternalError ) })?; - if status.is_client_error() { + if matches!(status, StatusCode::NOT_FOUND) || matches!(status, StatusCode::BAD_REQUEST) + { + error!( + status = ?status, + url = %url, + response = %text, + "Gmail API client error" + ); return Err(raise_error!( format!( - "Gmail API returned client error (status {}) for {}: historyId may be invalid or expired. Response: {}", + "Gmail API returned client error (status {}) for {}. Response: {}", status, url, text ), ErrorCode::GmailApiInvalidHistoryId diff --git a/src/modules/mailbox/create.rs b/src/modules/mailbox/create.rs index 9f91ef9..4a4da0a 100644 --- a/src/modules/mailbox/create.rs +++ b/src/modules/mailbox/create.rs @@ -8,7 +8,7 @@ use crate::{ }; pub async fn create_mailbox(account_id: u64, mailbox_name: &str) -> RustMailerResult<()> { - AccountV2::check_account_active(account_id).await?; + AccountV2::check_account_active(account_id, false).await?; let executor = RUST_MAIL_CONTEXT.imap(account_id).await?; executor .create_mailbox(encode_mailbox_name!(mailbox_name).as_str()) diff --git a/src/modules/mailbox/delete.rs b/src/modules/mailbox/delete.rs index 98ad23b..c6f16b6 100644 --- a/src/modules/mailbox/delete.rs +++ b/src/modules/mailbox/delete.rs @@ -2,12 +2,17 @@ // Licensed under RustMailer License Agreement v1.0 // Unauthorized copying, modification, or distribution is prohibited. -use crate::{encode_mailbox_name, modules::{ - account::v2::AccountV2, context::executors::RUST_MAIL_CONTEXT, error::RustMailerResult, -}}; +use crate::{ + encode_mailbox_name, + modules::{ + account::v2::AccountV2, context::executors::RUST_MAIL_CONTEXT, error::RustMailerResult, + }, +}; pub async fn delete_mailbox(account_id: u64, mailbox_name: &str) -> RustMailerResult<()> { - AccountV2::check_account_active(account_id).await?; + AccountV2::check_account_active(account_id, false).await?; let executor = RUST_MAIL_CONTEXT.imap(account_id).await?; - executor.delete_mailbox(encode_mailbox_name!(mailbox_name).as_str()).await + executor + .delete_mailbox(encode_mailbox_name!(mailbox_name).as_str()) + .await } diff --git a/src/modules/mailbox/list.rs b/src/modules/mailbox/list.rs index 62995be..850bef9 100644 --- a/src/modules/mailbox/list.rs +++ b/src/modules/mailbox/list.rs @@ -2,8 +2,14 @@ // Licensed under RustMailer License Agreement v1.0 // Unauthorized copying, modification, or distribution is prohibited. +use std::sync::Arc; + +use crate::modules::account::entity::MailerType; use crate::modules::account::v2::AccountV2; use crate::modules::cache::imap::mailbox::{Attribute, AttributeEnum, MailBox}; +use crate::modules::cache::vendor::gmail::model::labels::{Label, LabelDetail}; +use crate::modules::cache::vendor::gmail::sync::client::GmailClient; +use crate::modules::cache::vendor::gmail::sync::labels::GmailLabels; use crate::modules::context::executors::RUST_MAIL_CONTEXT; use crate::modules::error::code::ErrorCode; use crate::modules::error::{RustMailerError, RustMailerResult}; @@ -15,17 +21,23 @@ pub async fn get_account_mailboxes( account_id: u64, remote: bool, ) -> RustMailerResult> { - let account = AccountV2::check_account_active(account_id).await?; + let account = AccountV2::check_account_active(account_id, false).await?; let remote = remote || account.minimal_sync(); - if remote { - request_imap_all_mailbox_list(account_id).await - } else { - MailBox::list_all(account_id).await + + match (&account.mailer_type, remote) { + (MailerType::ImapSmtp, true) => request_imap_all_mailbox_list(account_id).await, + (MailerType::ImapSmtp, false) => MailBox::list_all(account_id).await, + + (MailerType::GmailApi, true) => request_gmail_label_list(&account).await, + (MailerType::GmailApi, false) => { + let labels = GmailLabels::list_all(account_id).await?; + Ok(labels.into_iter().map(Into::into).collect()) + } } } pub async fn list_subscribed_mailboxes(account_id: u64) -> RustMailerResult> { - AccountV2::check_account_active(account_id).await?; + AccountV2::check_account_active(account_id, true).await?; request_imap_subscribed_mailbox_list(account_id).await } @@ -43,6 +55,50 @@ pub async fn request_imap_all_mailbox_list(account_id: u64) -> RustMailerResult< convert_names_to_mailboxes(account_id, names.iter()).await } +pub async fn request_gmail_label_list(account: &AccountV2) -> RustMailerResult> { + let all_labels = GmailClient::list_labels(account.id, account.use_proxy).await?; + let visible_labels: Vec