feat(thread): Add support for fetching thread lists and thread messages from remote in Gmail and Graph API

This commit is contained in:
rustmailer
2025-12-03 02:43:02 +08:00
parent f89b8b1c2f
commit 49ba67a7d2
50 changed files with 1299 additions and 546 deletions
+1 -1
View File
@@ -930,7 +930,7 @@ async fn process_email_added_events(
message: message_content,
thread_name: envelope.thread_name,
reply_to: envelope.reply_to,
thread_id,
thread_id: thread_id.to_string(),
labels: vec![],
}),
),
+1 -1
View File
@@ -242,7 +242,7 @@ impl EmailThread {
let results: RustMailerResult<Vec<GmailEnvelope>> =
join_all(fetch_tasks).await.into_iter().collect();
let map = GmailClient::label_map(account.id, account.use_proxy).await?;
let map = GmailClient::for_get_label_name(account.id, account.use_proxy).await?;
let envelopes = results?
.into_iter()
.map(|e| e.into_envelope(&map))
+2 -2
View File
@@ -70,7 +70,7 @@ pub struct Envelope {
pub thread_name: Option<String>,
/// The identifier of the thread this email belongs to.
/// This is computed based on `in_reply_to` / `references` / `message_id`.
pub thread_id: u64,
pub thread_id: String,
/// The MIME version of the email (e.g., "1.0"), if specified.
pub mime_version: Option<String>,
/// A list of message IDs referenced by this email, used for threading.
@@ -162,7 +162,7 @@ impl From<EmailEnvelopeV3> for Envelope {
message_id: value.message_id,
subject: value.subject,
thread_name: value.thread_name,
thread_id: value.thread_id,
thread_id: value.thread_id.to_string(),
mime_version: value.mime_version,
references: value.references,
reply_to: value.reply_to,
+11
View File
@@ -7,7 +7,18 @@ use std::time::Duration;
use ahash::AHashMap;
use crate::modules::cache::vendor::gmail::model::thread::ThreadMessages;
use crate::modules::common::lru::TimedLruCache;
pub static GMAIL_LABELS_CACHE: LazyLock<TimedLruCache<u64, AHashMap<String, String>>> =
LazyLock::new(|| TimedLruCache::new(100, Duration::from_secs(60)));
// Gmail thread message cache.
// Key format: "{account_id}:{thread_id}"
// Stores the full message list of a thread to avoid repeated Gmail API calls.
pub static GMAIL_THREADS_CACHE: LazyLock<TimedLruCache<String, ThreadMessages>> =
LazyLock::new(|| TimedLruCache::new(2000, Duration::from_secs(180)));
pub fn thread_cache_key(account_id: u64, thread_id: &str) -> String {
format!("{}:{}", account_id, thread_id)
}
+1 -1
View File
@@ -9,7 +9,7 @@ pub struct HistoryList {
#[serde(default)]
pub history: Vec<History>,
#[serde(rename = "historyId")]
pub history_id: String,
pub history_id: String,//This is the current maximum history ID for the entire account.
#[serde(rename = "nextPageToken")]
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_page_token: Option<String>,
+2 -1
View File
@@ -4,4 +4,5 @@
pub mod history;
pub mod labels;
pub mod messages;
pub mod messages;
pub mod thread;
+168
View File
@@ -0,0 +1,168 @@
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use crate::{
modules::{
cache::{
model::Envelope,
vendor::gmail::{
cache::{thread_cache_key, GMAIL_THREADS_CACHE},
model::messages::MessageMeta,
sync::{client::GmailClient, envelope::GmailEnvelope},
},
},
error::{code::ErrorCode, RustMailerResult},
rest::response::CursorDataPage,
},
raise_error,
};
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ThreadIndex {
#[serde(rename = "historyId")]
pub history_id: String,
pub id: String,
#[serde(default)]
pub snippet: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ThreadList {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub threads: Option<Vec<ThreadIndex>>,
#[serde(rename = "nextPageToken")]
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_page_token: Option<String>,
#[serde(rename = "resultSizeEstimate")]
#[serde(default, skip_serializing_if = "Option::is_none")]
pub result_size_estimate: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ThreadMessages {
#[serde(rename = "historyId")]
pub history_id: String,
pub id: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub messages: Vec<MessageMeta>,
///A short part of the message text.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub snippet: Option<String>,
}
pub async fn list_threads_impl(
account_id: u64,
use_proxy: Option<u64>,
label_id: Option<&str>,
page_token: Option<&str>,
after: Option<&str>,
max_results: u64,
) -> RustMailerResult<CursorDataPage<Envelope>> {
let list = GmailClient::list_threads_internal(
account_id,
use_proxy,
label_id,
page_token,
after,
max_results,
)
.await?;
let threads = match list.threads {
Some(t) => t,
None => {
return Ok(CursorDataPage::new(
None,
Some(max_results),
0,
None,
vec![],
))
}
};
let label_map = GmailClient::for_get_label_name(account_id, use_proxy).await?;
let mut envelopes = Vec::with_capacity(threads.len());
for thread in threads {
let thread_id = &thread.id;
let cache_key = thread_cache_key(account_id, thread_id);
let thread_messages = if let Some(cached) = GMAIL_THREADS_CACHE.get(&cache_key).await {
(*cached).clone()
} else {
let fetched =
GmailClient::get_thread_messages(account_id, use_proxy, thread_id).await?;
GMAIL_THREADS_CACHE
.set(cache_key.clone(), Arc::new(fetched.clone()))
.await;
fetched
};
let thread_envelopes: Vec<GmailEnvelope> = thread_messages
.messages
.into_iter()
.map(GmailEnvelope::try_from)
.collect::<Result<_, _>>()?;
if let Some(mut envelope) = thread_envelopes
.into_iter()
.max_by_key(|env| env.internal_date)
{
envelope.account_id = account_id;
envelopes.push(envelope.into_envelope(&label_map));
} else {
tracing::warn!("No valid Gmail message found in thread {}", thread_id);
}
}
let total = list.result_size_estimate.ok_or_else(|| {
raise_error!(
"Missing 'resultSizeEstimate' in Gmail API response".into(),
ErrorCode::InternalError
)
})?;
let total_pages = (total as f64 / max_results as f64).ceil() as u64;
Ok(CursorDataPage::new(
list.next_page_token,
Some(max_results),
total,
Some(total_pages),
envelopes,
))
}
pub async fn get_thread_messages_impl(
account_id: u64,
use_proxy: Option<u64>,
thread_id: &str,
) -> RustMailerResult<Vec<Envelope>> {
let label_map = GmailClient::for_get_label_name(account_id, use_proxy).await?;
let cache_key = thread_cache_key(account_id, thread_id);
let thread_messages = if let Some(cached) = GMAIL_THREADS_CACHE.get(&cache_key).await {
(*cached).clone()
} else {
let fetched = GmailClient::get_thread_messages(account_id, use_proxy, thread_id).await?;
GMAIL_THREADS_CACHE
.set(cache_key.clone(), Arc::new(fetched.clone()))
.await;
fetched
};
let thread_envelopes: Vec<GmailEnvelope> = thread_messages
.messages
.into_iter()
.map(GmailEnvelope::try_from)
.collect::<Result<_, _>>()?;
let result = thread_envelopes
.into_iter()
.map(|mut e| {
e.account_id = account_id; // Labels are ignored in remote mode; label_id has no effect.
e.into_envelope(&label_map)
})
.collect();
Ok(result)
}
+77 -11
View File
@@ -16,6 +16,7 @@ use crate::{
history::HistoryList,
labels::{Label, LabelDetail, LabelList},
messages::{FullMessage, MessageList, MessageMeta, PartBody},
thread::{ThreadList, ThreadMessages},
},
},
common::http::HttpClient,
@@ -56,7 +57,7 @@ impl GmailClient {
Ok(list.labels)
}
pub async fn label_map(
pub async fn for_get_label_name(
account_id: u64,
use_proxy: Option<u64>,
) -> RustMailerResult<Arc<AHashMap<String, String>>> {
@@ -74,7 +75,7 @@ impl GmailClient {
Ok(map)
}
pub async fn reverse_label_map(
pub async fn for_lookup_label_id(
account_id: u64,
use_proxy: Option<u64>,
skip_cache: bool,
@@ -202,16 +203,20 @@ impl GmailClient {
pub async fn list_messages(
account_id: u64,
use_proxy: Option<u64>,
label_id: &str,
label_id: Option<&str>,
page_token: Option<&str>,
after: Option<&str>,
max_results: u32,
) -> RustMailerResult<MessageList> {
let mut url = format!(
"https://gmail.googleapis.com/gmail/v1/users/me/messages?labelIds={}&maxResults={}",
label_id, max_results
"https://gmail.googleapis.com/gmail/v1/users/me/messages?maxResults={}",
max_results
);
if let Some(label_id) = label_id {
url.push_str(&format!("&labelIds={}", label_id));
}
if let Some(after) = after {
url.push_str(&format!("&q=after:{}", after));
}
@@ -235,6 +240,66 @@ impl GmailClient {
Ok(list)
}
pub async fn list_threads_internal(
account_id: u64,
use_proxy: Option<u64>,
label_id: Option<&str>,
page_token: Option<&str>,
after: Option<&str>,
max_results: u64,
) -> RustMailerResult<ThreadList> {
let mut url = format!(
"https://gmail.googleapis.com/gmail/v1/users/me/threads?maxResults={}",
max_results
);
if let Some(label_id) = label_id {
url.push_str(&format!("&labelIds={}", label_id));
}
if let Some(after) = after {
url.push_str(&format!("&q=after:{}", after));
}
if let Some(page_token) = page_token {
url.push_str(&format!("&pageToken={}", page_token));
}
let client = HttpClient::new(use_proxy).await?;
let access_token = Self::get_access_token(account_id).await?;
let value = client.get(url.as_str(), &access_token).await?;
let list = serde_json::from_value::<ThreadList>(value).map_err(|e| {
raise_error!(
format!(
"Failed to deserialize Gmail API response into ThreadList: {:#?}. Possible model mismatch or API change.",
e
),
ErrorCode::InternalError
)
})?;
Ok(list)
}
pub async fn get_thread_messages(
account_id: u64,
use_proxy: Option<u64>,
thread_id: &str,
) -> RustMailerResult<ThreadMessages> {
let url = format!(
"https://gmail.googleapis.com/gmail/v1/users/me/threads/{}?format=metadata&metadataHeaders=Message-ID&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Cc&metadataHeaders=Bcc&metadataHeaders=Subject&metadataHeaders=Date&metadataHeaders=Mime-Version&metadataHeaders=Reply-To&metadataHeaders=In-Reply-To&metadataHeaders=References&metadataHeaders=Sender",
thread_id,
);
let client = HttpClient::new(use_proxy).await?;
let access_token = Self::get_access_token(account_id).await?;
let value = client.get(url.as_str(), &access_token).await?;
let messages = serde_json::from_value::<ThreadMessages>(value)
.map_err(|e| raise_error!(format!(
"Failed to deserialize Gmail API response into MessageMeta: {:#?}. Possible model mismatch or API change.",
e
), ErrorCode::InternalError))?;
Ok(messages)
}
pub async fn search_messages(
account_id: u64,
use_proxy: Option<u64>,
@@ -381,15 +446,16 @@ impl GmailClient {
pub async fn list_history(
account_id: u64,
use_proxy: Option<u64>,
label_id: &str,
label_id: Option<&str>,
start_history_id: &str,
page_token: Option<&str>,
max_results: u32,
) -> RustMailerResult<HistoryList> {
let mut url = format!(
"https://gmail.googleapis.com/gmail/v1/users/me/history?labelId={}&maxResults={}&startHistoryId={}",
label_id, max_results, start_history_id
);
let mut url = format!("https://gmail.googleapis.com/gmail/v1/users/me/history?maxResults={}&startHistoryId={}", max_results, start_history_id);
if let Some(label_id) = label_id {
url.push_str(&format!("&labelId={}", label_id));
}
if let Some(page_token) = page_token {
url.push_str(&format!("&pageToken={}", page_token));
@@ -425,7 +491,7 @@ impl GmailClient {
ErrorCode::InternalError
)
})?;
let map = Self::label_map(account_id, use_proxy).await?;
let map = Self::for_get_label_name(account_id, use_proxy).await?;
let name = map.get("DRAFT").ok_or_else(|| {
raise_error!(
"Cannot find 'DRAFT' label in Gmail account label map".into(),
+1 -1
View File
@@ -425,7 +425,7 @@ impl GmailEnvelope {
message_id: self.message_id,
subject: self.subject,
thread_name: None,
thread_id: self.thread_id,
thread_id: self.gmail_thread_id,
mime_version: self.mime_version,
references: self.references,
reply_to: self.reply_to,
+6 -79
View File
@@ -27,7 +27,7 @@ pub async fn fetch_and_save_since_date(
date: &str,
label: &GmailLabels,
initial: bool,
) -> RustMailerResult<(usize, Option<String>)> {
) -> RustMailerResult<usize> {
// let total_batches = total.div_ceil(page_size); // Calculate total number of batches, useful for tracking sync progress on UI
let mut inserted_count = 0;
let account_id = account.id;
@@ -37,14 +37,13 @@ pub async fn fetch_and_save_since_date(
let mut page_token: Option<String> = None;
let mut page = 1; // Used only for tracking sync progress
// let semaphore = Arc::new(Semaphore::new(1));
let mut history_ids = Vec::new();
let mut page_size = ENVELOPE_BATCH_SIZE;
let mut total_to_fetch = 100;
loop {
let resp = GmailClient::list_messages(
account_id,
use_proxy,
&label.label_id,
Some(&label.label_id),
page_token.as_deref(),
Some(date),
page_size,
@@ -129,10 +128,6 @@ pub async fn fetch_and_save_since_date(
})
.collect::<RustMailerResult<Vec<GmailEnvelope>>>()?;
inserted_count += envelopes.len();
let hid = compute_max_history_id(&envelopes);
if let Some(hid) = hid {
history_ids.push(hid.to_string());
}
GmailEnvelope::save_envelopes(envelopes).await?;
}
// Break if API response has no next page
@@ -145,8 +140,7 @@ pub async fn fetch_and_save_since_date(
}
page += 1;
}
let hid = max_history_id(&history_ids).map(|s| s.to_string());
Ok((inserted_count, hid))
Ok(inserted_count)
}
pub async fn fetch_and_save_full_label(
@@ -154,7 +148,7 @@ pub async fn fetch_and_save_full_label(
label: &GmailLabels,
total: u32,
initial: bool,
) -> RustMailerResult<(usize, Option<String>)> {
) -> RustMailerResult<usize> {
let folder_limit = account.folder_limit;
let total_to_fetch = match folder_limit {
Some(limit) if limit < total => total.min(limit.max(100)),
@@ -184,7 +178,6 @@ pub async fn fetch_and_save_full_label(
let mut page_token: Option<String> = None;
let mut page = 1; // Used only for tracking sync progress
// let semaphore = Arc::new(Semaphore::new(1));
let mut history_ids = Vec::new();
loop {
// Stop if we have already fetched enough messages
if inserted_count as u32 >= total_to_fetch {
@@ -196,7 +189,7 @@ pub async fn fetch_and_save_full_label(
let resp = GmailClient::list_messages(
account_id,
use_proxy,
&label.label_id,
Some(&label.label_id),
page_token.as_deref(),
None,
page_size,
@@ -250,10 +243,6 @@ pub async fn fetch_and_save_full_label(
})
.collect::<RustMailerResult<Vec<GmailEnvelope>>>()?;
inserted_count += envelopes.len();
let hid = compute_max_history_id(&envelopes);
if let Some(hid) = hid {
history_ids.push(hid.to_string());
}
GmailEnvelope::save_envelopes(envelopes).await?;
}
// Break if API response has no next page
@@ -262,68 +251,6 @@ pub async fn fetch_and_save_full_label(
}
page += 1;
}
let hid = max_history_id(&history_ids).map(|s| s.to_string());
Ok((inserted_count, hid))
}
fn max_history_id_fallback<'a>(a: &'a str, b: &'a str) -> &'a str {
match (a.parse::<u64>(), b.parse::<u64>()) {
(Ok(a_num), Ok(b_num)) => {
if a_num >= b_num {
a
} else {
b
}
}
_ => {
if a.len() > b.len() {
a
} else if b.len() > a.len() {
b
} else if a >= b {
a
} else {
b
}
}
}
}
pub fn max_history_id(ids: &[String]) -> Option<&str> {
ids.iter()
.map(|s| s.as_str())
.reduce(|a, b| max_history_id_fallback(a, b))
}
fn compute_max_history_id<'a>(envelopes: &'a [GmailEnvelope]) -> Option<&'a str> {
envelopes
.iter()
.map(|e| e.history_id.as_str())
.fold(None, |max_id, curr| {
Some(match max_id {
Some(m) => max_history_id_fallback(m, curr),
None => curr,
})
})
}
#[cfg(test)]
mod tests {
use crate::modules::cache::vendor::gmail::sync::flow::max_history_id_fallback;
#[tokio::test]
async fn test1() {
let ids = vec![
"2671855", "2671863", "2671871", "2671881", "2671891", "2671898", "100865", "81974",
"81967", "2671905", "531772", "531769", "3296", "1385924",
];
let max_id = ids
.iter()
.cloned()
.reduce(|a, b| max_history_id_fallback(a, b))
.unwrap();
assert_eq!(max_id, "2671905");
}
Ok(inserted_count)
}
+172 -85
View File
@@ -15,7 +15,6 @@ use crate::{
cleanup_single_label,
client::GmailClient,
envelope::GmailEnvelope,
flow::max_history_id,
labels::{GmailCheckPoint, GmailLabels},
rebuild::rebuild_single_label_cache,
},
@@ -43,14 +42,15 @@ pub async fn handle_history(
let use_proxy = account.use_proxy.clone();
let remote_labels = find_existing_remote_labels(local_labels, remote_labels);
let checkpoint = GmailCheckPoint::get(account_id).await?;
let mut history_ids = Vec::with_capacity(remote_labels.len());
let mut current_max_history_id: Option<u64> = None;
for remote in remote_labels {
let mut page_token = None;
loop {
let mut list = match GmailClient::list_history(
account_id,
use_proxy.clone(),
&remote.label_id,
Some(&remote.label_id),
&checkpoint.history_id,
page_token.as_deref(),
100, // 100 items per page
@@ -64,11 +64,10 @@ pub async fn handle_history(
location: _,
code,
} => {
if code == ErrorCode::GmailApiInvalidHistoryId {
let history_id = handle_invalid_history_id(account, &remote).await?;
if let Some(history_id) = history_id {
history_ids.push(history_id);
}
if code == ErrorCode::ApiCallFailed
&& message.contains("Requested entity was not found")
{
handle_invalid_history_id(account, &remote).await?;
break;
} else {
return Err(raise_error!(message, code));
@@ -76,29 +75,59 @@ pub async fn handle_history(
}
},
};
page_token = list.next_page_token.take();
let history_list: Vec<History> = list
.history
.into_iter()
.filter(|h| h.has_changes())
.collect();
let history_id = parse_history_id(&list.history_id)?;
page_token = list.next_page_token.take();
let history_items = list.history;
let history_list = match current_max_history_id {
Some(current) => {
if history_id > current {
history_items
.into_iter()
.filter(|h| {
h.has_changes()
&& h.id.parse::<u64>().map(|id| id <= current).unwrap_or(false)
})
.collect()
} else {
history_items
.into_iter()
.filter(|h| h.has_changes())
.collect()
}
}
None => {
current_max_history_id = Some(history_id);
history_items
.into_iter()
.filter(|h| h.has_changes())
.collect()
}
};
apply_history(account, &remote, history_list).await?;
if page_token.is_none() {
history_ids.push(list.history_id);
break;
}
}
GmailLabels::upsert(remote).await?;
}
let max = max_history_id(&history_ids);
if let Some(history_id) = max {
let checkpoint = GmailCheckPoint::new(account_id, history_id.to_string());
checkpoint.save().await?;
if let Some(current_max_history_id) = current_max_history_id {
GmailCheckPoint::new(account_id, current_max_history_id.to_string())
.save()
.await?;
}
Ok(())
}
fn parse_history_id(history_id: &str) -> RustMailerResult<u64> {
history_id.parse::<u64>().map_err(|e| {
raise_error!(
format!("Invalid Gmail historyId '{}': {}", history_id, e),
ErrorCode::InternalError
)
})
}
pub fn find_existing_remote_labels(
local_labels: &[GmailLabels],
remote_labels: &[GmailLabels],
@@ -127,79 +156,126 @@ pub async fn apply_history(
let mut label_changes: AHashMap<String, LabelChange> = AHashMap::new();
// -- labels_added
for item in history.labels_added {
let current =
GmailEnvelope::find(account.id, label.id, item.message.id.as_str()).await?;
match current {
Some(mut current) => {
let mut merged: HashSet<String> = current.label_ids.into_iter().collect();
let mut actually_added = Vec::new();
if let Some(to_add) = &item.label_ids {
for l in to_add {
if !merged.contains(l) {
actually_added.push(l.clone());
}
}
merged.extend(to_add.iter().cloned());
}
if !actually_added.is_empty() {
let entry = label_changes
.entry(item.message.id.clone())
.or_insert_with(|| LabelChange::default());
entry.added.extend(actually_added);
}
current.label_ids = merged.into_iter().collect();
GmailEnvelope::upsert(current).await?;
if account.minimal_sync() {
if let Some(to_add) = &item.label_ids {
let entry = label_changes
.entry(item.message.id.clone())
.or_insert_with(LabelChange::default);
entry.added.extend(to_add.iter().cloned());
}
None => {
warn!(
"Message {} not found in local cache, cannot merge labels.",
item.message.id
);
} else {
let current =
GmailEnvelope::find(account.id, label.id, item.message.id.as_str()).await?;
match current {
Some(mut current) => {
let mut merged: HashSet<String> = current.label_ids.into_iter().collect();
let mut actually_added = Vec::new();
if let Some(to_add) = &item.label_ids {
for l in to_add {
if !merged.contains(l) {
actually_added.push(l.clone());
}
}
merged.extend(to_add.iter().cloned());
}
if !actually_added.is_empty() {
let entry = label_changes
.entry(item.message.id.clone())
.or_insert_with(|| LabelChange::default());
entry.added.extend(actually_added);
}
current.label_ids = merged.into_iter().collect();
GmailEnvelope::upsert(current).await?;
}
None => {
warn!(
"Message {} not found in local cache, cannot merge labels.",
item.message.id
);
}
}
}
}
// -- labels_removed
for item in history.labels_removed {
let current =
GmailEnvelope::find(account.id, label.id, item.message.id.as_str()).await?;
match current {
Some(mut current) => {
if let Some(to_remove) = &item.label_ids {
let mut actually_removed = Vec::new();
if to_remove.contains(&label.id.to_string()) {
GmailEnvelope::delete(account.id, label.id, &current.id).await?;
} else {
current.label_ids.retain(|id| {
let keep = !to_remove.contains(id);
if !keep {
actually_removed.push(id.clone());
}
keep
});
GmailEnvelope::upsert(current).await?;
}
if !actually_removed.is_empty() {
let entry = label_changes
.entry(item.message.id.clone())
.or_insert_with(|| LabelChange::default());
entry.removed.extend(actually_removed);
if account.minimal_sync() {
if let Some(to_remove) = &item.label_ids {
let entry = label_changes
.entry(item.message.id.clone())
.or_insert_with(LabelChange::default);
entry.removed.extend(to_remove.iter().cloned());
}
} else {
let current =
GmailEnvelope::find(account.id, label.id, item.message.id.as_str()).await?;
match current {
Some(mut current) => {
if let Some(to_remove) = &item.label_ids {
let mut actually_removed = Vec::new();
if to_remove.contains(&label.id.to_string()) {
GmailEnvelope::delete(account.id, label.id, &current.id).await?;
} else {
current.label_ids.retain(|id| {
let keep = !to_remove.contains(id);
if !keep {
actually_removed.push(id.clone());
}
keep
});
GmailEnvelope::upsert(current).await?;
}
if !actually_removed.is_empty() {
let entry = label_changes
.entry(item.message.id.clone())
.or_insert_with(|| LabelChange::default());
entry.removed.extend(actually_removed);
}
}
}
}
None => {
warn!(
"Message {} not found in local cache, cannot merge labels.",
item.message.id
);
None => {
warn!(
"Message {} not found in local cache, cannot merge labels.",
item.message.id
);
}
}
}
}
if !label_changes.is_empty() {
if !account.minimal_sync()
&& EventHookTask::is_watching_email_flags_changed(account.id).await?
{
for entry in label_changes {
if !label_changes.is_empty()
&& EventHookTask::is_watching_email_flags_changed(account.id).await?
{
for entry in label_changes {
if account.minimal_sync() {
let message_data =
GmailClient::get_message(account.id, account.use_proxy, entry.0.as_str())
.await?;
let current: GmailEnvelope = message_data.try_into()?;
EVENT_CHANNEL
.queue(Event::new(
account.id,
&account.email,
RustMailerEvent::new(
EventType::EmailFlagsChanged,
EventPayload::EmailFlagsChanged(EmailFlagsChanged {
account_id: account.id,
account_email: account.email.clone(),
mailbox_name: label.name.clone(),
uid: None,
from: current.from,
to: current.to,
message_id: current.message_id,
subject: current.subject,
internal_date: Some(current.internal_date),
date: current.date,
flags_added: entry.1.added,
flags_removed: entry.1.removed,
mid: Some(entry.0),
}),
),
))
.await;
} else {
if let Some(current) =
GmailEnvelope::find(account.id, label.id, entry.0.as_str()).await?
{
@@ -280,7 +356,9 @@ pub async fn apply_history(
messages_added.len(),
&label.name
);
GmailEnvelope::save_envelopes(messages_added.clone()).await?;
if !account.minimal_sync() {
GmailEnvelope::save_envelopes(messages_added.clone()).await?;
}
if EventHookTask::is_watching_email_add_event(account.id).await? {
dispatch_new_email_notification(account, messages_added).await?;
}
@@ -306,13 +384,19 @@ async fn dispatch_new_email_notification(
account: &AccountModel,
messages: Vec<GmailEnvelope>,
) -> RustMailerResult<()> {
let label_map = GmailClient::label_map(account.id, account.use_proxy).await?;
let label_map = GmailClient::for_get_label_name(account.id, account.use_proxy).await?;
for message in messages {
let full_message =
GmailClient::get_full_messages(account.id, account.use_proxy, &message.id).await?;
let gmail_thread_id = full_message.thread_id.clone();
let message_content: FullMessageContent = full_message.try_into()?;
let mut envelope = message.into_envelope(&label_map);
envelope.thread_id = envelope.compute_thread_id();
envelope.thread_id = if account.minimal_sync() {
gmail_thread_id.unwrap_or_default()
} else {
envelope.compute_thread_id().to_string()
};
EVENT_CHANNEL
.queue(Event::new(
account.id,
@@ -355,7 +439,7 @@ async fn dispatch_new_email_notification(
async fn handle_invalid_history_id(
account: &AccountModel,
label: &GmailLabels,
) -> RustMailerResult<Option<String>> {
) -> RustMailerResult<()> {
info!(
"Account {}: Invalid history ID detected for label '{}'. Rebuilding local state...",
account.id, label.name
@@ -370,5 +454,8 @@ async fn handle_invalid_history_id(
"Account {}: Upserted label '{}' into local database",
account.id, label.name
);
rebuild_single_label_cache(account, label).await
if !account.minimal_sync() {
rebuild_single_label_cache(account, label).await?;
}
Ok(())
}
+60 -15
View File
@@ -18,13 +18,16 @@ use tracing::info;
use crate::modules::{
account::{entity::MailerType, migration::AccountModel, status::AccountRunningState},
cache::{
imap::{address::AddressEntity, thread::EmailThread}, sync_type::{determine_sync_type, SyncType}, vendor::gmail::sync::{
imap::{address::AddressEntity, thread::EmailThread},
sync_type::{determine_sync_type, SyncType},
vendor::gmail::sync::{
client::GmailClient,
envelope::GmailEnvelope,
history::handle_history,
labels::{GmailCheckPoint, GmailLabels},
rebuild::{rebuild_cache, rebuild_cache_since_date, rebuild_single_label_cache},
sync_labels::get_sync_labels,
}
},
},
error::RustMailerResult,
hook::{
@@ -96,6 +99,7 @@ pub async fn execute_gmail_sync(account: &AccountModel) -> RustMailerResult<()>
}
return Ok(());
}
handle_history(account, &local_labels, &remote_labels).await?;
let deleted_labels = find_deleted_labels(&local_labels, &remote_labels);
@@ -117,9 +121,11 @@ pub async fn execute_gmail_sync(account: &AccountModel) -> RustMailerResult<()>
"Inserting missing Gmail labels into database"
);
GmailLabels::batch_insert(&missing_labels).await?;
for label in &missing_labels {
//During incremental synchronization, if any labels are found missing or not fully synchronized, the checkpoint does not need to be updated.
rebuild_single_label_cache(account, label).await?;
if !account.minimal_sync() {
for label in &missing_labels {
//During incremental synchronization, if any labels are found missing or not fully synchronized, the checkpoint does not need to be updated.
rebuild_single_label_cache(account, label).await?;
}
}
}
AccountRunningState::set_incremental_sync_end(account.id).await?;
@@ -148,9 +154,12 @@ pub async fn should_rebuild_cache(
if checkpoint.is_some() {
GmailCheckPoint::clean(account.id).await?;
}
GmailEnvelope::clean_account(account.id).await?;
AddressEntity::clean_account(account.id).await?;
EmailThread::clean_account(account.id).await?;
if !account.minimal_sync() {
GmailEnvelope::clean_account(account.id).await?;
AddressEntity::clean_account(account.id).await?;
EmailThread::clean_account(account.id).await?;
}
info!(account_id = account.id, "Cache cleaning completed");
@@ -188,10 +197,13 @@ async fn cleanup_deleted_labels(
deleted_labels: &[GmailLabels],
) -> RustMailerResult<()> {
let start_time = Instant::now();
for label in deleted_labels {
GmailEnvelope::clean_label_envelopes(account.id, label.id).await?;
AddressEntity::clean_mailbox_envelopes(account.id, label.id).await?;
EmailThread::clean_mailbox_envelopes(account.id, label.id).await?;
if !account.minimal_sync() {
for label in deleted_labels {
GmailEnvelope::clean_label_envelopes(account.id, label.id).await?;
AddressEntity::clean_mailbox_envelopes(account.id, label.id).await?;
EmailThread::clean_mailbox_envelopes(account.id, label.id).await?;
}
}
GmailLabels::batch_delete(deleted_labels.to_vec()).await?;
let elapsed_time = start_time.elapsed().as_secs();
@@ -204,9 +216,11 @@ async fn cleanup_deleted_labels(
async fn cleanup_single_label(account: &AccountModel, label: &GmailLabels) -> RustMailerResult<()> {
let start_time = Instant::now();
GmailEnvelope::clean_label_envelopes(account.id, label.id).await?;
AddressEntity::clean_mailbox_envelopes(account.id, label.id).await?;
EmailThread::clean_mailbox_envelopes(account.id, label.id).await?;
if !account.minimal_sync() {
GmailEnvelope::clean_label_envelopes(account.id, label.id).await?;
AddressEntity::clean_mailbox_envelopes(account.id, label.id).await?;
EmailThread::clean_mailbox_envelopes(account.id, label.id).await?;
}
GmailLabels::delete(label.id).await?;
let elapsed_time = start_time.elapsed().as_secs();
info!(
@@ -215,3 +229,34 @@ async fn cleanup_single_label(account: &AccountModel, label: &GmailLabels) -> Ru
);
Ok(())
}
async fn save_latest_history_id(account: &AccountModel) -> RustMailerResult<()> {
// Fetch the most recent message
let result =
GmailClient::list_messages(account.id, account.use_proxy, None, None, None, 1).await?;
let first_message = match result.messages.as_ref().and_then(|m| m.first()) {
Some(msg) => msg,
None => return Ok(()),
};
// Retrieve the message details to get its history_id
let message =
GmailClient::get_message(account.id, account.use_proxy, &first_message.id).await?;
let start_history_id = &message.history_id;
// Fetch the latest history starting from the message's history_id
let history_list = GmailClient::list_history(
account.id,
account.use_proxy,
None,
start_history_id,
None,
1,
)
.await?;
// Save the latest history_id to ensure future incremental sync starts from the correct point
GmailCheckPoint::new(account.id, history_list.history_id)
.save()
.await
}
+64 -71
View File
@@ -3,10 +3,11 @@
// Unauthorized copying, modification, or distribution is prohibited.
use crate::modules::{
account::{since::DateSince, migration::AccountModel},
account::{migration::AccountModel, since::DateSince},
cache::vendor::gmail::sync::{
flow::{fetch_and_save_full_label, fetch_and_save_since_date, max_history_id},
labels::{GmailCheckPoint, GmailLabels},
flow::{fetch_and_save_full_label, fetch_and_save_since_date},
labels::GmailLabels,
save_latest_history_id,
},
error::RustMailerResult,
};
@@ -19,51 +20,44 @@ pub async fn rebuild_cache(
) -> RustMailerResult<()> {
let start_time = Instant::now();
let mut total_inserted = 0;
GmailLabels::batch_insert(remote_labels).await?;
let mut history_ids = Vec::with_capacity(remote_labels.len());
for label in remote_labels {
if label.exists == 0 {
info!(
if !account.minimal_sync() {
for label in remote_labels {
if label.exists == 0 {
info!(
"Account {}: Label '{}' on the remote server has no emails. Skipping fetch for this label.",
account.id, &label.name
);
continue;
}
match fetch_and_save_full_label(account, label, label.exists, true).await {
Ok((inserted, max_history_id)) => {
total_inserted += inserted;
if let Some(history_id) = max_history_id {
history_ids.push(history_id);
}
continue;
}
Err(e) => {
warn!(
match fetch_and_save_full_label(account, label, label.exists, true).await {
Ok(inserted) => {
total_inserted += inserted;
}
Err(e) => {
warn!(
"Account {}: Failed to sync label '{}'. Error: {:#?}. Removing label entry.",
account.id, &label.name, e
);
if let Err(del_err) = GmailLabels::delete(label.id).await {
error!(
"Account {}: Failed to delete label '{}' after sync error: {}",
account.id, &label.name, del_err
);
if let Err(del_err) = GmailLabels::delete(label.id).await {
error!(
"Account {}: Failed to delete label '{}' after sync error: {}",
account.id, &label.name, del_err
);
}
}
}
}
}
let max = max_history_id(&history_ids);
if let Some(history_id) = max {
let checkpoint = GmailCheckPoint::new(account.id, history_id.to_string());
checkpoint.save().await?;
}
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
save_latest_history_id(account).await?;
if !account.minimal_sync() {
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
This is a full data fetch as there was no local cache data available.",
total_inserted, elapsed_time
);
total_inserted, elapsed_time
);
}
Ok(())
}
@@ -77,67 +71,66 @@ pub async fn rebuild_cache_since_date(
let date = date_since.since_gmail_date()?;
GmailLabels::batch_insert(remote_labels).await?;
let mut history_ids = Vec::with_capacity(remote_labels.len());
for label in remote_labels {
if label.exists == 0 {
info!(
if !account.minimal_sync() {
for label in remote_labels {
if label.exists == 0 {
info!(
"Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
account.id, &label.name
);
continue;
}
match fetch_and_save_since_date(account, date.as_str(), label, true).await {
Ok((inserted, max_history_id)) => {
total_inserted += inserted;
if let Some(history_id) = max_history_id {
history_ids.push(history_id);
}
continue;
}
Err(e) => {
warn!(
match fetch_and_save_since_date(account, date.as_str(), label, true).await {
Ok(inserted) => {
total_inserted += inserted;
}
Err(e) => {
warn!(
"Account {}: Failed to sync mailbox '{}'. Error: {}. Removing mailbox entry.",
account.id, &label.name, e
);
if let Err(del_err) = GmailLabels::delete(label.id).await {
error!(
"Account {}: Failed to delete mailbox '{}' after sync error: {}",
account.id, &label.name, del_err
);
if let Err(del_err) = GmailLabels::delete(label.id).await {
error!(
"Account {}: Failed to delete mailbox '{}' after sync error: {}",
account.id, &label.name, del_err
);
}
}
}
}
}
let max = max_history_id(&history_ids);
if let Some(history_id) = max {
let checkpoint = GmailCheckPoint::new(account.id, history_id.to_string());
checkpoint.save().await?;
}
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
save_latest_history_id(account).await?;
if !account.minimal_sync() {
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
Data fetched from server starting from the specified date: {}.",
total_inserted, elapsed_time, date
);
total_inserted, elapsed_time, date
);
}
Ok(())
}
pub async fn rebuild_single_label_cache(
account: &AccountModel,
label: &GmailLabels,
) -> RustMailerResult<Option<String>> {
) -> RustMailerResult<()> {
if label.exists > 0 {
match &account.date_since {
Some(date_since) => {
let date = date_since.since_gmail_date()?;
match fetch_and_save_since_date(account, date.as_str(), label, true).await {
Ok((inserted, max_history_id)) => {
Ok(inserted) => {
info!(
"Account {}: Label '{}' synced successfully. {} messages inserted.",
account.id, label.name, inserted
);
return Ok(max_history_id);
return Ok(());
}
Err(e) => {
warn!(
@@ -154,12 +147,12 @@ pub async fn rebuild_single_label_cache(
}
}
None => match fetch_and_save_full_label(account, label, label.exists, true).await {
Ok((inserted, max_history_id)) => {
Ok(inserted) => {
info!(
"Account {}: Label '{}' synced successfully. {} messages inserted.",
account.id, label.name, inserted
);
return Ok(max_history_id);
return Ok(());
}
Err(e) => {
warn!(
@@ -176,5 +169,5 @@ pub async fn rebuild_single_label_cache(
},
}
}
Ok(None)
Ok(())
}
+83 -9
View File
@@ -21,6 +21,7 @@ use crate::{
model::{
history::HistoryList,
messages::{FullMessage, MessageList, MessageMeta, PartBody},
thread::{ThreadList, ThreadMessages},
},
sync::envelope::GmailEnvelope,
},
@@ -44,7 +45,7 @@ async fn access_token() -> String {
grpc_client.set_send_compressed(CompressionEncoding::GZIP);
let request = GetOAuth2TokensRequest {
account_id: 658031352292246,
account_id: 4727355769996270,
};
let mut request = poem_grpc::Request::new(request);
@@ -143,13 +144,13 @@ async fn test11() {
async fn test2() {
let access_token = access_token().await;
let url =
"https://gmail.googleapis.com/gmail/v1/users/me/messages?labelIds=INBOX&q=after:2025/08/28&maxResults=20";
"https://gmail.googleapis.com/gmail/v1/users/me/messages?labelIds=INBOX&q=before:2025/08/28&maxResults=20";
let mut builder = reqwest::ClientBuilder::new()
.user_agent(rustmailer_version!())
.timeout(Duration::from_secs(10))
.connect_timeout(Duration::from_secs(10));
let proxy_obj = reqwest::Proxy::all("socks5://127.0.0.1:22308").unwrap();
let proxy_obj = reqwest::Proxy::all("http://127.0.0.1:22307").unwrap();
builder = builder
.redirect(reqwest::redirect::Policy::none())
.proxy(proxy_obj);
@@ -175,13 +176,13 @@ async fn test2() {
async fn test3() {
let access_token = access_token().await;
let url =
"https://gmail.googleapis.com/gmail/v1/users/me/messages/198f6735682a3870?format=metadata&metadataHeaders=Message-Id&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Cc&metadataHeaders=Bcc&metadataHeaders=Subject&metadataHeaders=Date&metadataHeaders=Mime-Version&metadataHeaders=Reply-To&metadataHeaders=In-Reply-To&metadataHeaders=References";
"https://gmail.googleapis.com/gmail/v1/users/me/messages/19abb5fe57ca4856?format=metadata&metadataHeaders=Message-Id&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Cc&metadataHeaders=Bcc&metadataHeaders=Subject&metadataHeaders=Date&metadataHeaders=Mime-Version&metadataHeaders=Reply-To&metadataHeaders=In-Reply-To&metadataHeaders=References";
let mut builder = reqwest::ClientBuilder::new()
.user_agent(rustmailer_version!())
.timeout(Duration::from_secs(10))
.connect_timeout(Duration::from_secs(10));
let proxy_obj = reqwest::Proxy::all("socks5://127.0.0.1:22308").unwrap();
let proxy_obj = reqwest::Proxy::all("http://127.0.0.1:22307").unwrap();
builder = builder
.redirect(reqwest::redirect::Policy::none())
.proxy(proxy_obj);
@@ -210,13 +211,13 @@ async fn test3() {
async fn test4() {
let access_token = access_token().await;
let url =
"https://gmail.googleapis.com/gmail/v1/users/me/history?startHistoryId=42032&labelId=INBOX&maxResults=20";
"https://gmail.googleapis.com/gmail/v1/users/me/history?startHistoryId=45284&labelId=INBOX&maxResults=2";
let mut builder = reqwest::ClientBuilder::new()
.user_agent(rustmailer_version!())
.timeout(Duration::from_secs(10))
.connect_timeout(Duration::from_secs(10));
let proxy_obj = reqwest::Proxy::all("socks5://127.0.0.1:22308").unwrap();
let proxy_obj = reqwest::Proxy::all("http://127.0.0.1:22307").unwrap();
builder = builder
.redirect(reqwest::redirect::Policy::none())
.proxy(proxy_obj);
@@ -236,7 +237,11 @@ async fn test4() {
let list: HistoryList = serde_json::from_value(body).unwrap();
println!("Response = {:#?}", list);
} else {
eprintln!("Error: {} - {:?}", res.status(), res.text().await.unwrap());
//eprintln!("Error: {} - {:?}", res.status(), res.text().await.unwrap());
let text = res.text().await.unwrap();
if text.contains("Requested entity was not found") {
println!("history id is invalid");
}
}
}
@@ -428,4 +433,73 @@ async fn test9() {
} else {
eprintln!("Error: {} - {:?}", res.status(), res.text().await.unwrap());
}
}
}
#[tokio::test]
async fn list_threads() {
let access_token = access_token().await;
let url =
"https://gmail.googleapis.com/gmail/v1/users/me/threads?labelIds=INBOX&q=after:2025/07/28&maxResults=20";
let mut builder = reqwest::ClientBuilder::new()
.user_agent(rustmailer_version!())
.timeout(Duration::from_secs(10))
.connect_timeout(Duration::from_secs(10));
let proxy_obj = reqwest::Proxy::all("http://127.0.0.1:22307").unwrap();
builder = builder
.redirect(reqwest::redirect::Policy::none())
.proxy(proxy_obj);
let client = builder.build().unwrap();
let res = client
.get(url)
.header(AUTHORIZATION, format!("Bearer {}", access_token))
.header(CONTENT_TYPE, "application/json")
.send()
.await
.unwrap();
if res.status().is_success() {
let body: Value = res.json().await.unwrap();
//println!("Response = {:#?}", body);
let list: ThreadList = serde_json::from_value(body).unwrap();
println!("Response = {:#?}", list);
} else {
eprintln!("Error: {} - {:?}", res.status(), res.text().await.unwrap());
}
}
#[tokio::test]
async fn test13() {
let access_token = access_token().await;
let url =
"https://gmail.googleapis.com/gmail/v1/users/me/threads/19ad6ed3d0afba92?format=metadata&metadataHeaders=Message-Id&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Cc&metadataHeaders=Bcc&metadataHeaders=Subject&metadataHeaders=Date&metadataHeaders=Mime-Version&metadataHeaders=Reply-To&metadataHeaders=In-Reply-To&metadataHeaders=References";
let mut builder = reqwest::ClientBuilder::new()
.user_agent(rustmailer_version!())
.timeout(Duration::from_secs(10))
.connect_timeout(Duration::from_secs(10));
let proxy_obj = reqwest::Proxy::all("http://127.0.0.1:22307").unwrap();
builder = builder
.redirect(reqwest::redirect::Policy::none())
.proxy(proxy_obj);
let client = builder.build().unwrap();
let res = client
.get(url)
.header(AUTHORIZATION, format!("Bearer {}", access_token))
.header(CONTENT_TYPE, "application/json")
.send()
.await
.unwrap();
if res.status().is_success() {
let body: Value = res.json().await.unwrap();
println!("Response = {:#?}", body);
let messages: ThreadMessages = serde_json::from_value(body).unwrap();
println!("Response = {:#?}", messages);
// let envelope: GmailEnvelope = detail.try_into().unwrap();
// println!("Response = {:#?}", envelope);
} else {
eprintln!("Error: {} - {:?}", res.status(), res.text().await.unwrap());
}
}
+33
View File
@@ -182,6 +182,39 @@ impl OutlookClient {
Ok(list)
}
pub async fn get_thread_messages(
account_id: u64,
use_proxy: Option<u64>,
conversation_id: &str,
) -> RustMailerResult<MessageListResponse> {
let url = format!(
"https://graph.microsoft.com/v1.0/me/messages?\
$filter=conversationId eq '{}'&\
$select=id,isRead,conversationId,internetMessageId,from,body,toRecipients,ccRecipients,\
bccRecipients,replyTo,sender,subject,receivedDateTime,sentDateTime,isRead,bodyPreview,categories&\
$expand=attachments($select=id,name,contentType,size,isInline,microsoft.graph.fileAttachment/contentId)",
conversation_id
);
let client = HttpClient::new(use_proxy).await?;
let access_token = Self::get_access_token(account_id).await?;
let value = client.get(url.as_str(), &access_token).await?;
let list = match serde_json::from_value::<MessageListResponse>(value.clone()) {
Ok(res) => res,
Err(e) => {
error!(
"Failed to deserialize Graph API response into MessageListResponse: {:#?}",
e
);
error!("Original JSON: {}", value);
return Err(raise_error!(format!("Failed to deserialize Graph API response into MessageListResponse: {:#?}. Possible model mismatch or API change.",e),ErrorCode::InternalError));
}
};
Ok(list)
}
pub async fn get_delta_link(
account_id: u64,
use_proxy: Option<u64>,
+18 -6
View File
@@ -155,10 +155,14 @@ pub async fn handle_delta(
envelope.account_id = account_id;
envelope.folder_id = remote.id;
envelope.folder_name = remote.name.clone();
if envelope.exists().await? {
updated.push(envelope);
if account.minimal_sync() {
added.push((envelope, full_message)); //If the local system does not record the email ID, treat all messages—whether new or updated—as new and send them.
} else {
added.push((envelope, full_message));
if envelope.exists().await? {
updated.push(envelope);
} else {
added.push((envelope, full_message));
}
}
}
}
@@ -176,8 +180,10 @@ pub async fn handle_delta(
}
}
notify_outlook_envelopes(&account, &added).await?;
OutlookEnvelope::save_envelopes(added.into_iter().map(|t| t.0).collect()).await?;
OutlookEnvelope::update_envelopes(updated).await?;
if !account.minimal_sync() {
OutlookEnvelope::save_envelopes(added.into_iter().map(|t| t.0).collect()).await?;
OutlookEnvelope::update_envelopes(updated).await?;
}
OutlookFolder::upsert(remote).await?;
}
Ok(())
@@ -202,6 +208,12 @@ pub async fn notify_outlook_envelopes(
let account_id = account.id;
if EventHookTask::is_watching_email_add_event(account_id).await? {
for message in envelopes {
let thread_id = if account.minimal_sync() {
message.0.conversation_id.clone().unwrap()
} else {
message.0.thread_id.to_string()
};
EVENT_CHANNEL
.queue(Event::new(
account_id,
@@ -228,7 +240,7 @@ pub async fn notify_outlook_envelopes(
message: message.1.clone(),
thread_name: None,
reply_to: message.0.reply_to.clone(),
thread_id: message.0.thread_id,
thread_id,
labels: message.0.categories.clone(),
}),
),
+1 -1
View File
@@ -463,7 +463,7 @@ impl From<OutlookEnvelope> for Envelope {
message_id: value.message_id,
subject: value.subject,
thread_name: None,
thread_id: value.thread_id,
thread_id: value.thread_id.to_string(),
mime_version: value.mime_version,
references: value.references,
reply_to: value.reply_to,
+11 -7
View File
@@ -141,9 +141,11 @@ pub async fn should_rebuild_cache(
if has_delta {
FolderDeltaLink::clean(account.id).await?;
}
OutlookEnvelope::clean_account(account.id).await?;
AddressEntity::clean_account(account.id).await?;
EmailThread::clean_account(account.id).await?;
if !account.minimal_sync() {
OutlookEnvelope::clean_account(account.id).await?;
AddressEntity::clean_account(account.id).await?;
EmailThread::clean_account(account.id).await?;
}
info!(account_id = account.id, "Cache cleaning completed");
Ok(true)
}
@@ -179,10 +181,12 @@ async fn cleanup_deleted_folders(
deleted_folders: &[OutlookFolder],
) -> RustMailerResult<()> {
let start_time = Instant::now();
for folder in deleted_folders {
OutlookEnvelope::clean_folder_envelopes(account.id, folder.id).await?;
AddressEntity::clean_mailbox_envelopes(account.id, folder.id).await?;
EmailThread::clean_mailbox_envelopes(account.id, folder.id).await?;
if !account.minimal_sync() {
for folder in deleted_folders {
OutlookEnvelope::clean_folder_envelopes(account.id, folder.id).await?;
AddressEntity::clean_mailbox_envelopes(account.id, folder.id).await?;
EmailThread::clean_mailbox_envelopes(account.id, folder.id).await?;
}
}
OutlookFolder::batch_delete(deleted_folders.to_vec()).await?;
let elapsed_time = start_time.elapsed().as_secs();
+102 -88
View File
@@ -26,41 +26,46 @@ pub async fn rebuild_cache(
let use_proxy = account.use_proxy;
OutlookFolder::batch_insert(remote_folders).await?;
for folder in remote_folders {
if folder.exists > 0 {
match fetch_and_save_full_folder(account, folder, folder.exists, true).await {
Ok(inserted) => {
total_inserted += inserted;
}
Err(e) => {
warn!(
if !account.minimal_sync() {
if folder.exists > 0 {
match fetch_and_save_full_folder(account, folder, folder.exists, true).await {
Ok(inserted) => {
total_inserted += inserted;
}
Err(e) => {
warn!(
"Account {}: Failed to sync label '{}'. Error: {:#?}. Removing label entry.",
account.id, &folder.name, e
);
if let Err(del_err) = OutlookFolder::delete(folder.id).await {
error!(
"Account {}: Failed to delete label '{}' after sync error: {}",
account.id, &folder.name, del_err
);
if let Err(del_err) = OutlookFolder::delete(folder.id).await {
error!(
"Account {}: Failed to delete label '{}' after sync error: {}",
account.id, &folder.name, del_err
);
}
}
}
}
} else {
warn!(
} else {
warn!(
"Account {}: folder '{}' on the remote server has no emails. Skipping fetch for this folder.",
account.id, &folder.name
);
}
}
let delta_link =
OutlookClient::get_delta_link(account_id, use_proxy, &folder.folder_id).await?;
FolderDeltaLink::upsert(account_id, &folder.folder_id, &delta_link).await?;
}
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
This is a full data fetch as there was no local cache data available.",
total_inserted, elapsed_time
);
if !account.minimal_sync() {
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
This is a full data fetch as there was no local cache data available.",
total_inserted, elapsed_time
);
}
Ok(())
}
@@ -78,64 +83,17 @@ pub async fn rebuild_cache_since_date(
OutlookFolder::batch_insert(remote_folders).await?;
for folder in remote_folders {
if folder.exists > 0 {
match fetch_and_save_since_date(account, date.as_str(), folder, true).await {
Ok(inserted) => {
total_inserted += inserted;
}
Err(e) => {
warn!(
"Account {}: Failed to sync mailbox '{}'. Error: {}. Removing mailbox entry.",
account.id, &folder.name, e
);
if let Err(del_err) = OutlookFolder::delete(folder.id).await {
error!(
"Account {}: Failed to delete mailbox '{}' after sync error: {}",
account.id, &folder.name, del_err
);
}
}
}
} else {
warn!(
"Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
account.id, &folder.name
);
}
let delta_link =
OutlookClient::get_delta_link(account_id, use_proxy, &folder.folder_id).await?;
FolderDeltaLink::upsert(account_id, &folder.folder_id, &delta_link).await?;
}
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
Data fetched from server starting from the specified date: {}.",
total_inserted, elapsed_time, date
);
Ok(())
}
pub async fn rebuild_single_folder_cache(
account: &AccountModel,
folder: &OutlookFolder,
) -> RustMailerResult<()> {
if folder.exists > 0 {
match &account.date_since {
Some(date_since) => {
let date = date_since.since_outlook_date()?;
if !account.minimal_sync() {
if folder.exists > 0 {
match fetch_and_save_since_date(account, date.as_str(), folder, true).await {
Ok(inserted) => {
info!(
"Account {}: folder '{}' synced successfully. {} messages inserted.",
account.id, folder.name, inserted
);
return Ok(());
total_inserted += inserted;
}
Err(e) => {
warn!(
"Account {}: Failed to sync mailbox '{}'. Error: {}. Removing folder entry.",
account.id, &folder.name, e
);
"Account {}: Failed to sync mailbox '{}'. Error: {}. Removing mailbox entry.",
account.id, &folder.name, e
);
if let Err(del_err) = OutlookFolder::delete(folder.id).await {
error!(
"Account {}: Failed to delete mailbox '{}' after sync error: {}",
@@ -144,30 +102,86 @@ pub async fn rebuild_single_folder_cache(
}
}
}
} else {
warn!(
"Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
account.id, &folder.name
);
}
None => match fetch_and_save_full_folder(account, folder, folder.exists, true).await {
Ok(inserted) => {
info!(
}
let delta_link =
OutlookClient::get_delta_link(account_id, use_proxy, &folder.folder_id).await?;
FolderDeltaLink::upsert(account_id, &folder.folder_id, &delta_link).await?;
}
if !account.minimal_sync() {
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
Data fetched from server starting from the specified date: {}.",
total_inserted, elapsed_time, date
);
}
Ok(())
}
pub async fn rebuild_single_folder_cache(
account: &AccountModel,
folder: &OutlookFolder,
) -> RustMailerResult<()> {
if folder.exists > 0 {
if !account.minimal_sync() {
match &account.date_since {
Some(date_since) => {
let date = date_since.since_outlook_date()?;
match fetch_and_save_since_date(account, date.as_str(), folder, true).await {
Ok(inserted) => {
info!(
"Account {}: folder '{}' synced successfully. {} messages inserted.",
account.id, folder.name, inserted
);
return Ok(());
}
Err(e) => {
warn!(
"Account {}: Failed to sync mailbox '{}'. Error: {}. Removing folder entry.",
account.id, &folder.name, e
);
if let Err(del_err) = OutlookFolder::delete(folder.id).await {
error!(
"Account {}: Failed to delete mailbox '{}' after sync error: {}",
account.id, &folder.name, del_err
);
}
}
}
}
None => {
match fetch_and_save_full_folder(account, folder, folder.exists, true).await {
Ok(inserted) => {
info!(
"Account {}: folder '{}' synced successfully. {} messages inserted.",
account.id, folder.name, inserted
);
return Ok(());
}
Err(e) => {
warn!(
return Ok(());
}
Err(e) => {
warn!(
"Account {}: Failed to sync folder '{}'. Error: {:#?}. Removing folder entry.",
account.id, &folder.name, e
);
if let Err(del_err) = OutlookFolder::delete(folder.id).await {
error!(
"Account {}: Failed to delete folder '{}' after sync error: {}",
account.id, &folder.name, del_err
);
if let Err(del_err) = OutlookFolder::delete(folder.id).await {
error!(
"Account {}: Failed to delete folder '{}' after sync error: {}",
account.id, &folder.name, del_err
);
}
}
}
}
},
}
}
//重建完不要忘记将deltalink保存
let delta_link =
OutlookClient::get_delta_link(account.id, account.use_proxy, &folder.folder_id).await?;
FolderDeltaLink::upsert(account.id, &folder.folder_id, &delta_link).await?;
+42 -5
View File
@@ -31,7 +31,7 @@ async fn access_token() -> String {
grpc_client.set_send_compressed(CompressionEncoding::GZIP);
let request = GetOAuth2TokensRequest {
account_id: 1770162132609912,
account_id: 1409297407117997,
};
let mut request = poem_grpc::Request::new(request);
@@ -557,9 +557,6 @@ async fn copy_message() {
}
}
#[tokio::test]
async fn move_message() {
let access_token = access_token().await;
@@ -595,4 +592,44 @@ async fn move_message() {
} else {
eprintln!("Error: {} - {:?}", res.status(), res.text().await.unwrap());
}
}
}
#[tokio::test]
async fn get_thread_messages() {
let access_token = access_token().await;
let mut url =
"https://graph.microsoft.com/v1.0/me/mailFolders/inbox/messages?$top=3&\
$orderBy=receivedDateTime desc&\
$filter=receivedDateTime ge 2025-10-01T00:00:00Z&\
$select=id,isRead,conversationId,internetMessageId,from,body,toRecipients,ccRecipients,\
bccRecipients,replyTo,sender,subject,receivedDateTime,sentDateTime,isRead,bodyPreview,categories&\
$expand=attachments($select=id,name,contentType,size,isInline,microsoft.graph.fileAttachment/contentId)"
.to_string();
let client = reqwest::Client::builder()
.user_agent(rustmailer_version!())
.timeout(Duration::from_secs(10))
.connect_timeout(Duration::from_secs(10))
.proxy(reqwest::Proxy::all("http://127.0.0.1:22307").unwrap())
.redirect(reqwest::redirect::Policy::none())
.build()
.unwrap();
let mut nextlink_count = 0;
let res = client
.get(&url)
.header(AUTHORIZATION, format!("Bearer {}", access_token))
.header(CONTENT_TYPE, "application/json")
.send()
.await
.unwrap();
if res.status().is_success() {
let body: MessageListResponse = res.json().await.unwrap();
println!("{:#?}", body);
} else {
eprintln!("Error: {} - {:?}", res.status(), res.text().await.unwrap());
}
}