starred/rustmailer
1
0
Fork 0
mirror of https://github.com/rustmailer/rustmailer.git synced 2026-09-09 16:03:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(list-messages): switch from page-based to cursor-based pagination

Browse Source 
- Changed `list-messages` API from page/page_size to cursor/next_page_token
- Aligns pagination approach with Gmail API for easier integration
- Frontend updates to handle cursor-based navigation
This commit is contained in:
rustmailer
2025-09-28 07:39:40 +08:00
parent 08ef77c23c
commit 16c84abe67
15 changed files with 401 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/modules/message/list.rs
+204 -53
View File
@@ -3,7 +3,7 @@
// Unauthorized copying, modification, or distribution is prohibited.
use crate::{
encode_mailbox_name,
base64_encode_url_safe, encode_mailbox_name,
modules::{
account::{entity::MailerType, v2::AccountV2},
cache::{
@@ -12,10 +12,11 @@ use crate::{
client::GmailClient, envelope::GmailEnvelope, labels::GmailLabels,
},
},
common::{decode_page_token, parallel::run_with_limit},
context::executors::RUST_MAIL_CONTEXT,
envelope::extractor::extract_envelope,
error::{code::ErrorCode, RustMailerResult},
rest::response::DataPage,
rest::response::{CursorDataPage, DataPage},
},
raise_error,
};
@@ -24,19 +25,29 @@ use async_imap::types::Fetch;
pub async fn list_messages_in_mailbox(
account_id: u64,
mailbox_name: &str,
page: u64,
next_page_token: Option<&str>,
page_size: u64,
remote: bool,
desc: bool,
) -> RustMailerResult<DataPage<EmailEnvelopeV3>> {
) -> RustMailerResult<CursorDataPage<EmailEnvelopeV3>> {
let account = AccountV2::check_account_active(account_id, false).await?;
validate_pagination_params(page, page_size)?;
if page_size == 0 {
return Err(raise_error!(
"page_size must be greater than 0.".into(),
ErrorCode::InvalidParameter
));
}
if page_size > 500 {
return Err(raise_error!(
"The page_size exceeds the maximum allowed limit of 500.".into(),
ErrorCode::InvalidParameter
));
}
let remote = remote || account.minimal_sync();
if remote {
fetch_remote_messages(account_id, mailbox_name, page, page_size, desc).await
fetch_remote_messages(&account, mailbox_name, next_page_token, page_size, desc).await
} else {
fetch_local_messages(&account, mailbox_name, page, page_size, desc).await
fetch_local_messages(&account, mailbox_name, next_page_token, page_size, desc).await
}
}
@@ -47,9 +58,9 @@ fn validate_pagination_params(page: u64, page_size: u64) -> RustMailerResult<()>
ErrorCode::InvalidParameter
));
}
if page_size > 1000 {
if page_size > 500 {
return Err(raise_error!(
"The page_size exceeds the maximum allowed limit of 1000.".into(),
"The page_size exceeds the maximum allowed limit of 500.".into(),
ErrorCode::InvalidParameter
));
}
@@ -57,37 +68,126 @@ fn validate_pagination_params(page: u64, page_size: u64) -> RustMailerResult<()>
}
async fn fetch_remote_messages(
account_id: u64,
account: &AccountV2,
mailbox_name: &str,
page: u64,
next_page_token: Option<&str>,
page_size: u64,
desc: bool,
) -> RustMailerResult<DataPage<EmailEnvelopeV3>> {
let excutor = RUST_MAIL_CONTEXT.imap(account_id).await?;
let (mut fetches, total_items) = excutor
.retrieve_metadata_paginated(
page,
page_size,
encode_mailbox_name!(mailbox_name).as_str(),
desc,
false,
)
.await?;
) -> RustMailerResult<CursorDataPage<EmailEnvelopeV3>> {
match account.mailer_type {
MailerType::ImapSmtp => {
let page = decode_page_token(next_page_token)?;
let excutor = RUST_MAIL_CONTEXT.imap(account.id).await?;
let (mut fetches, total_items) = excutor
.retrieve_metadata_paginated(
page,
page_size,
encode_mailbox_name!(mailbox_name).as_str(),
desc,
false,
)
.await?;
if total_items == 0 {
return Ok(CursorDataPage::new(
None,
Some(page_size),
0,
Some(0),
vec![],
));
}
if desc {
fetches.reverse();
if desc {
fetches.reverse();
}
let total_pages = (total_items as f64 / page_size as f64).ceil() as u64;
let envelopes = process_fetches(fetches, account.id, mailbox_name).await?;
let next_page_token = if page == total_pages {
None
} else {
Some(base64_encode_url_safe!((page + 1).to_string()))
};
Ok(CursorDataPage::new(
next_page_token,
Some(page_size),
total_items,
Some(total_pages),
envelopes,
))
}
MailerType::GmailApi => {
let label_map =
GmailClient::reverse_label_map(account.id, account.use_proxy, true).await?;
let label_id = label_map.get(mailbox_name).ok_or_else(|| {
raise_error!(
format!("Label not found for mailbox: {}", mailbox_name),
ErrorCode::ResourceNotFound
)
})?;
let message_list = GmailClient::list_messages(
account.id,
account.use_proxy,
label_id,
next_page_token,
None,
page_size,
)
.await?;
let total = message_list.result_size_estimate.ok_or_else(|| {
raise_error!(
"Missing 'resultSizeEstimate' in Gmail API response".into(),
ErrorCode::InternalError
)
})?;
let messages = message_list.messages;
let messages = match messages {
Some(ref msgs) if !msgs.is_empty() => msgs,
_ => {
return Ok(CursorDataPage {
next_page_token: None,
page_size: Some(page_size),
total_items: 0,
items: vec![],
total_pages: Some(0),
})
}
};
let account_id = account.id;
let use_proxy = account.use_proxy;
let next_page_token = message_list.next_page_token;
let batch_messages =
run_with_limit(5, messages.iter().cloned(), move |index| async move {
GmailClient::get_message(account_id, use_proxy, &index.id).await
})
.await?;
let envelopes: Vec<EmailEnvelopeV3> = batch_messages
.into_iter()
.map(|m| {
let mut envelope: GmailEnvelope = m.try_into()?;
envelope.account_id = account_id;
envelope.label_name = mailbox_name.into();
Ok(envelope.into_v3(&label_map))
})
.collect::<RustMailerResult<Vec<EmailEnvelopeV3>>>()?;
let total_pages = (total as f64 / page_size as f64).ceil() as u64;
Ok(CursorDataPage {
next_page_token,
page_size: Some(page_size),
total_items: total,
items: envelopes,
total_pages: Some(total_pages),
})
}
}
let total_pages = (total_items as f64 / page_size as f64).ceil() as u64;
let envelopes = process_fetches(fetches, account_id, mailbox_name).await?;
Ok(DataPage::new(
Some(page),
Some(page_size),
total_items,
Some(total_pages),
envelopes,
))
}
async fn process_fetches(
@@ -106,10 +206,11 @@ async fn process_fetches(
async fn fetch_local_messages(
account: &AccountV2,
mailbox_name: &str,
page: u64,
next_page_token: Option<&str>,
page_size: u64,
desc: bool,
) -> RustMailerResult<DataPage<EmailEnvelopeV3>> {
) -> RustMailerResult<CursorDataPage<EmailEnvelopeV3>> {
let page = decode_page_token(next_page_token)?;
match account.mailer_type {
MailerType::ImapSmtp => {
let mailbox = MailBox::get(account.id, mailbox_name).await.map_err(|_| {
@@ -120,27 +221,77 @@ async fn fetch_local_messages(
ErrorCode::MailBoxNotCached
)
})?;
let DataPage {
current_page: _,
page_size,
total_items,
items,
total_pages,
} = EmailEnvelopeV3::list_messages_in_mailbox(mailbox.id, page, page_size, desc)
.await?;
EmailEnvelopeV3::list_messages_in_mailbox(mailbox.id, page, page_size, desc).await
if total_items == 0 {
Ok(CursorDataPage::new(None, page_size, 0, None, vec![]))
} else {
let total_pages = total_pages.ok_or_else(|| {
raise_error!(
"Internal error: total_pages is None (this should never happen)".into(),
ErrorCode::InternalError
)
})?;
let next_page_token = if page == total_pages {
None
} else {
Some(base64_encode_url_safe!((page + 1).to_string()))
};
Ok(CursorDataPage::new(
next_page_token,
page_size,
total_items,
Some(total_pages),
items,
))
}
}
MailerType::GmailApi => {
let target_label = GmailLabels::get_by_name(account.id, mailbox_name).await?;
let envelopes =
GmailEnvelope::list_messages_in_label(target_label.id, page, page_size, desc)
.await?;
let DataPage {
current_page: _,
page_size,
total_items,
items,
total_pages,
} = GmailEnvelope::list_messages_in_label(target_label.id, page, page_size, desc)
.await?;
let map = GmailClient::label_map(account.id, account.use_proxy).await?;
Ok(DataPage {
current_page: envelopes.current_page,
page_size: envelopes.page_size,
total_items: envelopes.total_items,
total_pages: envelopes.total_pages,
items: envelopes
.items
.into_iter()
.map(|e| e.into_v3(&map))
.collect(),
})
if total_items == 0 {
Ok(CursorDataPage::new(None, page_size, 0, None, vec![]))
} else {
let total_pages = total_pages.ok_or_else(|| {
raise_error!(
"Internal error: total_pages is None (this should never happen)".into(),
ErrorCode::InternalError
)
})?;
let next_page_token = if page == total_pages {
None
} else {
Some(base64_encode_url_safe!((page + 1).to_string()))
};
Ok(CursorDataPage::new(
next_page_token,
page_size,
total_items,
Some(total_pages),
items.into_iter().map(|e| e.into_v3(&map)).collect(),
))
}
}
}
}
src/modules/message/search/payload.rs
+3 -17
View File
@@ -2,19 +2,20 @@
// Licensed under RustMailer License Agreement v1.0
// Unauthorized copying, modification, or distribution is prohibited.
use crate::base64_encode_url_safe;
use crate::modules::account::entity::MailerType;
use crate::modules::cache::imap::address::AddressEntity;
use crate::modules::cache::imap::sync::flow::generate_uid_sequence_hashset;
use crate::modules::cache::imap::v2::EmailEnvelopeV3;
use crate::modules::cache::vendor::gmail::sync::client::GmailClient;
use crate::modules::cache::vendor::gmail::sync::envelope::GmailEnvelope;
use crate::modules::common::decode_page_token;
use crate::modules::common::paginated::paginate_vec;
use crate::modules::common::parallel::run_with_limit;
use crate::modules::database::Paginated;
use crate::modules::error::code::ErrorCode;
use crate::modules::message::search::cache::IMAP_SEARCH_CACHE;
use crate::modules::rest::response::CursorDataPage;
use crate::{base64_decode_url_safe, base64_encode_url_safe};
use crate::{
encode_mailbox_name,
modules::{
@@ -611,22 +612,7 @@ impl MessageSearchRequest {
));
}
let page = match next_page_token {
Some(next_page_token) => {
let decoded = base64_decode_url_safe!(next_page_token)
.ok()
.and_then(|bytes| String::from_utf8(bytes).ok())
.and_then(|s| s.parse::<u64>().ok());
decoded.ok_or_else(|| {
raise_error!(
"Invalid next_page_token: not a valid page token".into(),
ErrorCode::InvalidParameter
)
})?
}
None => 1,
};
let page = decode_page_token(next_page_token)?;
let mailbox = self.mailbox.as_deref().ok_or_else(|| {
raise_error!(
"IMAP accounts must specify a mailbox (e.g. INBOX, Sent, or custom folder)".into(),