feat(gmail): make search API

This commit is contained in:
rustmailer
2025-09-24 13:30:25 +08:00
parent 4c00e45657
commit 08ef77c23c
11 changed files with 365 additions and 69 deletions
+33 -7
View File
@@ -915,6 +915,9 @@ enum Conditions {
UNKEYWORD = 31;
// Messages that do not have the \Seen flag set.
UNSEEN = 32;
// This is a full Gmail search expression, only available for Gmail API accounts.
// Messages with a specific header containing the specified text
GMAIL_SEARCH = 33;
}
// Logic defines a logical operator (AND, OR, NOT) applied to child search conditions.
@@ -939,16 +942,22 @@ enum Operator {
message MessageSearchRequest {
// The ID of the account.
uint64 account_id = 1;
// The name of the mailbox to search within.
string mailbox_name = 2;
// The name of the mailbox to search in
// - For **IMAP accounts**, this field is **required** and specifies which mailbox
// (e.g. `INBOX`, `Sent`, or a custom folder) the search will run against.
// - For **Gmail API accounts**, this field is **optional**. If provided, it is treated
// as a label name and will override any label filter specified in the `query` string.
optional string mailbox_name = 2;
// The search query.
MessageSearch search = 3;
// The requested page number (1-based).
uint64 page = 4;
// The token for fetching the next page of results in pagination.
// - If `None`, this indicates that the first page should be returned.
// - If `Some(token)`, the page corresponding to this token will be fetched.
optional string next_page_token = 4;
// The number of messages to return per page.
uint64 page_size = 5;
// If true, results will be returned in descending order.
bool desc = 6;
// If true, results will be returned in descending order. imap account only
optional bool desc = 6;
}
/// Request structure for unified message search across accounts.
@@ -1034,6 +1043,23 @@ message PagedMessages {
optional uint64 total_pages = 5;
}
// CursorDataPage represents a collection of EmailEnvelope messages
// returned using cursor-based pagination rather than numeric page numbers.
message CursorDataPage {
// A cursor used for pagination (returned by the previous response).
// Pass this token to retrieve the next page of results.
// If `None`, there are no more pages available.
optional string next_page_token = 1;
// Optional: The number of items per page.
optional uint64 page_size = 2;
// The total number of items available across all pages.
uint64 total_items = 3;
// The list of EmailEnvelope items for the current page.
repeated EmailEnvelope items = 4;
// Optional: The total number of pages available.
optional uint64 total_pages = 5;
}
// A list of email envelopes.
// corresponding to an EmailEnvelope array.
message EmailEnvelopeList {
@@ -1138,7 +1164,7 @@ service MessageService {
// Fetches the complete raw EML content of an email message.
rpc FetchRawMessage(FetchRawMessageRequest) returns (ByteResponse);
// Searches for messages within a mailbox based on specified criteria.
rpc MessageSearch(MessageSearchRequest) returns (PagedMessages);
rpc MessageSearch(MessageSearchRequest) returns (CursorDataPage);
// Performs a unified search across mail accounts and messages.
rpc UnifiedSearch(UnifiedSearchRequest) returns (PagedMessages);
// Creates a reply draft email linked to an existing message thread.
+1 -1
View File
@@ -238,7 +238,7 @@ pub struct EmailEnvelopeV3 {
pub mid: Option<String>,
/// A list of labels applied to the message.
///
/// Each element is a string representing a Gmail label ID (e.g., "INBOX", "UNREAD").
/// Each element is a string representing a Gmail label name (e.g., "INBOX", "UNREAD").
/// This field reflects the current labels associated with the email.
///
/// Note: This field is populated only for Gmail API accounts. For other account types, it will be empty.
+1 -1
View File
@@ -10,4 +10,4 @@ use ahash::AHashMap;
use crate::modules::common::lru::TimedLruCache;
pub static GMAIL_LABELS_CACHE: LazyLock<TimedLruCache<u64, AHashMap<String, String>>> =
LazyLock::new(|| TimedLruCache::new(100, Duration::from_secs(3600)));
LazyLock::new(|| TimedLruCache::new(100, Duration::from_secs(60)));
+1 -1
View File
@@ -32,7 +32,7 @@ pub struct MessageList {
pub next_page_token: Option<String>,
#[serde(rename = "resultSizeEstimate")]
#[serde(default, skip_serializing_if = "Option::is_none")]
pub result_size_estimate: Option<i64>,
pub result_size_estimate: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+2
View File
@@ -2,6 +2,8 @@
// Licensed under RustMailer License Agreement v1.0
// Unauthorized copying, modification, or distribution is prohibited.
use crate::base64_encode_url_safe;
use super::error::code::ErrorCode;
use super::error::RustMailerError;
use mail_parser::{Addr as ImapAddr, Address as ImapAddress};
+16 -3
View File
@@ -9,7 +9,7 @@ use crate::modules::{
v2::EmailEnvelopeV3,
},
common::Addr,
grpc::service::rustmailer_grpc::{self, PagedMessages},
grpc::service::rustmailer_grpc::{self},
imap::section::{EmailBodyPart, Encoding, ImapAttachment, Param, PartType, SegmentPath},
message::{
append::AppendReplyToDraftRequest,
@@ -23,7 +23,7 @@ use crate::modules::{
},
transfer::MailboxTransferRequest,
},
rest::response::DataPage,
rest::response::{CursorDataPage, DataPage},
};
impl From<rustmailer_grpc::MailboxTransferRequest> for MailboxTransferRequest {
@@ -134,7 +134,19 @@ impl TryFrom<rustmailer_grpc::FlagMessageRequest> for FlagMessageRequest {
}
}
impl From<DataPage<EmailEnvelopeV3>> for PagedMessages {
impl From<CursorDataPage<EmailEnvelopeV3>> for rustmailer_grpc::CursorDataPage {
fn from(value: CursorDataPage<EmailEnvelopeV3>) -> Self {
Self {
next_page_token: value.next_page_token,
page_size: value.page_size,
total_items: value.total_items,
items: value.items.into_iter().map(Into::into).collect(),
total_pages: value.total_pages,
}
}
}
impl From<DataPage<EmailEnvelopeV3>> for rustmailer_grpc::PagedMessages {
fn from(value: DataPage<EmailEnvelopeV3>) -> Self {
Self {
current_page: value.current_page,
@@ -576,6 +588,7 @@ impl TryFrom<i32> for Conditions {
30 => Ok(Conditions::Unflagged),
31 => Ok(Conditions::Unkeyword),
32 => Ok(Conditions::Unseen),
33 => Ok(Conditions::GmailSeacrch),
_ => Err("Invalid value for Conditions"),
}
}
+13 -5
View File
@@ -8,8 +8,9 @@ use crate::modules::common::auth::ClientContext;
use crate::modules::error::code::ErrorCode;
use crate::modules::grpc::auth::require_account_access;
use crate::modules::grpc::service::rustmailer_grpc::{
AppendReplyToDraftRequest, ByteResponse, EmailEnvelopeList, GetThreadMessagesRequest,
ListThreadsRequest, MessageContentResponse, PagedMessages, UnifiedSearchRequest,
AppendReplyToDraftRequest, ByteResponse, CursorDataPage, EmailEnvelopeList,
GetThreadMessagesRequest, ListThreadsRequest, MessageContentResponse, PagedMessages,
UnifiedSearchRequest,
};
use crate::modules::grpc::service::rustmailer_grpc::{
Empty, FetchMessageAttachmentRequest, FetchMessageContentRequest, FetchRawMessageRequest,
@@ -162,17 +163,24 @@ impl MessageService for RustMailerMessageService {
async fn message_search(
&self,
request: Request<MessageSearchRequest>,
) -> Result<Response<PagedMessages>, Status> {
) -> Result<Response<CursorDataPage>, Status> {
let req = require_account_access(request, |r| r.account_id)?;
let account_id = req.account_id;
let page = req.page;
let next_page_token = req.next_page_token.clone();
let page_size = req.page_size;
let desc = req.desc;
let request: RustMailerMessageSearchRequest = req
.try_into()
.map_err(|e: &'static str| raise_error!(e.to_string(), ErrorCode::InvalidParameter))?;
let result = request.search(account_id, page, page_size, desc).await?;
let result = request
.search_impl(
account_id,
next_page_token.as_deref(),
page_size,
desc.unwrap_or(true),
)
.await?;
Ok(Response::new(result.into()))
}
+218 -40
View File
@@ -9,9 +9,12 @@ use crate::modules::cache::imap::v2::EmailEnvelopeV3;
use crate::modules::cache::vendor::gmail::sync::client::GmailClient;
use crate::modules::cache::vendor::gmail::sync::envelope::GmailEnvelope;
use crate::modules::common::paginated::paginate_vec;
use crate::modules::common::parallel::run_with_limit;
use crate::modules::database::Paginated;
use crate::modules::error::code::ErrorCode;
use crate::modules::message::search::cache::IMAP_SEARCH_CACHE;
use crate::modules::rest::response::CursorDataPage;
use crate::{base64_decode_url_safe, base64_encode_url_safe};
use crate::{
encode_mailbox_name,
modules::{
@@ -82,7 +85,10 @@ pub enum Conditions {
Flagged,
/// Messages with the specified text in the FROM field
From,
/// This is a full Gmail search expression, only available for Gmail API accounts.
/// Messages with a specific header containing the specified text
GmailSeacrch,
/// Search emails by a specific header value.
Header,
/// Messages with the specified keyword flag set
Keyword,
@@ -245,6 +251,12 @@ impl MessageSearch {
Conditions::Unflagged => "UNFLAGGED".into(),
Conditions::Unkeyword => format!("UNKEYWORD {}", Self::quote_value(value)?),
Conditions::Unseen => "UNSEEN".into(),
Conditions::GmailSeacrch => {
return Err(raise_error!(
"This condition is only supported for Gmail API accounts".into(),
ErrorCode::InvalidParameter
));
}
};
Ok(command)
}
@@ -309,6 +321,34 @@ impl MessageSearch {
}
}
pub fn to_gmail_api_search_command(&self) -> RustMailerResult<String> {
const ERR_MSG: &str = r#"Invalid GmailSeacrch condition format.
The JSON must include:
{
"type": "Condition",
"condition": "GmailSeacrch",
"value": "from:example@example.com OR subject:\"Invoice\" after:2025/01/01"
}
- "type" must be "Condition"
- "condition" must be "GmailSeacrch"
- "value" must be a full Gmail API search query
"#;
match self {
Self::Condition(condition) => match condition.condition {
Conditions::GmailSeacrch => {
let value = condition
.value
.as_deref()
.ok_or_else(|| raise_error!(ERR_MSG.into(), ErrorCode::InvalidParameter))?;
Ok(value.into())
}
_ => Err(raise_error!(ERR_MSG.into(), ErrorCode::InvalidParameter)),
},
Self::Logic(_) => Err(raise_error!(ERR_MSG.into(), ErrorCode::InvalidParameter)),
}
}
fn format_date(date: Option<&str>) -> RustMailerResult<String> {
let date = date.ok_or_else(|| {
raise_error!("Date value is required".into(), ErrorCode::InvalidParameter)
@@ -415,63 +455,194 @@ impl MessageSearch {
pub struct MessageSearchRequest {
/// The search criteria to apply (can be a simple condition or complex logical expression)
pub search: MessageSearch,
/// The name of the mailbox to search in
pub mailbox: String,
/// The name of the mailbox to search in
/// - For **IMAP accounts**, this field is **required** and specifies which mailbox
/// (e.g. `INBOX`, `Sent`, or a custom folder) the search will run against.
/// - For **Gmail API accounts**, this field is **optional**. If provided, it is treated
/// as a label name and will override any label filter specified in the `query` string.
pub mailbox: Option<String>,
}
impl MessageSearchRequest {
fn cache_key(
fn imap_search_cache_key(
&self,
account_id: u64,
page: u64,
page_size: u64,
desc: bool,
mailbox: &str,
search_query: &str,
) -> String {
format!(
"{}_{}_{}_{}_{}_{}",
account_id, self.mailbox, page, page_size, desc, search_query
"{}_{}_{}_{}_{}",
account_id, mailbox, page_size, desc, search_query
)
}
pub async fn search(
pub async fn search_impl(
&self,
account_id: u64,
page: u64,
next_page_token: Option<&str>,
page_size: u64,
desc: bool,
) -> RustMailerResult<DataPage<EmailEnvelopeV3>> {
) -> RustMailerResult<CursorDataPage<EmailEnvelopeV3>> {
let account = AccountV2::check_account_active(account_id, false).await?;
self.search_remote(&account, page, page_size, desc).await
match account.mailer_type {
MailerType::ImapSmtp => {
self.imap_search_impl(&account, next_page_token, page_size, desc)
.await
}
MailerType::GmailApi => {
self.gmail_api_search_impl(&account, next_page_token, page_size)
.await
}
}
}
async fn search_remote(
async fn gmail_api_search_impl(
&self,
account: &AccountV2,
page: u64,
next_page_token: Option<&str>,
page_size: u64,
desc: bool,
) -> RustMailerResult<DataPage<EmailEnvelopeV3>> {
// Validate page and page_size
if page == 0 || page_size == 0 {
) -> RustMailerResult<CursorDataPage<EmailEnvelopeV3>> {
if page_size == 0 {
return Err(raise_error!(
"Both page and page_size must be greater than 0.".into(),
"page_size must be greater than 0.".into(),
ErrorCode::InvalidParameter
));
}
if page_size > 1000 {
if page_size > 500 {
return Err(raise_error!(
"The page_size exceeds the maximum allowed limit of 1000.".into(),
"The page_size exceeds the maximum allowed limit of 500.".into(),
ErrorCode::InvalidParameter
));
}
let query = self.search.to_gmail_api_search_command()?;
let label_map: AHashMap<String, String> =
GmailClient::reverse_label_map(account.id, account.use_proxy, false).await?;
let label_id = match self.mailbox.as_deref() {
Some(name) => label_map.get(name).cloned().map(Some).ok_or_else(|| {
raise_error!(
format!("Label '{}' not found in Gmail account", name),
ErrorCode::InvalidParameter
)
})?,
None => None,
};
let message_list = GmailClient::search_messages(
account.id,
account.use_proxy,
label_id.as_deref(),
next_page_token,
Some(query.as_str()),
page_size,
)
.await?;
let total = message_list.result_size_estimate.ok_or_else(|| {
raise_error!(
"Missing 'resultSizeEstimate' in Gmail API response".into(),
ErrorCode::InternalError
)
})?;
let messages = message_list.messages;
let messages = match messages {
Some(ref msgs) if !msgs.is_empty() => msgs,
_ => {
return Ok(CursorDataPage {
next_page_token: None,
page_size: Some(page_size),
total_items: 0,
items: vec![],
total_pages: Some(0),
})
}
};
let account_id = account.id;
let use_proxy = account.use_proxy;
let next_page_token = message_list.next_page_token;
let batch_messages = run_with_limit(5, messages.iter().cloned(), move |index| async move {
GmailClient::get_message(account_id, use_proxy, &index.id).await
})
.await?;
let envelopes: Vec<EmailEnvelopeV3> = batch_messages
.into_iter()
.map(|m| {
let mut envelope: GmailEnvelope = m.try_into()?;
envelope.account_id = account_id;
Ok(envelope.into_v3(&label_map))
})
.collect::<RustMailerResult<Vec<EmailEnvelopeV3>>>()?;
let total_pages = (total as f64 / page_size as f64).ceil() as u64;
Ok(CursorDataPage {
next_page_token,
page_size: Some(page_size),
total_items: total,
items: envelopes,
total_pages: Some(total_pages),
})
}
async fn imap_search_impl(
&self,
account: &AccountV2,
next_page_token: Option<&str>,
page_size: u64,
desc: bool,
) -> RustMailerResult<CursorDataPage<EmailEnvelopeV3>> {
// Validate page and page_size
if page_size == 0 {
return Err(raise_error!(
"page_size must be greater than 0.".into(),
ErrorCode::InvalidParameter
));
}
if page_size > 500 {
return Err(raise_error!(
"The page_size exceeds the maximum allowed limit of 500.".into(),
ErrorCode::InvalidParameter
));
}
let page = match next_page_token {
Some(next_page_token) => {
let decoded = base64_decode_url_safe!(next_page_token)
.ok()
.and_then(|bytes| String::from_utf8(bytes).ok())
.and_then(|s| s.parse::<u64>().ok());
decoded.ok_or_else(|| {
raise_error!(
"Invalid next_page_token: not a valid page token".into(),
ErrorCode::InvalidParameter
)
})?
}
None => 1,
};
let mailbox = self.mailbox.as_deref().ok_or_else(|| {
raise_error!(
"IMAP accounts must specify a mailbox (e.g. INBOX, Sent, or custom folder)".into(),
ErrorCode::InvalidParameter
)
})?;
let search_query = self.search.to_imap_command(true)?;
info!(
"Executing remote search for account_id: {}, mailbox: {}, with query: {}",
account.id, self.mailbox, &search_query
account.id, mailbox, &search_query
);
let excutor = RUST_MAIL_CONTEXT.imap(account.id).await?;
let cache_key = self.cache_key(account.id, page, page_size, desc, &search_query);
let cache_key =
self.imap_search_cache_key(account.id, page_size, desc, mailbox, &search_query);
// Attempt to retrieve from cache
if let Some(v) = IMAP_SEARCH_CACHE.get(&cache_key).await {
@@ -480,8 +651,8 @@ impl MessageSearchRequest {
let total_pages = (total as f64 / page_size as f64).ceil() as u64;
if page > total_pages {
return Ok(DataPage::new(
Some(page),
return Ok(CursorDataPage::new(
None,
Some(page_size),
total,
Some(total_pages),
@@ -493,18 +664,24 @@ impl MessageSearchRequest {
let fetches = excutor
.uid_fetch_meta(
current_page_uids,
encode_mailbox_name!(&self.mailbox).as_str(),
encode_mailbox_name!(mailbox).as_str(),
false,
)
.await?;
let mut envelopes = Vec::new();
for fetch in fetches {
let envelope = extract_envelope(&fetch, account.id, &self.mailbox)?;
let envelope = extract_envelope(&fetch, account.id, mailbox)?;
envelopes.push(envelope);
}
return Ok(DataPage::new(
Some(page),
let next_page_token = if page == total_pages {
None
} else {
Some(base64_encode_url_safe!((page + 1).to_string()))
};
return Ok(CursorDataPage::new(
next_page_token,
Some(page_size),
total,
Some(total_pages),
@@ -514,14 +691,14 @@ impl MessageSearchRequest {
// Cache miss, perform search and fetch data
let uid_sets = excutor
.uid_search(&encode_mailbox_name!(self.mailbox), &search_query)
.uid_search(&encode_mailbox_name!(mailbox), &search_query)
.await?;
if uid_sets.is_empty() {
IMAP_SEARCH_CACHE
.set(cache_key, Arc::new((vec![], 0)))
.await;
return Ok(DataPage::new(
Some(page),
return Ok(CursorDataPage::new(
None,
Some(page_size),
0,
None,
@@ -538,8 +715,8 @@ impl MessageSearchRequest {
.await;
if page > total_pages {
return Ok(DataPage::new(
Some(page),
return Ok(CursorDataPage::new(
None,
Some(page_size),
total_items,
Some(total_pages),
@@ -549,20 +726,21 @@ impl MessageSearchRequest {
let current_page_uids = &pages[(page - 1) as usize];
let fetches = excutor
.uid_fetch_meta(
current_page_uids,
&encode_mailbox_name!(self.mailbox),
false,
)
.uid_fetch_meta(current_page_uids, &encode_mailbox_name!(mailbox), false)
.await?;
let mut envelopes = Vec::new();
for fetch in fetches {
let envelope = extract_envelope(&fetch, account.id, &self.mailbox)?;
let envelope = extract_envelope(&fetch, account.id, mailbox)?;
envelopes.push(envelope);
}
let next_page_token = if page == total_pages {
None
} else {
Some(base64_encode_url_safe!((page + 1).to_string()))
};
Ok(DataPage::new(
Some(page),
Ok(CursorDataPage::new(
next_page_token,
Some(page_size),
total_items,
Some(total_pages),
+2 -5
View File
@@ -4,11 +4,8 @@
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use crate::modules::{
cache::imap::sync::flow::generate_uid_sequence_hashset,
message::search::payload::{Condition, Conditions, Logic, MessageSearch, Operator},
use crate::modules::message::search::payload::{
Condition, Conditions, Logic, MessageSearch, Operator,
};
fn cond(condition: Conditions, value: &str) -> MessageSearch {
MessageSearch::Condition(Condition {
+9 -6
View File
@@ -21,7 +21,7 @@ use crate::modules::message::transfer::{
transfer_messages, MailboxTransferRequest, MessageTransfer,
};
use crate::modules::rest::api::ApiTags;
use crate::modules::rest::response::DataPage;
use crate::modules::rest::response::{CursorDataPage, DataPage};
use crate::modules::rest::ApiResult;
use poem::web::Path;
use poem::Body;
@@ -308,23 +308,26 @@ impl MessageApi {
&self,
/// The ID of the account owning the mailboxes.
account_id: Path<u64>,
/// The page number for pagination (1-based).
page: Query<u64>,
/// The token for fetching the next page of results in pagination.
///
/// - If `None`, this indicates that the first page should be returned.
/// - If `Some(token)`, the page corresponding to this token will be fetched.
next_page_token: Query<Option<String>>,
/// The number of messages per page.
page_size: Query<u64>,
/// If `true`, lists results in descending order; otherwise, ascending.
/// If `true`, lists results in descending order; otherwise, ascending. imap account only
desc: Query<Option<bool>>,
/// specifying the search criteria (e.g., keywords, flags).
payload: Json<MessageSearchRequest>,
context: ClientContext,
) -> ApiResult<Json<DataPage<EmailEnvelopeV3>>> {
) -> ApiResult<Json<CursorDataPage<EmailEnvelopeV3>>> {
let request = payload.0;
let desc = desc.0.unwrap_or(false);
let account_id = account_id.0;
context.require_account_access(account_id)?;
Ok(Json(
request
.search(account_id, page.0, page_size.0, desc)
.search_impl(account_id, next_page_token.0.as_deref(), page_size.0, desc)
.await?,
))
}
+69
View File
@@ -95,3 +95,72 @@ impl<
}
}
}
/// Represents a paginated response containing a subset of items along with pagination metadata.
///
/// This generic structure is commonly used to return paged data from list or search endpoints.
/// The type `S` represents the individual item type within the result set.
///
/// # Type Parameters
/// - `S`: The type of each item in the `items` list. Must implement several traits for serialization
/// and OpenAPI documentation.
///
/// # Fields
/// - `current_page`: The current page number (1-based). `None` if unspecified or not applicable.
/// - `page_size`: The number of items per page. `None` if unspecified.
/// - `total_items`: The total number of items matching the query.
/// - `items`: The list of items returned for the current page.
/// - `total_pages`: The total number of pages available. `None` if not calculated.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Object)]
pub struct CursorDataPage<S>
where
S: Serialize
+ std::fmt::Debug
+ std::marker::Unpin
+ Send
+ Sync
+ poem_openapi::types::Type
+ poem_openapi::types::ParseFromJSON
+ poem_openapi::types::ToJSON,
{
/// A cursor used for pagination (returned by the previous response).
/// Pass this token to retrieve the next page of results.
/// If `None`, there are no more pages available.
pub next_page_token: Option<String>,
/// The number of items per page.
pub page_size: Option<u64>,
/// The total number of items across all pages.
pub total_items: u64,
/// The list of items returned on the current page.
pub items: Vec<S>,
/// The total number of pages. This is optional and may not be set if not calculated.
pub total_pages: Option<u64>,
}
impl<
S: Serialize
+ std::fmt::Debug
+ std::marker::Unpin
+ Send
+ Sync
+ poem_openapi::types::Type
+ poem_openapi::types::ParseFromJSON
+ poem_openapi::types::ToJSON,
> CursorDataPage<S>
{
pub fn new(
next_page_token: Option<String>,
page_size: Option<u64>,
total_items: u64,
total_pages: Option<u64>,
items: Vec<S>,
) -> Self {
Self {
next_page_token,
page_size,
total_items,
total_pages,
items,
}
}
}