fix: resolve sync deadlock, add runtime watchdog, and relax disabled-account reads

- upgrade redb 2.6.2 -> 2.6.3 to fix the range() iterator hang that froze
    the whole process during stale-envelope cleanup
  - add runtime watchdog: independent OS thread aborts the process (exit 1)
    when the tokio heartbeat stalls, so orchestrators can restart it
  - make find_impl async via spawn_blocking so a stuck DB lock cannot pin
    API workers (configurable via RUSTMAILER_WATCHDOG_TIMEOUT_SECS)
  - oauth2: soft-disable token refresh when the OAuth2 config is disabled
    (keep token, stop refreshing) instead of failing with
    SecondaryKeyConstraintMismatch; remove delete_by_oauth2_id
  - allow local-cache reads for disabled accounts (mailboxes, message list,
    content, attachments); keep remote operations blocked
  - include disabled accounts in minimal_list; keepalive and gRPC still
    filter by enabled
  - frontend: 403 no longer navigates to /403 or retries in a loop; show a
    "account disabled" toast and stop loading content
This commit is contained in:
rustmailer
2026-08-21 05:14:23 +08:00
parent 8d0b4919c4
commit dd3d66d7cd
28 changed files with 223 additions and 83 deletions
Generated
+4 -4
View File
@@ -3815,7 +3815,7 @@ dependencies = [
"native_db_macro",
"native_model",
"redb 1.5.1",
"redb 2.6.2",
"redb 2.6.3",
"semver 1.0.27",
"serde",
"skeptic",
@@ -5340,9 +5340,9 @@ dependencies = [
[[package]]
name = "redb"
version = "2.6.2"
version = "2.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b38b05028f398f08bea4691640503ec25fcb60b82fb61ce1f8fd1f4fccd3f7"
checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01"
dependencies = [
"libc",
]
@@ -5814,7 +5814,7 @@ dependencies = [
"prost-types 0.14.3",
"pulldown-cmark 0.13.0",
"rand 0.9.2",
"redb 2.6.2",
"redb 2.6.3",
"regex",
"reqwest",
"ring",
+1 -1
View File
@@ -23,7 +23,7 @@ chrono = "0.4.42"
clap = { version = "4.5.51", features = ["derive", "env"] }
mimalloc = "0.1.48"
native_db = "0.8.2"
redb = "2.6.2"
redb = "2.6.3"
itertools = "0.14.0"
native_model = "0.4.20"
poem = { version = "3.1.12", features = ["embed", "compression", "rustls"] }
+4 -2
View File
@@ -4,7 +4,7 @@
use mimalloc::MiMalloc;
use modules::{
common::rustls::RustMailerTls,
common::{rustls::RustMailerTls, watchdog::init_watchdog},
context::{executors::EmailClientExecutors, Initialize},
error::{code::ErrorCode, RustMailerResult},
grpc::server::start_grpc_server,
@@ -47,7 +47,9 @@ async fn main() -> RustMailerResult<()> {
info!("Git: [{}]", env!("GIT_HASH"));
info!("Project: https://rustmailer.com");
info!("GitHub: https://github.com/rustmailer/rustmailer");
init_watchdog(SETTINGS.rustmailer_watchdog_timeout_secs);
if let Err(error) = initialize().await {
eprintln!("{:?}", error);
return Err(error);
-1
View File
@@ -488,7 +488,6 @@ impl AccountV3 {
let result = list_all_impl(DB_MANAGER.meta_db())
.await?
.into_iter()
.filter(|a: &AccountModel| a.enabled)
.map(|account: AccountModel| MinimalAccount {
id: account.id,
email: account.email,
+1 -1
View File
@@ -164,7 +164,7 @@ pub async fn extract_client_context(req: &Request) -> Result<ClientContext> {
})?;
// Check for root token
if let Ok(Some(root)) = SystemSetting::get(ROOT_TOKEN) {
if let Ok(Some(root)) = SystemSetting::get(ROOT_TOKEN).await {
if root.value == token {
return Ok(ClientContext {
ip_addr: Some(ip_addr),
+1
View File
@@ -33,6 +33,7 @@ pub mod signal;
pub mod timeout;
pub mod tls;
pub mod validator;
pub mod watchdog;
#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Object)]
pub struct Addr {
+61
View File
@@ -0,0 +1,61 @@
// Copyright © 2025-2026 rustmailer.com
// Licensed under RustMailer License Agreement v1.0
// Unauthorized copying, modification, or distribution is prohibited.
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use tracing::{error, info};
use crate::utc_now;
/// Last time the tokio runtime proved it was alive (milliseconds since UNIX_EPOCH).
/// Updated by the heartbeat task; checked by the watchdog thread.
pub static LAST_HEARTBEAT: AtomicU64 = AtomicU64::new(0);
/// Starts the runtime watchdog.
///
/// A tokio task refreshes `LAST_HEARTBEAT` on an interval. A native OS thread
/// (independent of the tokio runtime) aborts the process with a non-zero exit
/// code when the heartbeat is stale for longer than `timeout_secs`, so that
/// orchestrators (Docker restart policies, supervisors) can recover the
/// process from a wedged runtime. No-op when `timeout_secs` is 0.
pub fn init_watchdog(timeout_secs: u64) {
if timeout_secs == 0 {
info!("[watchdog] disabled (timeout=0)");
return;
}
let heartbeat_interval = Duration::from_secs(5);
let timeout = Duration::from_secs(timeout_secs);
tokio::spawn(async move {
let mut tick = tokio::time::interval(heartbeat_interval);
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tick.tick().await;
LAST_HEARTBEAT.store(now_ms(), Ordering::Relaxed);
}
});
std::thread::spawn(move || {
info!(
"[watchdog] started, heartbeat interval {:?}, timeout {:?}",
heartbeat_interval, timeout
);
loop {
std::thread::sleep(heartbeat_interval);
let last = LAST_HEARTBEAT.load(Ordering::Relaxed);
if last != 0 && now_ms().saturating_sub(last) > timeout.as_millis() as u64 {
error!(
"[watchdog] runtime heartbeat stalled for {:?}, aborting the process",
timeout
);
std::process::exit(1);
}
}
});
}
fn now_ms() -> u64 {
utc_now!() as u64
}
+2 -2
View File
@@ -36,10 +36,10 @@ impl RustMailTask for ImapHeartBeatTask {
}
async fn touch_connection() -> RustMailerResult<()> {
let accounts = AccountModel::minimal_list().await?;
let accounts = AccountModel::list_all().await?;
let imap_account_ids: Vec<u64> = accounts
.into_iter()
.filter(|a| matches!(a.mailer_type, MailerType::ImapSmtp))
.filter(|a| a.enabled && matches!(a.mailer_type, MailerType::ImapSmtp))
.map(|a| a.id)
.collect();
-15
View File
@@ -247,21 +247,6 @@ pub async fn async_find_impl<T: ToInput + Clone + Send + 'static>(
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
}
pub fn find_impl<T: ToInput + Clone + Send + 'static>(
database: &Arc<Database<'static>>,
key: &str,
) -> RustMailerResult<Option<T>> {
let db = database.clone();
let r_transaction = db
.r_transaction()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
let entity: Option<T> = r_transaction
.get()
.primary(key)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(entity)
}
pub async fn delete_impl<T: ToInput + Clone + Send + 'static>(
database: &Arc<Database<'static>>,
delete: impl FnOnce(&RwTransaction) -> RustMailerResult<T> + Send + 'static,
+11 -1
View File
@@ -4,6 +4,7 @@
use crate::modules::account::migration::AccountModel as RustMailerAccount;
use crate::modules::account::payload::filter_accessible_accounts;
use crate::modules::account::payload::MinimalAccount;
use crate::modules::account::payload::AccountCreateRequest as RustMailerAccountCreateRequest;
use crate::modules::account::payload::AccountUpdateRequest as RustMailerAccountUpdateRequest;
use crate::modules::account::status::AccountRunningState as RustMailerAccountRunningState;
@@ -169,7 +170,16 @@ impl AccountService for RustMailerAccountService {
})?;
let accessible_accounts = context.accessible_accounts()?;
let minimal_list = RustMailerAccount::minimal_list().await?;
let minimal_list: Vec<MinimalAccount> = RustMailerAccount::list_all()
.await?
.into_iter()
.filter(|a| a.enabled)
.map(|account| MinimalAccount {
id: account.id,
email: account.email,
mailer_type: account.mailer_type,
})
.collect();
let result = match accessible_accounts {
Some(set) => filter_accessible_accounts(&minimal_list, set),
+8 -1
View File
@@ -23,9 +23,16 @@ pub async fn get_account_mailboxes(
account_id: u64,
remote: bool,
) -> RustMailerResult<Vec<MailBox>> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
let remote = remote || account.minimal_sync();
if remote && !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
match (&account.mailer_type, remote) {
(MailerType::ImapSmtp, true) => request_imap_all_mailbox_list(account_id).await,
(MailerType::ImapSmtp, false) => MailBox::list_all(account_id).await,
+7 -1
View File
@@ -148,7 +148,13 @@ pub async fn retrieve_email_attachment(
account_id: u64,
request: AttachmentRequest,
) -> RustMailerResult<(cacache::Reader, Option<String>)> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
request.validate(&account)?;
match account.mailer_type {
MailerType::ImapSmtp => {
+8 -1
View File
@@ -391,7 +391,14 @@ pub async fn retrieve_email_content(
request: MessageContentRequest,
skip_cache: bool,
) -> RustMailerResult<FullMessageContent> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
request.validate(&account)?;
match account.mailer_type {
+9 -1
View File
@@ -47,7 +47,15 @@ pub async fn retrieve_raw_email(
mailbox: Option<&str>,
id: &str,
) -> RustMailerResult<cacache::Reader> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
match account.mailer_type {
MailerType::ImapSmtp => {
let mailbox = mailbox.ok_or_else(|| {
+26 -3
View File
@@ -38,7 +38,7 @@ pub async fn list_messages_in_mailbox(
remote: bool,
desc: bool,
) -> RustMailerResult<CursorDataPage<Envelope>> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if page_size == 0 {
return Err(raise_error!(
"page_size must be greater than 0.".into(),
@@ -52,6 +52,14 @@ pub async fn list_messages_in_mailbox(
));
}
let remote = remote || account.minimal_sync();
if remote && !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
if remote {
fetch_remote_messages(&account, mailbox_name, next_page_token, page_size, desc).await
} else {
@@ -426,7 +434,15 @@ pub async fn list_threads_in_mailbox(
remote: bool,
desc: bool,
) -> RustMailerResult<CursorDataPage<Envelope>> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if remote && !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
if page_size == 0 {
return Err(raise_error!(
"page_size must be greater than 0.".into(),
@@ -689,7 +705,14 @@ pub async fn get_thread_messages(
thread_id: String,
remote: bool,
) -> RustMailerResult<Vec<Envelope>> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if remote && !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
match account.mailer_type {
MailerType::ImapSmtp => {
+8 -1
View File
@@ -488,7 +488,14 @@ impl MessageSearchRequest {
page_size: u64,
desc: bool,
) -> RustMailerResult<CursorDataPage<Envelope>> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
match account.mailer_type {
MailerType::ImapSmtp => {
self.imap_search_impl(&account, next_page_token, page_size, desc)
+8 -1
View File
@@ -164,7 +164,14 @@ impl BatchTagRequest {
}
pub async fn tag_messages_impl(account_id: u64, payload: BatchTagRequest) -> RustMailerResult<()> {
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
let _ = &payload.validate(&account)?;
match account.mailer_type {
+8 -2
View File
@@ -47,8 +47,14 @@ pub async fn transfer_messages(
transfer: MessageTransfer,
) -> RustMailerResult<()> {
// Ensure the account exists before proceeding
let account = AccountModel::check_account_active(account_id, false).await?;
let account = AccountModel::get(account_id).await?;
if !account.enabled {
return Err(raise_error!(
format!("Account id='{account_id}' is disabled"),
ErrorCode::AccountDisabled
));
}
match account.mailer_type {
MailerType::ImapSmtp => {
if payload.ids.is_empty() {
+1 -5
View File
@@ -160,11 +160,7 @@ impl OAuth2Flow {
pub async fn refresh_access_token(&self, token: &OAuth2AccessToken) -> RustMailerResult<()> {
let entity = self.fetch_oauth2_entity().await?;
if !entity.enabled {
OAuth2AccessToken::delete_by_oauth2_id(token.oauth2_id).await?;
return Err(raise_error!(
"OAuth2 authentication is disabled for this client".into(),
ErrorCode::OAuth2ItemDisabled
));
return Ok(());
}
let client = self.build_oauth2_client(&entity)?;
let http_client = build_http_client(entity.use_proxy).await?;
+1 -6
View File
@@ -8,7 +8,7 @@ use crate::modules::oauth2::{flow::OAuth2Flow, token::OAuth2AccessToken};
use crate::modules::scheduler::periodic::PeriodicTask;
use crate::utc_now;
use std::time::Duration;
use tracing::{debug, error, info};
use tracing::{debug, error};
const TASK_INTERVAL: Duration = Duration::from_secs(60); // Interval set to 1 minute
const FIFTEEN_MINUTES: Duration = Duration::from_secs(45 * 60);
@@ -50,11 +50,6 @@ impl RustMailTask for OAuth2RefreshTask {
"Failed to refresh access token for {}: {}",
token.account_id, error
);
} else {
info!(
"Successfully refreshed access token for {}",
token.account_id
);
}
});
}
-10
View File
@@ -148,16 +148,6 @@ impl OAuth2AccessToken {
}).await
}
pub async fn delete_by_oauth2_id(oauth2_id: u64) -> RustMailerResult<()> {
delete_impl(DB_MANAGER.meta_db(), move |rw|{
rw.get().secondary::<OAuth2AccessToken>(OAuth2AccessTokenKey::oauth2_id, oauth2_id)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.ok_or_else(|| raise_error!(format!(
"The oauth2 access token entity with oauth2_id={oauth2_id} that you want to delete was not found."
),ErrorCode::ResourceNotFound))
}).await
}
pub async fn set_access_token(
account_id: u64,
access_token: String,
+1 -1
View File
@@ -12,7 +12,7 @@ use tracing::error;
/// on successful authentication.
#[handler]
pub async fn login(password: String) -> Response {
match check_root_password(&password) {
match check_root_password(&password).await {
Ok(root_token) => Response::builder()
.status(http::StatusCode::OK)
.content_type("text/plain")
+14
View File
@@ -217,6 +217,20 @@ pub struct Settings {
)]
pub rustmailer_envelope_cache_size: Option<usize>,
/// Abort the process when the tokio runtime stops producing heartbeats.
///
/// A watchdog thread exits with a non-zero code when no runtime heartbeat is
/// observed for this many seconds, letting orchestrators (Docker restart
/// policies, supervisors) recover the process from a wedged runtime.
/// Set to 0 to disable the watchdog.
#[clap(
long,
env,
default_value = "60",
help = "Abort the process if the runtime heartbeat is stale for this many seconds (0 disables the watchdog)"
)]
pub rustmailer_watchdog_timeout_secs: u64,
/// Enables or disables the access token mechanism for HTTP endpoints.
///
/// When set to `true`, HTTP requests will be subject to access token validation.
+5 -9
View File
@@ -3,7 +3,7 @@
// Unauthorized copying, modification, or distribution is prohibited.
use crate::modules::database::manager::DB_MANAGER;
use crate::modules::database::{find_impl, upsert_impl};
use crate::modules::database::{async_find_impl, upsert_impl};
use crate::modules::error::RustMailerResult;
use crate::utc_now;
use native_db::*;
@@ -35,16 +35,12 @@ impl SystemSetting {
upsert_impl(DB_MANAGER.meta_db(), self).await
}
pub fn get(key: &str) -> RustMailerResult<Option<SystemSetting>> {
find_impl(DB_MANAGER.meta_db(), key)
pub async fn get(key: &str) -> RustMailerResult<Option<SystemSetting>> {
async_find_impl(DB_MANAGER.meta_db(), key.to_string()).await
}
// pub async fn list() -> RustMailerResult<Vec<SystemSetting>> {
// list_all_impl(DB_MANAGER.metadata_db()).await
// }
pub fn get_existing_value(key: &str) -> RustMailerResult<Option<String>> {
let setting = Self::get(key)?;
pub async fn get_existing_value(key: &str) -> RustMailerResult<Option<String>> {
let setting = Self::get(key).await?;
Ok(setting.map(|s| s.value))
}
+11 -9
View File
@@ -24,7 +24,7 @@ async fn get_or_generate(
save_file_name: Option<&str>,
force: bool,
) -> RustMailerResult<String> {
if let Some(existing_value) = SystemSetting::get_existing_value(key)? {
if let Some(existing_value) = SystemSetting::get_existing_value(key).await? {
if force {
// If force is true, write the existing value to the file
if let Some(filename) = save_file_name {
@@ -77,8 +77,8 @@ async fn save_to_file(content: &str, filename: &str) -> RustMailerResult<()> {
Ok(())
}
pub fn check_root_password(password: &str) -> RustMailerResult<String> {
let stored_encrypted_password = SystemSetting::get_existing_value(ROOT_PASSWORD)?;
pub async fn check_root_password(password: &str) -> RustMailerResult<String> {
let stored_encrypted_password = SystemSetting::get_existing_value(ROOT_PASSWORD).await?;
let matched = match stored_encrypted_password {
Some(ref stored) => {
let decrypted = decrypt!(stored)?;
@@ -94,12 +94,14 @@ pub fn check_root_password(password: &str) -> RustMailerResult<String> {
));
}
let root_token = SystemSetting::get_existing_value(ROOT_TOKEN)?.ok_or_else(|| {
raise_error!(
"Root token not found — this should never happen".into(),
ErrorCode::InternalError
)
})?;
let root_token = SystemSetting::get_existing_value(ROOT_TOKEN)
.await?
.ok_or_else(|| {
raise_error!(
"Root token not found — this should never happen".into(),
ErrorCode::InternalError
)
})?;
Ok(root_token)
}
@@ -132,11 +132,17 @@ export function MailDisplayDrawer({ open, setOpen, onOpenChange, currentEnvelope
},
onError: (error) => {
setLoading(false)
const isDisabled = (error as any)?.response?.status === 403;
toast({
title: 'Failed to load email message.',
description: `${error.message}`,
title: isDisabled ? 'Account is disabled' : 'Failed to load email message.',
description: isDisabled
? 'This account is disabled. Message content is unavailable.'
: `${error.message}`,
variant: 'destructive'
})
if (isDisabled) {
setOpen(null);
}
},
});
+10 -2
View File
@@ -228,6 +228,9 @@ export function Mail({
} catch (error) {
setIsError(true);
setError(error);
// Stop retrying: clear the selected mailbox so dependency changes
// (page, remote, filter) do not re-trigger this effect in a loop.
setSelectedMailbox(undefined);
} finally {
setIsMessagesLoading(false)
}
@@ -346,10 +349,15 @@ export function Mail({
React.useEffect(() => {
if (isError && error) {
const isDisabled =
error?.response?.status === 403 ||
error?.message?.includes('disabled');
toast({
variant: "destructive",
title: "Failed to load messages",
description: error.message || "An unknown error occurred. Please try again.",
title: isDisabled ? "Account is disabled" : "Failed to load messages",
description: isDisabled
? "This account is disabled. Only cached messages are available."
: (error.message || "An unknown error occurred. Please try again."),
});
}
}, [isError, error]);
+5 -1
View File
@@ -41,7 +41,11 @@ const handleAxiosError = (error: any) => {
}
break;
case 403:
router.navigate({ to: '/403' });
toast({
variant: 'destructive',
title: 'Access Denied',
description: 'You do not have permission to perform this action.',
});
break;
case 500:
toast({