mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
28f2879a05 | ||
|
|
2474259c22 | ||
|
|
79c2c5948f |
@@ -321,7 +321,8 @@ impl Services {
|
||||
refresh_policy.clone(),
|
||||
token_client.clone(),
|
||||
)
|
||||
.with_refresh_lease_store(stores.refresh_leases.clone()),
|
||||
.with_refresh_lease_store(stores.refresh_leases.clone())
|
||||
.with_account_pool_sync(account_pool.clone()),
|
||||
);
|
||||
let upstream_client: Arc<dyn ModelCatalogSource> = codex.clone();
|
||||
let store: Arc<dyn ModelSnapshotStore> = Arc::new(stores.model_snapshots.clone());
|
||||
@@ -546,6 +547,7 @@ pub async fn run(config: BootstrapConfig) -> Result<(), Box<dyn Error + Send + S
|
||||
let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?;
|
||||
let (router, task_coordinator, connection_drain) = build_router(config).await?;
|
||||
let mut task_coordinator = Some(task_coordinator);
|
||||
let mut restart_path = None;
|
||||
|
||||
let serve_result = {
|
||||
let (graceful_shutdown_tx, graceful_shutdown_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -560,46 +562,63 @@ pub async fn run(config: BootstrapConfig) -> Result<(), Box<dyn Error + Send + S
|
||||
tokio::pin!(serve);
|
||||
tokio::select! {
|
||||
result = &mut serve => result,
|
||||
() = crate::bootstrap::shutdown::shutdown_signal() => {
|
||||
let _ = graceful_shutdown_tx.send(());
|
||||
let active_websocket_connections = connection_drain.begin_shutdown();
|
||||
tracing::info!(
|
||||
active_websocket_connections,
|
||||
"正在排空入站连接并关闭运行时任务"
|
||||
);
|
||||
let coordinator = task_coordinator.take();
|
||||
let connection_drain = connection_drain.clone();
|
||||
let runtime_shutdown = async move {
|
||||
let shutdown_tasks = async move {
|
||||
if let Some(coordinator) = coordinator {
|
||||
coordinator.shutdown().await;
|
||||
}
|
||||
};
|
||||
tokio::join!(shutdown_tasks, connection_drain.wait());
|
||||
};
|
||||
let drain_runtime = async {
|
||||
let (serve_result, ()) = tokio::join!(&mut serve, runtime_shutdown);
|
||||
serve_result
|
||||
};
|
||||
tokio::pin!(drain_runtime);
|
||||
tokio::select! {
|
||||
result = &mut drain_runtime => result,
|
||||
() = tokio::time::sleep(HTTP_DRAIN_TIMEOUT) => {
|
||||
tracing::warn!(
|
||||
timeout_secs = HTTP_DRAIN_TIMEOUT.as_secs(),
|
||||
"HTTP graceful shutdown timed out; cancelling remaining connections"
|
||||
);
|
||||
action = crate::bootstrap::shutdown::shutdown_signal() => {
|
||||
match action {
|
||||
crate::bootstrap::shutdown::ShutdownAction::Restart(executable_path) => {
|
||||
restart_path = Some(executable_path);
|
||||
Ok(())
|
||||
}
|
||||
() = crate::bootstrap::shutdown::shutdown_signal() => {
|
||||
tracing::warn!("Received a second shutdown signal; cancelling remaining connections");
|
||||
Ok(())
|
||||
crate::bootstrap::shutdown::ShutdownAction::Graceful => {
|
||||
let _ = graceful_shutdown_tx.send(());
|
||||
let active_websocket_connections = connection_drain.begin_shutdown();
|
||||
tracing::info!(
|
||||
active_websocket_connections,
|
||||
"正在排空入站连接并关闭运行时任务"
|
||||
);
|
||||
let coordinator = task_coordinator.take();
|
||||
let connection_drain = connection_drain.clone();
|
||||
let runtime_shutdown = async move {
|
||||
let shutdown_tasks = async move {
|
||||
if let Some(coordinator) = coordinator {
|
||||
coordinator.shutdown().await;
|
||||
}
|
||||
};
|
||||
tokio::join!(shutdown_tasks, connection_drain.wait());
|
||||
};
|
||||
let drain_runtime = async {
|
||||
let (serve_result, ()) = tokio::join!(&mut serve, runtime_shutdown);
|
||||
serve_result
|
||||
};
|
||||
tokio::pin!(drain_runtime);
|
||||
tokio::select! {
|
||||
result = &mut drain_runtime => result,
|
||||
() = tokio::time::sleep(HTTP_DRAIN_TIMEOUT) => {
|
||||
tracing::warn!(
|
||||
timeout_secs = HTTP_DRAIN_TIMEOUT.as_secs(),
|
||||
"HTTP graceful shutdown timed out; cancelling remaining connections"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
_ = crate::bootstrap::shutdown::shutdown_signal() => {
|
||||
tracing::warn!("Received a second shutdown signal; cancelling remaining connections");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(executable_path) = restart_path {
|
||||
tracing::info!(
|
||||
executable = %executable_path.display(),
|
||||
"正在以更新后的二进制替换当前进程"
|
||||
);
|
||||
drop(log_guard);
|
||||
return exec_replacement_process(executable_path);
|
||||
}
|
||||
|
||||
if let Some(coordinator) = task_coordinator.take() {
|
||||
let active_websocket_connections = connection_drain.begin_shutdown();
|
||||
tracing::info!(
|
||||
@@ -610,15 +629,6 @@ pub async fn run(config: BootstrapConfig) -> Result<(), Box<dyn Error + Send + S
|
||||
}
|
||||
serve_result?;
|
||||
|
||||
if let Some(executable_path) = crate::bootstrap::shutdown::restart_executable_path() {
|
||||
tracing::info!(
|
||||
executable = %executable_path.display(),
|
||||
"正在以更新后的二进制替换当前进程"
|
||||
);
|
||||
drop(log_guard);
|
||||
exec_replacement_process(executable_path)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
//! 优雅关闭信号处理。
|
||||
//! 进程关闭与重启信号处理。
|
||||
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
use std::{path::PathBuf, sync::OnceLock};
|
||||
|
||||
use tokio::signal;
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
static SHUTDOWN_REQUESTS: OnceLock<broadcast::Sender<()>> = OnceLock::new();
|
||||
static RESTART_REQUESTED: AtomicBool = AtomicBool::new(false);
|
||||
static RESTART_EXECUTABLE_PATH: OnceLock<PathBuf> = OnceLock::new();
|
||||
static SHUTDOWN_REQUESTS: OnceLock<broadcast::Sender<ShutdownAction>> = OnceLock::new();
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum ShutdownAction {
|
||||
Graceful,
|
||||
Restart(PathBuf),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct RuntimeProcessControl;
|
||||
@@ -24,12 +22,12 @@ impl crate::api::router::ProcessControl for RuntimeProcessControl {
|
||||
}
|
||||
|
||||
fn request_restart(&self, executable_path: PathBuf) {
|
||||
request_process_restart(executable_path);
|
||||
request_immediate_restart(executable_path);
|
||||
}
|
||||
}
|
||||
|
||||
/// 等待进程关闭信号(Ctrl+C 或 SIGTERM)。
|
||||
pub async fn shutdown_signal() {
|
||||
/// 等待进程关闭或重启信号。
|
||||
pub(crate) async fn shutdown_signal() -> ShutdownAction {
|
||||
let mut requested = shutdown_sender().subscribe();
|
||||
let ctrl_c = async {
|
||||
signal::ctrl_c()
|
||||
@@ -49,33 +47,25 @@ pub async fn shutdown_signal() {
|
||||
let terminate = std::future::pending::<()>();
|
||||
|
||||
tokio::select! {
|
||||
_ = requested.recv() => {},
|
||||
() = ctrl_c => {},
|
||||
() = terminate => {},
|
||||
action = requested.recv() => match action {
|
||||
Ok(action) => action,
|
||||
Err(_) => ShutdownAction::Graceful,
|
||||
},
|
||||
() = ctrl_c => ShutdownAction::Graceful,
|
||||
() = terminate => ShutdownAction::Graceful,
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求进程按优雅关闭路径退出。
|
||||
pub fn request_shutdown() {
|
||||
let _ = shutdown_sender().send(());
|
||||
let _ = shutdown_sender().send(ShutdownAction::Graceful);
|
||||
}
|
||||
|
||||
/// 请求进程完成优雅关闭后以新二进制替换当前进程。
|
||||
pub fn request_process_restart(executable_path: PathBuf) {
|
||||
let _ = RESTART_EXECUTABLE_PATH.set(executable_path);
|
||||
RESTART_REQUESTED.store(true, Ordering::SeqCst);
|
||||
request_shutdown();
|
||||
fn request_immediate_restart(executable_path: PathBuf) {
|
||||
let _ = shutdown_sender().send(ShutdownAction::Restart(executable_path));
|
||||
}
|
||||
|
||||
/// 获取待 exec 的重启目标。
|
||||
pub fn restart_executable_path() -> Option<PathBuf> {
|
||||
RESTART_REQUESTED
|
||||
.load(Ordering::SeqCst)
|
||||
.then(|| RESTART_EXECUTABLE_PATH.get().cloned())
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn shutdown_sender() -> &'static broadcast::Sender<()> {
|
||||
fn shutdown_sender() -> &'static broadcast::Sender<ShutdownAction> {
|
||||
SHUTDOWN_REQUESTS.get_or_init(|| {
|
||||
let (sender, _receiver) = broadcast::channel(16);
|
||||
sender
|
||||
|
||||
@@ -22,6 +22,7 @@ use uuid::Uuid;
|
||||
use super::TokenRefresher;
|
||||
use crate::fleet::{
|
||||
account::{Account, AccountStatus},
|
||||
pool::AccountPoolService,
|
||||
store::{
|
||||
AccountClaimsUpdate, AccountStore, PgAccountStore, PgAccountStoreError, StoredAccount,
|
||||
},
|
||||
@@ -48,6 +49,7 @@ where
|
||||
C: TokenRefresher,
|
||||
{
|
||||
store: PgAccountStore,
|
||||
account_pool: Option<Arc<AccountPoolService>>,
|
||||
scheduler: Arc<RefreshScheduler<C>>,
|
||||
policy: RuntimeRefreshPolicy,
|
||||
refresh_leases: Option<RedisRefreshLeaseStore>,
|
||||
@@ -193,6 +195,7 @@ where
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
store: self.store.clone(),
|
||||
account_pool: self.account_pool.clone(),
|
||||
scheduler: self.scheduler.clone(),
|
||||
policy: self.policy.clone(),
|
||||
refresh_leases: self.refresh_leases.clone(),
|
||||
@@ -215,6 +218,7 @@ where
|
||||
let policy = policy.into();
|
||||
Self {
|
||||
store,
|
||||
account_pool: None,
|
||||
scheduler: Arc::new(RefreshScheduler::new(policy.clone(), client)),
|
||||
policy,
|
||||
refresh_leases: None,
|
||||
@@ -233,6 +237,12 @@ where
|
||||
self
|
||||
}
|
||||
|
||||
/// 刷新后同步持久化账号到运行时池,避免旧 token 快照覆盖新状态。
|
||||
pub fn with_account_pool_sync(mut self, account_pool: Arc<AccountPoolService>) -> Self {
|
||||
self.account_pool = Some(account_pool);
|
||||
self
|
||||
}
|
||||
|
||||
/// 更新刷新策略。
|
||||
pub fn update_policy(&self, policy: RefreshPolicy) {
|
||||
self.policy.update(policy);
|
||||
@@ -587,6 +597,7 @@ where
|
||||
if attempt.refresh_token.is_none() {
|
||||
persist_status(&self.store, &attempt.id, AccountStatus::Expired).await?;
|
||||
persist_next_refresh_at(&self.store, &attempt.id, None).await?;
|
||||
self.sync_account_pool_after_update(&attempt.id).await;
|
||||
return Ok(TokenRefreshOutcome::StatusUpdated);
|
||||
}
|
||||
|
||||
@@ -601,6 +612,7 @@ where
|
||||
|
||||
persist_status(&self.store, &attempt.id, updated.status).await?;
|
||||
persist_next_refresh_at(&self.store, &attempt.id, None).await?;
|
||||
self.sync_account_pool_after_update(&attempt.id).await;
|
||||
return Ok(TokenRefreshOutcome::StatusUpdated);
|
||||
}
|
||||
Ok(mut updated) => {
|
||||
@@ -609,6 +621,7 @@ where
|
||||
updated.next_refresh_at = self.computed_refresh_at(&updated);
|
||||
let refresh_token_rotated = updated.refresh_token != attempt.refresh_token;
|
||||
persist_token_update(&self.store, &updated).await?;
|
||||
self.sync_account_pool_after_update(&updated.id).await;
|
||||
info!(
|
||||
account_id = %updated.id,
|
||||
attempt = attempt_index + 1,
|
||||
@@ -643,6 +656,19 @@ where
|
||||
Ok(TokenRefreshOutcome::Skipped)
|
||||
}
|
||||
|
||||
async fn sync_account_pool_after_update(&self, account_id: &str) {
|
||||
let Some(account_pool) = self.account_pool.as_ref() else {
|
||||
return;
|
||||
};
|
||||
if let Err(error) = account_pool.sync_account_from_store(account_id).await {
|
||||
warn!(
|
||||
account_id,
|
||||
error = %error,
|
||||
"Failed to sync runtime account pool after token refresh"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_refresh_attempt(
|
||||
&self,
|
||||
account: &Account,
|
||||
|
||||
@@ -174,6 +174,7 @@ update accounts
|
||||
set
|
||||
status = case
|
||||
when status in ('disabled', 'banned') then status
|
||||
when $4 = 'expired' and access_token_expires_at > $2 then status
|
||||
when (
|
||||
case
|
||||
when not $1 and quota_cooldown_until is not null and quota_cooldown_until > $2 then quota_limit_reached
|
||||
|
||||
@@ -9,12 +9,17 @@ use std::{
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, TimeZone, Utc};
|
||||
use codex_proxy_rs::fleet::account::AccountStatus;
|
||||
use codex_proxy_rs::fleet::refresh::{RedisRefreshLeaseStore, RefreshPolicy};
|
||||
use codex_proxy_rs::fleet::refresh::{RefreshFailure, TokenPair, TokenRefresher};
|
||||
use codex_proxy_rs::fleet::store::{
|
||||
AccountClaimsUpdate, NewAccount, PgAccountStore, StoredAccount,
|
||||
};
|
||||
use codex_proxy_rs::fleet::{
|
||||
account::AccountStatus,
|
||||
pool::{AccountPoolOptions, AccountPoolService},
|
||||
store::AccountStore,
|
||||
};
|
||||
use codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use serde_json::json;
|
||||
use tokio::{
|
||||
|
||||
@@ -1,5 +1,85 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn token_refresh_task_should_sync_refreshed_token_into_runtime_account_pool() {
|
||||
let (pool, _guard) = init_test_db("token-refresh-runtime-pool-sync").await;
|
||||
let store = PgAccountStore::new(pool.clone());
|
||||
let now = Utc.with_ymd_and_hms(2026, 6, 18, 7, 0, 0).unwrap();
|
||||
let new_expires_at = now + Duration::hours(1);
|
||||
let new_access_token = test_jwt(new_expires_at.timestamp());
|
||||
store
|
||||
.insert(NewAccount {
|
||||
id: "acct-refresh-runtime-pool".to_string(),
|
||||
email: Some("runtime-pool@example.com".to_string()),
|
||||
account_id: Some("chatgpt-runtime-pool".to_string()),
|
||||
user_id: Some("user-runtime-pool".to_string()),
|
||||
label: None,
|
||||
plan_type: Some("plus".to_string()),
|
||||
access_token: SecretString::new(
|
||||
test_jwt((now + Duration::seconds(30)).timestamp()).into(),
|
||||
),
|
||||
refresh_token: Some(SecretString::new("refresh-runtime-pool".to_string().into())),
|
||||
added_at: None,
|
||||
access_token_expires_at: Some(now + Duration::seconds(30)),
|
||||
status: AccountStatus::Active,
|
||||
})
|
||||
.await
|
||||
.expect("account should be inserted");
|
||||
let account_pool = Arc::new(AccountPoolService::new(
|
||||
Arc::new(store.clone()) as Arc<dyn AccountStore>,
|
||||
Arc::new(PgAccountUsageStore::new(pool)),
|
||||
AccountPoolOptions {
|
||||
max_concurrent_per_account: 1,
|
||||
..AccountPoolOptions::default()
|
||||
},
|
||||
0,
|
||||
));
|
||||
account_pool
|
||||
.restore_from_store()
|
||||
.await
|
||||
.expect("runtime account pool should restore");
|
||||
let refresher = StaticTokenRefresher {
|
||||
response: Arc::new(Mutex::new(Ok(TokenPair {
|
||||
access_token: new_access_token.clone(),
|
||||
refresh_token: None,
|
||||
}))),
|
||||
};
|
||||
let task = codex_proxy_rs::fleet::refresh::TokenRefreshService::new(
|
||||
store.clone(),
|
||||
RefreshPolicy {
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 1,
|
||||
},
|
||||
refresher,
|
||||
)
|
||||
.with_account_pool_sync(Arc::clone(&account_pool));
|
||||
|
||||
task.schedule_account_timers_once_at(now)
|
||||
.await
|
||||
.expect("refresh timer should be scheduled");
|
||||
let stored = wait_for_account(&store, "acct-refresh-runtime-pool", |account| {
|
||||
account.access_token.expose_secret() == new_access_token.as_str()
|
||||
})
|
||||
.await;
|
||||
let capacity = timeout(StdDuration::from_secs(2), async {
|
||||
loop {
|
||||
let capacity = account_pool
|
||||
.capacity_summary(now + Duration::minutes(1))
|
||||
.await;
|
||||
if capacity.available_slots == 1 {
|
||||
return capacity;
|
||||
}
|
||||
sleep(StdDuration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("refreshed account should remain available in the runtime pool");
|
||||
task.shutdown().await;
|
||||
|
||||
assert_eq!(stored.status, AccountStatus::Active);
|
||||
assert_eq!(capacity.available_slots, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn token_refresh_task_should_persist_refreshed_access_token_and_keep_refresh_token() {
|
||||
let (pool, _guard) = init_test_db("token-refresh").await;
|
||||
|
||||
@@ -793,6 +793,66 @@ async fn account_store_runtime_state_sync_should_not_clear_newer_future_quota_co
|
||||
assert_eq!(stored.3, Some(future_cooldown));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_store_runtime_state_sync_should_not_expire_a_newer_refreshed_token() {
|
||||
let (pool, _dir) = account_store_parts("accounts", 49).await;
|
||||
let repo = PgAccountStore::new(pool.clone());
|
||||
let stale_expires_at = pg_timestamp(Utc::now() - Duration::minutes(1));
|
||||
repo.insert(NewAccount {
|
||||
id: "acct_a".to_string(),
|
||||
email: Some("user@example.com".to_string()),
|
||||
account_id: Some("chatgpt-account".to_string()),
|
||||
user_id: Some("chatgpt-user".to_string()),
|
||||
label: None,
|
||||
plan_type: Some("plus".to_string()),
|
||||
access_token: SecretString::new("stale-access-token".to_string().into()),
|
||||
refresh_token: Some(SecretString::new("refresh-token".to_string().into())),
|
||||
access_token_expires_at: Some(stale_expires_at),
|
||||
status: AccountStatus::Active,
|
||||
added_at: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let mut stale_account = AccountStore::list_pool_accounts(&repo)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|account| account.id == "acct_a")
|
||||
.unwrap();
|
||||
let refreshed_expires_at = pg_timestamp(Utc::now() + Duration::hours(1));
|
||||
repo.update_from_claims(
|
||||
"acct_a",
|
||||
AccountClaimsUpdate {
|
||||
email: Some("user@example.com".to_string()),
|
||||
account_id: Some("chatgpt-account".to_string()),
|
||||
user_id: Some("chatgpt-user".to_string()),
|
||||
plan_type: Some("plus".to_string()),
|
||||
access_token: SecretString::new("refreshed-access-token".to_string().into()),
|
||||
refresh_token: None,
|
||||
access_token_expires_at: Some(refreshed_expires_at),
|
||||
next_refresh_at: Some(refreshed_expires_at - Duration::minutes(5)),
|
||||
status: AccountStatus::Active,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
stale_account.status = AccountStatus::Expired;
|
||||
|
||||
repo.sync_runtime_account_state(&stale_account)
|
||||
.await
|
||||
.unwrap();
|
||||
let stored = repo.get("acct_a").await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(stored.status, AccountStatus::Active);
|
||||
assert_eq!(
|
||||
(
|
||||
stored.access_token.expose_secret(),
|
||||
stored.access_token_expires_at
|
||||
),
|
||||
("refreshed-access-token", Some(refreshed_expires_at))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_store_runtime_state_sync_should_not_regress_newer_usage_window() {
|
||||
let (pool, _dir) = account_store_parts("accounts", 43).await;
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "2.5.1"
|
||||
version: "2.5.2"
|
||||
|
||||
Reference in New Issue
Block a user