fix(accounts): preserve quota when recovering disabled accounts

Enable scheduling without clearing observed quota, credential errors, or cooldowns for disabled accounts. Keep explicit recovery for enabled accounts and document both paths.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-09-20 18:41:22 +08:00
co-authored by Codex
parent 3a90d8d0b6
commit 6434edd554
4 changed files with 197 additions and 15 deletions
@@ -548,19 +548,39 @@ impl AccountsService for DefaultAccountsService {
context: &MutationContext,
account_id: ProviderAccountId,
) -> Result<AccountRefreshResult, AdminError> {
let (_, provider) = self.provider_for_account(&account_id).await?;
let result = self
.accounts
.recover_account(&account_id, context)
.await
.map_err(|error| map_store_error(error, "provider account recovery"))?;
let (stored, provider) = self.provider_for_account(&account_id).await?;
let config_revision = if stored.account.enabled {
self.accounts
.recover_account(&account_id, context)
.await
.map_err(|error| map_store_error(error, "provider account recovery"))?
.config_revision
} else {
// 停用只表示不参与调度,重新启用不能抹除已观测的额度、凭据或冷却事实。
self.accounts
.batch_update_accounts(
BatchUpdateAccounts {
account_ids: vec![account_id.to_string()],
enabled: Some(true),
concurrency_limit: None,
weight: None,
model_access: None,
group_ids: None,
outbound_proxy: None,
},
context,
)
.await
.map_err(|error| map_store_error(error, "enable provider account"))?
.config_revision
};
provider
.account_facts_changed(std::slice::from_ref(&account_id))
.await;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
publish_committed(self.snapshot.as_ref(), config_revision).await?;
let account = self.load_directory_item(&account_id, false).await?;
Ok(AccountRefreshResult {
config_revision: result.config_revision,
config_revision,
account,
})
}
@@ -935,6 +935,13 @@ impl AccountStore for FakeAccountStore {
self.record("store.batch_update_accounts");
self.record_context(context);
self.require_commit()?;
if let Some(enabled) = command.enabled {
for account in self.accounts.lock().expect("accounts").iter_mut() {
if command.account_ids.contains(&account.id) {
account.enabled = enabled;
}
}
}
Ok(AccountsUpdateResult {
config_revision: revision(2),
account_ids: command
@@ -1360,7 +1367,6 @@ async fn accounts_recover_should_commit_facts_then_return_normal_account() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let mut account = account_record("openai");
account.enabled = false;
account.credential_state = CredentialState::Invalid;
account.quota = QuotaState::exhausted(
QuotaEvidence::UsageLimitReached,
@@ -1407,6 +1413,110 @@ async fn accounts_recover_should_commit_facts_then_return_normal_account() {
assert_eq!(store.audit_requests(), ["recover-request"]);
}
#[tokio::test]
async fn accounts_recover_disabled_should_only_enable_scheduling_and_preserve_quota() {
for has_error in [false, true] {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let mut account = account_record("openai");
account.enabled = false;
if has_error {
account.credential_state = CredentialState::Invalid;
account.last_error_reason =
Some(gateway_core::account::AccountErrorReason::CredentialInvalid);
account.last_error_message = Some("invalid credential".to_owned());
account.quota = QuotaState::exhausted(
QuotaEvidence::UsageLimitReached,
std::time::SystemTime::now(),
None,
);
}
let quota = ProviderQuota {
observed_at: Some(Utc::now()),
windows: vec![ProviderQuotaWindow {
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(92.0),
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
limit_reached: false,
local_usage: None,
provider_data: None,
}],
..empty_quota()
};
provider.set_quota(quota.clone());
let store = FakeAccountStore::with_account(account.clone(), events.clone());
let services = accounts_service(provider.clone(), store.clone()).await;
let result = services
.accounts()
.recover(
&context("enable-request"),
ProviderAccountId::new("acct_test").expect("account ID"),
)
.await
.expect("enable disabled account");
account.enabled = true;
assert_eq!(result.config_revision, revision(2));
assert_eq!(result.account.account, account);
assert_eq!(result.account.quota, quota);
assert_eq!(
result.account.projection.status,
if has_error {
gateway_admin::model::accounts::AccountStatus::Error
} else {
gateway_admin::model::accounts::AccountStatus::Normal
}
);
assert_eq!(
recorded(&events),
[
"store.load_account",
"store.batch_update_accounts",
"provider.account_facts_changed",
"store.load_account",
]
);
assert_eq!(store.audit_requests(), ["enable-request"]);
let requests = provider.quota_requests();
assert_eq!(requests.len(), 1);
assert!(!requests[0].refresh);
}
}
#[tokio::test]
async fn accounts_recover_disabled_should_not_publish_when_enabling_fails() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let mut account = account_record("openai");
account.enabled = false;
let store = FakeAccountStore::with_account(account.clone(), events.clone());
store.fail_next_commit();
let services = accounts_service(provider, store.clone()).await;
let result = services
.accounts()
.recover(
&context("enable-failed"),
ProviderAccountId::new("acct_test").expect("account ID"),
)
.await;
assert!(result.is_err());
assert_eq!(*store.accounts.lock().expect("accounts"), [account]);
assert_eq!(
recorded(&events),
["store.load_account", "store.batch_update_accounts"]
);
}
#[tokio::test]
async fn accounts_update_should_commit_then_release_disabled_account_and_publish_facts() {
let events = events();
@@ -1519,7 +1519,7 @@ async fn invalid_account_notes_roll_back_scheduling_revision_and_audit() {
}
#[tokio::test]
async fn account_recovery_resets_status_facts_without_changing_credentials_or_scheduling() {
async fn account_enable_preserves_facts_and_explicit_recovery_clears_them() {
const GROUP_ID: &str = "grp_00000000000000000000000000000070";
let Some(database) = TestDatabase::create("provider_account_recovery").await else {
return;
@@ -1594,6 +1594,57 @@ async fn account_recovery_resets_status_facts_without_changing_credentials_or_sc
.await
.expect("load account before recovery");
let store = admin_account_store(&database.pool);
let facts_query = "select to_jsonb(a) - 'enabled' - 'updated_at'
from provider_accounts a where id = 'acct_recovery'";
let before_enable: serde_json::Value = sqlx::query_scalar(facts_query)
.fetch_one(&database.pool)
.await
.expect("load disabled account facts");
let enabled = store
.batch_update_accounts(
BatchUpdateAccounts {
account_ids: vec!["acct_recovery".to_owned()],
enabled: Some(true),
concurrency_limit: None,
weight: None,
model_access: None,
group_ids: None,
outbound_proxy: None,
},
&MutationContext {
actor: MutationActor::System,
request_id: "request_account_enable".to_owned(),
},
)
.await
.expect("enable account scheduling");
assert_eq!(enabled.config_revision.get(), 2);
assert!(
repository
.load_provider_account("acct_recovery")
.await
.unwrap()
.unwrap()
.summary
.enabled
);
assert_eq!(
sqlx::query_scalar::<_, serde_json::Value>(facts_query)
.fetch_one(&database.pool)
.await
.expect("load enabled account facts"),
before_enable
);
assert_eq!(
account_group_ids(&database.pool, "acct_recovery").await,
[GROUP_ID]
);
assert_eq!(
audit_count(&database.pool, "request_account_enable").await,
1
);
let result = store
.recover_account(
@@ -1606,7 +1657,7 @@ async fn account_recovery_resets_status_facts_without_changing_credentials_or_sc
.await
.expect("recover account");
assert_eq!(result.config_revision.get(), 2);
assert_eq!(result.config_revision.get(), 3);
let current = sqlx::query_as::<_, RecoveredAccountRow>(
"select enabled, credential_state, quota_access_state, quota_evidence,
last_error_message, provider_quota_json, concurrency_limit, weight,
+5 -4
View File
@@ -392,7 +392,7 @@ config 返回 `{ name, plaintextKey }`,仅读取服务端会话绑定的当前
| `GET` | `/api/admin/accounts/import-tasks/detail` | `taskId` | 任务摘要和逐条结果,不含原始凭据 |
| `POST` | `/api/admin/accounts/import-tasks/stop` | `{ taskId }` | 跳过未开始的条目,已开始的条目继续完成 |
| `POST` | `/api/admin/accounts/refresh` | `{ accountId }` | 手工刷新 OAuth credential(`idToken` / `accessToken` / `refreshToken`),不刷新额度 |
| `POST` | `/api/admin/accounts/recover` | `{ accountId }` | 管理员显式清除该账号的本地错误/额度/cooldown 事实并重新启用,不访问上游 |
| `POST` | `/api/admin/accounts/recover` | `{ accountId }` | 停用账号只启用调度;已启用账号强制清除本地错误/额度/cooldown 事实,不访问上游 |
| `POST` | `/api/admin/accounts/rotate` | OpenAI rotation 字段 | 更新指定 OpenAI 账号的 OAuth token 或 API Key 上游设置 |
| `POST` | `/api/admin/accounts/update` | `{ accountId, enabled, concurrencyLimit, weight, groupIds, notes?, modelAccess?, outboundProxyId?, outboundProxyUrl? }` | 一次更新账号备注、调度状态、并发上限(`null` 表示继承运行参数)、权重(1–100)、所属分组与出站代理 |
| `POST` | `/api/admin/accounts/batch-update` | `{ accountIds, enabled?, concurrencyLimit?, weight?, groupIds?, modelAccess?, outboundProxyId?, outboundProxyUrl? }` | 一次事务更新所选账号;仅修改提供的字段,至少提供一项修改 |
@@ -714,9 +714,10 @@ OAuth start 使用:
在该窗口的 `resetAt + 2 分钟` 后主动复核。后台每 30 秒检查触发条件;同一重置边界复核后仍未更新时
回到 30 分钟重试,避免旧 reset 持续触发请求。各窗口独立确认恢复,时间到期本身不会直接解除账号
耗尽或将展示用量归零。
- `POST /accounts/recover` 是管理员对本地事实的强制恢复:它清除 Redis cooldown 和已保存的额度/错误,
把账号重新启用并恢复为可调度 credential;它不验证上游账号是否已经恢复,下一次真实请求仍可重新写入
失败事实。
- `POST /accounts/recover` 对停用账号只将 `enabled` 改为 `true`,保留已有额度快照、凭据、错误和 Redis
cooldown;启用后仍按这些事实投影状态,不会把已有错误或耗尽改成正常。对已启用账号则执行强制恢复:
清除 Redis cooldown 和已保存的额度/错误,恢复为可调度 credential。两条路径均不访问上游;强制恢复
不验证上游账号是否已经恢复,下一次真实请求仍可重新写入失败事实。
- 成功额度观测会 revision-fenced 写入 quota;明确 `Allowed` 投影为 `normal`,明确耗尽投影为
`quota_exhausted`。额度观测不会清除凭据过期、无效或封禁事实;这些事实统一投影为 `error`,并由
`errorReason` 区分。额度接口的 401/403 也不足以判定 refresh token 永久失效,credential 终态只由