feat(settings): remove request history retention cap (#702)

* feat(settings): remove request history retention cap

Allow positive retention days beyond 31 throughout the form, domain rules,
and database constraint. Keep the existing default and saved settings.
Bound cleanup and account usage queries by stored data so large retention
windows remain safe without expanding empty history.

Co-authored-by: Codex <noreply@openai.com>

* test(admin): align retention policy coverage with positive windows

Accept long request retention windows while preserving zero rejection
for each target and independent event and audit retention settings.

Co-authored-by: Codex <noreply@openai.com>

---------

Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-11-13 17:29:21 +08:00
committed by GitHub
co-authored by Codex Zyy
parent f85d80d756
commit f61995e61d
16 changed files with 168 additions and 61 deletions
@@ -24,7 +24,7 @@ impl RetentionPolicy {
audit_days: u32,
) -> Result<(), &'static str> {
for (valid, field) in [
((1..=31).contains(&usage_days), "usage_retention_days"),
(usage_days > 0, "usage_retention_days"),
(ops_days > 0, "ops_event_retention_days"),
(audit_days > 0, "audit_retention_days"),
] {
@@ -38,7 +38,7 @@ impl RetentionPolicy {
pub fn try_new(usage_days: u32, ops_days: u32, audit_days: u32) -> Result<Self, AdminError> {
if Self::validate_values(usage_days, ops_days, audit_days).is_err() {
return Err(AdminError::invalid(
"请求保留期必须为 1~31 天,事件与审计保留期必须为正数",
"请求、事件与审计保留期必须为正整数天数",
));
}
Ok(Self {
@@ -3,25 +3,15 @@
use gateway_admin::model::retention::{RetentionPolicy, RetentionTarget};
#[test]
fn retention_policy_rejects_invalid_windows_and_preserves_independent_windows() {
for (usage, ops, audit) in [
(0, 1, 1),
(32, 1, 1),
(u32::MAX, 1, 1),
(1, 0, 1),
(1, 1, 0),
] {
fn retention_policy_rejects_zero_for_each_target() {
for (usage, ops, audit) in [(0, 1, 1), (1, 0, 1), (1, 1, 0)] {
assert!(RetentionPolicy::try_new(usage, ops, audit).is_err());
}
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
assert_eq!(policy.days(RetentionTarget::ModelRequests), 31);
assert_eq!(policy.days(RetentionTarget::OpsEvents), 7);
assert_eq!(policy.days(RetentionTarget::AdminAuditEvents), 90);
}
#[test]
fn request_retention_accepts_short_windows_without_changing_other_targets() {
for days in [1, 3, 7, 15, 30, 31] {
fn request_retention_accepts_positive_windows_without_changing_other_targets() {
for days in [1, 31, 32, 365, 3650, u32::MAX] {
let policy = RetentionPolicy::try_new(days, 30, 90).unwrap();
assert_eq!(policy.days(RetentionTarget::ModelRequests), days);
assert_eq!(policy.days(RetentionTarget::OpsEvents), 30);
@@ -357,11 +357,11 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
}
#[tokio::test]
async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it() {
async fn request_retention_round_trips_and_invalid_days_do_not_replace_it() {
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let mut body = update_body();
for days in [1, 3, 7, 15, 30, 31] {
for days in [1, 31, 32, 365, 3650, u32::MAX] {
body["usageRetentionDays"] = json!(days);
let response = app(fixture.state())
.oneshot(request(
@@ -378,8 +378,7 @@ async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it(
}
for (days, expected_status) in [
(json!(0), StatusCode::BAD_REQUEST),
(json!(32), StatusCode::BAD_REQUEST),
(json!(u32::MAX), StatusCode::BAD_REQUEST),
(json!(u64::from(u32::MAX) + 1), StatusCode::BAD_REQUEST),
(json!(-1), StatusCode::UNPROCESSABLE_ENTITY),
(json!(1.5), StatusCode::UNPROCESSABLE_ENTITY),
(json!(null), StatusCode::UNPROCESSABLE_ENTITY),
@@ -402,7 +401,7 @@ async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it(
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response_json(response).await["data"]["usageRetentionDays"],
31
u32::MAX
);
}
@@ -658,20 +658,41 @@ fn account_costs(state: &MetricState) -> StoreResult<Vec<AccountCost>> {
.collect())
}
async fn retained_account_usage_range(
snapshot: &mut AnalyticsSnapshot<'_>,
now: DateTime<Utc>,
) -> StoreResult<ObservabilityRange> {
// 只为实际存在的数据规划时间桶,超长保留期不展开空白历史或溢出日期
let (days, earliest): (i64, DateTime<Utc>) = sqlx::query_as(
"select usage_retention_days, least(
(select started_at from model_requests order by started_at, id limit 1),
(select started_at from request_analytics_facts order by started_at, id limit 1),
(select bucket_start from request_analytics_1h where collection = 'usage_contribution' order by bucket_start limit 1),
$1::timestamptz - interval '1 day'
) from runtime_settings where id = 1",
)
.bind(now)
.fetch_one(&mut *snapshot.tx)
.await
.map_err(|error| postgres_unavailable("read retained account usage range", error))?;
let duration =
TimeDelta::try_days(days).ok_or_else(|| invalid("invalid account usage retention"))?;
let start = now
.checked_sub_signed(duration)
.unwrap_or(earliest)
.max(earliest);
ObservabilityRange::new(start, now)
}
pub(crate) async fn account_usage_sort_values(
snapshot: &mut AnalyticsSnapshot<'_>,
now: DateTime<Utc>,
) -> StoreResult<(Vec<String>, Vec<i64>, Vec<Option<DateTime<Utc>>>)> {
let days: i32 =
sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1")
.fetch_one(&mut *snapshot.tx)
.await
.map_err(|error| postgres_unavailable("read account usage retention", error))?;
let range = retained_account_usage_range(snapshot, now).await?;
let ids: Vec<String> = sqlx::query_scalar("select id from provider_accounts")
.fetch_all(&mut *snapshot.tx)
.await
.map_err(|error| postgres_unavailable("read account usage sort identities", error))?;
let range = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?;
let mut totals = BTreeMap::<String, MetricState>::new();
for ((account, _), state) in snapshot.account_states(range, &ids, false).await? {
totals.entry(account).or_default().merge(&state)?;
@@ -699,11 +720,7 @@ pub(crate) async fn account_group_usage(
}
let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?;
let now = Utc::now();
let days: i32 =
sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1")
.fetch_one(&mut *snapshot.tx)
.await
.map_err(|error| postgres_unavailable("read group usage retention", error))?;
let retained = retained_account_usage_range(&mut snapshot, now).await?;
let memberships:Vec<(String,String)>=sqlx::query_as("select account_group_id,provider_account_id from account_group_accounts where account_group_id=any($1::text[])").bind(group_ids).fetch_all(&mut *snapshot.tx).await.map_err(|error|postgres_unavailable("read group usage membership",error))?;
let ids = memberships
.iter()
@@ -711,7 +728,6 @@ pub(crate) async fn account_group_usage(
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let retained = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?;
let today = ObservabilityRange::new(
timezone
.day_start(now)
@@ -2,7 +2,7 @@
use crate::{admin_store_error, postgres_unavailable};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use chrono::{DateTime, TimeDelta, Utc};
use gateway_admin::{
model::retention::{RetentionPolicy, RetentionTarget},
ports::{
@@ -54,15 +54,26 @@ impl RetentionStore for PgRetentionRepository {
policy: RetentionPolicy,
limit: NonZeroU32,
) -> AdminStoreResult<u64> {
let oldest_sql = match target {
RetentionTarget::ModelRequests => {
"select completed_at from model_requests where outcome <> 'running' order by completed_at limit 1"
}
RetentionTarget::OpsEvents => {
"select created_at from ops_events where model_request_id is null order by created_at limit 1"
}
RetentionTarget::AdminAuditEvents => {
"select created_at from admin_audit_events order by created_at limit 1"
}
};
let sql = match target {
RetentionTarget::ModelRequests => {
"delete from model_requests
where ctid in (
select ctid from model_requests
where outcome <> 'running'
and completed_at < $1 - ($2 * interval '1 day')
and completed_at < $1
order by completed_at
limit $3
limit $2
for update skip locked
)"
}
@@ -71,9 +82,9 @@ impl RetentionStore for PgRetentionRepository {
where ctid in (
select ctid from ops_events
where model_request_id is null
and created_at < $1 - ($2 * interval '1 day')
and created_at < $1
order by created_at
limit $3
limit $2
for update skip locked
)"
}
@@ -81,9 +92,9 @@ impl RetentionStore for PgRetentionRepository {
"delete from admin_audit_events
where ctid in (
select ctid from admin_audit_events
where created_at < $1 - ($2 * interval '1 day')
where created_at < $1
order by created_at
limit $3
limit $2
for update skip locked
)"
}
@@ -102,14 +113,25 @@ impl RetentionStore for PgRetentionRepository {
.execute(&mut *transaction)
.await
.map_err(|source| unavailable("set retention lock timeout", source))?;
let deleted = sqlx::query(sql)
.bind(now)
.bind(i64::from(policy.days(target)))
.bind(i64::from(limit.get()))
.execute(&mut *transaction)
let oldest: Option<DateTime<Utc>> = sqlx::query_scalar(oldest_sql)
.fetch_optional(&mut *transaction)
.await
.map(|result| result.rows_affected())
.map_err(|source| unavailable("delete expired records", source))?;
.map_err(|source| unavailable("read oldest retained record", source))?;
// 超长窗口覆盖全部现存记录时无需删除,也不向数据库传递越界日期
let cutoff = now
.checked_sub_signed(TimeDelta::days(i64::from(policy.days(target))))
.filter(|cutoff| oldest.is_some_and(|oldest| oldest < *cutoff));
let deleted = if let Some(cutoff) = cutoff {
sqlx::query(sql)
.bind(cutoff)
.bind(i64::from(limit.get()))
.execute(&mut *transaction)
.await
.map(|result| result.rows_affected())
.map_err(|source| unavailable("delete expired records", source))?
} else {
0
};
transaction
.commit()
.await
@@ -693,9 +693,16 @@ async fn group_costs_follow_current_members_and_pending_source_replacements() {
}
}
assert!(ready, "group fixture must publish rollups");
let costs = load_retained_group_costs(&groups).await;
assert_eq!(costs[MIXED_GROUP], "3.25");
assert_eq!(costs[EMPTY_GROUP], "0");
for days in [31_i64, 365, i64::from(u32::MAX)] {
sqlx::query("update runtime_settings set usage_retention_days=$1")
.bind(days)
.execute(&database.pool)
.await
.unwrap();
let costs = load_retained_group_costs(&groups).await;
assert_eq!(costs[MIXED_GROUP], "3.25");
assert_eq!(costs[EMPTY_GROUP], "0");
}
sqlx::query("update account_group_accounts set account_group_id=$1 where provider_account_id='acct_group_move'")
.bind(EMPTY_GROUP).execute(&database.pool).await.unwrap();
@@ -105,6 +105,11 @@ async fn quota_windows_reconcile_moved_requests_and_pending_deletions_without_wa
assert_eq!(new.usage.reasoning_output_tokens, None);
assert_eq!(new.usage.models[0].model, "new-model");
assert_eq!(new.usage.costs[0].amount.as_str(), "4.75");
sqlx::query("update runtime_settings set usage_retention_days=$1")
.bind(i64::from(u32::MAX))
.execute(&database.pool)
.await
.unwrap();
for field in [AccountSortField::Usage, AccountSortField::LastUsedAt] {
let ranked = store
.list_accounts(
@@ -1,4 +1,4 @@
//! 验证短保留期升级、分批清理边界与统计异步收敛期间的查询可用性
//! 验证保留期读写、分批清理边界与统计异步收敛期间的查询可用性
use std::{num::NonZeroU32, time::Duration};
@@ -20,6 +20,66 @@ use super::{
observability_query_budget, observability_repository,
};
#[tokio::test]
async fn long_retention_round_trips_and_cleans_only_expired_requests() {
use gateway_store::postgres::{PgRuntimeSettingsRepository, RuntimeSettingsRepository};
let Some(database) = TestDatabase::create("long_retention").await else {
return;
};
let settings = PgRuntimeSettingsRepository::new(database.pool.clone());
let retention = PgRetentionRepository::new(database.pool.clone());
let now = Utc::now();
for (id, age) in [("old", 400), ("month", 40), ("recent", 1)] {
seed(&database.pool, id, now - ChronoDuration::days(age), &[]).await;
}
for (days, expected_deleted) in [(u32::MAX, 0), (3650, 0), (365, 1), (32, 1), (1, 0)] {
let mut update = super::runtime_settings::settings_with_margin(300);
update.values.usage_retention_days = days;
settings.update_runtime_settings(update).await.unwrap();
assert_eq!(
settings
.load_runtime_settings()
.await
.unwrap()
.values
.usage_retention_days,
days
);
let policy = retention.load_policy().await.unwrap();
assert_eq!(policy.days(RetentionTarget::ModelRequests), days);
assert_eq!(
retention
.purge_batch(
RetentionTarget::ModelRequests,
now,
policy,
NonZeroU32::new(10).unwrap()
)
.await
.unwrap(),
expected_deleted
);
}
let mut invalid = super::runtime_settings::settings_with_margin(300);
invalid.values.usage_retention_days = 0;
assert!(settings.update_runtime_settings(invalid).await.is_err());
assert_eq!(
retention
.load_policy()
.await
.unwrap()
.days(RetentionTarget::ModelRequests),
1
);
let remaining: Vec<String> = sqlx::query_scalar("select id from model_requests order by id")
.fetch_all(&database.pool)
.await
.unwrap();
assert_eq!(remaining, ["recent"]);
database.close().await;
}
#[tokio::test]
async fn short_retention_keeps_statistics_readable_until_each_deletion_batch_is_projected() {
use gateway_admin::model::observability::{AnalyticsState, DashboardQuery, TimeRange};
+1
View File
@@ -39,3 +39,4 @@ b215ea90d5c8753aeb8192b17ffe65f7a95c1d620aaeaf9fc4337d8cf6f2634e 0037_retry_aft
a6b4e81d38e5d70776a75eba22749354573ebcb186160ff9eb45fd91157b3e0d 0039_queue_timeouts_quota_credits_and_installation_seed.sql
cd9e31400fc6b0d49a6515048c104341fca2cf44f290d89c401b41f94ead84a3 0040_account_scheduling_key_budgets_and_peer_routing.sql
aeff99fee7cef5a553ae49ffe27a48f49e31ccfb3fbc61a2482c9b109478202d 0041_account_model_catalogs.sql
19eba6b493647da51996831f72d7bc83e95416c06f207ea685f8f5e2edea7c64 0042_usage_retention_days.sql
@@ -0,0 +1,7 @@
-- 保留期仅要求正整数天数,沿用当前设置与默认值
alter table runtime_settings drop constraint runtime_settings_retention_ck;
alter table runtime_settings add constraint runtime_settings_retention_ck check (
usage_retention_days > 0
and ops_event_retention_days > 0
and audit_retention_days > 0
);
+1 -1
View File
@@ -2165,7 +2165,7 @@ HTTP 返回 `429`,`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly
### 历史保留
`usageRetentionDays` 控制请求明细及其用量统计的保留天数,默认 31,支持 1~31 的整数天数。
`usageRetentionDays` 控制请求明细及其用量统计的保留天数,默认 31,须为正整数,不设业务上限。
`opsEventRetentionDays` 与 `auditRetentionDays` 分别控制独立运维事件和管理审计,默认 30、90,均须为正整数
清理任务每轮读取已保存设置,按完成时间分批删除过期请求及关联事件,不删除运行中的请求;有界单轮结束后等待一分钟再检查。
+2 -2
View File
@@ -309,7 +309,7 @@
"retentionDaysLabel": "History retention (d)"
},
"retentionDays": "Retention (d)",
"enterDays": "Please enter 1–31 days",
"enterDays": "Enter retention days",
"day": "d",
"location": {
"requestLocationOverride": "Request location override",
@@ -626,7 +626,7 @@
"invalidBodyLimit": "Responses decompression limit must be a positive integer in MiB",
"invalidQuotaThreshold": "Remaining weekly quota threshold must be an integer from 0 to 100, leave blank for no limit",
"invalidResetLeadTime": "Reset credit lead time must be an integer from 1 to 1,440 minutes",
"invalidRetention": "History retention must be an integer from 1 to 31 days",
"invalidRetention": "History retention must be a positive integer",
"invalidQueueSettings": "Queue capacity must be an integer from 0 to 1,000 and timeout from 1 to 120 seconds",
"invalidProbeModel": "The probe model must be at most 128 characters without control characters",
"invalidWarmupCron": "Enter a valid Cron expression",
@@ -309,7 +309,7 @@
"retentionDaysLabel": "历史保留天数"
},
"retentionDays": "保留天数",
"enterDays": "请输入 1–31 天",
"enterDays": "请输入保留天数",
"day": "天",
"location": {
"requestLocationOverride": "请求位置覆盖",
@@ -626,7 +626,7 @@
"invalidBodyLimit": "Responses 解压上限应为有效的正整数(MiB)",
"invalidQuotaThreshold": "剩余周额度上限应为 0~100 的整数,留空表示不限制",
"invalidResetLeadTime": "重置卡提前使用时间应为 1~1440 分钟的整数",
"invalidRetention": "历史保留天数应为 1–31 的整数",
"invalidRetention": "历史保留天数应为正整数",
"invalidQueueSettings": "队列容量应为 0~1000 的整数,排队超时应为 1~120 秒的整数",
"invalidProbeModel": "探测模型最多 128 个字符,不能包含控制字符",
"invalidWarmupCron": "请输入有效的 Cron 表达式",
@@ -309,7 +309,7 @@
"retentionDaysLabel": "歷史保留天數"
},
"retentionDays": "保留天數",
"enterDays": "請輸入 1–31 天",
"enterDays": "請輸入保留天數",
"day": "天",
"location": {
"requestLocationOverride": "請求位置覆蓋",
@@ -626,7 +626,7 @@
"invalidBodyLimit": "Responses 解壓上限應爲有效的正整數(MiB)",
"invalidQuotaThreshold": "剩餘周額度上限應爲 0~100 的整數,留空表示不限制",
"invalidResetLeadTime": "重置卡提前使用時間應爲 1~1440 分鐘的整數",
"invalidRetention": "歷史保留天數應爲 1–31 的整數",
"invalidRetention": "歷史保留天數應爲正整數",
"invalidQueueSettings": "隊列容量應爲 0~1000 的整數,排隊超時應爲 1~120 秒的整數",
"invalidProbeModel": "探測模型最多 128 個字符,不能包含控制字符",
"invalidWarmupCron": "請輸入有效的 Cron 表達式",
@@ -12,7 +12,7 @@ const model = defineModel<string>({ required: true })
<ZCard :title="t('settings.history.title')" :description="t('settings.history.retentionHelp')">
<div class="grid min-w-0 gap-4.5 max-w-6xl sm:grid-cols-2 sm:items-end">
<ZFormItem :show-message="Boolean(model.trim())" prop="usageRetentionDays" :label="t('settings.retentionDays')" :description="t('settings.history.retentionConstraints')">
<ZInput v-model="model" :aria-label="t('settings.history.retentionDaysLabel')" type="number" min="1" max="31" step="1" :placeholder="t('settings.enterDays')">
<ZInput v-model="model" :aria-label="t('settings.history.retentionDaysLabel')" type="number" min="1" step="1" :placeholder="t('settings.enterDays')">
<template #suffix>
{{ t('settings.day') }}
</template>
@@ -9,7 +9,7 @@ type SettingsForm = ReturnType<typeof useSettingsForm>['form']
export function useSettingsFormRules(form: () => SettingsForm) {
const { t } = useI18n({ useScope: 'global' })
const integer = (min: number, max: number, message: string): FormRule => ({
const integer = (min: number, max: number | undefined, message: string): FormRule => ({
type: 'integer',
required: true,
min,
@@ -47,7 +47,7 @@ export function useSettingsFormRules(form: () => SettingsForm) {
...integer(0, 100, t('settings.form.invalidQuotaThreshold')),
required: false,
},
'usageRetentionDays': integer(1, 31, t('settings.form.invalidRetention')),
'usageRetentionDays': integer(1, undefined, t('settings.form.invalidRetention')),
'minCodexDesktopVersion': version(),
'minCodexCliVersion': version(),
'accountWarmupCron': [