diff --git a/backend/crates/gateway-admin/src/model/retention.rs b/backend/crates/gateway-admin/src/model/retention.rs index c7ffd16e1..2ee554379 100644 --- a/backend/crates/gateway-admin/src/model/retention.rs +++ b/backend/crates/gateway-admin/src/model/retention.rs @@ -24,7 +24,7 @@ impl RetentionPolicy { audit_days: u32, ) -> Result<(), &'static str> { for (valid, field) in [ - ((1..=31).contains(&usage_days), "usage_retention_days"), + (usage_days > 0, "usage_retention_days"), (ops_days > 0, "ops_event_retention_days"), (audit_days > 0, "audit_retention_days"), ] { @@ -38,7 +38,7 @@ impl RetentionPolicy { pub fn try_new(usage_days: u32, ops_days: u32, audit_days: u32) -> Result { if Self::validate_values(usage_days, ops_days, audit_days).is_err() { return Err(AdminError::invalid( - "请求保留期必须为 1～31 天，事件与审计保留期必须为正数", + "请求、事件与审计保留期必须为正整数天数", )); } Ok(Self { diff --git a/backend/crates/gateway-admin/tests/model/retention.rs b/backend/crates/gateway-admin/tests/model/retention.rs index 3618a9a17..2d2f158fd 100644 --- a/backend/crates/gateway-admin/tests/model/retention.rs +++ b/backend/crates/gateway-admin/tests/model/retention.rs @@ -3,25 +3,15 @@ use gateway_admin::model::retention::{RetentionPolicy, RetentionTarget}; #[test] -fn retention_policy_rejects_invalid_windows_and_preserves_independent_windows() { - for (usage, ops, audit) in [ - (0, 1, 1), - (32, 1, 1), - (u32::MAX, 1, 1), - (1, 0, 1), - (1, 1, 0), - ] { +fn retention_policy_rejects_zero_for_each_target() { + for (usage, ops, audit) in [(0, 1, 1), (1, 0, 1), (1, 1, 0)] { assert!(RetentionPolicy::try_new(usage, ops, audit).is_err()); } - let policy = RetentionPolicy::try_new(31, 7, 90).unwrap(); - assert_eq!(policy.days(RetentionTarget::ModelRequests), 31); - assert_eq!(policy.days(RetentionTarget::OpsEvents), 7); - assert_eq!(policy.days(RetentionTarget::AdminAuditEvents), 90); } #[test] -fn request_retention_accepts_short_windows_without_changing_other_targets() { - for days in [1, 3, 7, 15, 30, 31] { +fn request_retention_accepts_positive_windows_without_changing_other_targets() { + for days in [1, 31, 32, 365, 3650, u32::MAX] { let policy = RetentionPolicy::try_new(days, 30, 90).unwrap(); assert_eq!(policy.days(RetentionTarget::ModelRequests), days); assert_eq!(policy.days(RetentionTarget::OpsEvents), 30); diff --git a/backend/crates/gateway-api/tests/admin/settings.rs b/backend/crates/gateway-api/tests/admin/settings.rs index 030ad9e8f..0bc533241 100644 --- a/backend/crates/gateway-api/tests/admin/settings.rs +++ b/backend/crates/gateway-api/tests/admin/settings.rs @@ -357,11 +357,11 @@ fn settings_request_should_reject_unknown_rotation_strategy() { } #[tokio::test] -async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it() { +async fn request_retention_round_trips_and_invalid_days_do_not_replace_it() { let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); let mut body = update_body(); - for days in [1, 3, 7, 15, 30, 31] { + for days in [1, 31, 32, 365, 3650, u32::MAX] { body["usageRetentionDays"] = json!(days); let response = app(fixture.state()) .oneshot(request( @@ -378,8 +378,7 @@ async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it( } for (days, expected_status) in [ (json!(0), StatusCode::BAD_REQUEST), - (json!(32), StatusCode::BAD_REQUEST), - (json!(u32::MAX), StatusCode::BAD_REQUEST), + (json!(u64::from(u32::MAX) + 1), StatusCode::BAD_REQUEST), (json!(-1), StatusCode::UNPROCESSABLE_ENTITY), (json!(1.5), StatusCode::UNPROCESSABLE_ENTITY), (json!(null), StatusCode::UNPROCESSABLE_ENTITY), @@ -402,7 +401,7 @@ async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it( assert_eq!(response.status(), StatusCode::OK); assert_eq!( response_json(response).await["data"]["usageRetentionDays"], - 31 + u32::MAX ); } diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/accounts.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/accounts.rs index a91cb8b61..a7c4ed347 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/accounts.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/accounts.rs @@ -658,20 +658,41 @@ fn account_costs(state: &MetricState) -> StoreResult> { .collect()) } +async fn retained_account_usage_range( + snapshot: &mut AnalyticsSnapshot<'_>, + now: DateTime, +) -> StoreResult { + // 只为实际存在的数据规划时间桶，超长保留期不展开空白历史或溢出日期 + let (days, earliest): (i64, DateTime) = sqlx::query_as( + "select usage_retention_days, least( + (select started_at from model_requests order by started_at, id limit 1), + (select started_at from request_analytics_facts order by started_at, id limit 1), + (select bucket_start from request_analytics_1h where collection = 'usage_contribution' order by bucket_start limit 1), + $1::timestamptz - interval '1 day' + ) from runtime_settings where id = 1", + ) + .bind(now) + .fetch_one(&mut *snapshot.tx) + .await + .map_err(|error| postgres_unavailable("read retained account usage range", error))?; + let duration = + TimeDelta::try_days(days).ok_or_else(|| invalid("invalid account usage retention"))?; + let start = now + .checked_sub_signed(duration) + .unwrap_or(earliest) + .max(earliest); + ObservabilityRange::new(start, now) +} + pub(crate) async fn account_usage_sort_values( snapshot: &mut AnalyticsSnapshot<'_>, now: DateTime, ) -> StoreResult<(Vec, Vec, Vec>>)> { - let days: i32 = - sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1") - .fetch_one(&mut *snapshot.tx) - .await - .map_err(|error| postgres_unavailable("read account usage retention", error))?; + let range = retained_account_usage_range(snapshot, now).await?; let ids: Vec = sqlx::query_scalar("select id from provider_accounts") .fetch_all(&mut *snapshot.tx) .await .map_err(|error| postgres_unavailable("read account usage sort identities", error))?; - let range = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?; let mut totals = BTreeMap::::new(); for ((account, _), state) in snapshot.account_states(range, &ids, false).await? { totals.entry(account).or_default().merge(&state)?; @@ -699,11 +720,7 @@ pub(crate) async fn account_group_usage( } let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?; let now = Utc::now(); - let days: i32 = - sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1") - .fetch_one(&mut *snapshot.tx) - .await - .map_err(|error| postgres_unavailable("read group usage retention", error))?; + let retained = retained_account_usage_range(&mut snapshot, now).await?; let memberships:Vec<(String,String)>=sqlx::query_as("select account_group_id,provider_account_id from account_group_accounts where account_group_id=any($1::text[])").bind(group_ids).fetch_all(&mut *snapshot.tx).await.map_err(|error|postgres_unavailable("read group usage membership",error))?; let ids = memberships .iter() @@ -711,7 +728,6 @@ pub(crate) async fn account_group_usage( .collect::>() .into_iter() .collect::>(); - let retained = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?; let today = ObservabilityRange::new( timezone .day_start(now) diff --git a/backend/crates/gateway-store/src/postgres/retention.rs b/backend/crates/gateway-store/src/postgres/retention.rs index 5e4a66144..2cfa87fe3 100644 --- a/backend/crates/gateway-store/src/postgres/retention.rs +++ b/backend/crates/gateway-store/src/postgres/retention.rs @@ -2,7 +2,7 @@ use crate::{admin_store_error, postgres_unavailable}; use async_trait::async_trait; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, TimeDelta, Utc}; use gateway_admin::{ model::retention::{RetentionPolicy, RetentionTarget}, ports::{ @@ -54,15 +54,26 @@ impl RetentionStore for PgRetentionRepository { policy: RetentionPolicy, limit: NonZeroU32, ) -> AdminStoreResult { + let oldest_sql = match target { + RetentionTarget::ModelRequests => { + "select completed_at from model_requests where outcome <> 'running' order by completed_at limit 1" + } + RetentionTarget::OpsEvents => { + "select created_at from ops_events where model_request_id is null order by created_at limit 1" + } + RetentionTarget::AdminAuditEvents => { + "select created_at from admin_audit_events order by created_at limit 1" + } + }; let sql = match target { RetentionTarget::ModelRequests => { "delete from model_requests where ctid in ( select ctid from model_requests where outcome <> 'running' - and completed_at < $1 - ($2 * interval '1 day') + and completed_at < $1 order by completed_at - limit $3 + limit $2 for update skip locked )" } @@ -71,9 +82,9 @@ impl RetentionStore for PgRetentionRepository { where ctid in ( select ctid from ops_events where model_request_id is null - and created_at < $1 - ($2 * interval '1 day') + and created_at < $1 order by created_at - limit $3 + limit $2 for update skip locked )" } @@ -81,9 +92,9 @@ impl RetentionStore for PgRetentionRepository { "delete from admin_audit_events where ctid in ( select ctid from admin_audit_events - where created_at < $1 - ($2 * interval '1 day') + where created_at < $1 order by created_at - limit $3 + limit $2 for update skip locked )" } @@ -102,14 +113,25 @@ impl RetentionStore for PgRetentionRepository { .execute(&mut *transaction) .await .map_err(|source| unavailable("set retention lock timeout", source))?; - let deleted = sqlx::query(sql) - .bind(now) - .bind(i64::from(policy.days(target))) - .bind(i64::from(limit.get())) - .execute(&mut *transaction) + let oldest: Option> = sqlx::query_scalar(oldest_sql) + .fetch_optional(&mut *transaction) .await - .map(|result| result.rows_affected()) - .map_err(|source| unavailable("delete expired records", source))?; + .map_err(|source| unavailable("read oldest retained record", source))?; + // 超长窗口覆盖全部现存记录时无需删除，也不向数据库传递越界日期 + let cutoff = now + .checked_sub_signed(TimeDelta::days(i64::from(policy.days(target)))) + .filter(|cutoff| oldest.is_some_and(|oldest| oldest < *cutoff)); + let deleted = if let Some(cutoff) = cutoff { + sqlx::query(sql) + .bind(cutoff) + .bind(i64::from(limit.get())) + .execute(&mut *transaction) + .await + .map(|result| result.rows_affected()) + .map_err(|source| unavailable("delete expired records", source))? + } else { + 0 + }; transaction .commit() .await diff --git a/backend/crates/gateway-store/tests/postgres/account_groups.rs b/backend/crates/gateway-store/tests/postgres/account_groups.rs index 806095324..02b3f5d9c 100644 --- a/backend/crates/gateway-store/tests/postgres/account_groups.rs +++ b/backend/crates/gateway-store/tests/postgres/account_groups.rs @@ -693,9 +693,16 @@ async fn group_costs_follow_current_members_and_pending_source_replacements() { } } assert!(ready, "group fixture must publish rollups"); - let costs = load_retained_group_costs(&groups).await; - assert_eq!(costs[MIXED_GROUP], "3.25"); - assert_eq!(costs[EMPTY_GROUP], "0"); + for days in [31_i64, 365, i64::from(u32::MAX)] { + sqlx::query("update runtime_settings set usage_retention_days=$1") + .bind(days) + .execute(&database.pool) + .await + .unwrap(); + let costs = load_retained_group_costs(&groups).await; + assert_eq!(costs[MIXED_GROUP], "3.25"); + assert_eq!(costs[EMPTY_GROUP], "0"); + } sqlx::query("update account_group_accounts set account_group_id=$1 where provider_account_id='acct_group_move'") .bind(EMPTY_GROUP).execute(&database.pool).await.unwrap(); diff --git a/backend/crates/gateway-store/tests/postgres/provider_accounts/usage_projection.rs b/backend/crates/gateway-store/tests/postgres/provider_accounts/usage_projection.rs index a56332ca4..90b420f9d 100644 --- a/backend/crates/gateway-store/tests/postgres/provider_accounts/usage_projection.rs +++ b/backend/crates/gateway-store/tests/postgres/provider_accounts/usage_projection.rs @@ -105,6 +105,11 @@ async fn quota_windows_reconcile_moved_requests_and_pending_deletions_without_wa assert_eq!(new.usage.reasoning_output_tokens, None); assert_eq!(new.usage.models[0].model, "new-model"); assert_eq!(new.usage.costs[0].amount.as_str(), "4.75"); + sqlx::query("update runtime_settings set usage_retention_days=$1") + .bind(i64::from(u32::MAX)) + .execute(&database.pool) + .await + .unwrap(); for field in [AccountSortField::Usage, AccountSortField::LastUsedAt] { let ranked = store .list_accounts( diff --git a/backend/crates/gateway-store/tests/postgres/retention.rs b/backend/crates/gateway-store/tests/postgres/retention.rs index 4c0d87218..ba754e568 100644 --- a/backend/crates/gateway-store/tests/postgres/retention.rs +++ b/backend/crates/gateway-store/tests/postgres/retention.rs @@ -1,4 +1,4 @@ -//! 验证短保留期升级、分批清理边界与统计异步收敛期间的查询可用性 +//! 验证保留期读写、分批清理边界与统计异步收敛期间的查询可用性 use std::{num::NonZeroU32, time::Duration}; @@ -20,6 +20,66 @@ use super::{ observability_query_budget, observability_repository, }; +#[tokio::test] +async fn long_retention_round_trips_and_cleans_only_expired_requests() { + use gateway_store::postgres::{PgRuntimeSettingsRepository, RuntimeSettingsRepository}; + + let Some(database) = TestDatabase::create("long_retention").await else { + return; + }; + let settings = PgRuntimeSettingsRepository::new(database.pool.clone()); + let retention = PgRetentionRepository::new(database.pool.clone()); + let now = Utc::now(); + for (id, age) in [("old", 400), ("month", 40), ("recent", 1)] { + seed(&database.pool, id, now - ChronoDuration::days(age), &[]).await; + } + for (days, expected_deleted) in [(u32::MAX, 0), (3650, 0), (365, 1), (32, 1), (1, 0)] { + let mut update = super::runtime_settings::settings_with_margin(300); + update.values.usage_retention_days = days; + settings.update_runtime_settings(update).await.unwrap(); + assert_eq!( + settings + .load_runtime_settings() + .await + .unwrap() + .values + .usage_retention_days, + days + ); + let policy = retention.load_policy().await.unwrap(); + assert_eq!(policy.days(RetentionTarget::ModelRequests), days); + assert_eq!( + retention + .purge_batch( + RetentionTarget::ModelRequests, + now, + policy, + NonZeroU32::new(10).unwrap() + ) + .await + .unwrap(), + expected_deleted + ); + } + let mut invalid = super::runtime_settings::settings_with_margin(300); + invalid.values.usage_retention_days = 0; + assert!(settings.update_runtime_settings(invalid).await.is_err()); + assert_eq!( + retention + .load_policy() + .await + .unwrap() + .days(RetentionTarget::ModelRequests), + 1 + ); + let remaining: Vec = sqlx::query_scalar("select id from model_requests order by id") + .fetch_all(&database.pool) + .await + .unwrap(); + assert_eq!(remaining, ["recent"]); + database.close().await; +} + #[tokio::test] async fn short_retention_keeps_statistics_readable_until_each_deletion_batch_is_projected() { use gateway_admin::model::observability::{AnalyticsState, DashboardQuery, TimeRange}; diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index f2aad3152..c29d39c23 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -39,3 +39,4 @@ b215ea90d5c8753aeb8192b17ffe65f7a95c1d620aaeaf9fc4337d8cf6f2634e 0037_retry_aft a6b4e81d38e5d70776a75eba22749354573ebcb186160ff9eb45fd91157b3e0d 0039_queue_timeouts_quota_credits_and_installation_seed.sql cd9e31400fc6b0d49a6515048c104341fca2cf44f290d89c401b41f94ead84a3 0040_account_scheduling_key_budgets_and_peer_routing.sql aeff99fee7cef5a553ae49ffe27a48f49e31ccfb3fbc61a2482c9b109478202d 0041_account_model_catalogs.sql +19eba6b493647da51996831f72d7bc83e95416c06f207ea685f8f5e2edea7c64 0042_usage_retention_days.sql diff --git a/backend/migrations/0042_usage_retention_days.sql b/backend/migrations/0042_usage_retention_days.sql new file mode 100644 index 000000000..df894ec93 --- /dev/null +++ b/backend/migrations/0042_usage_retention_days.sql @@ -0,0 +1,7 @@ +-- 保留期仅要求正整数天数，沿用当前设置与默认值 +alter table runtime_settings drop constraint runtime_settings_retention_ck; +alter table runtime_settings add constraint runtime_settings_retention_ck check ( + usage_retention_days > 0 + and ops_event_retention_days > 0 + and audit_retention_days > 0 +); diff --git a/docs/api.md b/docs/api.md index 78d839bf8..ac2957769 100644 --- a/docs/api.md +++ b/docs/api.md @@ -2165,7 +2165,7 @@ HTTP 返回 `429`，`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly ### 历史保留 -`usageRetentionDays` 控制请求明细及其用量统计的保留天数，默认 31，支持 1～31 的整数天数。 +`usageRetentionDays` 控制请求明细及其用量统计的保留天数，默认 31，须为正整数，不设业务上限。 `opsEventRetentionDays` 与 `auditRetentionDays` 分别控制独立运维事件和管理审计，默认 30、90，均须为正整数 清理任务每轮读取已保存设置，按完成时间分批删除过期请求及关联事件，不删除运行中的请求；有界单轮结束后等待一分钟再检查。 diff --git a/frontend/src/i18n/locales/en/settings.json b/frontend/src/i18n/locales/en/settings.json index 31fa9fe7d..245e6a9d4 100644 --- a/frontend/src/i18n/locales/en/settings.json +++ b/frontend/src/i18n/locales/en/settings.json @@ -309,7 +309,7 @@ "retentionDaysLabel": "History retention (d)" }, "retentionDays": "Retention (d)", - "enterDays": "Please enter 1–31 days", + "enterDays": "Enter retention days", "day": "d", "location": { "requestLocationOverride": "Request location override", @@ -626,7 +626,7 @@ "invalidBodyLimit": "Responses decompression limit must be a positive integer in MiB", "invalidQuotaThreshold": "Remaining weekly quota threshold must be an integer from 0 to 100, leave blank for no limit", "invalidResetLeadTime": "Reset credit lead time must be an integer from 1 to 1,440 minutes", - "invalidRetention": "History retention must be an integer from 1 to 31 days", + "invalidRetention": "History retention must be a positive integer", "invalidQueueSettings": "Queue capacity must be an integer from 0 to 1,000 and timeout from 1 to 120 seconds", "invalidProbeModel": "The probe model must be at most 128 characters without control characters", "invalidWarmupCron": "Enter a valid Cron expression", diff --git a/frontend/src/i18n/locales/zh-Hans/settings.json b/frontend/src/i18n/locales/zh-Hans/settings.json index 540818447..622506a29 100644 --- a/frontend/src/i18n/locales/zh-Hans/settings.json +++ b/frontend/src/i18n/locales/zh-Hans/settings.json @@ -309,7 +309,7 @@ "retentionDaysLabel": "历史保留天数" }, "retentionDays": "保留天数", - "enterDays": "请输入 1–31 天", + "enterDays": "请输入保留天数", "day": "天", "location": { "requestLocationOverride": "请求位置覆盖", @@ -626,7 +626,7 @@ "invalidBodyLimit": "Responses 解压上限应为有效的正整数（MiB）", "invalidQuotaThreshold": "剩余周额度上限应为 0～100 的整数，留空表示不限制", "invalidResetLeadTime": "重置卡提前使用时间应为 1～1440 分钟的整数", - "invalidRetention": "历史保留天数应为 1–31 的整数", + "invalidRetention": "历史保留天数应为正整数", "invalidQueueSettings": "队列容量应为 0～1000 的整数，排队超时应为 1～120 秒的整数", "invalidProbeModel": "探测模型最多 128 个字符，不能包含控制字符", "invalidWarmupCron": "请输入有效的 Cron 表达式", diff --git a/frontend/src/i18n/locales/zh-Hant/settings.json b/frontend/src/i18n/locales/zh-Hant/settings.json index a4d09165e..de10a6c24 100644 --- a/frontend/src/i18n/locales/zh-Hant/settings.json +++ b/frontend/src/i18n/locales/zh-Hant/settings.json @@ -309,7 +309,7 @@ "retentionDaysLabel": "歷史保留天數" }, "retentionDays": "保留天數", - "enterDays": "請輸入 1–31 天", + "enterDays": "請輸入保留天數", "day": "天", "location": { "requestLocationOverride": "請求位置覆蓋", @@ -626,7 +626,7 @@ "invalidBodyLimit": "Responses 解壓上限應爲有效的正整數（MiB）", "invalidQuotaThreshold": "剩餘周額度上限應爲 0～100 的整數，留空表示不限制", "invalidResetLeadTime": "重置卡提前使用時間應爲 1～1440 分鐘的整數", - "invalidRetention": "歷史保留天數應爲 1–31 的整數", + "invalidRetention": "歷史保留天數應爲正整數", "invalidQueueSettings": "隊列容量應爲 0～1000 的整數，排隊超時應爲 1～120 秒的整數", "invalidProbeModel": "探測模型最多 128 個字符，不能包含控制字符", "invalidWarmupCron": "請輸入有效的 Cron 表達式", diff --git a/frontend/src/views/settings/components/RequestHistoryCard.vue b/frontend/src/views/settings/components/RequestHistoryCard.vue index cbaf6c51c..e7f68d754 100644 --- a/frontend/src/views/settings/components/RequestHistoryCard.vue +++ b/frontend/src/views/settings/components/RequestHistoryCard.vue @@ -12,7 +12,7 @@ const model = defineModel({ required: true })
- + diff --git a/frontend/src/views/settings/composables/useSettingsFormRules.ts b/frontend/src/views/settings/composables/useSettingsFormRules.ts index a606571bb..aca74a9e1 100644 --- a/frontend/src/views/settings/composables/useSettingsFormRules.ts +++ b/frontend/src/views/settings/composables/useSettingsFormRules.ts @@ -9,7 +9,7 @@ type SettingsForm = ReturnType['form'] export function useSettingsFormRules(form: () => SettingsForm) { const { t } = useI18n({ useScope: 'global' }) - const integer = (min: number, max: number, message: string): FormRule => ({ + const integer = (min: number, max: number | undefined, message: string): FormRule => ({ type: 'integer', required: true, min, @@ -47,7 +47,7 @@ export function useSettingsFormRules(form: () => SettingsForm) { ...integer(0, 100, t('settings.form.invalidQuotaThreshold')), required: false, }, - 'usageRetentionDays': integer(1, 31, t('settings.form.invalidRetention')), + 'usageRetentionDays': integer(1, undefined, t('settings.form.invalidRetention')), 'minCodexDesktopVersion': version(), 'minCodexCliVersion': version(), 'accountWarmupCron': [