Compare commits

...
19 Commits
Author SHA1 Message Date
zyycnandCodex bfd05722d5 chore: bump version to 2.4.0
Co-authored-by: Codex <noreply@openai.com>
2026-07-16 02:22:13 +08:00
dependabot[bot]andCodex 9dc2ea4a2f chore(deps): update backend dependency group
Update redis to 1.4.0, rustls to 0.23.42, and uuid to 1.23.5 after full backend, security, container, and Trivy validation.

Co-authored-by: Codex <noreply@openai.com>
2026-07-16 02:12:01 +08:00
dependabot[bot]andCodex 6fb27e869e chore(deps): refresh Rust 1.97 builder digest
Refresh the pinned rust:1.97-bookworm digest after a successful container build and Trivy scan.

Co-authored-by: Codex <noreply@openai.com>
2026-07-16 02:01:08 +08:00
dependabot[bot]andCodex 81c58b1cdc chore(deps): bump setup-node to 7.0.0
Update the pinned setup-node action to 7.0.0 across quality and security workflows.

Co-authored-by: Codex <noreply@openai.com>
2026-07-16 01:51:31 +08:00
zyycnandCodex c8dc2e293a fix(transport): eliminate websocket fallback race
Notify foreground and shared waiters before awaiting pool cleanup so an observed WebSocket opening error cannot be replaced by a fast-path timeout and HTTP fallback.

Drive the half-open breaker test through explicit state transitions instead of timing sleeps.

Co-authored-by: Codex <noreply@openai.com>
2026-07-16 01:46:33 +08:00
dependabot[bot] 25081de913 chore(deps): bump action-gh-release to 3.0.2
Update the pinned GitHub Release action from 3.0.1 to 3.0.2.\n\nCo-authored-by: Codex <noreply@openai.com>
2026-07-16 01:23:07 +08:00
zyycnandCodex 6eb0917ff1 feat(frontend): enrich interface micro-interactions
Add responsive hover reactions across dashboard, account, usage, settings, login, and navigation surfaces. Keep brand motion distinct, pair enter and leave states, and respect reduced-motion preferences.

Refine sidebar transitions, dashboard heartbeat, trend feedback, and 15-minute metric buckets without animating page content.

Co-authored-by: Codex <noreply@openai.com>
2026-07-16 01:07:33 +08:00
zyycnandCodex 72d892d5e3 feat(dashboard): improve request health visibility
Classify final request outcomes into success, failure, cancellation, and caller-error buckets so availability reflects only eligible requests. Expose structured 15-minute health points with focused coverage.

Redesign the health timeline, trend, wire-profile, and diagnostic surfaces with responsive shared interactions, and standardize configuration schema version 1.

Co-authored-by: Codex <noreply@openai.com>
2026-07-15 23:52:35 +08:00
ZyyandCodex 0b2da3087b refactor: replace fingerprint with wire profile
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 18:23:59 +08:00
zyycnandCodex 5391fdb31c fix(lifecycle): eliminate async finalization races
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 12:57:54 +08:00
zyycnandCodex 24036f1f84 docs: refine project README
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 12:27:27 +08:00
zyycnandCodex b043fced0c docs: streamline project guidance
Keep the public README terse and operational, and align the repository skills with the current lifecycle, transport, validation, and release boundaries.

Co-authored-by: Codex <noreply@openai.com>
2026-07-15 12:20:00 +08:00
zyycnandCodex 40a2b70bf2 chore: bump version to 2.3.2
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 12:07:00 +08:00
zyycnandCodex 1e6ac77d3c fix(frontend): scope usage record refresh
Refresh only the request detail table with an independent current-time snapshot, while preserving latest-request-wins ordering and leaving analytics cards untouched.

Co-authored-by: Codex <noreply@openai.com>
2026-07-15 12:06:38 +08:00
zyycnandCodex 3340934944 fix(transport): preserve websocket openings after fallback
Continue pool-owned WebSocket handshakes after the foreground budget expires, retain late successes for reuse, and cancel them on account eviction or shutdown. Keep breaker accounting and transport metrics aligned with the foreground decision.

Co-authored-by: Codex <noreply@openai.com>
2026-07-15 12:06:26 +08:00
zyycnandCodex 37a368de32 fix(ci): avoid duplicate pull request checks
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 11:33:11 +08:00
zyycnandCodex 79cd8ac965 chore(deps): delay fresh npm updates
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 11:13:50 +08:00
zyycnandCodex 57c6b95596 fix: report total dashboard billing
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 10:28:07 +08:00
zyycnandCodex a4f1843876 fix: unify account failures and usage diagnostics
Co-authored-by: Codex <noreply@openai.com>
2026-07-15 10:15:10 +08:00
142 changed files with 5059 additions and 3727 deletions
+2
View File
@@ -34,6 +34,8 @@ updates:
day: monday
time: '09:20'
timezone: Asia/Shanghai
cooldown:
default-days: 1
groups:
minor-and-patch:
update-types:
+1 -1
View File
@@ -93,7 +93,7 @@ jobs:
version: 11.7.0
- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 24
cache: pnpm
+3 -1
View File
@@ -3,8 +3,10 @@ name: CI
on:
push:
branches:
- '**'
- main
pull_request:
branches:
- main
permissions:
contents: read
+1 -1
View File
@@ -621,7 +621,7 @@ jobs:
cosign sign --yes "${images[@]}"
- name: Publish GitHub Release
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2
with:
tag_name: ${{ needs.release-info.outputs.tag }}
generate_release_notes: true
+4 -2
View File
@@ -3,8 +3,10 @@ name: Security Scan
on:
push:
branches:
- '**'
- main
pull_request:
branches:
- main
schedule:
- cron: '0 3 * * 1'
@@ -55,7 +57,7 @@ jobs:
version: 11.7.0
- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 24
cache: pnpm
+67 -122
View File
@@ -1,41 +1,41 @@
<!-- prettier-ignore -->
<div align="center">
<img src="frontend/public/favicon.svg" alt="Codex Proxy RS" width="88" height="88" />
<img src="frontend/public/favicon.svg" alt="Codex Proxy RS" width="80" height="80" />
# Codex Proxy RS
**基于 Rust 的高性能 Codex 多账号网关代理**
[![CI](https://github.com/zyycn/codex-proxy-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/zyycn/codex-proxy-rs/actions/workflows/ci.yml)
[![Release](https://img.shields.io/github/v/release/zyycn/codex-proxy-rs?display_name=tag&sort=semver)](https://github.com/zyycn/codex-proxy-rs/releases)
[![Rust 1.97](https://img.shields.io/badge/Rust-1.97-000?logo=rust&logoColor=white)](https://www.rust-lang.org/)
[![Vue 3.5](https://img.shields.io/badge/Vue-3.5-42b883?logo=vuedotjs&logoColor=white)](https://vuejs.org/)
[![Vite 8](https://img.shields.io/badge/Vite-8-646cff?logo=vite&logoColor=white)](https://vite.dev/)
[![GHCR](https://img.shields.io/badge/GHCR-codex--proxy--rs-2496ED?logo=docker&logoColor=white)](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
[![MIT](https://img.shields.io/badge/License-MIT-blue.svg)](#开源许可)
[![Release](https://img.shields.io/github/v/release/zyycn/codex-proxy-rs?display_name=tag&sort=semver&style=flat-square)](https://github.com/zyycn/codex-proxy-rs/releases)
[![GHCR](https://img.shields.io/badge/GHCR-codex--proxy--rs-2496ED?logo=docker&logoColor=white&style=flat-square)](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
[![MIT](https://img.shields.io/badge/License-MIT-blue.svg?style=flat-square)](https://opensource.org/license/mit)
[快速开始](#快速开始) · [首次使用](#首次使用) · [客户端接入](#客户端接入) · [常见问题](#常见问题) · [部署文档](deploy/README.md)
[快速开始](#快速开始) · [客户端接入](#客户端接入) · [运维](#运维) · [部署文档](deploy/README.md) · [架构](docs/architecture.md)
</div>
> [!NOTE]
> 当前兼容 OpenAI Responses API,不提供 `/v1/chat/completions`。
> 只支持 OpenAI Responses API,不提供 `/v1/chat/completions`。
## 功能
## 能力
- 支持 Responses HTTP、SSE 和官方 WebSocket 请求。
- 提供智能调度、额度重置优先、轮询和粘滞四种账号策略。
- 某个账号遇到额度、认证、模型或传输错误时,继续尝试其他账号。
- 在网页中管理账号、客户端 Key、模型别名和运行参数。
- 按请求、模型、账号或客户端 Key 查看 Token、费用、缓存、TTFT、延迟和错误。
- 支持账号 OAuth、Token 导入以及 CPR、Sub2API、CLIProxyAPI JSON 导入。
- 可直接生成 Codex CLI 配置,或通过 deeplink 导入 CCSwitch。
| 领域 | 实现 |
| -------- | ------------------------------------------------ |
| 协议 | Responses HTTP、SSE、官方 WebSocket |
| 传输 | 会话级 WebSocket 池、共享 HTTP/2 后备 |
| 调度 | 智能调度、额度优先、轮询、粘滞 |
| 故障处理 | 额度、认证、封禁、模型与传输故障切换 |
| 身份 | 账号身份隔离、稳定 installation ID、会话拓扑透传 |
| 管理 | 账号、客户端 Key、模型别名、运行设置与在线更新 |
| 遥测 | Token、费用、缓存、TTFT、延迟与错误 |
## 快速开始
Docker Compose 是最简单的启动方式。开始前请安装 Docker Engine 和 Docker Compose Plugin。
需要 Docker Engine 与 Docker Compose Plugin。
### 1. 准备配置
### 1. 准备
```bash
git clone https://github.com/zyycn/codex-proxy-rs.git
@@ -48,74 +48,63 @@ sudo chown "$(id -u):10001" deploy/config.yaml
chmod 0640 deploy/config.yaml
```
分别执行三次以下命令生成密码:
分别执行三次:
```bash
openssl rand -hex 24
```
把结果填入 `deploy/config.yaml` 的 `database.password`、`redis.password` 和
`admin.default_password`。PostgreSQL 与 Redis 密码必须是 48 位十六进制字符;管理员密码
至少 12 位且不能包含 `$`。
将结果写入 `deploy/config.yaml`:
> [!IMPORTANT]
> 应用容器以非 root 用户 `10001:10001` 运行。Linux Bind Mount 会保留宿主机权限,因此需要允许容器用户组读取配置并写入数据和日志目录:
>
> ```bash
> sudo chown -R "$(id -u):10001" .runtime/data .runtime/logs
> chmod 0770 .runtime/data .runtime/logs
> ```
>
> 配置文件通过 Compose `configs` 只读挂载;普通 Compose 会保留宿主机权限,因此由
> 当前用户持有、仅向容器组开放读取权限。
> macOS 和 Windows 的 Docker Desktop 通常也不需要调整数据目录权限。
| 配置 | 约束 |
| ------------------------------ | ------------------------ |
| `x-cpr.database.password` | 48 位十六进制 |
| `x-cpr.redis.password` | 48 位十六进制 |
| `x-cpr.admin.default_password` | 至少 12 位,不能包含 `$` |
### 2. 启动服务
Linux 需要允许容器组写入运行目录:
```bash
sudo chown -R "$(id -u):10001" .runtime/data .runtime/logs
chmod 0770 .runtime/data .runtime/logs
```
### 2. 启动
```bash
docker compose -f deploy/compose.yaml config --quiet
docker compose -f deploy/compose.yaml pull
docker compose -f deploy/compose.yaml up -d --no-build
```
启动后检查:
```bash
docker compose -f deploy/compose.yaml ps
curl -i http://127.0.0.1:8080/healthz
```
`/healthz` 返回 `204 No Content` 即表示服务正常。管理端地址为 <http://127.0.0.1:8080>。
`204 No Content` 表示应用、PostgreSQL 与 Redis 均可用。管理端地址:
`http://127.0.0.1:8080`。
## 首次使用
### 3. 初始化
1. 使用 `admin@cpr.local` 和 `x-cpr.admin.default_password` 中设置的密码登录。
2. 打开“账号”,用 OAuth、Access/Refresh Token 或 JSON 文件添加账号。
3. 点一次连接测试,能看到模型和额度即可。
4. 打开“API 密钥”,创建一个 `sk_...` 客户端 Key。
5. 点击“使用密钥”复制 Codex CLI 配置,或一键导入 CCSwitch。
1. 使用 `admin@cpr.local` 与初始密码登录。
2. 添加 Codex 账号并执行连接测试。
3. 创建 `sk_...` 客户端 Key。
4. 复制 Codex CLI 配置,或导入 CCSwitch。
> [!TIP]
> `admin.default_password` 只用于首次创建管理员。已有管理员后修改该字段不会重置登录密码。
> [!NOTE]
> `admin.default_password` 只在首次创建管理员时生效。
完整部署、权限和密码轮换规则见 [deploy/README.md](deploy/README.md)。
## 客户端接入
| 配置 | 值 |
| -------- | ---------------------------------------- |
| Base URL | `http://127.0.0.1:8080/v1` |
| API Key | 管理端创建的 `sk_...` 客户端 Key |
| API Key | 管理端创建的 `sk_...` |
| 鉴权 | `Authorization: Bearer <client-api-key>` |
先查询当前可用模型:
```bash
curl http://127.0.0.1:8080/v1/models \
-H 'Authorization: Bearer <client-api-key>'
```
发送一个非流式 Responses 请求:
```bash
curl http://127.0.0.1:8080/v1/responses \
-H 'Authorization: Bearer <client-api-key>' \
-H 'Content-Type: application/json' \
@@ -127,89 +116,45 @@ curl http://127.0.0.1:8080/v1/responses \
```
<details>
<summary>查看全部客户端 API 路由</summary>
<summary>API 路由</summary>
| 路由 | 说明 |
| -------------------------------- | ------------------------------- |
| `POST /v1/responses` | Responses 非流式或 SSE 流式请求 |
| `GET /v1/responses` | 官方 Responses WebSocket |
| `POST /v1/responses/review` | Review 请求 |
| `GET /v1/models` | OpenAI 格式的模型列表 |
| `GET /v1/models/catalog` | 完整模型目录 |
| `GET /v1/models/{model_id}` | 模型详情 |
| `GET /v1/models/{model_id}/info` | 模型运行信息 |
| 路由 | 用途 |
| --------------------------- | ------------------- |
| `POST /v1/responses` | JSON 或 SSE |
| `GET /v1/responses` | Responses WebSocket |
| `POST /v1/responses/review` | Review |
| `GET /v1/models` | 模型列表 |
| `GET /v1/models/catalog` | 模型目录 |
| `GET /v1/models/{model_id}` | 模型详情 |
所有 `/v1/*` 路由都需要客户端 API Key。
</details>
## 管理端
| 页面 | 内容 |
| -------- | ---------------------------------------------------------------------- |
| 概览 | 查看服务状态、请求趋势、Token、账号池容量、请求健康和最近调用 |
| 账号 | 添加、导入、导出、测试、启停账号,以及刷新 Token 和额度 |
| API 密钥 | 创建、复制、启停或删除 Key,生成 Codex CLI 配置并导入 CCSwitch |
| 使用统计 | 按时间、模型、账号和客户端 Key 查看用量、费用、缓存、延迟和错误 |
| 系统设置 | 配置模型别名、调度策略、单账号并发、请求间隔、刷新参数和管理员 API Key |
| 系统更新 | 查看版本和发布说明;Release 版本可以在线更新并重启 |
## 常用操作
### 升级
## 运维
```bash
# 升级
docker compose -f deploy/compose.yaml pull codex-proxy-rs
docker compose -f deploy/compose.yaml up -d --no-build
```
### 查看日志
```bash
# 日志
docker compose -f deploy/compose.yaml logs -f codex-proxy-rs
```
### 从当前源码构建
```bash
# 从源码构建
docker compose -f deploy/compose.yaml build codex-proxy-rs
docker compose -f deploy/compose.yaml up -d
```
> [!WARNING]
> PostgreSQL、Redis、应用数据和文件日志都保存在仓库根目录 `.runtime/`。普通
> `docker compose down` 不会删除这些绑定目录;删除 `.runtime/` 才会清除数据。升级前请备份
> 整个目录,数据库备份中包含客户端凭据,应按敏感数据保管。
> [!IMPORTANT]
> `.runtime/` 保存数据库、Redis、身份密钥和日志。删除该目录会永久清除运行状态。
## 常见问题
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
### `/healthz` 返回 `503`
## 文档
检查 PostgreSQL 和 Redis 容器是否健康:
```bash
docker compose -f deploy/compose.yaml ps
docker compose -f deploy/compose.yaml logs postgres redis
```
### 容器反复重启
检查 `deploy/config.yaml` 的三个密码是否填写完整,以及 `.runtime/data`、`.runtime/logs`
的权限是否正确。
### 需要从其他设备访问
Compose 默认仅绑定 `127.0.0.1`。请在服务前配置 HTTPS 反向代理,并正确转发 WebSocket 升级和真实客户端 IP 请求头。不要直接暴露 PostgreSQL 或 Redis。
## 相关链接
- [发布版本](https://github.com/zyycn/codex-proxy-rs/releases)
- [部署](deploy/README.md)
- [架构](docs/architecture.md)
- [Release](https://github.com/zyycn/codex-proxy-rs/releases)
- [容器镜像](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
- [详细部署说明](deploy/README.md)
- [架构说明](docs/architecture.md)
Docker 镜像支持 Linux amd64/arm64。Release 压缩包另有 macOS arm64 和 Windows amd64 版本。
## 开源许可
Codex Proxy RS 基于 [MIT License](https://opensource.org/license/mit) 开源。
+16 -6
View File
@@ -371,6 +371,7 @@ dependencies = [
"rand_core 0.6.4",
"redis",
"reqwest",
"roxmltree",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
@@ -1784,9 +1785,9 @@ dependencies = [
[[package]]
name = "redis"
version = "1.3.0"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fa6f8e4b491d7a8ef3a9550a4d71969bd0064f46e32b8dbbcc7fc60dad94fed"
checksum = "3cb5358643f48330db5c78856982faf39c93ba50c60d682b43d0344a3b649769"
dependencies = [
"arc-swap",
"arcstr",
@@ -1905,6 +1906,15 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "roxmltree"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
dependencies = [
"memchr",
]
[[package]]
name = "rustc-hash"
version = "2.1.3"
@@ -1926,9 +1936,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.36"
version = "0.23.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b"
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
dependencies = [
"once_cell",
"ring",
@@ -2882,9 +2892,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
version = "1.23.4"
version = "1.23.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a"
dependencies = [
"getrandom 0.4.3",
"js-sys",
+2 -1
View File
@@ -25,9 +25,10 @@ rand = "0.10.1"
# Enables password_hash::rand_core::OsRng for argon2 0.5.
rand_core = { version = "0.6.4", features = ["getrandom"] }
redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] }
roxmltree = "0.21.1"
# Pinned to the Codex Desktop 26.707.41301 HTTP stack; re-audit a newer Desktop before upgrading.
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "gzip", "brotli", "zstd", "deflate", "http2"] }
rustls = { version = "=0.23.36", default-features = false, features = ["ring", "std", "tls12"] }
rustls = { version = "=0.23.42", default-features = false, features = ["ring", "std", "tls12"] }
rustls-native-certs = "0.8.4"
rustls-pki-types = "1.14.1"
secrecy = { version = "0.10.3", features = ["serde"] }
+112 -103
View File
@@ -28,10 +28,7 @@ use crate::{
},
infra::{
format::{format_duration_ms, format_tokens},
time::{
china_datetime, china_datetime_rfc3339_str, china_day_start, china_quarter_hour_start,
china_relative_time,
},
time::{china_datetime, china_day_start, china_quarter_hour_start, china_relative_time},
},
telemetry::{
account_usage::query::AccountUsageTimeBucket,
@@ -41,11 +38,11 @@ use crate::{
dashboard_health_timeline_data, dashboard_trend_data,
},
usage::{
insights::RequestHealthTimeBucket,
query::{UsageQueryFilter, UsageRecordAccountUsage},
types::{UsageRecord, metadata_string},
},
},
upstream::openai::fingerprint::Fingerprint,
};
const DASHBOARD_ACCOUNT_USAGE_LIMIT: u32 = 4;
@@ -69,7 +66,7 @@ struct DashboardSummaryData {
trend: DashboardTrendData,
health_timeline: DashboardHealthTimelineData,
account_usage: Vec<DashboardAccountUsageData>,
service_statuses: Vec<DashboardServiceStatusData>,
wire_profile: DashboardWireProfileData,
usage_records: Vec<DashboardUsageRecordData>,
pool_summary: AccountPoolDiagnostics,
capacity_info: AccountCapacityDiagnostics,
@@ -114,11 +111,50 @@ struct DashboardUsageRecordData {
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardServiceStatusData {
label: String,
value: String,
detail: String,
tone: String,
struct DashboardWireProfileData {
originator: String,
codex_version: String,
desktop_version: String,
desktop_build: String,
target: DashboardWireTargetData,
user_agent: String,
verified_at: DateTime<Utc>,
release: DashboardDesktopReleaseData,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardWireTargetData {
os_type: String,
os_version: String,
arch: String,
terminal: String,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardDesktopReleaseData {
status: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
checked_at: Option<DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
latest_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
latest_build: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
published_at: Option<DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
minimum_system_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
hardware_requirements: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
download_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
download_size: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
signature_present: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
}
#[derive(Debug, Default, Serialize)]
@@ -142,35 +178,6 @@ struct AccountCapacityDiagnostics {
available_slots: usize,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct FingerprintDiagnostics {
source: &'static str,
originator: String,
app_version: String,
build_number: String,
platform: String,
arch: String,
chromium_version: String,
user_agent: String,
#[serde(skip_serializing_if = "Option::is_none")]
updated_at: Option<String>,
}
fn fingerprint_diagnostics(fingerprint: &Fingerprint) -> FingerprintDiagnostics {
FingerprintDiagnostics {
source: "runtime",
originator: fingerprint.originator.clone(),
app_version: fingerprint.app_version.clone(),
build_number: fingerprint.build_number.clone(),
platform: fingerprint.platform.clone(),
arch: fingerprint.arch.clone(),
chromium_version: fingerprint.chromium_version.clone(),
user_agent: fingerprint.user_agent(),
updated_at: fingerprint.updated_at.clone(),
}
}
impl From<AccountCapacitySummary> for AccountCapacityDiagnostics {
fn from(summary: AccountCapacitySummary) -> Self {
Self {
@@ -208,7 +215,7 @@ pub(crate) async fn dashboard_summary(
let retained_usage = state
.services
.usage
.retained_totals()
.retained_summary()
.await
.map_err(|error| dashboard_data_error("retained usage summary", &error))?;
let account_usage_records = state
@@ -224,7 +231,10 @@ pub(crate) async fn dashboard_summary(
.await
.map_err(|error| dashboard_data_error("recent usage records", &error))?;
let time_buckets = dashboard_time_buckets(&state, now).await?;
let (time_buckets, health_buckets) = tokio::try_join!(
dashboard_time_buckets(&state, now),
dashboard_health_buckets(&state, now),
)?;
let account_ids = accounts
.iter()
.map(|account| account.id.clone())
@@ -259,14 +269,14 @@ pub(crate) async fn dashboard_summary(
&retained_usage,
),
trend,
health_timeline: dashboard_health_timeline_data(&time_buckets),
health_timeline: dashboard_health_timeline_data(&health_buckets),
account_usage: account_usage_data(
&accounts,
&account_usage_records,
&quota_used_by_account,
now,
),
service_statuses: service_status_data(&state),
wire_profile: wire_profile_data(&state),
usage_records: dashboard_usage_records,
pool_summary,
capacity_info: AccountCapacityDiagnostics::from(capacity),
@@ -315,6 +325,18 @@ async fn dashboard_time_buckets(
.map_err(|error| dashboard_data_error("time buckets", &error))
}
async fn dashboard_health_buckets(
state: &AppState,
now: DateTime<Utc>,
) -> Result<Vec<RequestHealthTimeBucket>, AdminError> {
state
.services
.usage_records
.health_timeline(china_day_start(now), now)
.await
.map_err(|error| dashboard_data_error("health timeline", &error))
}
fn dashboard_account_counts(accounts: &[Account]) -> DashboardAccountCounts {
DashboardAccountCounts {
total: accounts.len() as u64,
@@ -554,66 +576,53 @@ fn dashboard_usage_record_data(
}
}
fn service_status_data(state: &AppState) -> Vec<DashboardServiceStatusData> {
let current_fingerprint = state.services.fingerprint.snapshot();
let fingerprint = fingerprint_diagnostics(&current_fingerprint);
vec![
DashboardServiceStatusData {
label: "客户端版本".to_string(),
value: empty_dash(fingerprint.app_version),
detail: if fingerprint.build_number.trim().is_empty() {
"-".to_string()
} else {
format!("Build {}", fingerprint.build_number)
},
tone: "info".to_string(),
},
DashboardServiceStatusData {
label: "平台架构".to_string(),
value: empty_dash(fingerprint.platform),
detail: empty_dash(fingerprint.arch),
tone: "info".to_string(),
},
DashboardServiceStatusData {
label: "Chromium".to_string(),
value: if fingerprint.chromium_version.trim().is_empty() {
"-".to_string()
} else {
format!("v{}", fingerprint.chromium_version)
},
detail: empty_dash(fingerprint.originator),
tone: "normal".to_string(),
},
DashboardServiceStatusData {
label: "更新时间".to_string(),
value: format_fingerprint_updated_at(fingerprint.updated_at),
detail: String::new(),
tone: "normal".to_string(),
},
DashboardServiceStatusData {
label: "User Agent".to_string(),
value: fingerprint.user_agent,
detail: String::new(),
tone: "normal".to_string(),
},
]
}
fn empty_dash(value: String) -> String {
if value.trim().is_empty() {
"-".to_string()
fn wire_profile_data(state: &AppState) -> DashboardWireProfileData {
let profile = &state.services.wire_profile;
let release_snapshot = state.services.desktop_release.snapshot();
let release_status = if release_snapshot.last_error.is_some() {
"check_failed"
} else if let Some(latest) = &release_snapshot.latest {
if latest.version == profile.desktop_version && latest.build == profile.desktop_build {
"aligned"
} else {
"review_required"
}
} else {
value
}
}
fn format_fingerprint_updated_at(value: Option<String>) -> String {
let Some(value) = value else {
return "-".to_string();
"unchecked"
};
let value = value.trim();
if value.is_empty() {
return "-".to_string();
let latest = release_snapshot.latest;
DashboardWireProfileData {
originator: profile.originator.clone(),
codex_version: profile.codex_version.clone(),
desktop_version: profile.desktop_version.clone(),
desktop_build: profile.desktop_build.clone(),
target: DashboardWireTargetData {
os_type: profile.os_type.clone(),
os_version: profile.os_version.clone(),
arch: profile.arch.clone(),
terminal: profile.terminal.clone(),
},
user_agent: profile.user_agent(),
verified_at: profile.verified_at,
release: DashboardDesktopReleaseData {
status: release_status,
checked_at: release_snapshot.checked_at,
latest_version: latest.as_ref().map(|release| release.version.clone()),
latest_build: latest.as_ref().map(|release| release.build.clone()),
published_at: latest.as_ref().and_then(|release| release.published_at),
minimum_system_version: latest
.as_ref()
.and_then(|release| release.minimum_system_version.clone()),
hardware_requirements: latest
.as_ref()
.and_then(|release| release.hardware_requirements.clone()),
download_url: latest
.as_ref()
.and_then(|release| release.download_url.clone()),
download_size: latest.as_ref().and_then(|release| release.download_size),
signature_present: latest.as_ref().map(|release| release.signature_present),
error: release_snapshot.last_error,
},
}
china_datetime_rfc3339_str(value)
}
+6 -2
View File
@@ -27,7 +27,10 @@ use crate::{
usage::query::UsageQueryService,
},
update::service::SystemUpdateService,
upstream::openai::{fingerprint::RuntimeFingerprint, token_client::OpenAiTokenClient},
upstream::openai::{
desktop_release::DesktopReleaseStatus, profile::CodexWireProfile,
token_client::OpenAiTokenClient,
},
};
/// HTTP API 所需的领域服务集合。
@@ -48,7 +51,8 @@ pub struct ApiServices {
pub token_refresh: Arc<TokenRefreshService<OpenAiTokenClient>>,
pub responses: Arc<ResponseDispatchService>,
pub session_affinity: Arc<SessionAffinityService>,
pub fingerprint: RuntimeFingerprint,
pub wire_profile: Arc<CodexWireProfile>,
pub desktop_release: DesktopReleaseStatus,
pub process_control: Arc<dyn ProcessControl>,
pub system_update: Arc<SystemUpdateService>,
pub connection_drain: crate::api::middleware::connection_drain::ConnectionDrain,
+55 -71
View File
@@ -1,10 +1,10 @@
use std::{
collections::{BTreeMap, HashSet},
collections::BTreeMap,
env,
path::{Path, PathBuf},
};
use axum::http::HeaderName;
use chrono::{DateTime, Utc};
use secrecy::{ExposeSecret, SecretBox};
use serde::{Deserialize, de::IgnoredAny};
use tracing_subscriber::EnvFilter;
@@ -106,7 +106,7 @@ impl BootstrapConfig {
runtime,
quota,
ws_pool,
fingerprint,
wire_profile,
admin,
logging,
telemetry,
@@ -122,7 +122,7 @@ impl BootstrapConfig {
redis: RedisConfig { url: redis.url },
runtime,
ws_pool,
fingerprint,
wire_profile,
admin: AdminConfig {
session_ttl_minutes: admin.session_ttl_minutes,
default_username: admin.default_username,
@@ -241,8 +241,8 @@ pub struct AppConfig {
pub runtime: RuntimePathsConfig,
/// WebSocket 连接池启动设置。
pub ws_pool: WebSocketPoolSettings,
/// 上游请求指纹默认配置。
pub fingerprint: FingerprintConfig,
/// 经审计固定的 Codex Desktop 上游请求画像。
pub wire_profile: WireProfileConfig,
/// 管理员初始化配置,不包含密码。
pub admin: AdminConfig,
/// 日志配置。
@@ -354,38 +354,28 @@ pub struct WebSocketPoolSettings {
pub initial_event_timeout_ms: u64,
}
/// 上游请求指纹默认配置。
/// 经审计固定的 Codex Desktop 上游请求画像配置。
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct FingerprintConfig {
/// 客户端来源名。
pub struct WireProfileConfig {
/// `originator` 请求头及 User-Agent 产品名。
pub originator: String,
/// 应用版本。
pub app_version: String,
/// 构建号。
pub build_number: String,
/// 平台名。
pub platform: String,
/// 架构名。
/// bundled Codex Core 版本。
pub codex_version: String,
/// Desktop 应用版本。
pub desktop_version: String,
/// Desktop 制品构建号。
pub desktop_build: String,
/// Codex Core UA 中的目标操作系统类型。
pub os_type: String,
/// Codex Core UA 中的目标操作系统版本。
pub os_version: String,
/// Codex Core UA 中的目标架构。
pub arch: String,
/// Chromium 主版本。
pub chromium_version: String,
/// User-Agent 模板。
pub user_agent_template: String,
/// 默认请求头。
pub default_headers: Vec<FingerprintHeaderConfig>,
/// 请求头排序优先级。
pub header_order: Vec<String>,
}
/// 指纹默认请求头配置项。
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct FingerprintHeaderConfig {
/// 请求头名称。
pub name: String,
/// 请求头值。
pub value: String,
/// Codex Core UA 中的终端标记。
pub terminal: String,
/// 此画像最后一次经制品与源码核验的时间。
pub verified_at: DateTime<Utc>,
}
/// 管理员初始化与会话配置,不包含初始化密码。
@@ -453,7 +443,7 @@ struct FileAppConfig {
runtime: RuntimePathsConfig,
quota: QuotaConfig,
ws_pool: WebSocketPoolSettings,
fingerprint: FingerprintConfig,
wire_profile: WireProfileConfig,
admin: FileAdminConfig,
logging: LoggingConfig,
telemetry: TelemetryConfig,
@@ -549,7 +539,7 @@ fn validate_file_config(config: &FileAppConfig) -> Result<(), ConfigError> {
if config.ws_pool.max_per_account == 0 {
return Err(ConfigError::InvalidField("ws_pool.max_per_account"));
}
validate_fingerprint(&config.fingerprint)?;
validate_wire_profile(&config.wire_profile)?;
validate_logging(&config.logging)?;
Ok(())
}
@@ -594,54 +584,48 @@ fn validate_admin_password(password: &str) -> Result<(), ConfigError> {
Ok(())
}
fn validate_fingerprint(config: &FingerprintConfig) -> Result<(), ConfigError> {
fn validate_wire_profile(config: &WireProfileConfig) -> Result<(), ConfigError> {
for (field, value) in [
("fingerprint.originator", config.originator.as_str()),
("fingerprint.app_version", config.app_version.as_str()),
("fingerprint.build_number", config.build_number.as_str()),
("fingerprint.platform", config.platform.as_str()),
("fingerprint.arch", config.arch.as_str()),
("wire_profile.originator", config.originator.as_str()),
("wire_profile.codex_version", config.codex_version.as_str()),
(
"fingerprint.chromium_version",
config.chromium_version.as_str(),
),
(
"fingerprint.user_agent_template",
config.user_agent_template.as_str(),
"wire_profile.desktop_version",
config.desktop_version.as_str(),
),
("wire_profile.desktop_build", config.desktop_build.as_str()),
("wire_profile.os_type", config.os_type.as_str()),
("wire_profile.os_version", config.os_version.as_str()),
("wire_profile.arch", config.arch.as_str()),
("wire_profile.terminal", config.terminal.as_str()),
] {
validate_nonempty(field, value)?;
}
if config.default_headers.is_empty() || config.header_order.is_empty() {
return Err(ConfigError::InvalidField("fingerprint"));
if semver::Version::parse(&config.codex_version).is_err() {
return Err(ConfigError::InvalidField("wire_profile.codex_version"));
}
let mut header_names = HashSet::new();
for header in &config.default_headers {
let name = header.name.trim().to_ascii_lowercase();
if name.is_empty()
|| header.value.is_empty()
|| HeaderName::from_bytes(name.as_bytes()).is_err()
|| !header_names.insert(name)
{
return Err(ConfigError::InvalidField("fingerprint.default_headers"));
}
if !is_numeric_dotted_version(&config.desktop_version) {
return Err(ConfigError::InvalidField("wire_profile.desktop_version"));
}
let mut ordered_headers = HashSet::new();
for header in &config.header_order {
let header = header.trim().to_ascii_lowercase();
if header.is_empty()
|| HeaderName::from_bytes(header.as_bytes()).is_err()
|| !ordered_headers.insert(header)
{
return Err(ConfigError::InvalidField("fingerprint.header_order"));
}
if !config
.desktop_build
.bytes()
.all(|byte| byte.is_ascii_digit())
{
return Err(ConfigError::InvalidField("wire_profile.desktop_build"));
}
Ok(())
}
fn is_numeric_dotted_version(value: &str) -> bool {
let mut parts = value.split('.');
let valid_parts = parts
.by_ref()
.filter(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit()))
.count();
valid_parts >= 2 && valid_parts == value.split('.').count()
}
fn validate_logging(config: &LoggingConfig) -> Result<(), ConfigError> {
EnvFilter::try_new(&config.level).map_err(|_| ConfigError::InvalidField("logging.level"))?;
if !config.stdout && !config.file.enabled {
+31 -37
View File
@@ -50,7 +50,8 @@ use crate::{
},
update::service::SystemUpdateService,
upstream::openai::{
fingerprint::{Fingerprint, PgFingerprintStore, RuntimeFingerprint},
desktop_release::DesktopReleaseStatus,
profile::CodexWireProfile,
token_client::{OpenAiTokenClient, TokenClientConfig, default_openai_token_client},
transport::{
CodexBackendClient, CodexModelCatalogClient, CodexWebSocketPool,
@@ -80,7 +81,6 @@ pub struct BackgroundTaskStores {
pub admin_users: PgAdminUserStore,
pub admin_sessions: RedisAdminSessionStore,
pub cookies: PgCookieStore,
pub fingerprints: PgFingerprintStore,
pub session_affinity: RedisSessionAffinityStore,
pub refresh_leases: RedisRefreshLeaseStore,
pub model_snapshots: RedisModelSnapshotStore,
@@ -117,7 +117,8 @@ pub struct Services {
pub session_affinity: Arc<SessionAffinityService>,
pub codex: Arc<CodexBackendClient>,
pub websocket_pool: Option<Arc<CodexWebSocketPool>>,
pub fingerprint: RuntimeFingerprint,
pub wire_profile: Arc<CodexWireProfile>,
pub desktop_release: DesktopReleaseStatus,
pub account_pseudonymizer: Arc<AccountPseudonymizer>,
pub system_update: Arc<SystemUpdateService>,
pub connection_drain: crate::api::middleware::connection_drain::ConnectionDrain,
@@ -145,21 +146,21 @@ impl Services {
pub fn new(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
wire_profile: Arc<CodexWireProfile>,
) -> Self {
Self::try_new(config, stores, fingerprint)
Self::try_new(config, stores, wire_profile)
.expect("failed to build runtime services with configured TLS transport")
}
pub fn try_new(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
wire_profile: Arc<CodexWireProfile>,
) -> Result<Self, CustomCaError> {
Self::try_with_usage_record_options(
config,
stores,
fingerprint,
wire_profile,
UsageRecordOptions::from_config(config),
)
}
@@ -167,7 +168,7 @@ impl Services {
pub fn try_with_usage_record_options(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
wire_profile: Arc<CodexWireProfile>,
usage_record_options: UsageRecordOptions,
) -> Result<Self, CustomCaError> {
let mut identity_secret = [0u8; 32];
@@ -175,7 +176,7 @@ impl Services {
Self::try_with_identity_secret_and_usage_record_options(
config,
stores,
fingerprint,
wire_profile,
identity_secret,
usage_record_options,
)
@@ -184,7 +185,7 @@ impl Services {
fn try_with_identity_secret_and_usage_record_options(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
wire_profile: Arc<CodexWireProfile>,
identity_secret: [u8; 32],
usage_record_options: UsageRecordOptions,
) -> Result<Self, CustomCaError> {
@@ -214,7 +215,7 @@ impl Services {
let client = CodexBackendClient::new(
build_reqwest_client()?,
config.api.base_url.clone(),
fingerprint.clone(),
wire_profile.clone(),
)
.with_websocket_initial_event_timeout(websocket_initial_event_timeout);
if let Some(pool) = &websocket_pool {
@@ -313,6 +314,8 @@ impl Services {
}));
let system_update = Arc::new(SystemUpdateService::from_env());
let desktop_release = DesktopReleaseStatus::default();
Ok(Self {
database,
redis,
@@ -333,7 +336,8 @@ impl Services {
session_affinity,
codex,
websocket_pool,
fingerprint,
wire_profile,
desktop_release,
account_pseudonymizer,
system_update,
connection_drain,
@@ -370,7 +374,8 @@ impl From<&Services> for crate::api::router::ApiServices {
token_refresh: services.token_refresh.clone(),
responses: services.responses.clone(),
session_affinity: services.session_affinity.clone(),
fingerprint: services.fingerprint.clone(),
wire_profile: services.wire_profile.clone(),
desktop_release: services.desktop_release.clone(),
process_control: Arc::new(crate::bootstrap::shutdown::RuntimeProcessControl),
system_update: services.system_update.clone(),
connection_drain: services.connection_drain.clone(),
@@ -419,24 +424,19 @@ pub fn apply_settings_to_config(config: &mut AppConfig, settings: &SettingsSnaps
config.auth.rotation_strategy = settings.rotation_strategy.clone();
}
pub fn fingerprint_from_config(
config: &crate::bootstrap::config::FingerprintConfig,
) -> Fingerprint {
Fingerprint {
pub fn wire_profile_from_config(
config: &crate::bootstrap::config::WireProfileConfig,
) -> CodexWireProfile {
CodexWireProfile {
originator: config.originator.clone(),
app_version: config.app_version.clone(),
build_number: config.build_number.clone(),
platform: config.platform.clone(),
codex_version: config.codex_version.clone(),
desktop_version: config.desktop_version.clone(),
desktop_build: config.desktop_build.clone(),
os_type: config.os_type.clone(),
os_version: config.os_version.clone(),
arch: config.arch.clone(),
chromium_version: config.chromium_version.clone(),
user_agent_template: config.user_agent_template.clone(),
default_headers: config
.default_headers
.iter()
.map(|header| (header.name.clone(), header.value.clone()))
.collect(),
header_order: config.header_order.clone(),
updated_at: None,
terminal: config.terminal.clone(),
verified_at: config.verified_at,
}
}
@@ -560,14 +560,12 @@ async fn build_router(
SettingsService::load_or_initialize(settings_snapshot_from_config(&config), &pool).await?;
apply_settings_to_config(&mut config, &settings);
let fingerprint_store = PgFingerprintStore::new(pool.clone());
let stores = BackgroundTaskStores {
redis: redis.clone(),
accounts: PgAccountStore::new(pool.clone()),
admin_users: PgAdminUserStore::new(pool.clone()),
admin_sessions: RedisAdminSessionStore::new(redis.clone()),
cookies: PgCookieStore::new(pool.clone()),
fingerprints: fingerprint_store.clone(),
session_affinity: RedisSessionAffinityStore::new(redis.clone()),
refresh_leases: RedisRefreshLeaseStore::new(redis.clone()),
model_snapshots: RedisModelSnapshotStore::new(redis.clone()),
@@ -580,16 +578,12 @@ async fn build_router(
let data_dir = ensure_data_dir(&config.runtime.data_directory)?;
let identity_secret = load_or_create_identity_secret(&data_dir)?;
let default_fingerprint = fingerprint_from_config(&config.fingerprint);
let runtime_fingerprint = fingerprint_store
.ensure_current_seed(&default_fingerprint)
.await?;
let runtime_fingerprint = RuntimeFingerprint::new(runtime_fingerprint);
let wire_profile = Arc::new(wire_profile_from_config(&config.wire_profile));
let runtime_config = crate::bootstrap::state::RuntimeConfig::from(&config);
let services = Services::try_with_identity_secret_and_usage_record_options(
&config,
stores,
runtime_fingerprint,
wire_profile,
identity_secret,
UsageRecordOptions::from_config(&config),
)?;
+12 -23
View File
@@ -1,19 +1,13 @@
//! 后台任务协调器。
use std::{future::Future, path::PathBuf, sync::Arc, time::Duration};
use std::{future::Future, sync::Arc, time::Duration};
use tokio::task::JoinHandle;
use super::{
cookie_cleanup::CookieCleanupTask,
fingerprint_update::{
CODEX_DESKTOP_APPCAST_URL, DEFAULT_EXTRACTED_FINGERPRINT_PATH,
start_fingerprint_update_task,
},
model_refresh::ModelRefreshTask,
quota_refresh::QuotaRefreshTask,
retention_trim::RetentionTrimTask,
token_refresh::TokenRefreshTask,
cookie_cleanup::CookieCleanupTask, desktop_release_update::start_desktop_release_update_task,
model_refresh::ModelRefreshTask, quota_refresh::QuotaRefreshTask,
retention_trim::RetentionTrimTask, token_refresh::TokenRefreshTask,
};
/// 后台任务协调器。
@@ -46,7 +40,6 @@ impl TaskCoordinator {
stores.usage_records.clone(),
stores.ops_errors.clone(),
stores.request_buckets.clone(),
stores.fingerprints.clone(),
)
.start(),
);
@@ -82,6 +75,7 @@ impl TaskCoordinator {
stores.accounts.clone(),
stores.account_usage.clone(),
services.codex.clone(),
services.account_pool.clone(),
services.account_pseudonymizer.clone(),
config
.quota
@@ -91,20 +85,15 @@ impl TaskCoordinator {
30 * 60,
)
.with_cookie_store(stores.cookies.clone())
.with_account_pool(services.account_pool.clone())
.start(),
);
coordinator.push("fingerprint_update", {
let fingerprint = services.fingerprint.snapshot();
start_fingerprint_update_task(
stores.fingerprints.clone(),
services.fingerprint.clone(),
CODEX_DESKTOP_APPCAST_URL.to_string(),
PathBuf::from(DEFAULT_EXTRACTED_FINGERPRINT_PATH),
fingerprint.app_version,
fingerprint.build_number,
)
});
coordinator.push(
"desktop_release_update",
start_desktop_release_update_task(
services.desktop_release.clone(),
crate::upstream::openai::desktop_release::CODEX_DESKTOP_APPCAST_URL.to_string(),
),
);
if let Some(pool) = &services.websocket_pool {
coordinator.push(
"websocket_pool",
@@ -0,0 +1,57 @@
//! Codex Desktop 官方发布观测任务接线。
use tracing::{info, warn};
use crate::upstream::openai::desktop_release::{
APPCAST_POLL_INTERVAL, DesktopReleaseChecker, DesktopReleaseStatus,
};
use super::{
coordinator::SchedulerHandle,
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, spawn_periodic_task},
};
/// 启动 Desktop 官方发布检查后台任务。
pub fn start_desktop_release_update_task(
status: DesktopReleaseStatus,
appcast_url: String,
) -> SchedulerHandle {
let checker = DesktopReleaseChecker::with_client(reqwest::Client::new(), appcast_url, status);
spawn_periodic_task(
DesktopReleaseUpdateTask {
checker,
first_tick: true,
},
PeriodicTaskConfig::new(
APPCAST_POLL_INTERVAL.as_secs(),
"Codex Desktop 发布检查器已启动",
"Codex Desktop 发布检查器已关闭",
),
)
}
struct DesktopReleaseUpdateTask {
checker: DesktopReleaseChecker,
first_tick: bool,
}
impl PeriodicTaskRunner for DesktopReleaseUpdateTask {
fn tick(&mut self) -> super::periodic::TaskFuture<'_, ()> {
Box::pin(async move {
let first_tick = std::mem::take(&mut self.first_tick);
match self.checker.check_and_record().await {
Ok(release) => info!(
version = %release.version,
build = %release.build,
"Codex Desktop 最新发布信息已刷新"
),
Err(error) if first_tick => {
warn!(error = %error, "Codex Desktop 首次发布检查失败");
}
Err(error) => {
warn!(error = %error, "Codex Desktop 定期发布检查失败");
}
}
})
}
}
@@ -1,85 +0,0 @@
//! 指纹更新任务接线。
use std::path::PathBuf;
use tracing::{info, warn};
use crate::upstream::openai::fingerprint::{
APPCAST_POLL_INTERVAL, PgFingerprintStore, RuntimeFingerprint, UpdateChecker,
};
use super::{
coordinator::SchedulerHandle,
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, spawn_periodic_task},
};
/// Codex Desktop 官方 appcast 地址。
pub const CODEX_DESKTOP_APPCAST_URL: &str =
"https://persistent.oaistatic.com/codex-app-prod/appcast.xml";
/// 本地可选完整指纹提取文件路径。
pub const DEFAULT_EXTRACTED_FINGERPRINT_PATH: &str = "data/extracted-fingerprint.json";
/// 启动指纹自动更新后台任务。
pub fn start_fingerprint_update_task(
store: PgFingerprintStore,
runtime_fingerprint: RuntimeFingerprint,
appcast_url: String,
extracted_fingerprint_path: PathBuf,
current_version: String,
current_build: String,
) -> SchedulerHandle {
let update_checker = UpdateChecker::with_client(
store,
reqwest::Client::new(),
appcast_url,
extracted_fingerprint_path,
current_version,
current_build,
);
spawn_periodic_task(
FingerprintUpdateTask {
checker: update_checker,
runtime_fingerprint,
first_tick: true,
},
PeriodicTaskConfig::new(
APPCAST_POLL_INTERVAL.as_secs(),
"Fingerprint 后台版本检查器已启动",
"Fingerprint 后台版本检查器已关闭",
),
)
}
struct FingerprintUpdateTask {
checker: UpdateChecker,
runtime_fingerprint: RuntimeFingerprint,
first_tick: bool,
}
impl PeriodicTaskRunner for FingerprintUpdateTask {
fn tick(&mut self) -> super::periodic::TaskFuture<'_, ()> {
Box::pin(async move {
let first_tick = std::mem::take(&mut self.first_tick);
match self.checker.check_and_apply_update().await {
Ok(Some(fingerprint)) => {
let app_version = fingerprint.app_version.clone();
let build_number = fingerprint.build_number.clone();
self.runtime_fingerprint.replace(fingerprint);
info!(
app_version = %app_version,
build_number = %build_number,
"Fingerprint 运行时版本已更新"
);
}
Ok(None) => {}
Err(error) => {
if first_tick {
warn!(error = %error, "Fingerprint 首次版本检查失败");
} else {
warn!(error = %error, "Fingerprint 定期版本检查失败");
}
}
}
})
}
}
+1 -1
View File
@@ -3,7 +3,7 @@
mod cleanup;
pub mod cookie_cleanup;
pub mod coordinator;
pub mod fingerprint_update;
pub mod desktop_release_update;
pub mod model_refresh;
mod periodic;
pub mod quota_refresh;
+2 -6
View File
@@ -35,6 +35,7 @@ impl QuotaRefreshTask {
store: PgAccountStore,
usage_store: PgAccountUsageStore,
codex: Arc<CodexBackendClient>,
account_pool: Arc<AccountPoolService>,
account_pseudonymizer: Arc<AccountPseudonymizer>,
interval_secs: u64,
min_refresh_interval_secs: u64,
@@ -44,6 +45,7 @@ impl QuotaRefreshTask {
store,
usage_store,
codex,
account_pool,
account_pseudonymizer,
min_refresh_interval_secs,
),
@@ -58,12 +60,6 @@ impl QuotaRefreshTask {
self
}
/// 设置运行时账号池,用于刷新后同步内存调度状态。
pub fn with_account_pool(mut self, account_pool: Arc<AccountPoolService>) -> Self {
self.service = self.service.with_account_pool(account_pool);
self
}
/// 启动后台刷新任务。
pub fn start(self) -> SchedulerHandle {
let config = PeriodicTaskConfig::new(
+3 -21
View File
@@ -3,12 +3,9 @@
use chrono::Utc;
use tracing::{info, warn};
use crate::{
telemetry::{
buckets::store::PgRequestBucketStore, ops::store::PgOpsErrorLogStore,
usage::store::PgUsageRecordStore,
},
upstream::openai::fingerprint::PgFingerprintStore,
use crate::telemetry::{
buckets::store::PgRequestBucketStore, ops::store::PgOpsErrorLogStore,
usage::store::PgUsageRecordStore,
};
use super::{
@@ -17,13 +14,11 @@ use super::{
};
const RETENTION_TRIM_INTERVAL_SECS: u64 = 60 * 60;
const FINGERPRINT_HISTORY_LIMIT: std::num::NonZeroU32 = std::num::NonZeroU32::new(100).unwrap();
pub struct RetentionTrimTask {
usage_records: PgUsageRecordStore,
ops_errors: PgOpsErrorLogStore,
buckets: PgRequestBucketStore,
fingerprints: PgFingerprintStore,
}
impl RetentionTrimTask {
@@ -31,13 +26,11 @@ impl RetentionTrimTask {
usage_records: PgUsageRecordStore,
ops_errors: PgOpsErrorLogStore,
buckets: PgRequestBucketStore,
fingerprints: PgFingerprintStore,
) -> Self {
Self {
usage_records,
ops_errors,
buckets,
fingerprints,
}
}
@@ -70,17 +63,6 @@ impl RetentionTrimTask {
Ok(_) => {}
Err(error) => warn!(error = %error, "清理请求时间桶失败"),
}
match self
.fingerprints
.trim_update_history_to_limit(FINGERPRINT_HISTORY_LIMIT)
.await
{
Ok(deleted) if deleted > 0 => {
info!(deleted, "已清理超限指纹更新历史")
}
Ok(_) => {}
Err(error) => warn!(error = %error, "清理超限指纹更新历史失败"),
}
}
}
@@ -2,19 +2,25 @@
use crate::{
dispatch::{
controllers::{
ControllerFailureFact,
account_state::{AccountStateEffect, AccountStateEffects},
},
controllers::ControllerFailureFact,
errors::ClientFailure,
failure::exhaustion::{AccountExhaustionRecord, ExhaustedAccountKind},
lifecycle::contract::{
AttemptDecision, AttemptObservation, AttemptObservationKind, CompleteResponseFacts,
},
transport::observation::{UpstreamFailureFacts, UpstreamFailureKind},
},
fleet::{account::AccountStatus, pool::AccountPoolService},
upstream::openai::{protocol::responses::ResponsesSseFailure, transport::CodexBackendClient},
fleet::{
account_failure::{
AccountFailureKind, AccountStateEffect, apply_account_state_effect_immediately,
classify_response_failure, classify_upstream_failure,
},
pool::AccountPoolService,
},
upstream::openai::{
failure::{UpstreamFailureFacts, UpstreamFailureKind},
protocol::responses::ResponsesSseFailure,
transport::CodexBackendClient,
},
};
pub(super) struct AccountFailureController;
@@ -46,7 +52,7 @@ impl AccountFailureController {
return;
};
if let Some(effect) = &classified.effect {
AccountStateEffects::apply(account_pool, codex, account_id, effect).await;
apply_account_state_effect_immediately(account_pool, codex, account_id, effect).await;
}
}
@@ -110,15 +116,7 @@ impl AccountFailureController {
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase();
let error_type = failure
.upstream_type
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase();
let message = failure.message.to_ascii_lowercase();
if is_model_unsupported(&code)
|| is_model_unsupported(&message)
|| code.contains("invalid_request")
if code.contains("invalid_request")
|| code.contains("not_found")
|| code.contains("context_window")
|| code.contains("invalid_prompt")
@@ -126,17 +124,18 @@ impl AccountFailureController {
{
return Some(ClientFailure::new(failure.clone(), 400, false));
}
if is_auth_failure(&code, &error_type, &message) {
return Some(ClientFailure::new(failure.clone(), 401, false));
}
if code.contains("forbidden") || code.contains("banned") || error_type == "permission_error"
{
return Some(ClientFailure::new(failure.clone(), 403, false));
}
if code.contains("server_overloaded") {
return Some(ClientFailure::new(failure.clone(), 503, true));
}
None
classify_response_failure(failure).map(|classified| {
let status = match classified.kind {
AccountFailureKind::ModelUnsupported => 400,
AccountFailureKind::Expired => 401,
AccountFailureKind::Disabled | AccountFailureKind::Banned => 403,
AccountFailureKind::QuotaExhausted | AccountFailureKind::RateLimited => 429,
};
ClientFailure::new(failure.clone(), status, false)
})
}
}
@@ -168,167 +167,42 @@ fn classify_failure(
}
fn classify_upstream(account_id: &str, facts: &UpstreamFailureFacts) -> Option<ClassifiedFailure> {
let status = facts.status_code;
let body = facts.body.as_str();
let code = facts.code.as_deref().unwrap_or_default();
let error_type = facts.error_type.as_deref().unwrap_or_default();
let message = facts.message.as_str();
if is_model_unsupported(code) || is_model_unsupported(message) || is_model_unsupported(body) {
return Some(model_unsupported_failure(account_id, body));
}
if let Some(account_status) = explicit_account_status(code, error_type, body) {
return Some(account_status_failure(
account_id,
account_status,
body,
status,
));
}
if status == Some(403) && !is_html(body) {
return Some(account_status_failure(
account_id,
AccountStatus::Banned,
body,
status,
));
}
if status == Some(401) {
return Some(account_status_failure(
account_id,
AccountStatus::Expired,
body,
status,
));
}
None
classified_failure(
account_id,
&facts.body,
facts.status_code,
classify_upstream_failure(facts)?,
)
}
fn classify_sse(account_id: &str, failure: &ResponsesSseFailure) -> Option<ClassifiedFailure> {
let code = failure.upstream_code.as_deref().unwrap_or_default();
let error_type = failure.upstream_type.as_deref().unwrap_or_default();
if is_model_unsupported(code) || is_model_unsupported(&failure.message) {
return Some(model_unsupported_failure(
account_id,
&crate::dispatch::failure::sse::sse_failure_error_body(failure),
));
}
let account_status =
explicit_account_status(code, error_type, &failure.message).or_else(|| {
(code.eq_ignore_ascii_case("forbidden") || error_type == "permission_error")
.then_some(AccountStatus::Banned)
})?;
Some(account_status_failure(
classified_failure(
account_id,
account_status,
&crate::dispatch::failure::sse::sse_failure_error_body(failure),
failure.explicit_status_code,
))
classify_response_failure(failure)?,
)
}
fn model_unsupported_failure(account_id: &str, message: &str) -> ClassifiedFailure {
ClassifiedFailure {
exhaustion: AccountExhaustionRecord::new(
account_id,
ExhaustedAccountKind::ModelUnsupported,
message,
),
effect: None,
}
}
fn account_status_failure(
fn classified_failure(
account_id: &str,
status: AccountStatus,
message: &str,
status_code: Option<u16>,
) -> ClassifiedFailure {
let kind = match status {
AccountStatus::Expired => ExhaustedAccountKind::Expired,
AccountStatus::Disabled => ExhaustedAccountKind::Disabled,
AccountStatus::Banned => ExhaustedAccountKind::Banned,
AccountStatus::Active | AccountStatus::QuotaExhausted => {
unreachable!("account failure controller only invalidates unavailable statuses")
}
classified: crate::fleet::account_failure::ClassifiedAccountFailure,
) -> Option<ClassifiedFailure> {
let kind = match classified.kind {
AccountFailureKind::ModelUnsupported => ExhaustedAccountKind::ModelUnsupported,
AccountFailureKind::Expired => ExhaustedAccountKind::Expired,
AccountFailureKind::Disabled => ExhaustedAccountKind::Disabled,
AccountFailureKind::Banned => ExhaustedAccountKind::Banned,
AccountFailureKind::QuotaExhausted | AccountFailureKind::RateLimited => return None,
};
let mut exhaustion = AccountExhaustionRecord::new(account_id, kind, message);
if let Some(status_code) = status_code {
exhaustion = exhaustion.with_status_code(status_code);
}
ClassifiedFailure {
Some(ClassifiedFailure {
exhaustion,
effect: Some(AccountStateEffect::SetStatus(status)),
}
}
fn explicit_account_status(code: &str, error_type: &str, message: &str) -> Option<AccountStatus> {
let code = code.trim().to_ascii_lowercase();
let error_type = error_type.trim().to_ascii_lowercase();
let message = message.to_ascii_lowercase();
if matches!(
code.as_str(),
"identity_verification_required" | "verification_required"
) || message.contains("identity verification is required")
{
return Some(AccountStatus::Disabled);
}
if matches!(
code.as_str(),
"account_banned"
| "account_deactivated"
| "account_disabled"
| "account_suspended"
| "deactivated_workspace"
| "organization_disabled"
| "workspace_deactivated"
) || message.contains("account is banned")
|| message.contains("account has been banned")
|| message.contains("account deactivated")
|| message.contains("account has been deactivated")
|| message.contains("account disabled")
|| message.contains("account has been disabled")
|| message.contains("account suspended")
|| message.contains("organization has been disabled")
|| message.contains("workspace has been deactivated")
|| message.contains("deactivated_workspace")
{
return Some(AccountStatus::Banned);
}
is_auth_failure(&code, &error_type, &message).then_some(AccountStatus::Expired)
}
fn is_auth_failure(code: &str, error_type: &str, message: &str) -> bool {
matches!(
code,
"token_invalid"
| "token_invalidated"
| "token_expired"
| "token_revoked"
| "refresh_token_invalidated"
| "unauthorized"
| "invalid_api_key"
| "authentication_error"
) || error_type == "authentication_error"
|| message.contains("token revoked")
|| message.contains("token invalidated")
|| message.contains("token invalid")
|| message.contains("token expired")
|| message.contains("unauthorized")
|| message.contains("invalid api key")
}
fn is_model_unsupported(value: &str) -> bool {
let value = value.to_ascii_lowercase();
value.contains("model_not_supported")
|| value.contains("model_not_available")
|| (value.contains("model")
&& (value.contains("not supported")
|| value.contains("not available")
|| value.contains("not_supported")
|| value.contains("not_available")))
}
fn is_html(value: &str) -> bool {
let value = value.trim_start().to_ascii_lowercase();
value.starts_with("<!doctype") || value.starts_with("<html") || value.contains("<html")
effect: classified.effect,
})
}
@@ -1,38 +0,0 @@
//! 明确账号事实触发的运行时失效与连接驱逐。
use chrono::{DateTime, Utc};
use crate::{
fleet::{account::AccountStatus, pool::AccountPoolService},
upstream::openai::transport::CodexBackendClient,
};
pub(super) enum AccountStateEffect {
SetStatus(AccountStatus),
MarkQuotaLimitedUntil(DateTime<Utc>),
}
pub(super) struct AccountStateEffects;
impl AccountStateEffects {
pub(super) async fn apply(
account_pool: &AccountPoolService,
codex: &CodexBackendClient,
account_id: &str,
effect: &AccountStateEffect,
) {
match effect {
AccountStateEffect::SetStatus(status) => {
account_pool
.set_status_immediately(account_id, *status)
.await;
}
AccountStateEffect::MarkQuotaLimitedUntil(until) => {
account_pool
.mark_quota_limited_until_immediately(account_id, *until)
.await;
}
}
codex.evict_websocket_account(account_id).await;
}
}
@@ -9,13 +9,13 @@ use crate::{
AttemptDecision, AttemptObservation, AttemptObservationKind, AttemptReturnKind,
},
lifecycle::stream::StreamTerminal,
transport::observation::UpstreamFailureFacts,
},
fleet::{
account::AccountStatus,
cookies::{CloudflareChallengeCooldownTracker, CloudflarePathBlockTracker, PgCookieStore},
pool::AccountPoolService,
},
upstream::openai::failure::UpstreamFailureFacts,
};
pub(super) struct CloudflareController;
@@ -15,9 +15,11 @@ use crate::{
controllers::ControllerFailureFact,
errors::ClientFailure,
lifecycle::contract::{AttemptDecision, AttemptObservation, AttemptReturnKind},
transport::observation::UpstreamFailureFacts,
},
upstream::openai::protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
upstream::openai::{
failure::UpstreamFailureFacts,
protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
},
};
use self::{store::Store, types::SessionState};
+1 -1
View File
@@ -13,9 +13,9 @@ use crate::{
AttemptDecision, AttemptObservation, AttemptObservationKind, AttemptReturnKind,
AttemptRoutingFacts, CompleteResponseFacts, PinnedCandidateAcquireFailureKind,
},
transport::observation::{UpstreamFailureFacts, UpstreamFailureKind},
},
upstream::openai::{
failure::{UpstreamFailureFacts, UpstreamFailureKind},
protocol::responses::{CodexResponsesRequest, PreviousResponseScope, StreamCommitPolicy},
transport::CodexBackendTransport,
transport::websocket::PreviousResponseUnavailableReason,
+4 -3
View File
@@ -49,14 +49,15 @@ use crate::{
},
stream::StreamTerminal,
},
transport::observation::UpstreamFailureFacts,
},
fleet::pool::AccountPoolService,
upstream::openai::protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
upstream::openai::{
failure::UpstreamFailureFacts,
protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
},
};
mod account_failure;
mod account_state;
mod affinity;
pub(crate) mod cloudflare;
pub(crate) mod cyber_policy;
+25 -123
View File
@@ -1,23 +1,22 @@
//! 请求前配额验证规则 owner。
use chrono::{Duration, Utc};
use serde_json::Value;
use crate::{
dispatch::{
affinity::AccountIdentityService,
controllers::{
ControllerFailureFact,
account_state::{AccountStateEffect, AccountStateEffects},
},
controllers::ControllerFailureFact,
errors::ClientFailure,
failure::exhaustion::{AccountExhaustionRecord, ExhaustedAccountKind},
lifecycle::contract::{
AttemptDecision, AttemptObservation, AttemptObservationKind, AttemptReturnKind,
},
transport::observation::UpstreamFailureFacts,
},
fleet::{
account_failure::{
AccountFailureKind, AccountStateEffect, apply_account_state_effect_immediately,
classify_response_failure, classify_upstream_failure,
},
pool::{AccountLease, AccountPoolService},
quota::{
quota_from_usage, quota_snapshot_limit_reached, quota_snapshot_limit_window_seconds,
@@ -25,13 +24,13 @@ use crate::{
},
},
upstream::openai::{
failure::UpstreamFailureFacts,
protocol::responses::ResponsesSseFailure,
transport::{CodexBackendClient, CodexRequestContext},
},
};
const LIMIT_REACHED_MESSAGE: &str = "Upstream usage quota still reports limit_reached";
const DEFAULT_RATE_LIMIT_RETRY_SECONDS: u64 = 60;
pub(super) enum QuotaEnterOutcome {
Ready(Box<AccountLease>),
@@ -167,7 +166,7 @@ impl QuotaController {
return;
};
if let Some(effect) = &classified.effect {
AccountStateEffects::apply(account_pool, codex, account_id, effect).await;
apply_account_state_effect_immediately(account_pool, codex, account_id, effect).await;
}
}
@@ -201,131 +200,34 @@ fn classify_upstream(
account_id: &str,
facts: &UpstreamFailureFacts,
) -> Option<ClassifiedQuotaFailure> {
let signaled_kind = facts
.code
.as_deref()
.and_then(classify_quota_signal)
.or_else(|| facts.error_type.as_deref().and_then(classify_quota_signal))
.or_else(|| classify_quota_message(&facts.message));
match signaled_kind.or(match facts.status_code {
Some(429) => Some(StreamQuotaKind::RateLimited),
Some(402) => Some(StreamQuotaKind::QuotaExhausted),
_ => None,
})? {
StreamQuotaKind::RateLimited => Some(rate_limited_failure(
account_id,
facts.body.clone(),
facts.retry_after_seconds,
)),
StreamQuotaKind::QuotaExhausted => {
Some(quota_exhausted_failure(account_id, facts.body.clone()))
}
}
classified_quota_failure(account_id, &facts.body, classify_upstream_failure(facts)?)
}
fn classify_stream(
account_id: &str,
failure: &ResponsesSseFailure,
) -> Option<ClassifiedQuotaFailure> {
let kind = classify_stream_signal(failure).or(match failure.explicit_status_code {
Some(429) => Some(StreamQuotaKind::RateLimited),
Some(402) => Some(StreamQuotaKind::QuotaExhausted),
_ => None,
})?;
let body = crate::dispatch::failure::sse::sse_failure_error_body(failure);
Some(match kind {
StreamQuotaKind::RateLimited => {
rate_limited_failure(account_id, body, failure.retry_after_seconds)
}
StreamQuotaKind::QuotaExhausted => quota_exhausted_failure(account_id, body),
})
classified_quota_failure(account_id, &body, classify_response_failure(failure)?)
}
#[derive(Clone, Copy)]
enum StreamQuotaKind {
RateLimited,
QuotaExhausted,
}
fn classify_stream_signal(failure: &ResponsesSseFailure) -> Option<StreamQuotaKind> {
failure
.upstream_code
.as_deref()
.and_then(classify_quota_signal)
.or_else(|| {
failure
.upstream_type
.as_deref()
.and_then(classify_quota_signal)
})
.or_else(|| classify_quota_message(&failure.message))
}
fn classify_quota_signal(signal: &str) -> Option<StreamQuotaKind> {
let signal = signal.trim().to_ascii_lowercase();
match signal.as_str() {
"usage_limit_reached"
| "rate_limit_exceeded"
| "rate_limit_reached"
| "rate_limit_error"
| "workspace_owner_usage_limit_reached"
| "workspace_member_usage_limit_reached" => Some(StreamQuotaKind::RateLimited),
"quota_exhausted"
| "quota_exceeded"
| "payment_required"
| "insufficient_quota"
| "workspace_owner_credits_depleted"
| "workspace_member_credits_depleted" => Some(StreamQuotaKind::QuotaExhausted),
signal if signal.starts_with("billing_limit") => Some(StreamQuotaKind::QuotaExhausted),
_ => None,
}
}
fn classify_quota_message(message: &str) -> Option<StreamQuotaKind> {
let message = message.to_ascii_lowercase();
if message.contains("rate limit") || message.contains("usage limit") {
return Some(StreamQuotaKind::RateLimited);
}
if message.contains("quota")
|| message.contains("payment required")
|| message.contains("billing limit")
{
return Some(StreamQuotaKind::QuotaExhausted);
}
None
}
fn rate_limited_failure(
fn classified_quota_failure(
account_id: &str,
message: String,
retry_after_seconds: Option<u64>,
) -> ClassifiedQuotaFailure {
let seconds = retry_after_seconds
.unwrap_or(DEFAULT_RATE_LIMIT_RETRY_SECONDS)
.min(i64::MAX as u64) as i64;
ClassifiedQuotaFailure {
exhaustion: AccountExhaustionRecord::new(
account_id,
ExhaustedAccountKind::RateLimited,
message,
),
effect: Some(AccountStateEffect::MarkQuotaLimitedUntil(
Utc::now() + Duration::seconds(seconds),
)),
}
}
fn quota_exhausted_failure(account_id: &str, message: String) -> ClassifiedQuotaFailure {
ClassifiedQuotaFailure {
exhaustion: AccountExhaustionRecord::new(
account_id,
ExhaustedAccountKind::QuotaExhausted,
message,
),
effect: Some(AccountStateEffect::SetStatus(
crate::fleet::account::AccountStatus::QuotaExhausted,
)),
}
message: &str,
classified: crate::fleet::account_failure::ClassifiedAccountFailure,
) -> Option<ClassifiedQuotaFailure> {
let kind = match classified.kind {
AccountFailureKind::RateLimited => ExhaustedAccountKind::RateLimited,
AccountFailureKind::QuotaExhausted => ExhaustedAccountKind::QuotaExhausted,
AccountFailureKind::ModelUnsupported
| AccountFailureKind::Expired
| AccountFailureKind::Disabled
| AccountFailureKind::Banned => return None,
};
Some(ClassifiedQuotaFailure {
exhaustion: AccountExhaustionRecord::new(account_id, kind, message),
effect: classified.effect,
})
}
fn acquired_with_verified_quota(mut acquired: AccountLease, quota: &Value) -> AccountLease {
@@ -546,6 +546,9 @@ pub(super) async fn record_live_response_stream_event(
status_code,
body,
);
if !is_success_usage_event(level, status_code) {
ensure_stream_failure_class(&mut metadata);
}
log_live_response_stream_finalized(context, status_code, level, message, &metadata);
enrich_usage_record_identity(
&mut metadata,
@@ -601,6 +604,19 @@ fn is_success_usage_event(level: UsageRecordLevel, status_code: i64) -> bool {
level != UsageRecordLevel::Error && status_code < 400
}
fn ensure_stream_failure_class(metadata: &mut Value) {
let failure_class = ["failureClass", "upstreamCode", "terminal"]
.into_iter()
.find_map(|field| metadata_string_field(metadata, field))
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string);
let (Some(failure_class), Some(object)) = (failure_class, metadata.as_object_mut()) else {
return;
};
object.insert("failureClass".to_string(), Value::String(failure_class));
}
fn log_live_response_stream_finalized(
context: &StreamContext<'_>,
status_code: i64,
+1 -1
View File
@@ -21,7 +21,6 @@ use crate::{
create_response_with_account, prepare_response_transport_with_account,
},
canonical::{CanonicalStreamDecoder, normalize_complete_response},
observation::upstream_failure_facts,
prefetch::{PrefetchedStream, StreamPrefetchError, prefetch_until_commit},
},
},
@@ -30,6 +29,7 @@ use crate::{
pool::{AccountCandidateLease, AccountPoolService},
},
upstream::openai::{
failure::upstream_failure_facts,
protocol::responses::{
CodexResponsesRequest, CollectedResponse, TransportRequirement, transport_requirement,
},
+2 -4
View File
@@ -6,13 +6,11 @@ use crate::{
dispatch::{
errors::{ClientFailure, ResponseDispatchError},
failure::exhaustion::{AccountExhaustionRecord, ExhaustedAccount},
transport::{
canonical::{CanonicalStreamBatch, CanonicalStreamDecoder},
observation::UpstreamFailureFacts,
},
transport::canonical::{CanonicalStreamBatch, CanonicalStreamDecoder},
},
fleet::{account::Account, pool::AccountLease},
upstream::openai::{
failure::UpstreamFailureFacts,
protocol::{
responses::{CodexResponsesRequest, CollectedResponse, ResponsesSseFailure},
sse::SseError,
+5 -2
View File
@@ -69,14 +69,17 @@ impl StreamFinalizer {
controllers
.leave_stream(controller_scope, controller_context, &summary, &body)
.await;
if timeout(LEASE_FINALIZE_TIMEOUT, account_lease.complete())
let mut lease_completion = Box::pin(account_lease.complete());
if timeout(LEASE_FINALIZE_TIMEOUT, lease_completion.as_mut())
.await
.is_err()
{
tracing::warn!(
timeout_ms = LEASE_FINALIZE_TIMEOUT.as_millis(),
"Timed out finalizing response account lease"
"Response account lease is continuing finalization in the background"
);
// 超时只约束客户端收尾,不取消已经开始的 slot 释放与请求用量持久化。
drop(tokio::spawn(lease_completion));
}
finish_client_stream(sender, event_sender, &mut summary).await;
}
+1 -2
View File
@@ -1,6 +1,5 @@
//! Attempt 阶段的账号作用域上游调用与事实规范化。
//! Attempt 阶段的账号作用域上游调用与流规范化。
pub(in crate::dispatch) mod account;
pub(crate) mod canonical;
pub(in crate::dispatch) mod observation;
pub(in crate::dispatch) mod prefetch;
+317
View File
@@ -0,0 +1,317 @@
//! 上游失败到账号状态 effect 的唯一领域分类。
use chrono::{Duration, Utc};
use crate::{
fleet::{account::AccountStatus, pool::AccountPoolService},
upstream::openai::{
failure::{UpstreamFailureFacts, upstream_failure_facts},
protocol::responses::ResponsesSseFailure,
transport::{CodexBackendClient, CodexClientError},
},
};
const DEFAULT_RATE_LIMIT_RETRY_SECONDS: u64 = 60;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountFailureKind {
ModelUnsupported,
Expired,
Disabled,
Banned,
QuotaExhausted,
RateLimited,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AccountStateEffect {
SetStatus(AccountStatus),
MarkQuotaLimitedUntil(chrono::DateTime<Utc>),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ClassifiedAccountFailure {
pub kind: AccountFailureKind,
pub effect: Option<AccountStateEffect>,
}
impl ClassifiedAccountFailure {
pub fn account_status(&self) -> Option<AccountStatus> {
match self.effect.as_ref()? {
AccountStateEffect::SetStatus(status) => Some(*status),
AccountStateEffect::MarkQuotaLimitedUntil(_) => None,
}
}
}
pub fn classify_client_failure(error: &CodexClientError) -> Option<ClassifiedAccountFailure> {
classify_upstream_failure(&upstream_failure_facts(error))
}
pub fn classify_upstream_failure(facts: &UpstreamFailureFacts) -> Option<ClassifiedAccountFailure> {
classify_failure(FailureFields {
status_code: facts.status_code,
code: facts.code.as_deref(),
error_type: facts.error_type.as_deref(),
message: &facts.message,
body: &facts.body,
retry_after_seconds: facts.retry_after_seconds,
})
}
pub fn classify_response_failure(
failure: &ResponsesSseFailure,
) -> Option<ClassifiedAccountFailure> {
classify_failure(FailureFields {
status_code: failure.explicit_status_code,
code: failure.upstream_code.as_deref(),
error_type: failure.upstream_type.as_deref(),
message: &failure.message,
body: &failure.message,
retry_after_seconds: failure.retry_after_seconds,
})
}
pub async fn apply_account_state_effect_immediately(
account_pool: &AccountPoolService,
codex: &CodexBackendClient,
account_id: &str,
effect: &AccountStateEffect,
) {
match effect {
AccountStateEffect::SetStatus(status) => {
account_pool
.set_status_immediately(account_id, *status)
.await;
}
AccountStateEffect::MarkQuotaLimitedUntil(until) => {
account_pool
.mark_quota_limited_until_immediately(account_id, *until)
.await;
}
}
codex.evict_websocket_account(account_id).await;
}
pub async fn apply_account_state_effect(
account_pool: &AccountPoolService,
codex: &CodexBackendClient,
account_id: &str,
effect: &AccountStateEffect,
) -> bool {
let updated = match effect {
AccountStateEffect::SetStatus(status) => account_pool.set_status(account_id, *status).await,
AccountStateEffect::MarkQuotaLimitedUntil(until) => {
account_pool
.mark_quota_limited_until(account_id, *until)
.await
}
};
codex.evict_websocket_account(account_id).await;
updated
}
pub async fn apply_client_failure_effect(
account_pool: &AccountPoolService,
codex: &CodexBackendClient,
account_id: &str,
error: &CodexClientError,
) -> Option<ClassifiedAccountFailure> {
let classified = classify_client_failure(error)?;
if let Some(effect) = &classified.effect {
apply_account_state_effect(account_pool, codex, account_id, effect).await;
}
Some(classified)
}
struct FailureFields<'a> {
status_code: Option<u16>,
code: Option<&'a str>,
error_type: Option<&'a str>,
message: &'a str,
body: &'a str,
retry_after_seconds: Option<u64>,
}
fn classify_failure(fields: FailureFields<'_>) -> Option<ClassifiedAccountFailure> {
let code = fields.code.unwrap_or_default();
let error_type = fields.error_type.unwrap_or_default();
if [code, fields.message, fields.body]
.into_iter()
.any(is_model_unsupported)
{
return Some(classified(AccountFailureKind::ModelUnsupported, None));
}
if let Some(status) = explicit_account_status(code, error_type, fields.message)
.or_else(|| explicit_account_status(code, error_type, fields.body))
{
return Some(status_failure(status));
}
if fields.status_code == Some(403) && !is_html(fields.body) {
return Some(status_failure(AccountStatus::Banned));
}
if fields.status_code == Some(401) {
return Some(status_failure(AccountStatus::Expired));
}
let quota_kind = [code, error_type]
.into_iter()
.find_map(classify_quota_signal)
.or_else(|| classify_quota_message(fields.message))
.or_else(|| classify_quota_message(fields.body))
.or(match fields.status_code {
Some(429) => Some(AccountFailureKind::RateLimited),
Some(402) => Some(AccountFailureKind::QuotaExhausted),
_ => None,
})?;
Some(match quota_kind {
AccountFailureKind::RateLimited => {
let seconds = fields
.retry_after_seconds
.unwrap_or(DEFAULT_RATE_LIMIT_RETRY_SECONDS)
.min(i64::MAX as u64) as i64;
classified(
AccountFailureKind::RateLimited,
Some(AccountStateEffect::MarkQuotaLimitedUntil(
Utc::now() + Duration::seconds(seconds),
)),
)
}
AccountFailureKind::QuotaExhausted => classified(
AccountFailureKind::QuotaExhausted,
Some(AccountStateEffect::SetStatus(AccountStatus::QuotaExhausted)),
),
AccountFailureKind::ModelUnsupported
| AccountFailureKind::Expired
| AccountFailureKind::Disabled
| AccountFailureKind::Banned => unreachable!("quota classifier returned account status"),
})
}
fn status_failure(status: AccountStatus) -> ClassifiedAccountFailure {
let kind = match status {
AccountStatus::Expired => AccountFailureKind::Expired,
AccountStatus::Disabled => AccountFailureKind::Disabled,
AccountStatus::Banned => AccountFailureKind::Banned,
AccountStatus::Active | AccountStatus::QuotaExhausted => {
unreachable!("explicit account failure must make the account unavailable")
}
};
classified(kind, Some(AccountStateEffect::SetStatus(status)))
}
fn classified(
kind: AccountFailureKind,
effect: Option<AccountStateEffect>,
) -> ClassifiedAccountFailure {
ClassifiedAccountFailure { kind, effect }
}
fn explicit_account_status(code: &str, error_type: &str, message: &str) -> Option<AccountStatus> {
let code = code.trim().to_ascii_lowercase();
let error_type = error_type.trim().to_ascii_lowercase();
let message = message.to_ascii_lowercase();
if matches!(
code.as_str(),
"identity_verification_required" | "verification_required"
) || message.contains("identity verification is required")
{
return Some(AccountStatus::Disabled);
}
if matches!(
code.as_str(),
"account_banned"
| "account_deactivated"
| "account_disabled"
| "account_suspended"
| "deactivated_workspace"
| "organization_disabled"
| "workspace_deactivated"
) || message.contains("account is banned")
|| message.contains("account has been banned")
|| message.contains("account deactivated")
|| message.contains("account has been deactivated")
|| message.contains("account disabled")
|| message.contains("account has been disabled")
|| message.contains("account suspended")
|| message.contains("organization has been disabled")
|| message.contains("workspace has been deactivated")
|| message.contains("deactivated_workspace")
|| code == "forbidden"
|| error_type == "permission_error"
{
return Some(AccountStatus::Banned);
}
is_auth_failure(&code, &error_type, &message).then_some(AccountStatus::Expired)
}
fn is_auth_failure(code: &str, error_type: &str, message: &str) -> bool {
matches!(
code,
"token_invalid"
| "token_invalidated"
| "token_expired"
| "token_revoked"
| "refresh_token_invalidated"
| "unauthorized"
| "invalid_api_key"
| "authentication_error"
) || error_type == "authentication_error"
|| message.contains("token revoked")
|| message.contains("token invalidated")
|| message.contains("token invalid")
|| message.contains("token expired")
|| message.contains("unauthorized")
|| message.contains("invalid api key")
}
fn classify_quota_signal(signal: &str) -> Option<AccountFailureKind> {
let signal = signal.trim().to_ascii_lowercase();
match signal.as_str() {
"usage_limit_reached"
| "rate_limit_exceeded"
| "rate_limit_reached"
| "rate_limit_error"
| "workspace_owner_usage_limit_reached"
| "workspace_member_usage_limit_reached" => Some(AccountFailureKind::RateLimited),
"quota_exhausted"
| "quota_exceeded"
| "payment_required"
| "insufficient_quota"
| "workspace_owner_credits_depleted"
| "workspace_member_credits_depleted" => Some(AccountFailureKind::QuotaExhausted),
signal if signal.starts_with("billing_limit") => Some(AccountFailureKind::QuotaExhausted),
_ => None,
}
}
fn classify_quota_message(message: &str) -> Option<AccountFailureKind> {
let message = message.to_ascii_lowercase();
if message.contains("rate limit") || message.contains("usage limit") {
return Some(AccountFailureKind::RateLimited);
}
if message.contains("quota")
|| message.contains("payment required")
|| message.contains("billing limit")
{
return Some(AccountFailureKind::QuotaExhausted);
}
None
}
fn is_model_unsupported(value: &str) -> bool {
let value = value.to_ascii_lowercase();
value.contains("model_not_supported")
|| value.contains("model_not_available")
|| (value.contains("model")
&& (value.contains("not supported")
|| value.contains("not available")
|| value.contains("not_supported")
|| value.contains("not_available")))
}
fn is_html(value: &str) -> bool {
let value = value.trim_start().to_ascii_lowercase();
value.starts_with("<!doctype") || value.starts_with("<html") || value.contains("<html")
}
+8 -49
View File
@@ -7,16 +7,18 @@ use serde_json::{Value, json};
use tokio::sync::mpsc;
use crate::{
fleet::{account::AccountStatus, store::StoredAccount},
fleet::{
account::AccountStatus,
account_failure::{classify_client_failure, classify_response_failure},
store::StoredAccount,
},
models::{service::ModelRefreshPlanAccount, types::CodexModelInfo},
upstream::openai::{
protocol::{
responses::{CodexResponsesRequest, ResponsesSseFailure},
sse::{SseEvent, encode_sse_event, parse_sse_events},
},
transport::{
CodexClientError, CodexRequestContext, is_banned_auth_signal, is_banned_upstream_error,
},
transport::{CodexClientError, CodexRequestContext},
},
};
@@ -366,54 +368,11 @@ async fn process_sse_event(
}
fn account_status_from_client_error(error: &CodexClientError) -> Option<AccountStatus> {
if is_banned_upstream_error(error) {
Some(AccountStatus::Banned)
} else {
match error {
CodexClientError::Upstream { status, .. } if status.as_u16() == 402 => {
Some(AccountStatus::QuotaExhausted)
}
CodexClientError::Upstream { status, body, .. } if status.as_u16() == 401 => {
Some(if is_banned_auth_signal(body) {
AccountStatus::Banned
} else {
AccountStatus::Expired
})
}
_ => None,
}
}
classify_client_failure(error)?.account_status()
}
fn sse_failure_account_status(failure: &ResponsesSseFailure) -> Option<AccountStatus> {
let code = failure
.upstream_code
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase();
let message = failure.message.to_ascii_lowercase();
if matches!(code.as_str(), "quota_exceeded" | "insufficient_quota") || message.contains("quota")
{
return Some(AccountStatus::QuotaExhausted);
}
let authentication_failed = matches!(
code.as_str(),
"token_invalid"
| "token_expired"
| "token_revoked"
| "account_deactivated"
| "unauthorized"
| "invalid_api_key"
) || message.contains("token revoked")
|| message.contains("token invalid")
|| message.contains("token expired");
authentication_failed.then(|| {
if is_banned_auth_signal(&failure.message) {
AccountStatus::Banned
} else {
AccountStatus::Expired
}
})
classify_response_failure(failure)?.account_status()
}
fn take_sse_frame(buffer: &mut String) -> Option<String> {
+9 -5
View File
@@ -1,6 +1,7 @@
use secrecy::ExposeSecret;
use super::{AccountManageService, types::AccountManageError};
use crate::fleet::account_failure::apply_client_failure_effect;
impl AccountManageService {
async fn usage_cookie_header(&self, account_id: &str) -> Option<String> {
@@ -53,11 +54,14 @@ impl AccountManageService {
client_request_id: None,
turn_id: None,
};
let raw = self
.codex
.fetch_usage(context)
.await
.map_err(|e| AccountManageError::FetchQuota(e.to_string()))?;
let raw = match self.codex.fetch_usage(context).await {
Ok(raw) => raw,
Err(error) => {
apply_client_failure_effect(&self.account_pool, &self.codex, account_id, &error)
.await;
return Err(AccountManageError::FetchQuota(error.to_string()));
}
};
let normalized = crate::fleet::quota::quota_from_usage(&raw);
self.account_pool
.apply_quota_snapshot(account_id, &normalized)
+4 -5
View File
@@ -7,6 +7,7 @@ use thiserror::Error;
use crate::{
fleet::{
account::{AccountStatus, AccountStatus as AcctStatus},
account_failure::classify_client_failure,
store::StoredAccountMetadata,
},
upstream::openai::{token_client::RefreshFailure, transport::CodexClientError},
@@ -323,11 +324,9 @@ fn normalized_plan_type(value: &str) -> Option<String> {
}
pub(super) fn import_status_from_usage_error(error: &CodexClientError) -> Option<AccountStatus> {
if crate::upstream::openai::transport::is_banned_upstream_error(error) {
Some(AccountStatus::Banned)
} else {
None
}
classify_client_failure(error)?
.account_status()
.filter(|status| *status == AccountStatus::Banned)
}
pub(super) fn parse_account_status(status: &str) -> Result<AcctStatus, AccountManageError> {
+1
View File
@@ -1,6 +1,7 @@
//! 账号、调度、配额与刷新领域。
pub mod account;
pub mod account_failure;
pub mod cookies;
pub mod import;
pub mod manage;
+14 -13
View File
@@ -13,6 +13,7 @@ use tracing::warn;
use crate::fleet::{
account::{Account, AccountStatus},
account_failure::apply_client_failure_effect,
pool::AccountPoolService,
quota::{
quota_from_usage, quota_snapshot_limit_reached, quota_snapshot_limit_window_seconds,
@@ -41,7 +42,7 @@ pub struct QuotaRefreshService {
request_spacing: Duration,
account_pseudonymizer: Arc<crate::infra::identity::AccountPseudonymizer>,
cookie_store: Option<PgCookieStore>,
account_pool: Option<Arc<AccountPoolService>>,
account_pool: Arc<AccountPoolService>,
}
impl QuotaRefreshService {
@@ -50,6 +51,7 @@ impl QuotaRefreshService {
store: PgAccountStore,
usage_store: PgAccountUsageStore,
codex: Arc<CodexBackendClient>,
account_pool: Arc<AccountPoolService>,
account_pseudonymizer: Arc<crate::infra::identity::AccountPseudonymizer>,
) -> Self {
Self {
@@ -60,7 +62,7 @@ impl QuotaRefreshService {
request_spacing: Self::default_request_spacing(),
account_pseudonymizer,
cookie_store: None,
account_pool: None,
account_pool,
}
}
@@ -73,6 +75,7 @@ impl QuotaRefreshService {
store: PgAccountStore,
usage_store: PgAccountUsageStore,
codex: Arc<CodexBackendClient>,
account_pool: Arc<AccountPoolService>,
account_pseudonymizer: Arc<crate::infra::identity::AccountPseudonymizer>,
min_refresh_interval_secs: u64,
) -> Self {
@@ -84,7 +87,7 @@ impl QuotaRefreshService {
request_spacing: Self::default_request_spacing(),
account_pseudonymizer,
cookie_store: None,
account_pool: None,
account_pool,
}
}
@@ -100,12 +103,6 @@ impl QuotaRefreshService {
self
}
/// 设置运行时账号池,用于刷新后同步内存调度状态。
pub fn with_account_pool(mut self, account_pool: Arc<AccountPoolService>) -> Self {
self.account_pool = Some(account_pool);
self
}
/// 执行一次配额锁定或待验证账号刷新,复用调用方传入的最近刷新记录。
pub async fn refresh_locked_accounts(
&self,
@@ -211,6 +208,13 @@ impl QuotaRefreshService {
self.sync_runtime_pool_account(&account.id).await;
}
Err(error) => {
apply_client_failure_effect(
&self.account_pool,
&self.codex,
&account.id,
&error,
)
.await;
warn!(
account_id = %account.id,
error = %error,
@@ -238,10 +242,7 @@ impl QuotaRefreshService {
}
async fn sync_runtime_pool_account(&self, account_id: &str) {
let Some(account_pool) = &self.account_pool else {
return;
};
if let Err(error) = account_pool.sync_account_from_store(account_id).await {
if let Err(error) = self.account_pool.sync_account_from_store(account_id).await {
warn!(
account_id = %account_id,
error = %error,
@@ -231,35 +231,3 @@ create index idx_account_cookies_account_domain
on account_cookies(account_id, domain);
create index idx_account_cookies_expires
on account_cookies(expires_at) where expires_at is not null;
create table fingerprints (
id text primary key,
originator text not null,
app_version text not null,
build_number text not null,
platform text not null,
arch text not null,
chromium_version text not null,
user_agent_template text not null,
default_headers_json jsonb not null,
header_order_json jsonb not null,
source text not null,
created_at timestamptz not null,
updated_at timestamptz not null
);
create table fingerprint_update_history (
id text primary key,
current_fingerprint_id text not null references fingerprints(id) on delete cascade,
app_version text not null,
build_number text not null,
chromium_version text,
source text not null,
manifest_json jsonb,
created_at timestamptz not null
);
create index idx_fingerprint_update_history_created_id
on fingerprint_update_history(created_at desc, id desc);
create index idx_fingerprint_update_history_fingerprint
on fingerprint_update_history(current_fingerprint_id);
+26 -31
View File
@@ -42,12 +42,25 @@ impl AccountUsageQueryService {
.map_err(|_| AccountUsageQueryError::List)
}
/// 汇总请求时间桶保留期内的全局用量。
pub async fn retained_totals(&self) -> Result<UsageBucketTotals, AccountUsageQueryError> {
self.buckets
.retained_totals()
/// 汇总请求时间桶保留期内的全局用量与计费。
pub async fn retained_summary(&self) -> Result<RetainedUsageSummary, AccountUsageQueryError> {
let buckets = self
.buckets
.retained_usage_buckets()
.await
.map_err(|_| AccountUsageQueryError::RetainedTotals)
.map_err(|_| AccountUsageQueryError::RetainedSummary)?;
let mut summary = RetainedUsageSummary::default();
for bucket in buckets {
summary.billing_amount_usd += billing::calculate_billing_amount(
nonnegative_i64_to_u64(bucket.totals.input_tokens),
nonnegative_i64_to_u64(bucket.totals.output_tokens),
nonnegative_i64_to_u64(bucket.totals.cached_tokens),
&bucket.model,
bucket.service_tier.as_deref(),
);
summary.totals.add(bucket.totals);
}
Ok(summary)
}
/// 列出指定时间范围内的聚合时间桶。
@@ -95,11 +108,18 @@ pub enum AccountUsageQueryError {
#[error("failed to list account usage")]
List,
#[error("failed to summarize retained usage")]
RetainedTotals,
RetainedSummary,
#[error("failed to list usage time buckets")]
TimeBuckets,
}
/// 请求时间桶保留期内的全局用量与计费汇总。
#[derive(Debug, Clone, Default, PartialEq)]
pub struct RetainedUsageSummary {
pub totals: UsageBucketTotals,
pub billing_amount_usd: f64,
}
/// 管理端账号用量记录。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountUsageRecord {
@@ -145,12 +165,10 @@ pub struct AccountUsageTimeBucket {
pub latency_count: i64,
pub max_latency_ms: i64,
pub min_latency_ms: i64,
pub billing_amount_usd: Option<f64>,
}
impl From<UsageTimeBucketRecord> for AccountUsageTimeBucket {
fn from(record: UsageTimeBucketRecord) -> Self {
let billing_amount_usd = time_bucket_billing_amount_usd(&record);
Self {
bucket_start: record.bucket_start,
model: record.model,
@@ -166,33 +184,10 @@ impl From<UsageTimeBucketRecord> for AccountUsageTimeBucket {
latency_count: record.latency_count,
max_latency_ms: record.max_latency_ms,
min_latency_ms: record.min_latency_ms,
billing_amount_usd,
}
}
}
fn time_bucket_billing_amount_usd(record: &UsageTimeBucketRecord) -> Option<f64> {
let input_tokens = nonnegative_i64_to_u64(record.input_tokens);
let output_tokens = nonnegative_i64_to_u64(record.output_tokens);
let cached_tokens = nonnegative_i64_to_u64(record.cached_tokens);
if input_tokens == 0 && output_tokens == 0 && cached_tokens == 0 {
return None;
}
let model = record.model.trim();
if model.is_empty() {
return None;
}
Some(billing::calculate_billing_amount(
input_tokens,
output_tokens,
cached_tokens,
model,
record.service_tier.as_deref(),
))
}
impl From<UsageListRecord> for AccountUsageRecord {
fn from(usage: UsageListRecord) -> Self {
Self {
+33 -13
View File
@@ -29,13 +29,16 @@ where bucket_start >= $1 and bucket_start <= $2
group by bucket_start, model, service_tier
order by bucket_start asc, model asc, service_tier asc";
const RETAINED_USAGE_TOTALS_SQL: &str = r"
const RETAINED_USAGE_BUCKETS_SQL: &str = r"
select
model,
nullif(service_tier, '__unknown__') as service_tier,
coalesce(sum(success_count + error_count), 0)::bigint as request_count,
coalesce(sum(input_tokens), 0)::bigint as input_tokens,
coalesce(sum(output_tokens), 0)::bigint as output_tokens,
coalesce(sum(cached_tokens), 0)::bigint as cached_tokens
from request_time_buckets";
from request_time_buckets
group by bucket_start, model, service_tier";
/// 请求时间桶查询错误。
#[derive(Debug, Error)]
@@ -80,8 +83,16 @@ pub struct UsageBucketTotals {
pub cached_tokens: i64,
}
/// 保留期内单个计费桶的用量事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetainedUsageBucket {
pub model: String,
pub service_tier: Option<String>,
pub totals: UsageBucketTotals,
}
impl UsageBucketTotals {
fn add(&mut self, other: Self) {
pub(crate) fn add(&mut self, other: Self) {
self.request_count = self.request_count.saturating_add(other.request_count);
self.input_tokens = self.input_tokens.saturating_add(other.input_tokens);
self.output_tokens = self.output_tokens.saturating_add(other.output_tokens);
@@ -135,17 +146,26 @@ impl PgRequestBucketQuery {
Ok(rows.iter().map(usage_time_bucket_from_row).collect())
}
/// 汇总请求时间桶保留期内的全局用量,不受当前账号集合影响。
pub async fn retained_totals(&self) -> Result<UsageBucketTotals, PgRequestBucketQueryError> {
let row = sqlx::query(RETAINED_USAGE_TOTALS_SQL)
.fetch_one(&self.pool)
/// 列出请求时间桶保留期内的全局计费桶,不受当前账号集合影响。
pub async fn retained_usage_buckets(
&self,
) -> Result<Vec<RetainedUsageBucket>, PgRequestBucketQueryError> {
let rows = sqlx::query(RETAINED_USAGE_BUCKETS_SQL)
.fetch_all(&self.pool)
.await?;
Ok(UsageBucketTotals {
request_count: row.get("request_count"),
input_tokens: row.get("input_tokens"),
output_tokens: row.get("output_tokens"),
cached_tokens: row.get("cached_tokens"),
})
Ok(rows
.iter()
.map(|row| RetainedUsageBucket {
model: row.get("model"),
service_tier: row.get("service_tier"),
totals: UsageBucketTotals {
request_count: row.get("request_count"),
input_tokens: row.get("input_tokens"),
output_tokens: row.get("output_tokens"),
cached_tokens: row.get("cached_tokens"),
},
})
.collect())
}
pub async fn usage_by_windows(
+134 -63
View File
@@ -11,11 +11,14 @@ use crate::infra::{
time::{china_day_start, china_quarter_hour_start},
};
use super::{account_usage::query::AccountUsageTimeBucket, buckets::query::UsageBucketTotals};
use super::account_usage::query::{AccountUsageTimeBucket, RetainedUsageSummary};
use super::usage::insights::RequestHealthTimeBucket;
const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15;
const HEALTH_TIMELINE_SLOTS: i64 = 24 * 4;
const HEALTH_TIMELINE_STABLE_SUCCESS_THRESHOLD: u64 = 3;
const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10;
const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3;
const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 99.0;
const TREND_SLOT_MINUTES: i64 = 15;
#[derive(Debug, Clone, Copy, Default)]
@@ -40,7 +43,35 @@ pub struct DashboardHealthTimelineData {
title: String,
description: String,
reliability_display: String,
points: String,
status: DashboardHealthTimelineStatus,
success_requests: u64,
failed_requests: u64,
cancelled_requests: u64,
caller_error_requests: u64,
points: Vec<DashboardHealthTimelinePointData>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardHealthTimelinePointData {
time: String,
status: DashboardHealthTimelineStatus,
reliability_display: String,
success_requests: u64,
failed_requests: u64,
cancelled_requests: u64,
caller_error_requests: u64,
}
#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
enum DashboardHealthTimelineStatus {
Future,
NoData,
Unavailable,
Unstable,
LowSample,
Stable,
}
#[derive(Debug, Serialize)]
@@ -79,7 +110,7 @@ struct DashboardTokenCardData {
today_tokens_value: u64,
yesterday_tokens_value: u64,
total_tokens: String,
today_billing_amount_usd: String,
total_billing_amount_usd: String,
}
#[derive(Debug, Serialize)]
@@ -148,6 +179,14 @@ struct UsageWindow {
min_first_token_bucket_latency: u64,
}
#[derive(Debug, Clone, Copy, Default)]
struct HealthWindow {
success_requests: u64,
failed_requests: u64,
cancelled_requests: u64,
caller_error_requests: u64,
}
impl UsageWindow {
fn tokens(self) -> u64 {
self.input_tokens + self.output_tokens
@@ -174,7 +213,7 @@ impl UsageWindow {
pub fn dashboard_cards(
accounts: DashboardAccountCounts,
buckets: &[AccountUsageTimeBucket],
retained_usage: &UsageBucketTotals,
retained_usage: &RetainedUsageSummary,
) -> DashboardCardsData {
dashboard_cards_at(accounts, buckets, retained_usage, Utc::now())
}
@@ -182,18 +221,17 @@ pub fn dashboard_cards(
pub fn dashboard_cards_at(
accounts: DashboardAccountCounts,
buckets: &[AccountUsageTimeBucket],
retained_usage: &UsageBucketTotals,
retained_usage: &RetainedUsageSummary,
now: DateTime<Utc>,
) -> DashboardCardsData {
let today_start = china_day_start(now);
let yesterday_start = today_start - Duration::days(1);
let today = usage_window(buckets, today_start, now);
let yesterday = usage_window(buckets, yesterday_start, today_start);
let today_billing_amount = billing_window(buckets, today_start, now).unwrap_or(0.0);
let total_requests = nonnegative_i64_to_u64(retained_usage.request_count);
let total_tokens = nonnegative_i64_to_u64(retained_usage.input_tokens)
.saturating_add(nonnegative_i64_to_u64(retained_usage.output_tokens));
let total_cached_tokens = nonnegative_i64_to_u64(retained_usage.cached_tokens);
let total_requests = nonnegative_i64_to_u64(retained_usage.totals.request_count);
let total_tokens = nonnegative_i64_to_u64(retained_usage.totals.input_tokens)
.saturating_add(nonnegative_i64_to_u64(retained_usage.totals.output_tokens));
let total_cached_tokens = nonnegative_i64_to_u64(retained_usage.totals.cached_tokens);
DashboardCardsData {
accounts: DashboardAccountsCardData {
@@ -215,7 +253,7 @@ pub fn dashboard_cards_at(
today_tokens_value: today.tokens(),
yesterday_tokens_value: yesterday.tokens(),
total_tokens: format_tokens(total_tokens),
today_billing_amount_usd: format_billing_amount(today_billing_amount),
total_billing_amount_usd: format_billing_amount(retained_usage.billing_amount_usd),
},
cache: DashboardCacheCardData {
today_hit_rate: format_rate(today.cache_hit_rate()),
@@ -321,13 +359,13 @@ pub fn dashboard_trend_data_at(
}
pub fn dashboard_health_timeline_data(
buckets: &[AccountUsageTimeBucket],
buckets: &[RequestHealthTimeBucket],
) -> DashboardHealthTimelineData {
dashboard_health_timeline_data_at(buckets, Utc::now())
}
pub fn dashboard_health_timeline_data_at(
records: &[AccountUsageTimeBucket],
records: &[RequestHealthTimeBucket],
now: DateTime<Utc>,
) -> DashboardHealthTimelineData {
let current_slot = china_quarter_hour_start(now);
@@ -336,7 +374,7 @@ pub fn dashboard_health_timeline_data_at(
.map(|index| {
(
start + Duration::minutes(HEALTH_TIMELINE_SLOT_MINUTES * index),
UsageWindow::default(),
HealthWindow::default(),
)
})
.collect::<Vec<_>>();
@@ -349,36 +387,62 @@ pub fn dashboard_health_timeline_data_at(
.iter_mut()
.find(|(bucket_start, _)| *bucket_start == record_slot)
{
apply_bucket(bucket, record);
bucket.success_requests = bucket
.success_requests
.saturating_add(record.success_requests);
bucket.failed_requests = bucket
.failed_requests
.saturating_add(record.failed_requests);
bucket.cancelled_requests = bucket
.cancelled_requests
.saturating_add(record.cancelled_requests);
bucket.caller_error_requests = bucket
.caller_error_requests
.saturating_add(record.caller_error_requests);
}
}
let (requests, errors) = buckets
let totals = buckets
.iter()
.filter(|(bucket_start, _)| *bucket_start <= current_slot)
.fold((0, 0), |(requests, errors), (_, bucket)| {
(requests + bucket.requests, errors + bucket.errors)
.fold(HealthWindow::default(), |mut totals, (_, bucket)| {
totals.success_requests = totals
.success_requests
.saturating_add(bucket.success_requests);
totals.failed_requests = totals
.failed_requests
.saturating_add(bucket.failed_requests);
totals.cancelled_requests = totals
.cancelled_requests
.saturating_add(bucket.cancelled_requests);
totals.caller_error_requests = totals
.caller_error_requests
.saturating_add(bucket.caller_error_requests);
totals
});
let reliability = (requests > 0)
.then(|| ((requests - errors) as f64 / requests as f64 * 1000.0).round() / 10.0);
DashboardHealthTimelineData {
title: "请求健康时间线".to_string(),
description: "请求可靠性".to_string(),
reliability_display: reliability
.map(|value| format!("{value:.1}%"))
.unwrap_or_else(|| "-".to_string()),
description: "有效请求可用性".to_string(),
reliability_display: format_health_reliability(totals),
status: health_status(totals, false),
success_requests: totals.success_requests,
failed_requests: totals.failed_requests,
cancelled_requests: totals.cancelled_requests,
caller_error_requests: totals.caller_error_requests,
points: buckets
.into_iter()
.map(|(bucket_start, bucket)| {
let success_rate = if bucket.requests > 0 {
(bucket.requests.saturating_sub(bucket.errors) as f64 / bucket.requests as f64
* 1000.0)
.round()
/ 10.0
} else {
0.0
};
health_tone(bucket, success_rate, bucket_start > current_slot)
.enumerate()
.map(|(index, (bucket_start, bucket))| {
let elapsed_minutes = index as i64 * HEALTH_TIMELINE_SLOT_MINUTES;
DashboardHealthTimelinePointData {
time: format!("{:02}:{:02}", elapsed_minutes / 60, elapsed_minutes % 60),
status: health_status(bucket, bucket_start > current_slot),
reliability_display: format_health_reliability(bucket),
success_requests: bucket.success_requests,
failed_requests: bucket.failed_requests,
cancelled_requests: bucket.cancelled_requests,
caller_error_requests: bucket.caller_error_requests,
}
})
.collect(),
}
@@ -487,19 +551,6 @@ fn usage_window(
})
}
fn billing_window(
records: &[AccountUsageTimeBucket],
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Option<f64> {
let billing_amounts = records
.iter()
.filter(|record| record.bucket_start >= start && record.bucket_start < end)
.filter_map(|record| record.billing_amount_usd)
.collect::<Vec<_>>();
(!billing_amounts.is_empty()).then(|| billing_amounts.into_iter().sum())
}
fn apply_bucket(window: &mut UsageWindow, record: &AccountUsageTimeBucket) {
window.requests += nonnegative_i64_to_u64(record.request_count);
window.input_tokens += nonnegative_i64_to_u64(record.input_tokens);
@@ -524,27 +575,47 @@ fn first_token_bucket_latency(record: &AccountUsageTimeBucket) -> Option<u64> {
.filter(|latency| *latency > 0)
}
fn health_tone(bucket: UsageWindow, success_rate: f64, is_future: bool) -> char {
let successes = bucket.requests.saturating_sub(bucket.errors);
fn health_status(bucket: HealthWindow, is_future: bool) -> DashboardHealthTimelineStatus {
let eligible_requests = bucket
.success_requests
.saturating_add(bucket.failed_requests);
if is_future {
'0'
} else if bucket.requests == 0 {
'1'
} else if successes == 0 {
'2'
} else if success_rate < 90.0 {
'3'
} else if successes < HEALTH_TIMELINE_STABLE_SUCCESS_THRESHOLD {
'4'
DashboardHealthTimelineStatus::Future
} else if eligible_requests == 0 {
DashboardHealthTimelineStatus::NoData
} else if bucket.success_requests == 0
&& bucket.failed_requests >= HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD
{
DashboardHealthTimelineStatus::Unavailable
} else if eligible_requests < HEALTH_TIMELINE_MIN_SAMPLE_SIZE {
DashboardHealthTimelineStatus::LowSample
} else if health_reliability(bucket)
.is_some_and(|reliability| reliability < HEALTH_TIMELINE_STABLE_RELIABILITY)
{
DashboardHealthTimelineStatus::Unstable
} else {
'5'
DashboardHealthTimelineStatus::Stable
}
}
fn summary_cache_hit_rate(summary: &UsageBucketTotals) -> Option<f64> {
let input_tokens = nonnegative_i64_to_u64(summary.input_tokens);
fn health_reliability(bucket: HealthWindow) -> Option<f64> {
let eligible_requests = bucket
.success_requests
.saturating_add(bucket.failed_requests);
(eligible_requests > 0)
.then(|| bucket.success_requests as f64 / eligible_requests as f64 * 100.0)
}
fn format_health_reliability(bucket: HealthWindow) -> String {
health_reliability(bucket)
.map(|value| format!("{value:.1}%"))
.unwrap_or_else(|| "-".to_string())
}
fn summary_cache_hit_rate(summary: &RetainedUsageSummary) -> Option<f64> {
let input_tokens = nonnegative_i64_to_u64(summary.totals.input_tokens);
(input_tokens > 0)
.then(|| nonnegative_i64_to_u64(summary.cached_tokens) as f64 / input_tokens as f64)
.then(|| nonnegative_i64_to_u64(summary.totals.cached_tokens) as f64 / input_tokens as f64)
}
fn format_optional_duration_ms(value: Option<u64>) -> String {
+107 -10
View File
@@ -19,7 +19,8 @@ success_terminals as (
select distinct on (request_key)
request_key, id, request_id, terminal_at, true as is_success,
client_api_key_id, provider, account_id, transport, model_name,
null::text as failure_class, latency_ms, first_token_ms, billing_model,
null::text as failure_class, false as is_client_cancelled,
false as is_caller_error, latency_ms, first_token_ms, billing_model,
service_tier, input_tokens, output_tokens, cached_tokens
from (
select
@@ -40,7 +41,8 @@ error_terminals as (
select distinct on (request_key)
request_key, id, request_id, terminal_at, false as is_success,
client_api_key_id, provider, account_id, transport, model_name,
failure_class, latency_ms, null::bigint as first_token_ms,
failure_class, is_client_cancelled, is_caller_error,
latency_ms, null::bigint as first_token_ms,
null::text as billing_model, null::text as service_tier,
null::bigint as input_tokens, null::bigint as output_tokens,
null::bigint as cached_tokens
@@ -49,7 +51,30 @@ error_terminals as (
case when request_id is null then 'event:' || id else 'request:' || request_id end as request_key,
id, request_id, created_at as terminal_at, client_api_key_id, provider,
account_id, transport, coalesce(nullif(model, ''), '未知模型') as model_name,
coalesce(nullif(failure_class, ''), '未分类错误') as failure_class,
coalesce(
nullif(failure_class, ''),
nullif(metadata_json->>'failureClass', ''),
nullif(metadata_json->>'upstreamCode', ''),
nullif(metadata_json->>'terminal', ''),
'未分类错误'
) as failure_class,
coalesce((
coalesce(
nullif(failure_class, ''),
nullif(metadata_json->>'failureClass', ''),
nullif(metadata_json->>'upstreamCode', ''),
nullif(metadata_json->>'terminal', ''),
''
) in ('cancelled', 'downstream_closed', 'client_cancelled', 'consumer_dropped')
or (
(status_code = 499 or client_status_code = 499)
and (
metadata_json->>'cancelled' = 'true'
or message = 'v1 responses stream cancelled'
)
)
), false) as is_client_cancelled,
coalesce(metadata_json->>'failureSource' = 'client', false) as is_caller_error,
latency_ms
from ops_error_logs
where created_at >= $1 and created_at < $2
@@ -166,9 +191,7 @@ impl UsageDiagnosticsDimension {
Self::Account => {
"coalesce(nullif(concat_ws(' → ', nullif(diagnostic_accounts.email, ''), nullif(terminal_requests.account_id, '')), ''), '未知账号')"
}
Self::ApiKey => {
"coalesce(nullif(terminal_requests.client_api_key_id, ''), '未知 API Key')"
}
Self::ApiKey => "coalesce(nullif(diagnostic_api_keys.name, ''), '未知密钥')",
Self::Provider => "coalesce(nullif(terminal_requests.provider, ''), '未知 Provider')",
Self::Transport => "coalesce(nullif(terminal_requests.transport, ''), '未知传输')",
Self::FailureClass => "terminal_requests.failure_class",
@@ -180,7 +203,10 @@ impl UsageDiagnosticsDimension {
Self::Account => {
"terminal_requests left join accounts diagnostic_accounts on diagnostic_accounts.id = terminal_requests.account_id"
}
Self::Model | Self::ApiKey | Self::Provider | Self::Transport | Self::FailureClass => {
Self::ApiKey => {
"terminal_requests left join client_api_keys diagnostic_api_keys on diagnostic_api_keys.id = terminal_requests.client_api_key_id"
}
Self::Model | Self::Provider | Self::Transport | Self::FailureClass => {
"terminal_requests"
}
}
@@ -202,6 +228,8 @@ struct UsageHealthInsights {
total_requests: u64,
success_requests: u64,
failed_requests: u64,
cancelled_requests: u64,
caller_error_requests: u64,
success_rate: f64,
request_change_rate: Option<f64>,
success_rate_change: Option<f64>,
@@ -215,9 +243,21 @@ struct UsageHealthPoint {
label: String,
success_requests: u64,
failed_requests: u64,
cancelled_requests: u64,
caller_error_requests: u64,
error_rate: f64,
}
/// Dashboard 请求健康时间桶,仅统计每个请求的最终终态。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RequestHealthTimeBucket {
pub bucket_start: DateTime<Utc>,
pub success_requests: u64,
pub failed_requests: u64,
pub cancelled_requests: u64,
pub caller_error_requests: u64,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct UsagePerformanceInsights {
@@ -380,6 +420,47 @@ pub async fn diagnostics(
})
}
/// 按 15 分钟聚合最终请求终态;客户端取消与调用方错误保持独立,不污染可用性。
pub async fn health_timeline(
pool: &PgPool,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> PgUsageRecordStoreResult<Vec<RequestHealthTimeBucket>> {
let bucket = InsightsGranularity::QuarterHour.bucket_expression("terminal_at");
let sql = format!(
"with {TERMINAL_REQUESTS_CTE}\n\
select {bucket} as bucket_start,\n\
count(*) filter (where is_success)::bigint as success_requests,\n\
count(*) filter (\n\
where not is_success and not is_client_cancelled and not is_caller_error\n\
)::bigint as failed_requests,\n\
count(*) filter (where is_client_cancelled)::bigint as cancelled_requests,\n\
count(*) filter (\n\
where is_caller_error and not is_client_cancelled\n\
)::bigint as caller_error_requests\n\
from terminal_requests\n\
group by bucket_start\n\
order by bucket_start"
);
// SAFETY: 时间桶表达式与 CTE 都是模块内固定 SQL,时间范围仍使用绑定参数。
let rows = sqlx::query(AssertSqlSafe(sql))
.bind(start)
.bind(end)
.fetch_all(pool)
.await?;
Ok(rows
.into_iter()
.map(|row| RequestHealthTimeBucket {
bucket_start: row.get("bucket_start"),
success_requests: nonnegative(row.get("success_requests")),
failed_requests: nonnegative(row.get("failed_requests")),
cancelled_requests: nonnegative(row.get("cancelled_requests")),
caller_error_requests: nonnegative(row.get("caller_error_requests")),
})
.collect())
}
async fn load_health(
pool: &PgPool,
start: DateTime<Utc>,
@@ -394,7 +475,13 @@ async fn load_health(
select case when terminal_at >= $3 then 'current' else 'previous' end as period,\n\
{bucket} as bucket_start,\n\
count(*) filter (where is_success)::bigint as success_requests,\n\
count(*) filter (where not is_success)::bigint as failed_requests\n\
count(*) filter (\n\
where not is_success and not is_client_cancelled and not is_caller_error\n\
)::bigint as failed_requests,\n\
count(*) filter (where is_client_cancelled)::bigint as cancelled_requests,\n\
count(*) filter (\n\
where is_caller_error and not is_client_cancelled\n\
)::bigint as caller_error_requests\n\
from terminal_requests\n\
group by period, bucket_start\n\
order by bucket_start"
@@ -410,17 +497,23 @@ async fn load_health(
let mut current = HashMap::new();
let mut current_success = 0_u64;
let mut current_failed = 0_u64;
let mut current_cancelled = 0_u64;
let mut current_caller_error = 0_u64;
let mut previous_success = 0_u64;
let mut previous_failed = 0_u64;
for row in rows {
let success = nonnegative(row.get("success_requests"));
let failed = nonnegative(row.get("failed_requests"));
let cancelled = nonnegative(row.get("cancelled_requests"));
let caller_error = nonnegative(row.get("caller_error_requests"));
if row.get::<String, _>("period") == "current" {
current_success = current_success.saturating_add(success);
current_failed = current_failed.saturating_add(failed);
current_cancelled = current_cancelled.saturating_add(cancelled);
current_caller_error = current_caller_error.saturating_add(caller_error);
current.insert(
row.get::<DateTime<Utc>, _>("bucket_start"),
(success, failed),
(success, failed, cancelled, caller_error),
);
} else {
previous_success = previous_success.saturating_add(success);
@@ -434,7 +527,7 @@ async fn load_health(
let points = bucket_starts(start, end, granularity)
.into_iter()
.map(|bucket| {
let (success_requests, failed_requests) =
let (success_requests, failed_requests, cancelled_requests, caller_error_requests) =
current.get(&bucket).copied().unwrap_or_default();
let point_total = success_requests.saturating_add(failed_requests);
UsageHealthPoint {
@@ -442,6 +535,8 @@ async fn load_health(
label: bucket_label(bucket, granularity),
success_requests,
failed_requests,
cancelled_requests,
caller_error_requests,
error_rate: rate(failed_requests, point_total),
}
})
@@ -450,6 +545,8 @@ async fn load_health(
total_requests: total,
success_requests: current_success,
failed_requests: current_failed,
cancelled_requests: current_cancelled,
caller_error_requests: current_caller_error,
success_rate,
request_change_rate: relative_change(total, previous_total),
success_rate_change: (previous_total > 0).then_some(success_rate - previous_success_rate),
+13 -2
View File
@@ -11,8 +11,8 @@ use crate::{
telemetry::{
billing,
usage::insights::{
UsageDiagnosticsDimension, UsageDiagnosticsInsights, UsageInsightsOverview,
diagnostics, overview,
RequestHealthTimeBucket, UsageDiagnosticsDimension, UsageDiagnosticsInsights,
UsageInsightsOverview, diagnostics, health_timeline, overview,
},
usage::store::{
PgUsageRecordStore, PgUsageRecordStoreResult, UsageRecordFilter, push_filter,
@@ -239,6 +239,17 @@ impl UsageQueryService {
.map_err(|_| UsageQueryError::List)
}
/// 读取按最终请求终态去重的 15 分钟健康时间桶。
pub async fn health_timeline(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<Vec<RequestHealthTimeBucket>, UsageQueryError> {
health_timeline(self.store.pool(), start, end)
.await
.map_err(|_| UsageQueryError::List)
}
/// 按维度读取请求热点诊断。
pub async fn insights_diagnostics(
&self,
@@ -0,0 +1,204 @@
//! Codex Desktop 官方发布制品观测。
use std::{
sync::{Arc, RwLock},
time::Duration,
};
use chrono::{DateTime, Utc};
use roxmltree::{Document, Node};
const APPCAST_TIMEOUT: Duration = Duration::from_secs(30);
/// Codex Desktop 官方 appcast 地址。
pub const CODEX_DESKTOP_APPCAST_URL: &str =
"https://persistent.oaistatic.com/codex-app-prod/appcast.xml";
/// 官方发布检查周期。
pub const APPCAST_POLL_INTERVAL: Duration = Duration::from_hours(24);
/// appcast 中的最新 Desktop 完整制品信息。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DesktopRelease {
pub version: String,
pub build: String,
pub published_at: Option<DateTime<Utc>>,
pub minimum_system_version: Option<String>,
pub hardware_requirements: Option<String>,
pub download_url: Option<String>,
pub download_size: Option<u64>,
pub signature_present: bool,
}
/// 最近一次 Desktop 发布检查的内存快照。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DesktopReleaseSnapshot {
pub checked_at: Option<DateTime<Utc>>,
pub latest: Option<DesktopRelease>,
pub last_error: Option<String>,
}
/// Dashboard 与后台检查器共享的 Desktop 发布观测状态。
#[derive(Debug, Clone, Default)]
pub struct DesktopReleaseStatus {
snapshot: Arc<RwLock<DesktopReleaseSnapshot>>,
}
impl DesktopReleaseStatus {
pub fn snapshot(&self) -> DesktopReleaseSnapshot {
self.snapshot
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
pub fn record_success(&self, checked_at: DateTime<Utc>, latest: DesktopRelease) {
*self
.snapshot
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = DesktopReleaseSnapshot {
checked_at: Some(checked_at),
latest: Some(latest),
last_error: None,
};
}
pub fn record_failure(&self, checked_at: DateTime<Utc>, error: String) {
let mut snapshot = self
.snapshot
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
snapshot.checked_at = Some(checked_at);
snapshot.last_error = Some(error);
}
}
/// 读取官方 appcast 并更新发布观测状态。
#[derive(Clone)]
pub struct DesktopReleaseChecker {
client: reqwest::Client,
appcast_url: String,
status: DesktopReleaseStatus,
}
impl DesktopReleaseChecker {
pub fn with_client(
client: reqwest::Client,
appcast_url: impl Into<String>,
status: DesktopReleaseStatus,
) -> Self {
Self {
client,
appcast_url: appcast_url.into(),
status,
}
}
pub async fn check_and_record(&self) -> Result<DesktopRelease, DesktopReleaseError> {
let checked_at = Utc::now();
let result = self.fetch_latest().await;
match result {
Ok(release) => {
self.status.record_success(checked_at, release.clone());
Ok(release)
}
Err(error) => {
self.status.record_failure(checked_at, error.to_string());
Err(error)
}
}
}
async fn fetch_latest(&self) -> Result<DesktopRelease, DesktopReleaseError> {
let response = self
.client
.get(&self.appcast_url)
.timeout(APPCAST_TIMEOUT)
.send()
.await?;
if !response.status().is_success() {
return Err(DesktopReleaseError::AppcastFetch(
response.status().as_u16(),
));
}
parse_latest_desktop_release(&response.text().await?)
}
}
/// 解析 appcast 中按顺序出现的首个完整发布项。
pub fn parse_latest_desktop_release(xml: &str) -> Result<DesktopRelease, DesktopReleaseError> {
let document = Document::parse(xml)?;
let item = document
.descendants()
.find(|node| node.is_element() && node.tag_name().name() == "item")
.ok_or(DesktopReleaseError::MissingItem)?;
let enclosure = item
.children()
.find(|node| node.is_element() && node.tag_name().name() == "enclosure");
let version = child_text(item, "shortVersionString")
.or_else(|| local_attribute(enclosure, "shortVersionString"))
.ok_or(DesktopReleaseError::MissingField("shortVersionString"))?;
let build = child_text(item, "version")
.or_else(|| local_attribute(enclosure, "version"))
.ok_or(DesktopReleaseError::MissingField("version"))?;
let published_at = child_text(item, "pubDate")
.map(|value| {
DateTime::parse_from_rfc2822(&value)
.map(|value| value.to_utc())
.map_err(|_| DesktopReleaseError::InvalidField("pubDate"))
})
.transpose()?;
let download_size = local_attribute(enclosure, "length")
.map(|value| {
value
.parse::<u64>()
.map_err(|_| DesktopReleaseError::InvalidField("enclosure.length"))
})
.transpose()?;
Ok(DesktopRelease {
version,
build,
published_at,
minimum_system_version: child_text(item, "minimumSystemVersion"),
hardware_requirements: child_text(item, "hardwareRequirements"),
download_url: local_attribute(enclosure, "url"),
download_size,
signature_present: local_attribute(enclosure, "edSignature").is_some(),
})
}
fn child_text(parent: Node<'_, '_>, local_name: &str) -> Option<String> {
parent
.children()
.find(|node| node.is_element() && node.tag_name().name() == local_name)
.and_then(|node| node.text())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
}
fn local_attribute(node: Option<Node<'_, '_>>, local_name: &str) -> Option<String> {
node?
.attributes()
.find(|attribute| attribute.name() == local_name)
.map(|attribute| attribute.value().trim())
.filter(|value| !value.is_empty())
.map(ToString::to_string)
}
#[derive(Debug, thiserror::Error)]
pub enum DesktopReleaseError {
#[error("Codex Desktop appcast request failed: {0}")]
Http(#[from] reqwest::Error),
#[error("Codex Desktop appcast returned HTTP {0}")]
AppcastFetch(u16),
#[error("Codex Desktop appcast XML is invalid: {0}")]
Xml(#[from] roxmltree::Error),
#[error("Codex Desktop appcast contains no release item")]
MissingItem,
#[error("Codex Desktop appcast release is missing `{0}`")]
MissingField(&'static str),
#[error("Codex Desktop appcast field `{0}` is invalid")]
InvalidField(&'static str),
}
@@ -1,4 +1,4 @@
//! 上游原始错误到 attempt 通用事实的单次规范化。
//! Codex 上游原始错误到稳定失败事实的规范化。
use serde_json::Value;
@@ -8,7 +8,7 @@ use crate::upstream::openai::transport::{
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(in crate::dispatch) enum UpstreamFailureKind {
pub enum UpstreamFailureKind {
HttpStatus,
HttpConnect,
HttpTimeout,
@@ -25,7 +25,7 @@ pub(in crate::dispatch) enum UpstreamFailureKind {
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(in crate::dispatch) struct UpstreamFailureFacts {
pub struct UpstreamFailureFacts {
pub kind: UpstreamFailureKind,
pub status_code: Option<u16>,
pub code: Option<String>,
@@ -35,9 +35,7 @@ pub(in crate::dispatch) struct UpstreamFailureFacts {
pub retry_after_seconds: Option<u64>,
}
pub(in crate::dispatch) fn upstream_failure_facts(
error: &CodexClientError,
) -> UpstreamFailureFacts {
pub fn upstream_failure_facts(error: &CodexClientError) -> UpstreamFailureFacts {
match error {
CodexClientError::Http(error) => UpstreamFailureFacts {
kind: if error.is_connect() {
@@ -156,6 +154,7 @@ fn error_fields(body: &str) -> (Option<String>, Option<String>, Option<String>)
let error = value
.pointer("/response/error")
.or_else(|| value.get("error"))
.or_else(|| value.get("detail"))
.unwrap_or(&value);
(
error
@@ -1,11 +0,0 @@
//! OpenAI 客户端指纹。
mod runtime;
mod store;
mod types;
mod updater;
pub use runtime::*;
pub use store::*;
pub use types::*;
pub use updater::*;
@@ -1,40 +0,0 @@
//! OpenAI 客户端运行时指纹快照。
use std::sync::{Arc, RwLock};
use super::types::Fingerprint;
#[derive(Debug, Clone)]
pub struct RuntimeFingerprint {
current: Arc<RwLock<Arc<Fingerprint>>>,
}
impl RuntimeFingerprint {
pub fn new(fingerprint: Fingerprint) -> Self {
Self {
current: Arc::new(RwLock::new(Arc::new(fingerprint))),
}
}
pub fn current(&self) -> Arc<Fingerprint> {
self.current
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
pub fn snapshot(&self) -> Fingerprint {
self.current().as_ref().clone()
}
pub fn replace(&self, fingerprint: Fingerprint) {
*self
.current
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(fingerprint);
}
}
// ---------------------------------------------------------------------------
// 指纹历史记录与仓储
// ---------------------------------------------------------------------------
@@ -1,245 +0,0 @@
//! OpenAI 客户端指纹 PostgreSQL 存储。
use std::num::NonZeroU32;
use chrono::{DateTime, Utc};
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use sqlx::{PgPool, Row};
use uuid::Uuid;
use super::types::Fingerprint;
pub const CURRENT_FINGERPRINT_ID: &str = "current";
const AUTO_UPDATE_SOURCE: &str = "auto_update";
const CONFIG_SEED_SOURCE: &str = "config_seed";
#[derive(Debug, Clone, Serialize, Deserialize)]
struct StoredHeader {
name: String,
value: String,
}
/// 指纹自动更新状态。
#[derive(Clone)]
pub struct PgFingerprintStore {
pool: PgPool,
}
impl PgFingerprintStore {
/// 使用给定连接池构造仓储。
pub fn new(pool: PgPool) -> Self {
Self { pool }
}
/// 写入当前指纹默认值;如果当前槽位已存在,直接读取数据库值。
pub async fn ensure_current_seed(
&self,
default_fingerprint: &Fingerprint,
) -> Result<Fingerprint, sqlx::Error> {
if let Some(fingerprint) = self.load_current().await? {
return Ok(fingerprint);
}
let now = Utc::now();
sqlx::query(
r"
insert into fingerprints (
id,
originator,
app_version,
build_number,
platform,
arch,
chromium_version,
user_agent_template,
default_headers_json,
header_order_json,
source,
created_at,
updated_at
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $12)
on conflict (id) do nothing
",
)
.bind(CURRENT_FINGERPRINT_ID)
.bind(&default_fingerprint.originator)
.bind(&default_fingerprint.app_version)
.bind(&default_fingerprint.build_number)
.bind(&default_fingerprint.platform)
.bind(&default_fingerprint.arch)
.bind(&default_fingerprint.chromium_version)
.bind(&default_fingerprint.user_agent_template)
.bind(sqlx::types::Json(encode_default_headers(
&default_fingerprint.default_headers,
)))
.bind(sqlx::types::Json(&default_fingerprint.header_order))
.bind(CONFIG_SEED_SOURCE)
.bind(now)
.execute(&self.pool)
.await?;
self.load_current().await?.ok_or(sqlx::Error::RowNotFound)
}
/// 更新当前指纹中的自动更新版本字段。
pub async fn update_current_version(
&self,
app_version: &str,
build_number: &str,
chromium_version: Option<&str>,
) -> Result<(), sqlx::Error> {
let resolved_chromium_version = match chromium_version {
Some(version) => version.to_string(),
None => {
sqlx::query_scalar::<_, String>(
"select chromium_version from fingerprints where id = $1",
)
.bind(CURRENT_FINGERPRINT_ID)
.fetch_one(&self.pool)
.await?
}
};
let result = sqlx::query(
"update fingerprints set app_version = $1, build_number = $2, chromium_version = $3, source = $4, updated_at = $5 where id = $6",
)
.bind(app_version)
.bind(build_number)
.bind(resolved_chromium_version)
.bind(AUTO_UPDATE_SOURCE)
.bind(Utc::now())
.bind(CURRENT_FINGERPRINT_ID)
.execute(&self.pool)
.await?;
if result.rows_affected() == 0 {
return Err(sqlx::Error::RowNotFound);
}
Ok(())
}
/// 读取当前运行时指纹。
pub async fn load_current(&self) -> Result<Option<Fingerprint>, sqlx::Error> {
let row = sqlx::query(
r"
select
originator,
app_version,
build_number,
platform,
arch,
chromium_version,
user_agent_template,
default_headers_json,
header_order_json,
updated_at
from fingerprints
where id = $1
",
)
.bind(CURRENT_FINGERPRINT_ID)
.fetch_optional(&self.pool)
.await?;
row.map(|row| fingerprint_from_row(&row)).transpose()
}
/// 记录自动更新历史。
pub async fn insert_update_history(
&self,
app_version: &str,
build_number: &str,
chromium_version: Option<&str>,
manifest_json: Option<&str>,
) -> Result<(), sqlx::Error> {
let manifest = manifest_json
.map(serde_json::from_str::<serde_json::Value>)
.transpose()
.map_err(json_error)?;
sqlx::query(
"insert into fingerprint_update_history (id, current_fingerprint_id, app_version, build_number, chromium_version, source, manifest_json, created_at) values ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(Uuid::new_v4().to_string())
.bind(CURRENT_FINGERPRINT_ID)
.bind(app_version)
.bind(build_number)
.bind(chromium_version)
.bind(AUTO_UPDATE_SOURCE)
.bind(manifest.map(sqlx::types::Json))
.bind(Utc::now())
.execute(&self.pool)
.await?;
Ok(())
}
/// 将指纹更新历史裁剪为最新的指定条数。
pub async fn trim_update_history_to_limit(
&self,
max_entries: NonZeroU32,
) -> Result<u64, sqlx::Error> {
let result = sqlx::query(
r"
delete from fingerprint_update_history as target
using (
select id
from (
select
id,
row_number() over (order by created_at desc, id desc) as position
from fingerprint_update_history
) as ranked
where position > $1
) as expired
where target.id = expired.id
",
)
.bind(i64::from(max_entries.get()))
.execute(&self.pool)
.await?;
Ok(result.rows_affected())
}
}
fn fingerprint_from_row(row: &sqlx::postgres::PgRow) -> Result<Fingerprint, sqlx::Error> {
let default_headers = row
.try_get::<sqlx::types::Json<Vec<StoredHeader>>, _>("default_headers_json")?
.0;
let header_order = row
.try_get::<sqlx::types::Json<Vec<String>>, _>("header_order_json")?
.0;
let updated_at = row.try_get::<DateTime<Utc>, _>("updated_at")?;
Ok(Fingerprint {
originator: row.try_get("originator")?,
app_version: row.try_get("app_version")?,
build_number: row.try_get("build_number")?,
platform: row.try_get("platform")?,
arch: row.try_get("arch")?,
chromium_version: row.try_get("chromium_version")?,
user_agent_template: row.try_get("user_agent_template")?,
default_headers: decode_default_headers(default_headers),
header_order,
updated_at: Some(updated_at.to_rfc3339()),
})
}
fn encode_default_headers(headers: &IndexMap<String, String>) -> Vec<StoredHeader> {
headers
.iter()
.map(|(name, value)| StoredHeader {
name: name.clone(),
value: value.clone(),
})
.collect()
}
fn decode_default_headers(headers: Vec<StoredHeader>) -> IndexMap<String, String> {
headers
.into_iter()
.map(|header| (header.name, header.value))
.collect()
}
fn json_error(error: serde_json::Error) -> sqlx::Error {
sqlx::Error::Decode(Box::new(error))
}
@@ -1,92 +0,0 @@
//! OpenAI 客户端指纹领域类型。
use chrono::{DateTime, Utc};
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Fingerprint {
/// 客户端来源名。
pub originator: String,
/// 应用版本。
pub app_version: String,
/// 构建号。
pub build_number: String,
/// 平台名。
pub platform: String,
/// 架构名。
pub arch: String,
/// Chromium 主版本。
pub chromium_version: String,
/// User-Agent 模板。
pub user_agent_template: String,
/// 默认请求头。
pub default_headers: IndexMap<String, String>,
/// 请求头顺序。
pub header_order: Vec<String>,
/// DB 最后更新时间。
#[serde(skip_serializing_if = "Option::is_none")]
pub updated_at: Option<String>,
}
impl Fingerprint {
/// 根据模板展开最终 User-Agent。
pub fn user_agent(&self) -> String {
self.user_agent_template
.replace("{version}", &self.app_version)
.replace("{platform}", &self.platform)
.replace("{arch}", &self.arch)
}
/// 生成 `sec-ch-ua` 头值。
pub fn sec_ch_ua(&self) -> String {
format!(
"\"Chromium\";v=\"{}\", \"Not:A-Brand\";v=\"24\"",
self.chromium_version
)
}
}
/// 运行时共享指纹快照。
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UpdateState {
/// 最近检查时间。
pub last_check: DateTime<Utc>,
/// 最新版本。
pub latest_version: Option<String>,
/// 最新构建号。
pub latest_build: Option<String>,
/// 下载地址。
pub download_url: Option<String>,
/// 是否有可用更新。
pub update_available: bool,
/// 当前版本。
pub current_version: String,
/// 当前构建号。
pub current_build: String,
}
/// Appcast 检查错误。
#[derive(Debug, Error)]
pub enum UpdateError {
/// HTTP 请求失败。
#[error("HTTP 请求失败: {0}")]
Http(#[from] reqwest::Error),
/// 获取 appcast 失败。
#[error("获取 appcast 失败,状态码: {0}")]
AppcastFetch(u16),
/// 解析 appcast 失败。
#[error("解析 appcast 失败")]
AppcastParse,
/// JSON 序列化失败。
#[error("JSON 序列化失败: {0}")]
Json(#[from] serde_json::Error),
/// 文件操作失败。
#[error("文件操作失败: {0}")]
Io(#[from] std::io::Error),
/// 数据库存储失败。
#[error("数据库操作失败: {0}")]
Database(#[from] sqlx::Error),
}
@@ -1,215 +0,0 @@
//! OpenAI 客户端指纹更新检查器。
use std::{path::PathBuf, sync::Arc, time::Duration};
use chrono::Utc;
use tokio::sync::Mutex;
use tracing::info;
use super::{
store::PgFingerprintStore,
types::{Fingerprint, UpdateError, UpdateState},
};
const APPCAST_TIMEOUT: Duration = Duration::from_secs(30);
pub const APPCAST_POLL_INTERVAL: Duration = Duration::from_hours(24);
type AppcastFields = (Option<String>, Option<String>, Option<String>);
/// Appcast 指纹更新检查器。
#[derive(Clone)]
pub struct UpdateChecker {
store: PgFingerprintStore,
client: reqwest::Client,
appcast_url: String,
extracted_fingerprint_path: PathBuf,
state: Arc<Mutex<InternalState>>,
}
struct InternalState {
current_version: String,
current_build: String,
}
impl UpdateChecker {
/// 使用自定义 client 和路径构造检查器。
pub fn with_client(
store: PgFingerprintStore,
client: reqwest::Client,
appcast_url: impl Into<String>,
extracted_fingerprint_path: PathBuf,
current_version: impl Into<String>,
current_build: impl Into<String>,
) -> Self {
Self {
store,
client,
appcast_url: appcast_url.into(),
extracted_fingerprint_path,
state: Arc::new(Mutex::new(InternalState {
current_version: current_version.into(),
current_build: current_build.into(),
})),
}
}
/// 检查 appcast 是否存在更新。
pub async fn check_for_update(&self) -> Result<UpdateState, UpdateError> {
let state = self.state.lock().await;
let current_version = state.current_version.clone();
let current_build = state.current_build.clone();
drop(state);
let xml = self.fetch_appcast().await?;
let (version, build, download_url) = parse_appcast(&xml)?;
let update_available = version
.as_ref()
.is_some_and(|value| value != &current_version)
|| build.as_ref().is_some_and(|value| value != &current_build);
Ok(UpdateState {
last_check: Utc::now(),
latest_version: version,
latest_build: build,
download_url,
update_available,
current_version,
current_build,
})
}
/// 检查并在需要时应用更新。
pub async fn check_and_apply_update(&self) -> Result<Option<Fingerprint>, UpdateError> {
let update_state = self.check_for_update().await?;
if !update_state.update_available {
return Ok(None);
}
let (Some(version), Some(build)) = (
update_state.latest_version.as_deref(),
update_state.latest_build.as_deref(),
) else {
return Ok(None);
};
let state = self.state.lock().await;
let current_version = state.current_version.clone();
let current_build = state.current_build.clone();
drop(state);
info!(
version = %version,
build = %build,
current_version = %current_version,
current_build = %current_build,
"发现新的 fingerprint 版本"
);
let updated_fingerprint = self.apply_version_update(version, build).await?;
{
let mut state = self.state.lock().await;
state.current_version = version.to_string();
state.current_build = build.to_string();
}
Ok(Some(updated_fingerprint))
}
async fn fetch_appcast(&self) -> Result<String, UpdateError> {
let response = self
.client
.get(&self.appcast_url)
.timeout(APPCAST_TIMEOUT)
.send()
.await?;
if !response.status().is_success() {
return Err(UpdateError::AppcastFetch(response.status().as_u16()));
}
response.text().await.map_err(UpdateError::from)
}
async fn apply_version_update(
&self,
version: &str,
build: &str,
) -> Result<Fingerprint, UpdateError> {
let chromium_version =
load_matching_chromium_version(&self.extracted_fingerprint_path, version, build);
self.store
.update_current_version(version, build, chromium_version.as_deref())
.await?;
self.store
.insert_update_history(version, build, chromium_version.as_deref(), None)
.await?;
let updated = self
.store
.load_current()
.await?
.ok_or(sqlx::Error::RowNotFound)?;
Ok(updated)
}
}
fn parse_appcast(xml: &str) -> Result<AppcastFields, UpdateError> {
let item_start = xml.find("<item>").ok_or(UpdateError::AppcastParse)?;
let item_end = xml.find("</item>").ok_or(UpdateError::AppcastParse)?;
let item = &xml[item_start..item_end];
let version = extract_sparkle_field(item, "shortVersionString");
let build = extract_sparkle_field(item, "version");
let download_url = item.find("url=\"").and_then(|position| {
let start = position + 5;
item[start..]
.find('"')
.map(|end| item[start..start + end].to_string())
});
Ok((version, build, download_url))
}
fn extract_sparkle_field(item: &str, field: &str) -> Option<String> {
let attr_pattern = format!("sparkle:{}=\"", field);
if let Some(position) = item.find(&attr_pattern) {
let start = position + attr_pattern.len();
if let Some(end) = item[start..].find('"') {
return Some(item[start..start + end].to_string());
}
}
let elem_start = format!("<sparkle:{}>", field);
let elem_end = format!("</sparkle:{}>", field);
if let Some(start_position) = item.find(&elem_start) {
let content_start = start_position + elem_start.len();
if let Some(end_position) = item[content_start..].find(&elem_end) {
return Some(item[content_start..content_start + end_position].to_string());
}
}
None
}
fn load_matching_chromium_version(
extracted_fingerprint_path: &PathBuf,
version: &str,
build: &str,
) -> Option<String> {
let content = std::fs::read_to_string(extracted_fingerprint_path).ok()?;
let parsed: serde_json::Value = serde_json::from_str(&content).ok()?;
let extracted_version = parsed.get("app_version")?.as_str()?;
let extracted_build = parsed.get("build_number")?.as_str()?;
if extracted_version == version && extracted_build == build {
parsed
.get("chromium_version")?
.as_str()
.map(ToString::to_string)
} else {
None
}
}
+3 -1
View File
@@ -1,6 +1,8 @@
//! OpenAI/Codex 上游客户端。
pub mod fingerprint;
pub mod desktop_release;
pub mod failure;
pub mod profile;
pub mod protocol;
pub mod token_client;
pub mod transport;
+46
View File
@@ -0,0 +1,46 @@
//! Codex Desktop 上游请求画像。
use chrono::{DateTime, Utc};
/// 经制品与 Codex 源码核验后固定的上游请求身份。
///
/// 该画像是启动配置,不会被在线发布检查自动改写。Desktop 发布变化后应先重新审计
/// bundled Codex Core,再整体更新此画像。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CodexWireProfile {
/// `originator` 请求头及 User-Agent 产品名。
pub originator: String,
/// bundled Codex Core 版本;同时用于 `/codex/models?client_version=`。
pub codex_version: String,
/// Desktop 应用版本,用于 app-server `clientInfo.version` 对应的 UA 后缀。
pub desktop_version: String,
/// Desktop 制品构建号,仅用于发布对齐诊断。
pub desktop_build: String,
/// Codex Core UA 中的目标操作系统类型。
pub os_type: String,
/// Codex Core UA 中的目标操作系统版本。
pub os_version: String,
/// Codex Core UA 中的目标架构。
pub arch: String,
/// Codex Core UA 中的终端标记。
pub terminal: String,
/// 此画像最后一次经制品与源码核验的时间。
pub verified_at: DateTime<Utc>,
}
impl CodexWireProfile {
/// 按 Codex Core 的官方格式生成最终 User-Agent。
pub fn user_agent(&self) -> String {
format!(
"{}/{} ({} {}; {}) {} ({}; {})",
self.originator,
self.codex_version,
self.os_type,
self.os_version,
self.arch,
self.terminal,
self.originator,
self.desktop_version,
)
}
}
@@ -13,15 +13,15 @@ use futures::{Stream, StreamExt, TryStreamExt};
use reqwest::{
Client, Response as ReqwestResponse, StatusCode,
header::{
ACCEPT, ACCEPT_ENCODING, AUTHORIZATION, CONTENT_TYPE, COOKIE, HeaderMap, HeaderName,
HeaderValue, RETRY_AFTER, USER_AGENT,
ACCEPT, AUTHORIZATION, CONTENT_TYPE, COOKIE, HeaderMap, HeaderName, HeaderValue,
RETRY_AFTER, USER_AGENT,
},
};
use serde_json::{Value, map::Map};
use thiserror::Error;
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
use crate::upstream::openai::fingerprint::{Fingerprint, RuntimeFingerprint};
use crate::upstream::openai::profile::CodexWireProfile;
use crate::upstream::openai::protocol::events::{extract_sse_usage, retry_after_seconds_from_body};
use crate::upstream::openai::protocol::responses::{
CodexResponsesRequest, TransportRequirement, transport_requirement,
@@ -33,10 +33,7 @@ use crate::upstream::openai::protocol::websocket::{
use super::diagnostics::CodexUpstreamDiagnostics;
use super::endpoints::{CODEX_RESPONSES_PATH, endpoint_url};
use super::headers::{
build_ordered_codex_base_headers, insert_optional_header, insert_ordered_headers,
websocket_header_pairs,
};
use super::headers::{build_codex_base_headers, insert_optional_header, websocket_header_pairs};
use super::response_meta;
use super::tls::{CustomCaError, build_reqwest_client_with_custom_ca, custom_ca_env_cache_key};
use super::websocket::{
@@ -166,39 +163,6 @@ impl CodexClientError {
}
}
/// 判断上游错误正文是否表示账号已封禁或停用。
pub fn is_banned_auth_signal(value: &str) -> bool {
let value = value.to_ascii_lowercase();
value.contains("account_deactivated")
|| value.contains("account deactivated")
|| value.contains("account has been deactivated")
|| value.contains("deactivated")
|| value.contains("banned")
}
/// 判断 402 错误正文是否带有 OpenAI 工作区停用标记。
pub fn is_deactivated_workspace_error_body(value: &str) -> bool {
serde_json::from_str::<Value>(value).is_ok_and(|value| {
value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace")
})
}
/// 判断 Codex 上游错误是否表示账号已封禁或停用。
pub fn is_banned_upstream_error(error: &CodexClientError) -> bool {
matches!(
error,
CodexClientError::Upstream { status, body, .. }
if (status.as_u16() == 403
&& !is_html_error_body(body))
|| (status.as_u16() == 402 && is_deactivated_workspace_error_body(body))
)
}
fn is_html_error_body(value: &str) -> bool {
let value = value.trim_start().to_ascii_lowercase();
value.starts_with("<!doctype") || value.starts_with("<html") || value.contains("<html")
}
/// Codex 客户端结果类型。
pub type CodexClientResult<T> = Result<T, CodexClientError>;
@@ -342,7 +306,7 @@ pub enum CodexTransportDecision {
ConnectedWebSocket,
ExactWebSocket,
RequiredWebSocket,
Http2WebSocketSlow,
Http2WebSocketBudgetExhausted,
Http2BreakerOpen,
Http2PoolUnavailable,
Http2PreSendFailure,
@@ -356,7 +320,7 @@ impl CodexTransportDecision {
Self::ConnectedWebSocket => "ws_connected_fast",
Self::ExactWebSocket => "ws_exact_required",
Self::RequiredWebSocket => "ws_required",
Self::Http2WebSocketSlow => "http2_ws_slow",
Self::Http2WebSocketBudgetExhausted => "http2_ws_budget_exhausted",
Self::Http2BreakerOpen => "http2_breaker_open",
Self::Http2PoolUnavailable => "http2_pool_unavailable",
Self::Http2PreSendFailure => "http2_ws_pre_send_failure",
@@ -418,7 +382,7 @@ pub struct CodexBackendStreamingResponse {
pub struct CodexBackendClient {
pub(super) client: Client,
pub(super) base_url: String,
fingerprint: RuntimeFingerprint,
profile: Arc<CodexWireProfile>,
websocket_pool: Option<Arc<CodexWebSocketPool>>,
websocket_initial_event_timeout: Option<Duration>,
websocket_fast_path_budget: Duration,
@@ -660,7 +624,7 @@ fn websocket_success_decision(
fn http_fallback_decision(error: &CodexWebSocketExchangeError) -> CodexTransportDecision {
match error {
CodexWebSocketExchangeError::FastPathTimeout { .. } => {
CodexTransportDecision::Http2WebSocketSlow
CodexTransportDecision::Http2WebSocketBudgetExhausted
}
CodexWebSocketExchangeError::OriginCircuitOpen
| CodexWebSocketExchangeError::OriginHalfOpenBusy => {
@@ -5,14 +5,14 @@ impl CodexBackendClient {
pub fn new(
client: Client,
base_url: impl Into<String>,
fingerprint: RuntimeFingerprint,
profile: Arc<CodexWireProfile>,
) -> Self {
let base_url = base_url.into().trim_end_matches('/').to_string();
Self {
client,
websocket_origin_key: websocket_origin_key(&base_url),
base_url,
fingerprint,
profile,
websocket_pool: None,
websocket_initial_event_timeout: Some(DEFAULT_INITIAL_EVENT_TIMEOUT),
websocket_fast_path_budget: WEBSOCKET_FAST_PATH_BUDGET,
@@ -314,11 +314,7 @@ impl CodexBackendClient {
route: PreparedResponseRoute::Http,
metrics: CodexTransportMetrics {
decision: Some(decision),
ws_connect_ms: matches!(
error,
CodexWebSocketExchangeError::FastPathTimeout { .. }
)
.then_some(wait_ms),
ws_connect_ms: None,
transport_decision_wait_ms: Some(wait_ms),
..CodexTransportMetrics::default()
},
@@ -504,13 +500,12 @@ impl CodexBackendClient {
&self,
context: CodexRequestContext<'_>,
) -> CodexClientResult<Vec<Value>> {
let fingerprint = self.fingerprint.current();
let endpoint = endpoint_url(&self.base_url, "codex/models");
let headers = self.auxiliary_request_headers(&fingerprint, context)?;
let headers = self.auxiliary_request_headers(context)?;
let response = self
.client
.get(endpoint)
.query(&[("client_version", fingerprint.app_version.as_str())])
.query(&[("client_version", self.profile.codex_version.as_str())])
.headers(headers)
.send()
.await?;
@@ -575,16 +570,9 @@ impl CodexBackendClient {
}
fn request_headers(&self, context: CodexRequestContext<'_>) -> CodexClientResult<HeaderMap> {
let fingerprint = self.fingerprint.current();
let request_id = context.client_request_id.unwrap_or(context.request_id);
let ordered_headers = build_ordered_codex_base_headers(
&fingerprint,
context.access_token,
context.account_id,
);
let mut headers = HeaderMap::new();
insert_ordered_headers(&mut headers, &ordered_headers)?;
let mut headers =
build_codex_base_headers(&self.profile, context.access_token, context.account_id)?;
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream"));
@@ -629,19 +617,14 @@ impl CodexBackendClient {
fn auxiliary_request_headers(
&self,
fingerprint: &Fingerprint,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let ordered_headers =
build_ordered_codex_base_headers(fingerprint, context.access_token, context.account_id);
let mut headers = HeaderMap::new();
insert_ordered_headers(&mut headers, &ordered_headers)?;
let mut headers =
build_codex_base_headers(&self.profile, context.access_token, context.account_id)?;
if let Some(cookie_header) = context.cookie_header {
headers.insert(COOKIE, HeaderValue::from_str(cookie_header)?);
}
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate"));
insert_optional_header(
&mut headers,
"x-codex-installation-id",
@@ -654,11 +637,10 @@ impl CodexBackendClient {
&self,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let fingerprint = self.fingerprint.current();
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_str(&fingerprint.user_agent())?,
HeaderValue::from_str(&self.profile.user_agent())?,
);
headers.insert(
AUTHORIZATION,
@@ -666,7 +648,7 @@ impl CodexBackendClient {
);
headers.insert(
HeaderName::from_static("originator"),
HeaderValue::from_str(&fingerprint.originator)?,
HeaderValue::from_str(&self.profile.originator)?,
);
insert_optional_header(&mut headers, "chatgpt-account-id", context.account_id)?;
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
@@ -1,102 +1,49 @@
use indexmap::IndexMap;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use reqwest::header::{ACCEPT, AUTHORIZATION, HeaderMap, HeaderName, HeaderValue, USER_AGENT};
use crate::upstream::openai::fingerprint::Fingerprint;
use crate::upstream::openai::profile::CodexWireProfile;
use super::client::CodexClientResult;
/// 构造标准 Codex 请求头。
/// 构造 Codex Core 为模型请求设置的稳定身份请求头。
pub fn build_codex_base_headers(
fingerprint: &Fingerprint,
profile: &CodexWireProfile,
access_token: &str,
account_id: Option<&str>,
) -> IndexMap<String, String> {
let mut headers = IndexMap::new();
) -> CodexClientResult<HeaderMap> {
let mut headers = HeaderMap::new();
headers.insert(
"authorization".to_string(),
format!("Bearer {access_token}"),
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {access_token}"))?,
);
if let Some(account_id) = account_id {
headers.insert("chatgpt-account-id".to_string(), account_id.to_string());
}
headers.insert("originator".to_string(), fingerprint.originator.clone());
headers.insert("user-agent".to_string(), fingerprint.user_agent());
headers.insert("sec-ch-ua".to_string(), fingerprint.sec_ch_ua());
for (key, value) in &fingerprint.default_headers {
let key_lower = key.to_ascii_lowercase();
if !headers.contains_key(&key_lower) {
headers.insert(key_lower, value.clone());
}
}
headers
insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?;
headers.insert(
HeaderName::from_static("originator"),
HeaderValue::from_str(&profile.originator)?,
);
headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?);
Ok(headers)
}
/// 构造包含标准请求头和额外请求头值的完整请求头集合。
/// 构造含通用请求上下文的 Codex 模型请求头。
pub fn build_codex_headers(
fingerprint: &Fingerprint,
profile: &CodexWireProfile,
access_token: &str,
account_id: Option<&str>,
turn_state: Option<&str>,
request_id: &str,
) -> IndexMap<String, String> {
let mut headers = build_codex_base_headers(fingerprint, access_token, account_id);
) -> CodexClientResult<HeaderMap> {
let mut headers = build_codex_base_headers(profile, access_token, account_id)?;
headers.insert(
"x-openai-internal-codex-residency".to_string(),
"us".to_string(),
HeaderName::from_static("x-openai-internal-codex-residency"),
HeaderValue::from_static("us"),
);
headers.insert("x-client-request-id".to_string(), request_id.to_string());
if let Some(turn_state) = turn_state {
headers.insert("x-codex-turn-state".to_string(), turn_state.to_string());
}
headers.insert("accept".to_string(), "text/event-stream".to_string());
headers
}
/// 按给定顺序重排请求头。
pub fn order_headers(
headers: IndexMap<String, String>,
order: &[String],
) -> IndexMap<String, String> {
let mut ordered = IndexMap::new();
for key in order {
if let Some(value) = headers.get(key) {
ordered.insert(key.clone(), value.clone());
}
}
for (key, value) in headers {
if !ordered.contains_key(&key) {
ordered.insert(key, value);
}
}
ordered
}
/// 构造并按指纹顺序重排完整请求头。
pub fn build_ordered_codex_headers(
fingerprint: &Fingerprint,
access_token: &str,
account_id: Option<&str>,
turn_state: Option<&str>,
request_id: &str,
) -> IndexMap<String, String> {
let headers = build_codex_headers(
fingerprint,
access_token,
account_id,
turn_state,
request_id,
headers.insert(
HeaderName::from_static("x-client-request-id"),
HeaderValue::from_str(request_id)?,
);
order_headers(headers, &fingerprint.header_order)
}
/// 构造并按指纹顺序重排基础请求头(不含请求 ID 和 turn state)。
pub fn build_ordered_codex_base_headers(
fingerprint: &Fingerprint,
access_token: &str,
account_id: Option<&str>,
) -> IndexMap<String, String> {
let headers = build_codex_base_headers(fingerprint, access_token, account_id);
order_headers(headers, &fingerprint.header_order)
insert_optional_header(&mut headers, "x-codex-turn-state", turn_state)?;
headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream"));
Ok(headers)
}
pub(super) fn insert_optional_header(
@@ -111,19 +58,6 @@ pub(super) fn insert_optional_header(
Ok(())
}
pub(super) fn insert_ordered_headers(
headers: &mut HeaderMap,
ordered_headers: &IndexMap<String, String>,
) -> CodexClientResult<()> {
for (name, value) in ordered_headers {
headers.insert(
HeaderName::from_bytes(name.as_bytes())?,
HeaderValue::from_str(value)?,
);
}
Ok(())
}
fn header_pairs(headers: &HeaderMap) -> Vec<(String, String)> {
headers
.iter()
+1 -5
View File
@@ -20,17 +20,13 @@ pub use self::{
CodexModelCatalogClient, CodexModelCatalogClientError, CodexModelCatalogRequest,
CodexRateLimitHeaderUpdates, CodexRequestContext, CodexResponseMetadata,
CodexTransportDecision, CodexTransportMetrics, CodexTurnStateUpdate, build_reqwest_client,
is_banned_auth_signal, is_banned_upstream_error, is_deactivated_workspace_error_body,
},
diagnostics::CodexUpstreamDiagnostics,
endpoints::{
CODEX_RESPONSES_PATH, CODEX_USAGE_API_PATH, CODEX_USAGE_PATH, WHAM_USAGE_PATH,
endpoint_request_path, endpoint_url, usage_endpoint_urls,
},
headers::{
build_codex_base_headers, build_codex_headers, build_ordered_codex_base_headers,
build_ordered_codex_headers, order_headers,
},
headers::{build_codex_base_headers, build_codex_headers},
websocket_pool::{
CodexWebSocketPool, CodexWebSocketPoolConfig, CodexWebSocketPoolKey, WebSocketPoolDecision,
},
@@ -12,7 +12,7 @@ use bytes::Bytes;
use chrono::Utc;
use futures::Stream;
use thiserror::Error;
use tokio::sync::mpsc;
use tokio::sync::{mpsc, oneshot};
use tokio::{sync::Mutex, time::timeout};
use tokio_tungstenite::{Connector, connect_async_tls_with_config};
use tungstenite::{
@@ -39,12 +39,13 @@ use crate::upstream::openai::protocol::websocket::{
use super::websocket_breaker::{
WebSocketOriginBreaker, WebSocketOriginBreakerDecision, WebSocketOriginBreakerPermit,
WebSocketOriginFastPathReporter,
};
use super::websocket_pool::{
CodexWebSocketConnectionMetadata, CodexWebSocketPool, CodexWebSocketPoolKey,
PooledWebSocketConnection, WebSocketContinuationState, WebSocketPoolAcquire,
WebSocketPoolBypassReason, WebSocketPoolConnectOutcome, WebSocketPoolDecision,
WebSocketPoolLease,
WebSocketPoolBypassReason, WebSocketPoolConnectLease, WebSocketPoolConnectOutcome,
WebSocketPoolDecision, WebSocketPoolLease,
};
use super::websocket_pump::{PumpExitReason, PumpKeepalive, PumpedWebSocket, RawWsStream};
use super::{client::CodexResponseMetadata, diagnostics::CodexUpstreamDiagnostics, response_meta};
@@ -576,7 +577,7 @@ async fn prepare_unpooled_websocket(
async fn prepare_pooled_websocket(
request: &CodexWebSocketRequest,
pool: &CodexWebSocketPool,
connect_lease: super::websocket_pool::WebSocketPoolConnectLease,
connect_lease: WebSocketPoolConnectLease,
breaker: &WebSocketOriginBreaker,
origin_key: &str,
fast_path_budget: Option<Duration>,
@@ -589,27 +590,21 @@ async fn prepare_pooled_websocket(
return Err(error);
}
};
let connected =
connect_with_budget(request.connection(), pool.keepalive(), fast_path_budget).await;
let (connection, connect_elapsed) = match finish_breaker_attempt(permit, connected) {
Ok(connected) => connected,
Err(error) => {
connect_lease.failed().await;
return Err(error);
}
};
let connected = match connect_lease.connected(connection).await {
Ok(connected) => connected,
Err(connection) => {
connection.websocket.close().await;
return Err(continuation_unavailable(
PreviousResponseUnavailableReason::PoolUnavailable,
));
}
let mut waiter = start_pooled_websocket_connect(
request.connection().clone(),
pool.clone(),
connect_lease,
permit,
);
let connect_elapsed = waiter.wait(fast_path_budget).await?;
let Some((connection, lease)) = pool.take_idle(waiter.key()).await else {
return Err(continuation_unavailable(
PreviousResponseUnavailableReason::PoolUnavailable,
));
};
Ok(PreparedWebSocket {
connection: *connected.connection,
lease: Some(connected.lease),
connection: *connection,
lease: Some(lease),
reused: false,
pool_decision: Some(WebSocketPoolDecision::new()),
connect_elapsed: Some(connect_elapsed),
@@ -618,6 +613,132 @@ async fn prepare_pooled_websocket(
})
}
struct PooledWebSocketConnectWaiter {
key: CodexWebSocketPoolKey,
started_at: tokio::time::Instant,
receiver: oneshot::Receiver<Result<Duration, CodexWebSocketExchangeError>>,
fast_path_reporter: WebSocketOriginFastPathReporter,
}
impl PooledWebSocketConnectWaiter {
fn key(&self) -> &CodexWebSocketPoolKey {
&self.key
}
async fn wait(
&mut self,
fast_path_budget: Option<Duration>,
) -> Result<Duration, CodexWebSocketExchangeError> {
let received = match fast_path_budget {
Some(budget) => {
let remaining = budget.saturating_sub(self.started_at.elapsed());
match timeout(remaining, &mut self.receiver).await {
Ok(received) => received,
Err(_) => {
self.fast_path_reporter.missed();
return Err(CodexWebSocketExchangeError::FastPathTimeout {
timeout: budget,
});
}
}
}
None => (&mut self.receiver).await,
};
received.map_err(|_| CodexWebSocketExchangeError::SharedConnectFailed)?
}
}
fn start_pooled_websocket_connect(
connection: CodexWebSocketConnection,
pool: CodexWebSocketPool,
connect_lease: WebSocketPoolConnectLease,
permit: WebSocketOriginBreakerPermit,
) -> PooledWebSocketConnectWaiter {
let key = connect_lease.key().clone();
let started_at = connect_lease.started_at();
let cancellation = connect_lease.cancellation_token();
let fast_path_reporter = permit.fast_path_reporter();
let keepalive = pool.keepalive();
let task_key = key.clone();
let (sender, receiver) = oneshot::channel();
pool.spawn_connect_task(async move {
let connected = tokio::select! {
biased;
_ = cancellation.cancelled() => {
permit.cancel();
let _ = sender.send(Err(CodexWebSocketExchangeError::SharedConnectFailed));
connect_lease.failed().await;
tracing::info!(
account_id = task_key.account_id(),
conversation_id_hash = task_key.conversation_id_hash(),
ws_preconnect_duration_ms = duration_millis_u64(started_at.elapsed()),
ws_preconnect_outcome = "cancelled",
"WebSocket pool connect finished"
);
return;
}
result = connect_with_budget(&connection, keepalive, None) => result,
};
match finish_breaker_attempt(permit, connected) {
Ok((connection, connect_elapsed)) => {
match connect_lease.connected_idle(connection).await {
Ok(()) => {
let foreground_waiting = sender.send(Ok(connect_elapsed)).is_ok();
tracing::info!(
account_id = task_key.account_id(),
conversation_id_hash = task_key.conversation_id_hash(),
ws_preconnect_duration_ms = duration_millis_u64(connect_elapsed),
foreground_waiting,
ws_preconnect_outcome = "ready",
"WebSocket pool connect finished"
);
}
Err(connection) => {
connection.websocket.close().await;
let _ = sender.send(Err(continuation_unavailable(
PreviousResponseUnavailableReason::PoolUnavailable,
)));
tracing::info!(
account_id = task_key.account_id(),
conversation_id_hash = task_key.conversation_id_hash(),
ws_preconnect_duration_ms = duration_millis_u64(connect_elapsed),
ws_preconnect_outcome = "rejected",
"WebSocket pool connect finished"
);
}
}
}
Err(error) => {
let error_message = error.to_string();
// 先交付 opening 原始错误,避免连接池清理侵占前台 fast-path 预算。
let foreground_waiting = sender.send(Err(error)).is_ok();
connect_lease.failed().await;
tracing::warn!(
account_id = task_key.account_id(),
conversation_id_hash = task_key.conversation_id_hash(),
ws_preconnect_duration_ms = duration_millis_u64(started_at.elapsed()),
foreground_waiting,
error = %error_message,
ws_preconnect_outcome = "failed",
"WebSocket pool connect finished"
);
}
}
});
PooledWebSocketConnectWaiter {
key,
started_at,
receiver,
fast_path_reporter,
}
}
fn duration_millis_u64(duration: Duration) -> u64 {
u64::try_from(duration.as_millis())
.unwrap_or(u64::MAX)
.max(1)
}
async fn connect_with_budget(
connection: &CodexWebSocketConnection,
keepalive: PumpKeepalive,
@@ -683,11 +804,11 @@ fn finish_breaker_attempt(
async fn wait_for_shared_connect(
waiter: super::websocket_pool::WebSocketPoolConnectWaiter,
fast_path_budget: Option<Duration>,
decision_started_at: Instant,
_decision_started_at: Instant,
) -> Result<WebSocketPoolConnectOutcome, CodexWebSocketExchangeError> {
match fast_path_budget {
Some(budget) => {
let remaining = budget.saturating_sub(decision_started_at.elapsed());
let remaining = waiter.remaining_budget(budget);
timeout(remaining, waiter.wait())
.await
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })
@@ -2,7 +2,10 @@
use std::{
collections::{HashMap, VecDeque},
sync::{Arc, Mutex},
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant},
};
@@ -73,8 +76,8 @@ impl WebSocketOriginBreaker {
.or_insert_with(CircuitState::closed);
match state {
CircuitState::Closed { fast_timeouts } => {
retain_window(fast_timeouts, now, self.config.failure_window);
CircuitState::Closed { failures } => {
retain_window(failures, now, self.config.failure_window);
WebSocketOriginBreakerDecision::Allowed(WebSocketOriginBreakerPermit::new(
self.clone(),
origin_key.to_string(),
@@ -105,7 +108,7 @@ impl WebSocketOriginBreaker {
}
}
fn record_fast_timeout(&self, origin_key: &str, probe_id: Option<Uuid>) {
fn record_failure(&self, origin_key: &str, probe_id: Option<Uuid>) {
let now = Instant::now();
let mut circuits = self
.inner
@@ -127,35 +130,17 @@ impl WebSocketOriginBreaker {
let state = circuits
.entry(origin_key.to_string())
.or_insert_with(CircuitState::closed);
let CircuitState::Closed { fast_timeouts } = state else {
let CircuitState::Closed { failures } = state else {
return;
};
retain_window(fast_timeouts, now, self.config.failure_window);
fast_timeouts.push_back(now);
if fast_timeouts.len() >= self.config.failure_threshold {
retain_window(failures, now, self.config.failure_window);
failures.push_back(now);
if failures.len() >= self.config.failure_threshold {
*state = CircuitState::Open {
until: now + self.config.open_duration,
};
}
}
fn record_failure(&self, origin_key: &str, probe_id: Option<Uuid>) {
let Some(probe_id) = probe_id else {
return;
};
let mut circuits = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if permit_still_owns_state(circuits.get(origin_key), Some(probe_id)) {
circuits.insert(
origin_key.to_string(),
CircuitState::Open {
until: Instant::now() + self.config.open_duration,
},
);
}
}
}
/// 一次新的 WebSocket opening 许可;消费式完成保证 half-open 探针不会泄漏。
@@ -163,6 +148,7 @@ pub struct WebSocketOriginBreakerPermit {
breaker: WebSocketOriginBreaker,
origin_key: String,
probe_id: Option<Uuid>,
degradation_recorded: Arc<AtomicBool>,
armed: bool,
}
@@ -172,6 +158,7 @@ impl WebSocketOriginBreakerPermit {
breaker,
origin_key,
probe_id,
degradation_recorded: Arc::new(AtomicBool::new(false)),
armed: true,
}
}
@@ -180,33 +167,71 @@ impl WebSocketOriginBreakerPermit {
self.probe_id.is_some()
}
/// 创建可跨前台预算与后台 opening 生命周期共享的快路径观察器。
pub(crate) fn fast_path_reporter(&self) -> WebSocketOriginFastPathReporter {
WebSocketOriginFastPathReporter {
breaker: self.breaker.clone(),
origin_key: self.origin_key.clone(),
probe_id: self.probe_id,
degradation_recorded: Arc::clone(&self.degradation_recorded),
}
}
pub fn succeed(mut self) {
self.breaker.record_success(&self.origin_key, self.probe_id);
if !self.degradation_recorded.load(Ordering::Acquire) {
self.breaker.record_success(&self.origin_key, self.probe_id);
}
self.armed = false;
}
pub fn fast_timeout(mut self) {
self.breaker
.record_fast_timeout(&self.origin_key, self.probe_id);
self.fast_path_reporter().missed();
self.armed = false;
}
pub fn fail(mut self) {
self.breaker.record_failure(&self.origin_key, self.probe_id);
self.record_failure_once();
self.armed = false;
}
/// opening 因账号驱逐或服务关闭而取消,不污染 origin 可用性。
pub(crate) fn cancel(mut self) {
self.armed = false;
}
fn record_failure_once(&self) {
if !self.degradation_recorded.swap(true, Ordering::AcqRel) {
self.breaker.record_failure(&self.origin_key, self.probe_id);
}
}
}
impl Drop for WebSocketOriginBreakerPermit {
fn drop(&mut self) {
if self.armed {
self.record_failure_once();
}
}
}
/// 前台等待结束后仍可报告同一次后台 opening 错过快路径预算。
pub(crate) struct WebSocketOriginFastPathReporter {
breaker: WebSocketOriginBreaker,
origin_key: String,
probe_id: Option<Uuid>,
degradation_recorded: Arc<AtomicBool>,
}
impl WebSocketOriginFastPathReporter {
pub(crate) fn missed(&self) {
if !self.degradation_recorded.swap(true, Ordering::AcqRel) {
self.breaker.record_failure(&self.origin_key, self.probe_id);
}
}
}
enum CircuitState {
Closed { fast_timeouts: VecDeque<Instant> },
Closed { failures: VecDeque<Instant> },
Open { until: Instant },
HalfOpen { probe_id: Uuid },
}
@@ -214,7 +239,7 @@ enum CircuitState {
impl CircuitState {
fn closed() -> Self {
Self::Closed {
fast_timeouts: VecDeque::new(),
failures: VecDeque::new(),
}
}
}
@@ -1,6 +1,6 @@
//! Codex WebSocket 连接池。
use std::{collections::HashMap, sync::Arc, time::Duration};
use std::{collections::HashMap, future::Future, sync::Arc, time::Duration};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
@@ -8,6 +8,10 @@ use tokio::{
sync::{Mutex, watch},
time::Instant,
};
use tokio_util::{
sync::CancellationToken,
task::{TaskTracker, task_tracker::TaskTrackerToken},
};
use uuid::Uuid;
use super::websocket_pump::{PumpKeepalive, PumpedWebSocket};
@@ -63,6 +67,8 @@ impl CodexWebSocketPoolKey {
pub struct CodexWebSocketPool {
inner: Arc<Mutex<WebSocketPoolState>>,
config: CodexWebSocketPoolConfig,
tasks: TaskTracker,
shutdown: CancellationToken,
}
impl Default for CodexWebSocketPool {
@@ -138,6 +144,8 @@ impl CodexWebSocketPool {
let pool = Self {
inner: Arc::new(Mutex::new(WebSocketPoolState::default())),
config,
tasks: TaskTracker::new(),
shutdown: CancellationToken::new(),
};
pool.spawn_maintenance_task();
pool
@@ -153,6 +161,11 @@ impl CodexWebSocketPool {
self.config.initial_event_timeout
}
/// 注册由连接池生命周期托管的 opening 任务。
pub(crate) fn spawn_connect_task(&self, future: impl Future<Output = ()> + Send + 'static) {
drop(self.tasks.spawn(future));
}
pub(crate) async fn acquire(&self, key: &CodexWebSocketPoolKey) -> WebSocketPoolAcquire {
let mut expired_connection = None;
let acquire = {
@@ -167,6 +180,7 @@ impl CodexWebSocketPool {
Some(WebSocketPoolSlot::Connecting(connecting)) => {
return WebSocketPoolAcquire::Wait(WebSocketPoolConnectWaiter {
receiver: connecting.outcome.subscribe(),
started_at: connecting.started_at,
});
}
Some(WebSocketPoolSlot::Idle(_)) => {
@@ -199,8 +213,9 @@ impl CodexWebSocketPool {
key.clone(),
WebSocketPoolSlot::Connecting(WebSocketPoolConnecting {
id: lease.id,
started_at: Instant::now(),
started_at: lease.started_at,
outcome: lease.outcome.clone(),
cancellation: lease.cancellation.clone(),
}),
);
WebSocketPoolAcquire::Connect(lease)
@@ -216,6 +231,39 @@ impl CodexWebSocketPool {
acquire
}
/// 只租用已经处于 Idle 的连接,不创建新的 Connecting slot。
pub(crate) async fn take_idle(
&self,
key: &CodexWebSocketPoolKey,
) -> Option<(Box<PooledWebSocketConnection>, WebSocketPoolLease)> {
let mut expired_connection = None;
let acquired = {
let mut state = self.inner.lock().await;
if !self.config.enabled || state.shutting_down {
return None;
}
let Some(WebSocketPoolSlot::Idle(connection)) = state.slots.remove(key) else {
return None;
};
if connection.created_at.elapsed() < self.config.max_age
&& !connection.websocket.is_closed()
{
let lease = WebSocketPoolLease::reserve(self.clone(), key.clone());
state
.slots
.insert(key.clone(), WebSocketPoolSlot::Busy(lease.reservation));
Some((connection, lease))
} else {
expired_connection = Some(*connection);
None
}
};
if let Some(connection) = expired_connection {
close_pooled_connection(connection).await;
}
acquired
}
async fn put_reserved(
&self,
key: &CodexWebSocketPoolKey,
@@ -256,26 +304,22 @@ impl CodexWebSocketPool {
}
}
async fn finish_connect(
async fn finish_connect_idle(
&self,
key: &CodexWebSocketPoolKey,
connect_id: Uuid,
connection: PooledWebSocketConnection,
) -> Result<WebSocketPoolConnected, Box<PooledWebSocketConnection>> {
) -> Result<(), Box<PooledWebSocketConnection>> {
let mut state = self.inner.lock().await;
let owns_connect = matches!(
state.slots.get(key),
Some(WebSocketPoolSlot::Connecting(connecting)) if connecting.id == connect_id
);
if owns_connect && !state.shutting_down && self.config.enabled {
let lease = WebSocketPoolLease::reserve(self.clone(), key.clone());
state
.slots
.insert(key.clone(), WebSocketPoolSlot::Busy(lease.reservation));
Ok(WebSocketPoolConnected {
connection: Box::new(connection),
lease,
})
.insert(key.clone(), WebSocketPoolSlot::Idle(Box::new(connection)));
Ok(())
} else {
if owns_connect {
state.slots.remove(key);
@@ -294,7 +338,7 @@ impl CodexWebSocketPool {
}
}
/// 驱逐指定账号的 idle 连接,并阻止已占用 slot 回收到池中。
/// 驱逐指定账号的全部 slot,取消 opening,并阻止 busy 连接回收到池中。
pub async fn evict_account(&self, account_id: &str) {
let mut idle_connections = Vec::new();
{
@@ -311,6 +355,7 @@ impl CodexWebSocketPool {
idle_connections.push(*connection);
}
Some(WebSocketPoolSlot::Connecting(connecting)) => {
connecting.cancellation.cancel();
connecting
.outcome
.send_replace(WebSocketPoolConnectOutcome::Failed);
@@ -322,8 +367,10 @@ impl CodexWebSocketPool {
close_pooled_connections(idle_connections).await;
}
/// 关闭连接池,关闭所有 idle 连接,并让后续 acquire 直接绕过池。
/// 关闭连接池,取消受管任务、关闭 idle 连接,并让后续 acquire 直接绕过池。
pub async fn shutdown(&self) {
self.tasks.close();
self.shutdown.cancel();
let idle_connections = {
let mut state = self.inner.lock().await;
state.shutting_down = true;
@@ -333,6 +380,7 @@ impl CodexWebSocketPool {
.filter_map(|(_, slot)| match slot {
WebSocketPoolSlot::Idle(connection) => Some(*connection),
WebSocketPoolSlot::Connecting(connecting) => {
connecting.cancellation.cancel();
connecting
.outcome
.send_replace(WebSocketPoolConnectOutcome::Failed);
@@ -343,6 +391,7 @@ impl CodexWebSocketPool {
.collect::<Vec<_>>()
};
close_pooled_connections(idle_connections).await;
self.tasks.wait().await;
}
/// 维护池 slot:清扫已死亡或超龄的 idle 连接,以及异常残留的 Busy reservation。
@@ -364,20 +413,30 @@ impl CodexWebSocketPool {
};
let inner = Arc::downgrade(&self.inner);
let config = self.config;
tokio::spawn(async move {
let tasks = self.tasks.clone();
let shutdown = self.shutdown.clone();
drop(self.tasks.spawn(async move {
let mut interval = tokio::time::interval(interval_duration);
loop {
interval.tick().await;
tokio::select! {
_ = shutdown.cancelled() => break,
_ = interval.tick() => {}
}
let Some(inner) = inner.upgrade() else {
break;
};
let pool = CodexWebSocketPool { inner, config };
let pool = CodexWebSocketPool {
inner,
config,
tasks: tasks.clone(),
shutdown: shutdown.clone(),
};
if pool.is_shutdown().await {
break;
}
pool.maintain_idle_connections().await;
}
});
}));
}
/// 返回连接池是否已进入关闭状态。
@@ -428,6 +487,7 @@ impl CodexWebSocketPool {
);
}
Some(WebSocketPoolSlot::Connecting(connecting)) => {
connecting.cancellation.cancel();
connecting
.outcome
.send_replace(WebSocketPoolConnectOutcome::Failed);
@@ -503,6 +563,7 @@ struct WebSocketPoolConnecting {
id: Uuid,
started_at: Instant,
outcome: watch::Sender<WebSocketPoolConnectOutcome>,
cancellation: CancellationToken,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -514,9 +575,14 @@ pub(crate) enum WebSocketPoolConnectOutcome {
pub(crate) struct WebSocketPoolConnectWaiter {
receiver: watch::Receiver<WebSocketPoolConnectOutcome>,
started_at: Instant,
}
impl WebSocketPoolConnectWaiter {
pub(crate) fn remaining_budget(&self, budget: Duration) -> Duration {
budget.saturating_sub(self.started_at.elapsed())
}
pub(crate) async fn wait(mut self) -> WebSocketPoolConnectOutcome {
loop {
let outcome = *self.receiver.borrow_and_update();
@@ -534,34 +600,50 @@ pub(crate) struct WebSocketPoolConnectLease {
pool: CodexWebSocketPool,
key: CodexWebSocketPoolKey,
id: Uuid,
started_at: Instant,
outcome: watch::Sender<WebSocketPoolConnectOutcome>,
cancellation: CancellationToken,
// slot 分配时即注册,封闭 acquire 与后台 task spawn 之间的 shutdown 竞态。
_task_registration: TaskTrackerToken,
armed: bool,
}
pub(crate) struct WebSocketPoolConnected {
pub(crate) connection: Box<PooledWebSocketConnection>,
pub(crate) lease: WebSocketPoolLease,
}
impl WebSocketPoolConnectLease {
fn reserve(pool: CodexWebSocketPool, key: CodexWebSocketPoolKey) -> Self {
let (outcome, _) = watch::channel(WebSocketPoolConnectOutcome::Pending);
let cancellation = pool.shutdown.child_token();
let task_registration = pool.tasks.token();
Self {
pool,
key,
id: Uuid::new_v4(),
started_at: Instant::now(),
outcome,
cancellation,
_task_registration: task_registration,
armed: true,
}
}
pub(crate) async fn connected(
pub(crate) fn started_at(&self) -> Instant {
self.started_at
}
pub(crate) fn key(&self) -> &CodexWebSocketPoolKey {
&self.key
}
pub(crate) fn cancellation_token(&self) -> CancellationToken {
self.cancellation.clone()
}
pub(crate) async fn connected_idle(
mut self,
connection: PooledWebSocketConnection,
) -> Result<WebSocketPoolConnected, Box<PooledWebSocketConnection>> {
) -> Result<(), Box<PooledWebSocketConnection>> {
let result = self
.pool
.finish_connect(&self.key, self.id, connection)
.finish_connect_idle(&self.key, self.id, connection)
.await;
let outcome = if result.is_ok() {
WebSocketPoolConnectOutcome::Ready
@@ -574,9 +656,11 @@ impl WebSocketPoolConnectLease {
}
pub(crate) async fn failed(mut self) {
self.pool.fail_connect(&self.key, self.id).await;
self.cancellation.cancel();
// 先唤醒共享等待者,pool mutex 清理不得延迟前台 transport 决策。
self.outcome
.send_replace(WebSocketPoolConnectOutcome::Failed);
self.pool.fail_connect(&self.key, self.id).await;
self.armed = false;
}
}
@@ -586,6 +670,7 @@ impl Drop for WebSocketPoolConnectLease {
if !self.armed {
return;
}
self.cancellation.cancel();
self.outcome
.send_replace(WebSocketPoolConnectOutcome::Failed);
let Ok(runtime) = tokio::runtime::Handle::try_current() else {
@@ -18,12 +18,8 @@ async fn admin_accounts_import_should_store_cpr_account_tokens() {
);
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state);
@@ -618,12 +614,8 @@ async fn admin_accounts_import_should_fetch_usage_to_complete_missing_plan_and_q
let mut config = test_config(test_database_url());
config.api.base_url = format!("{}/backend-api", server.uri());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state);
let access_token = test_jwt_with_exp(
@@ -830,12 +822,8 @@ async fn admin_accounts_import_should_require_admin_session_cookie() {
let redis = create_test_redis("admin-accounts-import-auth").await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state);
@@ -11,12 +11,8 @@ async fn admin_accounts_lifecycle_should_update_and_delete_accounts() {
seed_account_related_rows(&pool, "acct_lifecycle").await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis.clone());
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let session_affinity =
codex_proxy_rs::dispatch::affinity::RedisSessionAffinityStore::new(redis);
@@ -873,12 +869,8 @@ async fn admin_account_create_should_derive_claims_and_store_plain_tokens() {
);
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state.clone());
@@ -907,12 +899,8 @@ async fn admin_account_manual_create_should_reject_missing_invalid_expired_or_un
seed_admin_session(&pool, &redis, "session_1").await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state);
@@ -15,12 +15,8 @@ async fn admin_accounts_list_should_not_expose_account_tokens() {
.execute(&pool).await.unwrap();
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state);
@@ -23,9 +23,9 @@ use crate::support::jwt::unsigned_jwt;
use crate::support::{
admin::seed_admin_session,
config::test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::wire_profile,
};
mod export_routes;
@@ -196,12 +196,8 @@ async fn admin_accounts_test_app_with_overrides(
config.auth.oauth_token_endpoint = oauth_token_endpoint;
}
let stores = background_task_stores(pool.clone(), redis.clone());
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state.clone());
(
@@ -195,3 +195,60 @@ async fn admin_account_quota_should_send_usage_cookie() {
"2K"
);
}
#[tokio::test]
async fn admin_account_quota_should_ban_deactivated_workspace() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/backend-api/wham/usage"))
.respond_with(
ResponseTemplate::new(402)
.set_body_json(json!({ "detail": { "code": "deactivated_workspace" } })),
)
.expect(1)
.mount(&server)
.await;
let (app, _state, pool, _dir) = admin_accounts_test_app_with_api_base_url(
"admin-account-quota-deactivated",
89,
format!("{}/backend-api", server.uri()),
)
.await;
seed_account(
&pool,
NewAccount {
id: "acct_quota_deactivated".to_string(),
email: Some("quota-deactivated@example.com".to_string()),
account_id: Some("chatgpt-quota-deactivated".to_string()),
user_id: None,
label: None,
plan_type: Some("plus".to_string()),
access_token: SecretString::new("access-quota-deactivated".to_string().into()),
refresh_token: None,
access_token_expires_at: None,
status: AccountStatus::Active,
added_at: None,
},
)
.await;
let response = app
.oneshot(
Request::builder()
.method("GET")
.uri("/api/admin/accounts/quota?id=acct_quota_deactivated")
.header("cookie", "cpr_admin_session=session_1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status: String = sqlx::query_scalar("select status from accounts where id = $1")
.bind("acct_quota_deactivated")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
assert_eq!(status, "banned");
}
@@ -9,9 +9,9 @@ use tower::util::ServiceExt;
use crate::support::{
config::test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::wire_profile,
};
#[tokio::test]
@@ -237,11 +237,11 @@ async fn admin_login_test_app(
seed_admin_user(&pool, password).await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(codex_proxy_rs::bootstrap::services::Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
wire_profile(profile),
));
let state = codex_proxy_rs::api::AppState::from(services.as_ref());
(
+114 -114
View File
@@ -8,7 +8,7 @@ use codex_proxy_rs::{
bootstrap::services::Services,
infra::time::{china_datetime, china_day_start, china_quarter_hour_start},
telemetry::{usage::store::PgUsageRecordStore, usage::types::UsageRecord},
upstream::openai::fingerprint::Fingerprint,
upstream::openai::profile::CodexWireProfile,
};
use serde_json::{Value, json};
use sqlx::PgPool;
@@ -16,49 +16,54 @@ use tower::util::ServiceExt;
use crate::support::{
admin::seed_admin_session,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::test_wire_profile,
};
#[tokio::test]
async fn dashboard_summary_should_render_fingerprint_updated_at_as_beijing_time() {
let (app, _store, _pool, _dir) = dashboard_test_app(
"dashboard-fingerprint-updated",
crate::support::fingerprint::test_fingerprint_with_updated_at(Some(
"2026-06-23T11:36:46.965574590+00:00",
)),
)
.await;
async fn dashboard_summary_should_expose_effective_wire_profile() {
let (app, _store, _pool, _dir) =
dashboard_test_app("dashboard-wire-profile", test_wire_profile()).await;
let body = dashboard_summary(app).await;
assert_eq!(
service_status_value(&body, "更新时间"),
"2026-06-23 19:36:46"
body["data"]["wireProfile"]["userAgent"],
"Codex Desktop/0.144.2 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.707.72221)"
);
assert_eq!(body["data"]["wireProfile"]["codexVersion"], "0.144.2");
assert_eq!(
body["data"]["wireProfile"]["desktopVersion"],
"26.707.72221"
);
assert_eq!(body["data"]["wireProfile"]["desktopBuild"], "5307");
assert_eq!(body["data"]["wireProfile"]["target"]["osType"], "Mac OS");
assert_eq!(body["data"]["wireProfile"]["target"]["arch"], "arm64");
assert_eq!(
body["data"]["wireProfile"]["verifiedAt"],
"2026-07-14T18:25:50Z"
);
}
#[tokio::test]
async fn dashboard_summary_should_render_dash_when_fingerprint_updated_at_is_missing() {
let (app, _store, _pool, _dir) = dashboard_test_app(
"dashboard-fingerprint-missing",
crate::support::fingerprint::test_fingerprint(),
)
.await;
async fn dashboard_summary_should_keep_release_observation_separate_from_wire_profile() {
let (app, _store, _pool, _dir) =
dashboard_test_app("dashboard-release-unchecked", test_wire_profile()).await;
let body = dashboard_summary(app).await;
assert_eq!(service_status_value(&body, "更新时间"), "-");
assert_eq!(
body["data"]["wireProfile"]["release"]["status"],
"unchecked"
);
assert!(body["data"].get("serviceStatuses").is_none());
}
#[tokio::test]
async fn dashboard_summary_should_report_database_failure() {
let (app, _store, pool, _dir) = dashboard_test_app(
"dashboard-summary-database-failure",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, _store, pool, _dir) =
dashboard_test_app("dashboard-summary-database-failure", test_wire_profile()).await;
pool.close().await;
let response = dashboard_response(app, "/api/admin/dashboard/summary").await;
@@ -72,11 +77,8 @@ async fn dashboard_summary_should_report_database_failure() {
#[tokio::test]
async fn dashboard_trend_should_report_database_failure() {
let (app, _store, pool, _dir) = dashboard_test_app(
"dashboard-trend-database-failure",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, _store, pool, _dir) =
dashboard_test_app("dashboard-trend-database-failure", test_wire_profile()).await;
pool.close().await;
let response = dashboard_response(app, "/api/admin/dashboard/trend?kind=usage").await;
@@ -91,11 +93,8 @@ async fn dashboard_trend_should_report_database_failure() {
#[tokio::test]
async fn dashboard_summary_should_calculate_today_traffic_and_average_first_token_latency_from_time_buckets()
{
let (app, store, pool, _dir) = dashboard_test_app(
"dashboard-traffic-latency",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, pool, _dir) =
dashboard_test_app("dashboard-traffic-latency", test_wire_profile()).await;
let now = Utc::now();
let today_start = china_day_start(now);
let today_record_at = if now - today_start > Duration::minutes(1) {
@@ -151,7 +150,7 @@ async fn dashboard_summary_should_calculate_today_traffic_and_average_first_toke
async fn dashboard_summary_should_keep_trend_after_usage_records_are_cleared() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-time-buckets-survive-usage-record-clear",
crate::support::fingerprint::test_fingerprint(),
test_wire_profile(),
)
.await;
let mut record = usage_record_with_tokens(Utc::now(), 12);
@@ -177,11 +176,8 @@ async fn dashboard_summary_should_keep_trend_after_usage_records_are_cleared() {
#[tokio::test]
async fn dashboard_trend_should_bucket_usage_by_china_quarter_hour() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-trend-china-quarter-hour",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, _pool, _dir) =
dashboard_test_app("dashboard-trend-china-quarter-hour", test_wire_profile()).await;
let record_at = Utc::now();
store
.append(&usage_record_with_token_parts(record_at, 40, 60, 10))
@@ -205,11 +201,8 @@ async fn dashboard_trend_should_bucket_usage_by_china_quarter_hour() {
#[tokio::test]
async fn dashboard_trend_should_only_include_china_today() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-trend-china-today",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, _pool, _dir) =
dashboard_test_app("dashboard-trend-china-today", test_wire_profile()).await;
let now = Utc::now();
let today_start = china_day_start(now);
store
@@ -237,11 +230,8 @@ async fn dashboard_trend_should_only_include_china_today() {
#[tokio::test]
async fn dashboard_latency_trend_should_use_first_token_latency() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-trend-first-token-latency",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, _pool, _dir) =
dashboard_test_app("dashboard-trend-first-token-latency", test_wire_profile()).await;
let now = Utc::now();
let mut slow_completion = usage_record_with_tokens(now, 10);
slow_completion.latency_ms = Some(10_000);
@@ -277,11 +267,8 @@ async fn dashboard_latency_trend_should_use_first_token_latency() {
#[tokio::test]
async fn dashboard_latency_trend_should_serialize_missing_samples_as_null() {
let (app, _store, _pool, _dir) = dashboard_test_app(
"dashboard-trend-empty-latency",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, _store, _pool, _dir) =
dashboard_test_app("dashboard-trend-empty-latency", test_wire_profile()).await;
let body = dashboard_trend(app, "latency").await;
let point = &body["data"]["points"][0];
@@ -312,11 +299,8 @@ async fn dashboard_latency_trend_should_serialize_missing_samples_as_null() {
#[tokio::test]
async fn dashboard_error_trend_should_serialize_missing_success_rate_as_null() {
let (app, _store, _pool, _dir) = dashboard_test_app(
"dashboard-trend-empty-success-rate",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, _store, _pool, _dir) =
dashboard_test_app("dashboard-trend-empty-success-rate", test_wire_profile()).await;
let body = dashboard_trend(app, "errors").await;
let point = &body["data"]["points"][0];
@@ -346,7 +330,7 @@ async fn dashboard_error_trend_should_serialize_missing_success_rate_as_null() {
async fn dashboard_summary_should_return_requested_trend_kind() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-summary-requested-trend-kind",
crate::support::fingerprint::test_fingerprint(),
test_wire_profile(),
)
.await;
let now = Utc::now();
@@ -370,7 +354,7 @@ async fn dashboard_summary_should_return_requested_trend_kind() {
#[tokio::test]
async fn dashboard_summary_should_calculate_priority_billing_from_event_service_tier() {
let billing_amount = dashboard_summary_today_billing_for_usage_record(
let billing_amount = dashboard_summary_total_billing_for_usage_record(
"dashboard-billing-priority",
"gpt-5.5",
json!({
@@ -389,7 +373,7 @@ async fn dashboard_summary_should_calculate_priority_billing_from_event_service_
#[tokio::test]
async fn dashboard_summary_should_charge_cached_input_at_cache_read_price() {
let billing_amount = dashboard_summary_today_billing_for_usage_record(
let billing_amount = dashboard_summary_total_billing_for_usage_record(
"dashboard-billing-cached",
"gpt-5.5",
json!({
@@ -407,7 +391,7 @@ async fn dashboard_summary_should_charge_cached_input_at_cache_read_price() {
#[tokio::test]
async fn dashboard_summary_should_apply_long_context_prices() {
let billing_amount = dashboard_summary_today_billing_for_usage_record(
let billing_amount = dashboard_summary_total_billing_for_usage_record(
"dashboard-billing-long-context",
"gpt-5.5",
json!({
@@ -425,7 +409,7 @@ async fn dashboard_summary_should_apply_long_context_prices() {
#[tokio::test]
async fn dashboard_summary_should_match_versioned_model_names_for_billing() {
let billing_amount = dashboard_summary_today_billing_for_usage_record(
let billing_amount = dashboard_summary_total_billing_for_usage_record(
"dashboard-billing-codex-model",
"gpt-5.6-terra-2026-07-01",
json!({
@@ -443,11 +427,8 @@ async fn dashboard_summary_should_match_versioned_model_names_for_billing() {
#[tokio::test]
async fn dashboard_summary_should_return_backend_formatted_time_fields() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-backend-formatted-time",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, _pool, _dir) =
dashboard_test_app("dashboard-backend-formatted-time", test_wire_profile()).await;
let record_at = Utc::now() - Duration::seconds(1);
let mut record = usage_record_with_tokens(record_at, 10);
record.metadata = json!({
@@ -470,11 +451,8 @@ async fn dashboard_summary_should_return_backend_formatted_time_fields() {
#[tokio::test]
async fn dashboard_summary_should_only_return_today_account_usage_and_usage_records() {
let (app, store, pool, _dir) = dashboard_test_app(
"dashboard-summary-today-records",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, pool, _dir) =
dashboard_test_app("dashboard-summary-today-records", test_wire_profile()).await;
seed_dashboard_account(&pool, "acct_today", "today@example.com").await;
seed_dashboard_account(&pool, "acct_yesterday", "yesterday@example.com").await;
let now = Utc::now();
@@ -519,11 +497,8 @@ async fn dashboard_summary_should_only_return_today_account_usage_and_usage_reco
#[tokio::test]
async fn dashboard_summary_should_order_account_usage_by_recent_usage() {
let (app, store, pool, _dir) = dashboard_test_app(
"dashboard-account-usage-recent-order",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, pool, _dir) =
dashboard_test_app("dashboard-account-usage-recent-order", test_wire_profile()).await;
seed_dashboard_account(&pool, "acct_old_heavy", "old-heavy@example.com").await;
seed_dashboard_account(&pool, "acct_recent_light", "recent-light@example.com").await;
let now = Utc::now();
@@ -563,7 +538,7 @@ async fn dashboard_summary_should_order_account_usage_by_recent_usage() {
async fn dashboard_summary_should_use_five_hour_quota_window_for_account_usage() {
let (app, store, pool, _dir) = dashboard_test_app(
"dashboard-account-usage-five-hour-quota",
crate::support::fingerprint::test_fingerprint(),
test_wire_profile(),
)
.await;
seed_dashboard_account(&pool, "acct_quota_five_hour", "quota-five-hour@example.com").await;
@@ -612,11 +587,8 @@ async fn dashboard_summary_should_use_five_hour_quota_window_for_account_usage()
#[tokio::test]
async fn dashboard_summary_should_fall_back_to_weekly_quota_window_when_five_hour_missing() {
let (app, store, pool, _dir) = dashboard_test_app(
"dashboard-account-usage-weekly-quota",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, pool, _dir) =
dashboard_test_app("dashboard-account-usage-weekly-quota", test_wire_profile()).await;
seed_dashboard_account(&pool, "acct_quota_weekly", "quota-weekly@example.com").await;
store
.append(&usage_record_with_account_tokens(
@@ -663,11 +635,8 @@ async fn dashboard_summary_should_fall_back_to_weekly_quota_window_when_five_hou
#[tokio::test]
async fn dashboard_summary_should_return_today_health_timeline() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-health-timeline",
crate::support::fingerprint::test_fingerprint(),
)
.await;
let (app, store, _pool, _dir) =
dashboard_test_app("dashboard-health-timeline", test_wire_profile()).await;
store
.append(&usage_record_with_tokens(Utc::now(), 10))
.await
@@ -681,21 +650,66 @@ async fn dashboard_summary_should_return_today_health_timeline() {
.unwrap();
let body = dashboard_summary(app).await;
let points = body["data"]["healthTimeline"]["points"].as_str().unwrap();
let points = body["data"]["healthTimeline"]["points"].as_array().unwrap();
assert_eq!(body["data"]["healthTimeline"]["title"], "请求健康时间线");
assert_eq!(body["data"]["healthTimeline"]["description"], "请求可靠性");
assert_eq!(
body["data"]["healthTimeline"]["description"],
"有效请求可用性"
);
assert_eq!(points.len(), 96);
assert_eq!(points.matches('4').count(), 1);
assert_eq!(
points
.iter()
.filter(|point| point["status"] == "low_sample")
.count(),
1
);
}
async fn dashboard_summary_today_billing_for_usage_record(
#[tokio::test]
async fn dashboard_summary_should_include_retained_history_in_total_billing() {
let (app, store, _pool, _dir) = dashboard_test_app(
"dashboard-total-billing-retained-history",
test_wire_profile(),
)
.await;
let metadata = json!({
"usage": {
"inputTokens": 100_000u64,
"outputTokens": 100_000u64,
"cachedTokens": 0u64
},
"serviceTier": "priority"
});
for created_at in [Utc::now(), Utc::now() - Duration::days(8)] {
let mut record = usage_record(created_at, "gpt-5.5", metadata.clone());
record.input_tokens = Some(100_000);
record.output_tokens = Some(100_000);
record.cached_tokens = Some(0);
record.service_tier = Some("priority".to_string());
store.append(&record).await.unwrap();
}
let body = dashboard_summary(app).await;
assert_eq!(
body["data"]["cards"]["tokens"]["totalBillingAmountUsd"],
"$17.50"
);
assert!(
body["data"]["cards"]["tokens"]
.get("todayBillingAmountUsd")
.is_none()
);
}
async fn dashboard_summary_total_billing_for_usage_record(
db_name: &str,
model: &str,
metadata: Value,
) -> String {
let (app, store, _pool, _dir) =
dashboard_test_app(db_name, crate::support::fingerprint::test_fingerprint()).await;
let (app, store, _pool, _dir) = dashboard_test_app(db_name, test_wire_profile()).await;
let mut record = usage_record(Utc::now(), model, metadata.clone());
record.input_tokens = metadata
.pointer("/usage/inputTokens")
@@ -716,7 +730,7 @@ async fn dashboard_summary_today_billing_for_usage_record(
store.append(&record).await.unwrap();
let body = dashboard_summary(app).await;
body["data"]["cards"]["tokens"]["todayBillingAmountUsd"]
body["data"]["cards"]["tokens"]["totalBillingAmountUsd"]
.as_str()
.unwrap()
.to_string()
@@ -724,7 +738,7 @@ async fn dashboard_summary_today_billing_for_usage_record(
async fn dashboard_test_app(
db_name: &str,
fingerprint: Fingerprint,
wire_profile: std::sync::Arc<CodexWireProfile>,
) -> (
axum::Router,
PgUsageRecordStore,
@@ -736,11 +750,7 @@ async fn dashboard_test_app(
seed_admin_session(&pool, &redis, "session_1").await;
let config = crate::support::config::test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let services = std::sync::Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let services = std::sync::Arc::new(Services::new(&config, stores, wire_profile));
let state = AppState::from(services.as_ref());
(
codex_proxy_rs::api::router::router().with_state(state),
@@ -851,16 +861,6 @@ async fn set_account_quota_json(pool: &PgPool, account_id: &str, quota_json: Val
.unwrap();
}
fn service_status_value<'a>(body: &'a Value, label: &str) -> &'a str {
body["data"]["serviceStatuses"]
.as_array()
.unwrap()
.iter()
.find(|status| status["label"] == label)
.and_then(|status| status["value"].as_str())
.unwrap()
}
fn usage_record_with_tokens(created_at: chrono::DateTime<Utc>, total_tokens: u64) -> UsageRecord {
usage_record_with_token_parts(created_at, total_tokens, 0, 0)
}
+3 -7
View File
@@ -10,9 +10,9 @@ use tower::util::ServiceExt;
use crate::support::{
admin::seed_admin_session,
config::test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::wire_profile,
};
mod authorization;
@@ -30,12 +30,8 @@ async fn admin_client_key_test_app(
seed_admin_session(&pool, &redis, "session_1").await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = std::sync::Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
(
codex_proxy_rs::api::router::router().with_state(state),
@@ -25,9 +25,9 @@ use tower::util::ServiceExt;
use crate::support::{
admin::seed_admin_session,
config::test_config as base_test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::wire_profile,
};
#[tokio::test]
@@ -261,8 +261,8 @@ async fn admin_settings_update_should_persist_runtime_settings_to_database() {
let config = test_config(test_database_url());
seed_admin_session(&pool, &redis, "session_1").await;
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = Services::new(&config, stores, runtime_fingerprint(fingerprint));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = Services::new(&config, stores, wire_profile(profile));
let services = std::sync::Arc::new(services);
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state);
@@ -356,12 +356,8 @@ async fn admin_settings_update_should_apply_runtime_services() {
seed_schedulable_account(&pool, "acct_runtime_a").await;
seed_schedulable_account(&pool, "acct_runtime_b").await;
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = std::sync::Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(Services::new(&config, stores, wire_profile(profile)));
let settings_tasks = TaskCoordinator::start_settings_subscriptions(&services);
services.account_pool.restore_from_store().await.unwrap();
let state = AppState::from(services.as_ref());
@@ -466,12 +462,8 @@ async fn admin_settings_update_should_reject_unsupported_or_invalid_fields() {
let config = test_config(test_database_url());
seed_admin_session(&pool, &redis, "session_1").await;
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = std::sync::Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
let app = codex_proxy_rs::api::router::router().with_state(state);
@@ -501,12 +493,8 @@ async fn admin_settings_test_app(
seed_admin_session(&pool, &redis, "session_1").await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = std::sync::Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(Services::new(&config, stores, wire_profile(profile)));
let state = AppState::from(services.as_ref());
(codex_proxy_rs::api::router::router().with_state(state), dir)
}
+2 -2
View File
@@ -28,8 +28,8 @@ use wiremock::{
use crate::support::{
admin::seed_admin_session,
config::test_config,
fingerprint::{runtime_fingerprint, test_fingerprint},
http::response_json,
wire_profile::{test_wire_profile_value, wire_profile},
};
const VERSION_OLD: &str = "0.1.0";
@@ -1269,7 +1269,7 @@ async fn admin_system_test_app_with_process_control(
let services = Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(test_fingerprint()),
wire_profile(test_wire_profile_value()),
));
let mut state = AppState::from(services.as_ref());
state.services.system_update = Arc::new(SystemUpdateService::new(update_config));
+37 -6
View File
@@ -16,9 +16,9 @@ use tower::util::ServiceExt;
use crate::support::{
admin::seed_admin_session,
config::test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::wire_profile,
};
#[tokio::test]
@@ -138,10 +138,24 @@ async fn usage_insights_should_collapse_retry_errors_to_terminal_requests() {
.execute(usage.pool())
.await
.unwrap();
sqlx::query(
"insert into client_api_keys
(id, name, prefix, key, enabled, created_at, last_used_at)
values ($1, $2, $3, $4, true, $5, null)",
)
.bind("key_diagnostic")
.bind("生产密钥")
.bind("sk-diagnostic")
.bind("sk-diagnostic-secret")
.bind(now)
.execute(usage.pool())
.await
.unwrap();
let mut success = success_record("eventually succeeded");
success.id = "usage_insights_success".to_string();
success.request_id = Some("req_retried".to_string());
success.client_api_key_id = Some("key_diagnostic".to_string());
success.input_tokens = Some(100);
success.output_tokens = Some(20);
success.cached_tokens = Some(20);
@@ -157,14 +171,18 @@ async fn usage_insights_should_collapse_retry_errors_to_terminal_requests() {
retried_error.created_at = now - chrono::Duration::seconds(2);
ops.append(&retried_error).await.unwrap();
for (id, failure_class, seconds) in [
("ops_insights_failed_older", "upstream", 2),
("ops_insights_failed_latest", "response_failed", 1),
for (id, failure_class, seconds, persist_class) in [
("ops_insights_failed_older", "upstream", 2, true),
("ops_insights_failed_latest", "response_failed", 1, false),
] {
let mut error = OpsErrorLog::new("upstream", failure_class);
error.id = id.to_string();
error.request_id = Some("req_failed".to_string());
error.failure_class = Some(failure_class.to_string());
if persist_class {
error.failure_class = Some(failure_class.to_string());
} else {
error.metadata = json!({ "upstreamCode": failure_class });
}
error.created_at = now - chrono::Duration::seconds(seconds);
ops.append(&error).await.unwrap();
}
@@ -211,6 +229,19 @@ async fn usage_insights_should_collapse_retry_errors_to_terminal_requests() {
.iter()
.any(|item| item["name"] == "usage-account@example.com → acct_usage")
);
let api_key_diagnostics = admin_get(
&app,
"/api/admin/usage/records/insights/diagnostics?dimension=apiKey",
)
.await;
assert!(
api_key_diagnostics["data"]["items"]
.as_array()
.unwrap()
.iter()
.any(|item| item["name"] == "生产密钥")
);
}
#[tokio::test]
@@ -334,7 +365,7 @@ async fn monitoring_test_app(
let services = Services::new(
&config,
stores,
runtime_fingerprint(crate::support::fingerprint::test_fingerprint()),
wire_profile(crate::support::wire_profile::test_wire_profile_value()),
);
let app = codex_proxy_rs::api::router::router().with_state(AppState::from(&services));
(
+3 -3
View File
@@ -12,12 +12,12 @@ use tower::ServiceExt;
use crate::support::{
config::test_config as base_test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{
TestDatabaseGuard, background_task_stores, create_test_redis, init_test_db,
test_database_url,
},
wire_profile::wire_profile,
};
#[tokio::test]
@@ -185,11 +185,11 @@ async fn router_with_dist_and_database(
let redis = create_test_redis(label).await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(codex_proxy_rs::bootstrap::services::Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
wire_profile(profile),
));
let state = codex_proxy_rs::api::AppState::from(services.as_ref());
(
+3 -7
View File
@@ -11,9 +11,9 @@ use tower::util::ServiceExt;
use crate::support::{
client_keys::insert_client_api_key,
config::test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::wire_profile,
};
#[tokio::test]
@@ -154,12 +154,8 @@ async fn build_test_app(
let redis = create_test_redis(label).await;
seed_model_snapshot(&redis).await;
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = std::sync::Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(Services::new(&config, stores, wire_profile(profile)));
services
.initialize_hot_path_state()
.await
+3 -7
View File
@@ -8,9 +8,9 @@ use tower::util::ServiceExt;
use crate::support::{
client_keys::insert_client_api_key,
config::test_config,
fingerprint::runtime_fingerprint,
http::response_json,
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
wire_profile::wire_profile,
};
mod websocket;
@@ -165,12 +165,8 @@ pub(super) async fn test_app_with_client_api_key() -> (
insert_model_snapshot(&redis).await;
let config = test_config(test_database_url());
let stores = background_task_stores(pool.clone(), redis);
let fingerprint = crate::support::fingerprint::test_fingerprint();
let services = std::sync::Arc::new(Services::new(
&config,
stores,
runtime_fingerprint(fingerprint),
));
let profile = crate::support::wire_profile::test_wire_profile_value();
let services = std::sync::Arc::new(Services::new(&config, stores, wire_profile(profile)));
services
.initialize_hot_path_state()
.await
+20 -11
View File
@@ -22,8 +22,9 @@ fn config_loader_should_load_complete_example() {
config.server.host.as_str(),
config.server.port,
config.api.base_url.as_str(),
config.fingerprint.app_version.as_str(),
config.fingerprint.build_number.as_str(),
config.wire_profile.codex_version.as_str(),
config.wire_profile.desktop_version.as_str(),
config.wire_profile.desktop_build.as_str(),
config.quota.refresh_interval_minutes,
config.ws_pool.initial_event_timeout_ms,
config.admin.default_username.as_str(),
@@ -33,8 +34,9 @@ fn config_loader_should_load_complete_example() {
"127.0.0.1",
8080,
"https://chatgpt.com/backend-api",
"26.707.51957",
"5175",
"0.144.2",
"26.707.72221",
"5307",
5,
20_000,
"admin@cpr.local",
@@ -158,7 +160,7 @@ fn config_loader_should_reject_missing_explicit_fields() {
#[test]
fn config_loader_should_reject_unsupported_schema_version() {
let yaml = complete_config().replace("schema_version: 1", "schema_version: 2");
let yaml = complete_config().replace("schema_version: 1", "schema_version: 0");
let directory = write_config(&yaml);
let error = BootstrapConfig::load_from_path(directory.path().join("config.yaml")).unwrap_err();
@@ -223,18 +225,25 @@ fn config_loader_should_reject_admin_password_with_compose_interpolation() {
}
#[test]
fn config_loader_should_reject_duplicate_fingerprint_headers() {
let yaml = complete_config().replace(
" - name: 'Accept-Language'",
" - name: 'Accept-Encoding'",
);
fn config_loader_should_reject_legacy_fingerprint_section() {
let yaml = complete_config().replace(" wire_profile:", " fingerprint:");
let directory = write_config(&yaml);
let error = BootstrapConfig::load_from_path(directory.path().join("config.yaml")).unwrap_err();
assert!(matches!(error, ConfigError::InvalidDocument { .. }));
}
#[test]
fn config_loader_should_reject_invalid_codex_core_version() {
let yaml = complete_config().replace("codex_version: '0.144.2'", "codex_version: 'Desktop'");
let directory = write_config(&yaml);
let error = BootstrapConfig::load_from_path(directory.path().join("config.yaml")).unwrap_err();
assert_eq!(
error,
ConfigError::InvalidField("fingerprint.default_headers")
ConfigError::InvalidField("wire_profile.codex_version")
);
}
+3 -3
View File
@@ -21,7 +21,7 @@ async fn services_try_new_should_use_configured_tls_transport_builder() {
let Err(error) = Services::try_new(
&config,
stores,
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
) else {
panic!("invalid custom CA should fail service transport construction");
};
@@ -59,7 +59,7 @@ async fn services_try_new_should_expose_websocket_pool_when_enabled() {
let services = Services::try_new(
&config,
background_task_stores(pool, redis),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
)
.expect("services should build");
@@ -75,7 +75,7 @@ async fn services_try_new_should_not_expose_websocket_pool_when_disabled() {
let services = Services::try_new(
&config,
background_task_stores(pool, redis),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
)
.expect("services should build");
@@ -0,0 +1,86 @@
use super::*;
#[tokio::test]
async fn desktop_release_update_task_should_start_background_checker() {
let status = DesktopReleaseStatus::default();
let handle =
codex_proxy_rs::bootstrap::tasks::desktop_release_update::start_desktop_release_update_task(
status,
"https://example.invalid/appcast.xml".to_string(),
);
handle.shutdown().await;
}
#[tokio::test]
async fn desktop_release_update_task_should_record_initial_appcast_observation() {
let server = MockServer::start().await;
mount_appcast(
&server,
r#"
<rss xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle">
<channel>
<item>
<pubDate>Tue, 14 Jul 2026 07:56:40 +0000</pubDate>
<sparkle:shortVersionString>26.900.1</sparkle:shortVersionString>
<sparkle:version>7001</sparkle:version>
<sparkle:minimumSystemVersion>12.0</sparkle:minimumSystemVersion>
<sparkle:hardwareRequirements>arm64</sparkle:hardwareRequirements>
<enclosure url="https://example.invalid/download" length="1234"
sparkle:edSignature="signature" />
</item>
</channel>
</rss>
"#,
)
.await;
let status = DesktopReleaseStatus::default();
let handle =
codex_proxy_rs::bootstrap::tasks::desktop_release_update::start_desktop_release_update_task(
status.clone(),
format!("{}/appcast.xml", server.uri()),
);
let snapshot = wait_for_latest_desktop_release(&status, "26.900.1").await;
handle.shutdown().await;
let release = snapshot.latest.expect("latest release");
assert_eq!(release.build, "7001");
assert_eq!(release.minimum_system_version.as_deref(), Some("12.0"));
assert_eq!(release.hardware_requirements.as_deref(), Some("arm64"));
assert_eq!(release.download_size, Some(1234));
assert!(release.signature_present);
assert!(snapshot.checked_at.is_some());
assert!(snapshot.last_error.is_none());
}
#[tokio::test]
async fn desktop_release_update_task_should_record_fetch_failure_without_mutating_wire_profile() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/appcast.xml"))
.respond_with(ResponseTemplate::new(503))
.mount(&server)
.await;
let profile = crate::support::wire_profile::test_wire_profile();
let original_user_agent = profile.user_agent();
let status = DesktopReleaseStatus::default();
let handle =
codex_proxy_rs::bootstrap::tasks::desktop_release_update::start_desktop_release_update_task(
status.clone(),
format!("{}/appcast.xml", server.uri()),
);
wait_for_appcast_requests(&server, 1).await;
let deadline = tokio::time::Instant::now() + StdDuration::from_secs(2);
while status.snapshot().last_error.is_none() {
assert!(tokio::time::Instant::now() < deadline);
tokio::time::sleep(StdDuration::from_millis(25)).await;
}
handle.shutdown().await;
assert_eq!(profile.user_agent(), original_user_agent);
assert!(status.snapshot().latest.is_none());
}
@@ -1,131 +0,0 @@
use super::*;
#[tokio::test]
async fn fingerprint_update_task_should_start_background_checker() {
let dir = tempfile::tempdir().expect("temp dir");
let (pool, _guard) = init_test_db("fingerprint-task-start").await;
let repo = PgFingerprintStore::new(pool.clone());
repo.ensure_current_seed(&crate::support::fingerprint::test_fingerprint())
.await
.expect("seed current fingerprint");
let runtime_fingerprint = runtime_fingerprint_from_repo(&repo).await;
let handle =
codex_proxy_rs::bootstrap::tasks::fingerprint_update::start_fingerprint_update_task(
repo,
runtime_fingerprint,
"https://example.invalid/appcast.xml".to_string(),
dir.path().join("extracted-fingerprint.json"),
"26.800.1".to_string(),
"6001".to_string(),
);
handle.shutdown().await;
}
#[tokio::test]
async fn fingerprint_update_task_should_apply_initial_appcast_update_to_store() {
let server = MockServer::start().await;
mount_appcast(
&server,
r#"
<rss>
<channel>
<item>
<enclosure url="https://example.invalid/download" sparkle:shortVersionString="26.900.1" sparkle:version="7001" />
</item>
</channel>
</rss>
"#,
)
.await;
let dir = tempfile::tempdir().expect("temp dir");
let (pool, _guard) = init_test_db("fingerprint-task-update").await;
let repo = PgFingerprintStore::new(pool.clone());
repo.ensure_current_seed(&crate::support::fingerprint::test_fingerprint())
.await
.expect("seed current fingerprint");
let runtime_fingerprint = runtime_fingerprint_from_repo(&repo).await;
let extracted_path = dir.path().join("extracted-fingerprint.json");
std::fs::write(
&extracted_path,
r#"{"app_version":"26.900.1","build_number":"7001","chromium_version":"147"}"#,
)
.expect("extracted fingerprint should be written");
let handle =
codex_proxy_rs::bootstrap::tasks::fingerprint_update::start_fingerprint_update_task(
repo.clone(),
runtime_fingerprint.clone(),
format!("{}/appcast.xml", server.uri()),
extracted_path,
"26.800.1".to_string(),
"6001".to_string(),
);
let stored = wait_for_current_fingerprint_version(&repo).await;
handle.shutdown().await;
assert_eq!(
(
stored.app_version.as_str(),
stored.build_number.as_str(),
stored.chromium_version.as_str()
),
("26.900.1", "7001", "147")
);
let runtime_snapshot = runtime_fingerprint.snapshot();
assert_eq!(
(
runtime_snapshot.app_version.as_str(),
runtime_snapshot.build_number.as_str(),
runtime_snapshot.chromium_version.as_str()
),
("26.900.1", "7001", "147")
);
}
#[tokio::test]
async fn fingerprint_update_task_should_not_persist_when_appcast_matches_current_fingerprint() {
let server = MockServer::start().await;
mount_appcast(
&server,
r#"
<rss>
<channel>
<item>
<enclosure url="https://example.invalid/download" sparkle:shortVersionString="26.900.1" sparkle:version="7001" />
</item>
</channel>
</rss>
"#,
)
.await;
let dir = tempfile::tempdir().expect("temp dir");
let (pool, _guard) = init_test_db("fingerprint-task-no-update").await;
let repo = PgFingerprintStore::new(pool.clone());
repo.ensure_current_seed(&crate::support::fingerprint::test_fingerprint())
.await
.expect("seed current fingerprint");
let runtime_fingerprint = runtime_fingerprint_from_repo(&repo).await;
let handle =
codex_proxy_rs::bootstrap::tasks::fingerprint_update::start_fingerprint_update_task(
repo.clone(),
runtime_fingerprint,
format!("{}/appcast.xml", server.uri()),
dir.path().join("missing-extracted-fingerprint.json"),
"26.900.1".to_string(),
"7001".to_string(),
);
wait_for_appcast_requests(&server, 1).await;
handle.shutdown().await;
let history_count: i64 = sqlx::query_scalar("select count(*) from fingerprint_update_history")
.fetch_one(&pool)
.await
.expect("fingerprint history count should load");
assert_eq!(history_count, 0);
}
+16 -22
View File
@@ -11,7 +11,9 @@ use codex_proxy_rs::telemetry::account_usage::store::{
AccountUsageDelta, AccountUsageSnapshot, AccountUsageStore, AccountUsageStoreError,
AccountUsageWindow,
};
use codex_proxy_rs::upstream::openai::fingerprint::{PgFingerprintStore, RuntimeFingerprint};
use codex_proxy_rs::upstream::openai::desktop_release::{
DesktopReleaseSnapshot, DesktopReleaseStatus,
};
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{method, path},
@@ -21,7 +23,7 @@ use crate::support::storage::init_test_db;
mod cleanup;
mod coordinator;
mod fingerprint_update;
mod desktop_release_update;
mod model_refresh;
mod retention_trim;
@@ -124,37 +126,29 @@ async fn mount_appcast(server: &MockServer, body: &str) {
.await;
}
async fn wait_for_current_fingerprint_version(
repo: &PgFingerprintStore,
) -> codex_proxy_rs::upstream::openai::fingerprint::Fingerprint {
async fn wait_for_latest_desktop_release(
status: &DesktopReleaseStatus,
expected_version: &str,
) -> DesktopReleaseSnapshot {
let deadline = tokio::time::Instant::now() + StdDuration::from_secs(2);
loop {
if let Some(stored) = repo
.load_current()
.await
.expect("current fingerprint should load")
.filter(|stored| stored.app_version == "26.900.1")
let snapshot = status.snapshot();
if snapshot
.latest
.as_ref()
.is_some_and(|release| release.version == expected_version)
{
return stored;
return snapshot;
}
assert!(
tokio::time::Instant::now() < deadline,
"fingerprint update task did not persist auto update before timeout"
"Desktop release task did not refresh appcast state before timeout"
);
tokio::time::sleep(StdDuration::from_millis(25)).await;
}
}
async fn runtime_fingerprint_from_repo(repo: &PgFingerprintStore) -> RuntimeFingerprint {
RuntimeFingerprint::new(
repo.load_current()
.await
.expect("current fingerprint should load")
.expect("current fingerprint should exist"),
)
}
async fn wait_for_appcast_requests(server: &MockServer, expected_count: usize) {
let deadline = tokio::time::Instant::now() + StdDuration::from_secs(2);
loop {
@@ -171,7 +165,7 @@ async fn wait_for_appcast_requests(server: &MockServer, expected_count: usize) {
assert!(
tokio::time::Instant::now() < deadline,
"fingerprint update task did not request appcast before timeout"
"Desktop release task did not request appcast before timeout"
);
tokio::time::sleep(StdDuration::from_millis(25)).await;
}
@@ -6,7 +6,6 @@ use codex_proxy_rs::{
ops::{store::PgOpsErrorLogStore, types::OpsErrorLog},
usage::{store::PgUsageRecordStore, types::UsageRecord},
},
upstream::openai::fingerprint::PgFingerprintStore,
};
use crate::support::storage::init_test_db;
@@ -28,11 +27,6 @@ async fn retention_trim_task_should_trim_all_growth_tables_in_one_run() {
let usage_store = PgUsageRecordStore::new(pool.clone());
let ops_store = PgOpsErrorLogStore::new(pool.clone());
let bucket_store = PgRequestBucketStore::new(pool.clone());
let fingerprint_store = PgFingerprintStore::new(pool.clone());
fingerprint_store
.ensure_current_seed(&crate::support::fingerprint::test_fingerprint())
.await
.unwrap();
let now = Utc::now();
let mut old_usage = UsageRecord::new("request", "old", "acct", "gpt-old", 200);
@@ -55,19 +49,7 @@ async fn retention_trim_task_should_trim_all_growth_tables_in_one_run() {
current_error.created_at = now;
ops_store.append(&current_error).await.unwrap();
sqlx::query(
"insert into fingerprint_update_history (
id, current_fingerprint_id, app_version, build_number, source, created_at
)
select 'history_' || value, 'current', '1.0.' || value, value::text, 'test',
now() + value * interval '1 microsecond'
from generate_series(1, 101) as value",
)
.execute(&pool)
.await
.unwrap();
RetentionTrimTask::new(usage_store, ops_store, bucket_store, fingerprint_store)
RetentionTrimTask::new(usage_store, ops_store, bucket_store)
.run_once()
.await;
@@ -85,13 +67,7 @@ async fn retention_trim_task_should_trim_all_growth_tables_in_one_run() {
.fetch_one(&pool)
.await
.unwrap();
let history_count: i64 = sqlx::query_scalar("select count(*) from fingerprint_update_history")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(usage_ids, vec!["usage_current".to_string()]);
assert_eq!(ops_ids, vec!["ops_current".to_string()]);
assert_eq!(old_buckets, 0);
assert_eq!(history_count, 100);
}
+2 -1
View File
@@ -143,7 +143,8 @@ fn controller_best_effort_io_should_share_one_bounded_owner() {
"controller I/O {operation} must use the shared best-effort timeout"
);
}
assert!(finalizer.contains("timeout(LEASE_FINALIZE_TIMEOUT, account_lease.complete())"));
assert!(finalizer.contains("timeout(LEASE_FINALIZE_TIMEOUT, lease_completion.as_mut())"));
assert!(finalizer.contains("tokio::spawn(lease_completion)"));
}
#[test]
+3 -1
View File
@@ -181,8 +181,10 @@ fn live_stream_exit_paths_should_share_one_consuming_finalizer() {
.find(".leave_stream(")
.expect("controllers should leave from the finalizer");
let lease = finalizer
.find("timeout(LEASE_FINALIZE_TIMEOUT, account_lease.complete())")
.find("let mut lease_completion = Box::pin(account_lease.complete())")
.expect("the account lease should finalize");
assert!(finalizer.contains("timeout(LEASE_FINALIZE_TIMEOUT, lease_completion.as_mut())"));
assert!(finalizer.contains("tokio::spawn(lease_completion)"));
let client = finalizer
.find("finish_client_stream(")
.expect("client termination should run after internal finalization");
+27 -1
View File
@@ -243,7 +243,7 @@ async fn test_app_state_with_storage(
let services = Services::try_with_usage_record_options(
&config,
stores,
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
usage_record_options,
)
.expect("failed to build runtime services with configured TLS transport");
@@ -1501,6 +1501,32 @@ async fn response_usage_record_count(pool: &PgPool) -> i64 {
count
}
async fn wait_for_account_usage(
pool: &PgPool,
account_id: &str,
expected: (i64, i64, i64),
) -> (i64, i64, i64) {
let deadline = tokio::time::Instant::now() + StdDuration::from_secs(5);
loop {
let usage: Option<(i64, i64, i64)> = sqlx::query_as(
"select request_count, input_tokens, output_tokens from account_usage where account_id = $1",
)
.bind(account_id)
.fetch_optional(pool)
.await
.unwrap();
let usage = usage.unwrap_or_default();
if usage == expected {
return usage;
}
assert!(
tokio::time::Instant::now() < deadline,
"account usage for {account_id} did not settle: expected {expected:?}, got {usage:?}"
);
tokio::time::sleep(StdDuration::from_millis(10)).await;
}
}
fn assert_rate_limit_header(metadata: &Value, name: &str, value: &str) {
let headers = metadata["rateLimitHeaders"]
.as_array()
@@ -2010,20 +2010,8 @@ async fn responses_stream_should_fallback_to_next_account_after_rate_limit() {
.fetch_one(&pool)
.await
.unwrap();
let primary_usage: (i64, i64, i64) = sqlx::query_as(
"select request_count, input_tokens, output_tokens from account_usage where account_id = $1",
)
.bind("acct_primary")
.fetch_one(&pool)
.await
.unwrap();
let secondary_usage: (i64, i64, i64) = sqlx::query_as(
"select request_count, input_tokens, output_tokens from account_usage where account_id = $1",
)
.bind("acct_secondary")
.fetch_one(&pool)
.await
.unwrap();
let primary_usage = wait_for_account_usage(&pool, "acct_primary", (1, 0, 0)).await;
let secondary_usage = wait_for_account_usage(&pool, "acct_secondary", (1, 4, 1)).await;
assert_eq!(status, StatusCode::OK);
assert!(content_type.starts_with("text/event-stream"));
@@ -553,7 +553,7 @@ async fn responses_should_record_empty_response_attempt() {
}
#[tokio::test]
async fn responses_stream_should_proxy_sse_and_record_usage() {
async fn responses_stream_should_proxy_sse_and_finish_slow_usage_persistence() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/codex/responses"))
@@ -568,6 +568,27 @@ async fn responses_stream_should_proxy_sse_and_record_usage() {
.await;
let (app, api_key, pool, _dir) = test_app_with_account_and_pool(server.uri()).await;
// 强制请求计数持久化越过 stream finalizer 的 100ms 客户端收尾上限,
// 验证超时后原 future 会继续完成,而不是被取消并永久丢失计数。
sqlx::raw_sql(
r#"
create function delay_request_count_update() returns trigger as $$
begin
if new.request_count > old.request_count then
perform pg_sleep(0.2);
end if;
return new;
end;
$$ language plpgsql;
create trigger delay_request_count_update
before update of request_count on account_usage
for each row execute function delay_request_count_update();
"#,
)
.execute(&pool)
.await
.unwrap();
let response = app
.oneshot(
Request::builder()
@@ -596,13 +617,7 @@ async fn responses_stream_should_proxy_sse_and_record_usage() {
.unwrap_or_default()
.to_string();
let body = response_text(response).await;
let usage: (i64, i64, i64) = sqlx::query_as(
"select request_count, input_tokens, output_tokens from account_usage where account_id = $1",
)
.bind("acct_chat")
.fetch_one(&pool)
.await
.unwrap();
let usage = wait_for_account_usage(&pool, "acct_chat", (1, 3, 5)).await;
assert_eq!(status, StatusCode::OK);
assert!(content_type.starts_with("text/event-stream"));
@@ -796,6 +811,7 @@ async fn responses_stream_should_record_usage_record_after_late_disconnect() {
assert_eq!(metadata["stream"], true);
assert_eq!(metadata["failed"], true);
assert_eq!(metadata["upstreamCode"], "stream_disconnected");
assert_eq!(event.failure_class.as_deref(), Some("stream_disconnected"));
assert_eq!(metadata["failureSource"], "proxy");
assert_eq!(metadata["synthetic"], true);
assert_rate_limit_header(&metadata, "retry-after", "11");
+79
View File
@@ -0,0 +1,79 @@
use codex_proxy_rs::{
fleet::{
account::AccountStatus,
account_failure::{AccountFailureKind, AccountStateEffect, classify_client_failure},
},
upstream::openai::transport::{
CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics,
},
};
use reqwest::StatusCode;
#[test]
fn deactivated_workspace_should_be_banned_before_generic_payment_classification() {
let failure = classify(402, r#"{"detail":{"code":"deactivated_workspace"}}"#);
assert_eq!(failure.kind, AccountFailureKind::Banned);
assert_eq!(
failure.effect,
Some(AccountStateEffect::SetStatus(AccountStatus::Banned))
);
}
#[test]
fn generic_payment_required_should_exhaust_quota() {
let failure = classify(402, r#"{"detail":{"code":"payment_required"}}"#);
assert_eq!(failure.kind, AccountFailureKind::QuotaExhausted);
assert_eq!(
failure.effect,
Some(AccountStateEffect::SetStatus(AccountStatus::QuotaExhausted))
);
}
#[test]
fn unauthorized_should_expire_account() {
let failure = classify(401, r#"{"error":{"message":"unauthorized"}}"#);
assert_eq!(failure.kind, AccountFailureKind::Expired);
assert_eq!(
failure.effect,
Some(AccountStateEffect::SetStatus(AccountStatus::Expired))
);
}
#[test]
fn rate_limit_should_create_temporary_quota_effect() {
let failure = classify(429, r#"{"error":{"code":"rate_limit_exceeded"}}"#);
assert_eq!(failure.kind, AccountFailureKind::RateLimited);
assert!(matches!(
failure.effect,
Some(AccountStateEffect::MarkQuotaLimitedUntil(_))
));
}
#[test]
fn html_forbidden_page_should_not_ban_account() {
let error = upstream_error(403, "<!doctype html><html>temporary edge failure</html>");
assert!(classify_client_failure(&error).is_none());
}
fn classify(
status: u16,
body: &str,
) -> codex_proxy_rs::fleet::account_failure::ClassifiedAccountFailure {
classify_client_failure(&upstream_error(status, body)).expect("failure should be classified")
}
fn upstream_error(status: u16, body: &str) -> CodexClientError {
CodexClientError::Upstream {
status: StatusCode::from_u16(status).unwrap(),
body: body.to_string(),
retry_after_seconds: None,
diagnostics: CodexUpstreamDiagnostics::with_status(status),
set_cookie_headers: Vec::new(),
transport: CodexBackendTransport::HttpSse,
}
}
+1
View File
@@ -1,3 +1,4 @@
pub(crate) mod account_failure;
pub(crate) mod cookies;
pub(crate) mod pool;
pub(crate) mod quota;
+75 -13
View File
@@ -1,13 +1,16 @@
use std::{collections::HashMap, sync::Arc, time::Duration as StdDuration};
use chrono::{Duration, Utc};
use codex_proxy_rs::fleet::account::AccountStatus;
use codex_proxy_rs::fleet::cookies::PgCookieStore;
use codex_proxy_rs::fleet::quota::{
QuotaRefreshService, quota_from_usage, quota_snapshot_limit_reached,
quota_snapshot_limit_window_seconds, quota_snapshot_reset_at,
use codex_proxy_rs::fleet::{
account::AccountStatus,
cookies::PgCookieStore,
pool::{AccountPoolOptions, AccountPoolService},
quota::{
QuotaRefreshService, quota_from_usage, quota_snapshot_limit_reached,
quota_snapshot_limit_window_seconds, quota_snapshot_reset_at,
},
store::{AccountStore, NewAccount, PgAccountStore},
};
use codex_proxy_rs::fleet::store::{NewAccount, PgAccountStore};
use codex_proxy_rs::infra::identity::AccountPseudonymizer;
use codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore;
use codex_proxy_rs::upstream::openai::transport::CodexBackendClient;
@@ -24,6 +27,15 @@ fn test_account_pseudonymizer() -> Arc<AccountPseudonymizer> {
Arc::new(AccountPseudonymizer::new([7; 32]))
}
fn test_account_pool(pool: &sqlx::PgPool) -> Arc<AccountPoolService> {
Arc::new(AccountPoolService::new(
Arc::new(PgAccountStore::new(pool.clone())) as Arc<dyn AccountStore>,
Arc::new(PgAccountUsageStore::new(pool.clone())),
AccountPoolOptions::default(),
0,
))
}
#[test]
fn quota_from_usage_should_preserve_spend_control_individual_limit() {
let usage = json!({
@@ -315,12 +327,13 @@ async fn quota_refresh_service_should_send_usage_cookie_when_cookie_store_is_con
let codex = CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
);
let service = QuotaRefreshService::new(
store,
PgAccountUsageStore::new(pool.clone()),
Arc::new(codex),
test_account_pool(&pool),
test_account_pseudonymizer(),
)
.with_cookie_store(cookies);
@@ -397,12 +410,13 @@ async fn quota_refresh_service_should_fetch_usage_for_quota_locked_accounts_and_
let codex = CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
);
let service = QuotaRefreshService::new(
store.clone(),
PgAccountUsageStore::new(pool.clone()),
Arc::new(codex),
test_account_pool(&pool),
test_account_pseudonymizer(),
);
@@ -435,6 +449,49 @@ async fn quota_refresh_service_should_fetch_usage_for_quota_locked_accounts_and_
);
}
#[tokio::test]
async fn quota_refresh_service_should_ban_deactivated_workspace() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.respond_with(
ResponseTemplate::new(402)
.set_body_json(json!({ "detail": { "code": "deactivated_workspace" } })),
)
.expect(1)
.mount(&server)
.await;
let (pool, _guard) = init_test_db("quota-refresh-deactivated-workspace").await;
let store = PgAccountStore::new(pool.clone());
insert_quota_locked_account(&store, &pool, "acct-deactivated", "access-token").await;
let codex = Arc::new(CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::wire_profile::test_wire_profile(),
));
let service = QuotaRefreshService::new(
store,
PgAccountUsageStore::new(pool.clone()),
codex,
test_account_pool(&pool),
test_account_pseudonymizer(),
);
let summary = service
.refresh_locked_accounts(&mut HashMap::new())
.await
.expect("quota refresh should complete");
let status: String = sqlx::query_scalar("select status from accounts where id = $1")
.bind("acct-deactivated")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(summary.failed, 1);
assert_eq!(status, "banned");
}
#[tokio::test]
async fn quota_refresh_service_should_refresh_quota_exhausted_accounts() {
let server = MockServer::start().await;
@@ -473,12 +530,13 @@ async fn quota_refresh_service_should_refresh_quota_exhausted_accounts() {
let codex = CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
);
let service = QuotaRefreshService::new(
store,
PgAccountUsageStore::new(pool.clone()),
Arc::new(codex),
test_account_pool(&pool),
test_account_pseudonymizer(),
);
@@ -509,12 +567,13 @@ async fn quota_refresh_service_should_skip_recent_locked_account_before_cooldown
let codex = CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
);
let service = QuotaRefreshService::with_min_refresh_interval_secs(
store,
PgAccountUsageStore::new(pool.clone()),
Arc::new(codex),
test_account_pool(&pool),
test_account_pseudonymizer(),
1_800,
);
@@ -549,12 +608,13 @@ async fn quota_refresh_service_should_skip_recent_locked_account_inside_cooldown
let codex = CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
);
let service = QuotaRefreshService::with_min_refresh_interval_secs(
store,
PgAccountUsageStore::new(pool.clone()),
Arc::new(codex),
test_account_pool(&pool),
test_account_pseudonymizer(),
1_800,
);
@@ -606,12 +666,13 @@ async fn quota_refresh_service_should_bypass_recent_skip_after_cooldown_grace()
let codex = CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
);
let service = QuotaRefreshService::with_min_refresh_interval_secs(
store,
PgAccountUsageStore::new(pool.clone()),
Arc::new(codex),
test_account_pool(&pool),
test_account_pseudonymizer(),
1_800,
);
@@ -661,12 +722,13 @@ async fn quota_refresh_service_should_stagger_multiple_locked_account_requests()
let codex = CodexBackendClient::new(
reqwest::Client::new(),
server.uri(),
crate::support::fingerprint::runtime_test_fingerprint(),
crate::support::wire_profile::test_wire_profile(),
);
let service = QuotaRefreshService::new(
store,
PgAccountUsageStore::new(pool.clone()),
Arc::new(codex),
test_account_pool(&pool),
test_account_pseudonymizer(),
)
.with_request_spacing(StdDuration::from_millis(200));
-2
View File
@@ -25,8 +25,6 @@ async fn postgres_schema_is_terminal_v1_without_unused_storage() {
"accounts",
"admin_users",
"client_api_keys",
"fingerprint_update_history",
"fingerprints",
"ops_error_logs",
"request_time_buckets",
"runtime_settings",
+1 -1
View File
@@ -42,7 +42,7 @@ pub(crate) fn test_config(database_url: String) -> AppConfig {
max_per_account: 8,
initial_event_timeout_ms: 20_000,
},
fingerprint: crate::support::fingerprint::test_fingerprint_config(),
wire_profile: crate::support::wire_profile::test_wire_profile_config(),
admin: AdminConfig {
session_ttl_minutes: 1440,
default_username: "admin".to_string(),
-75
View File
@@ -1,75 +0,0 @@
use codex_proxy_rs::{
bootstrap::{
config::{FingerprintConfig, FingerprintHeaderConfig},
services::fingerprint_from_config,
},
upstream::openai::fingerprint::{Fingerprint, RuntimeFingerprint},
};
pub(crate) fn test_fingerprint_config() -> FingerprintConfig {
let header = |name: &str, value: &str| FingerprintHeaderConfig {
name: name.to_string(),
value: value.to_string(),
};
FingerprintConfig {
originator: "Codex Desktop".to_string(),
app_version: "26.707.51957".to_string(),
build_number: "5175".to_string(),
platform: "darwin".to_string(),
arch: "arm64".to_string(),
chromium_version: "146".to_string(),
user_agent_template: "Codex Desktop/{version} ({platform}; {arch})".to_string(),
default_headers: vec![
header("Accept-Encoding", "gzip, deflate, br, zstd"),
header("Accept-Language", "en-US,en;q=0.9"),
header("sec-ch-ua-mobile", "?0"),
header("sec-ch-ua-platform", "\"macOS\""),
header("sec-fetch-site", "same-origin"),
header("sec-fetch-mode", "cors"),
header("sec-fetch-dest", "empty"),
],
header_order: [
"authorization",
"chatgpt-account-id",
"originator",
"x-openai-internal-codex-residency",
"x-client-request-id",
"x-codex-installation-id",
"x-codex-turn-state",
"openai-beta",
"user-agent",
"sec-ch-ua",
"sec-ch-ua-mobile",
"sec-ch-ua-platform",
"accept-encoding",
"accept-language",
"sec-fetch-site",
"sec-fetch-mode",
"sec-fetch-dest",
"content-type",
"accept",
"cookie",
]
.map(str::to_string)
.to_vec(),
}
}
pub(crate) fn test_fingerprint() -> Fingerprint {
fingerprint_from_config(&test_fingerprint_config())
}
pub(crate) fn test_fingerprint_with_updated_at(updated_at: Option<&str>) -> Fingerprint {
let mut fingerprint = test_fingerprint();
fingerprint.updated_at = updated_at.map(ToString::to_string);
fingerprint
}
pub(crate) fn runtime_test_fingerprint() -> RuntimeFingerprint {
RuntimeFingerprint::new(test_fingerprint())
}
pub(crate) fn runtime_fingerprint(fingerprint: Fingerprint) -> RuntimeFingerprint {
RuntimeFingerprint::new(fingerprint)
}
+1 -1
View File
@@ -3,7 +3,7 @@ pub(crate) mod admin;
pub(crate) mod assertions;
pub(crate) mod client_keys;
pub(crate) mod config;
pub(crate) mod fingerprint;
pub(crate) mod http;
pub(crate) mod jwt;
pub(crate) mod storage;
pub(crate) mod wire_profile;
+27 -5
View File
@@ -3,7 +3,7 @@ use std::{env, sync::OnceLock, thread};
use chrono::{DateTime, Utc};
use codex_proxy_rs::{
auth::store::{PgAdminUserStore, RedisAdminSessionStore},
bootstrap::services::BackgroundTaskStores,
bootstrap::{config::BootstrapConfig, services::BackgroundTaskStores},
dispatch::affinity::RedisSessionAffinityStore,
fleet::{cookies::PgCookieStore, refresh::RedisRefreshLeaseStore, store::PgAccountStore},
infra::{database::migrate, redis::RedisConnection},
@@ -13,7 +13,6 @@ use codex_proxy_rs::{
account_usage::store::PgAccountUsageStore, buckets::store::PgRequestBucketStore,
ops::store::PgOpsErrorLogStore, usage::store::PgUsageRecordStore,
},
upstream::openai::fingerprint::PgFingerprintStore,
};
use sqlx::{Connection, PgConnection, PgPool, postgres::PgPoolOptions};
use uuid::Uuid;
@@ -24,6 +23,7 @@ const DEFAULT_TEST_REDIS_URL: &str = "redis://127.0.0.1:6379";
static DATABASE_INIT_LIMIT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(2);
static REDIS_TEST_CLEANUP: tokio::sync::OnceCell<()> = tokio::sync::OnceCell::const_new();
static REDIS_TEST_RUN_ID: OnceLock<String> = OnceLock::new();
static LOCAL_BOOTSTRAP_CONFIG: OnceLock<Option<BootstrapConfig>> = OnceLock::new();
pub(crate) struct TestDatabaseGuard {
database_name: Option<String>,
@@ -173,7 +173,6 @@ pub(crate) fn background_task_stores(pool: PgPool, redis: RedisConnection) -> Ba
admin_users: PgAdminUserStore::new(pool.clone()),
admin_sessions: RedisAdminSessionStore::new(redis.clone()),
cookies: PgCookieStore::new(pool.clone()),
fingerprints: PgFingerprintStore::new(pool.clone()),
session_affinity: RedisSessionAffinityStore::new(redis.clone()),
refresh_leases: RedisRefreshLeaseStore::new(redis.clone()),
model_snapshots: RedisModelSnapshotStore::new(redis),
@@ -186,11 +185,34 @@ pub(crate) fn background_task_stores(pool: PgPool, redis: RedisConnection) -> Ba
}
pub(crate) fn test_database_url() -> String {
env::var("CPR_TEST_DATABASE_URL").unwrap_or_else(|_| DEFAULT_TEST_DATABASE_URL.to_string())
// 显式测试拓扑优先;本地开发默认跟随 Compose 配置;无本地配置的 CI 使用固定测试值。
env::var("CPR_TEST_DATABASE_URL")
.ok()
.or_else(|| local_bootstrap_config().map(|config| config.database_url().to_string()))
.unwrap_or_else(|| DEFAULT_TEST_DATABASE_URL.to_string())
}
pub(crate) fn test_redis_url() -> String {
env::var("CPR_TEST_REDIS_URL").unwrap_or_else(|_| DEFAULT_TEST_REDIS_URL.to_string())
env::var("CPR_TEST_REDIS_URL")
.ok()
.or_else(|| local_bootstrap_config().map(|config| config.redis_url().to_string()))
.unwrap_or_else(|| DEFAULT_TEST_REDIS_URL.to_string())
}
fn local_bootstrap_config() -> Option<&'static BootstrapConfig> {
LOCAL_BOOTSTRAP_CONFIG
.get_or_init(|| {
let path =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../deploy/config.yaml");
if !path.is_file() {
return None;
}
Some(
BootstrapConfig::load_from_path(path)
.unwrap_or_else(|error| panic!("load local test service config: {error}")),
)
})
.as_ref()
}
pub(crate) fn timestamp(value: &str) -> DateTime<Utc> {
+34
View File
@@ -0,0 +1,34 @@
use std::sync::Arc;
use codex_proxy_rs::{
bootstrap::{config::WireProfileConfig, services::wire_profile_from_config},
upstream::openai::profile::CodexWireProfile,
};
use crate::support::storage::timestamp;
pub(crate) fn test_wire_profile_config() -> WireProfileConfig {
WireProfileConfig {
originator: "Codex Desktop".to_string(),
codex_version: "0.144.2".to_string(),
desktop_version: "26.707.72221".to_string(),
desktop_build: "5307".to_string(),
os_type: "Mac OS".to_string(),
os_version: "15.7.1".to_string(),
arch: "arm64".to_string(),
terminal: "unknown".to_string(),
verified_at: timestamp("2026-07-14T18:25:50Z"),
}
}
pub(crate) fn test_wire_profile_value() -> CodexWireProfile {
wire_profile_from_config(&test_wire_profile_config())
}
pub(crate) fn test_wire_profile() -> Arc<CodexWireProfile> {
Arc::new(test_wire_profile_value())
}
pub(crate) fn wire_profile(profile: CodexWireProfile) -> Arc<CodexWireProfile> {
Arc::new(profile)
}
+65
View File
@@ -0,0 +1,65 @@
use chrono::{Duration, TimeZone, Utc};
use codex_proxy_rs::{
infra::time::china_day_start,
telemetry::{
dashboard::dashboard_health_timeline_data_at, usage::insights::RequestHealthTimeBucket,
},
};
use serde_json::json;
#[test]
fn dashboard_health_timeline_should_classify_only_eligible_final_requests() {
let now = Utc.with_ymd_and_hms(2026, 7, 15, 5, 0, 0).unwrap();
let start = china_day_start(now);
let records = vec![
health_bucket(start, 0, 0, 7, 2),
health_bucket(start + Duration::minutes(15), 9, 0, 0, 0),
health_bucket(start + Duration::minutes(30), 0, 3, 0, 0),
health_bucket(start + Duration::minutes(45), 98, 2, 0, 0),
health_bucket(start + Duration::minutes(60), 99, 1, 0, 0),
];
let value = serde_json::to_value(dashboard_health_timeline_data_at(&records, now)).unwrap();
let points = value["points"].as_array().unwrap();
assert_eq!(
json!({
"description": value["description"],
"reliabilityDisplay": value["reliabilityDisplay"],
"status": value["status"],
"successRequests": value["successRequests"],
"failedRequests": value["failedRequests"],
"cancelledRequests": value["cancelledRequests"],
"callerErrorRequests": value["callerErrorRequests"],
"pointCount": points.len(),
"statuses": points.iter().take(5).map(|point| &point["status"]).collect::<Vec<_>>(),
}),
json!({
"description": "有效请求可用性",
"reliabilityDisplay": "97.2%",
"status": "unstable",
"successRequests": 206,
"failedRequests": 6,
"cancelledRequests": 7,
"callerErrorRequests": 2,
"pointCount": 96,
"statuses": ["no_data", "low_sample", "unavailable", "unstable", "stable"],
})
);
}
fn health_bucket(
bucket_start: chrono::DateTime<Utc>,
success_requests: u64,
failed_requests: u64,
cancelled_requests: u64,
caller_error_requests: u64,
) -> RequestHealthTimeBucket {
RequestHealthTimeBucket {
bucket_start,
success_requests,
failed_requests,
cancelled_requests,
caller_error_requests,
}
}
+1
View File
@@ -1,4 +1,5 @@
pub(crate) mod account_usage;
mod dashboard;
pub(crate) mod ops;
pub(crate) mod rebuild;
pub(crate) mod usage;
+108
View File
@@ -0,0 +1,108 @@
use chrono::{Duration, TimeZone, Utc};
use codex_proxy_rs::telemetry::{
ops::{store::PgOpsErrorLogStore, types::OpsErrorLog},
usage::{
insights::{RequestHealthTimeBucket, health_timeline},
store::PgUsageRecordStore,
types::UsageRecord,
},
};
use serde_json::json;
use crate::support::storage::init_test_db;
#[tokio::test]
async fn health_timeline_should_use_final_request_outcomes_and_keep_neutral_counts_separate() {
let (pool, _guard) = init_test_db("usage-health-timeline-final-outcomes").await;
let usage_store = PgUsageRecordStore::new(pool.clone());
let error_store = PgOpsErrorLogStore::new(pool.clone());
let start = Utc.with_ymd_and_hms(2026, 7, 15, 12, 0, 0).unwrap();
error_store
.append(&service_error(
"ops_recovered",
"req_recovered",
start + Duration::minutes(1),
))
.await
.unwrap();
usage_store
.append(&success(
"usage_recovered",
"req_recovered",
start + Duration::minutes(2),
))
.await
.unwrap();
usage_store
.append(&success(
"usage_success",
"req_success",
start + Duration::minutes(3),
))
.await
.unwrap();
for (id, minute) in [("ops_failed_first", 4), ("ops_failed_latest", 5)] {
error_store
.append(&service_error(
id,
"req_failed",
start + Duration::minutes(minute),
))
.await
.unwrap();
}
let mut cancelled = service_error(
"ops_cancelled",
"req_cancelled",
start + Duration::minutes(6),
);
cancelled.status_code = Some(499);
cancelled.failure_class = None;
cancelled.metadata = json!({ "terminal": "cancelled" });
error_store.append(&cancelled).await.unwrap();
let mut caller_error = service_error(
"ops_caller_error",
"req_caller_error",
start + Duration::minutes(7),
);
caller_error.status_code = Some(400);
caller_error.metadata = json!({ "failureSource": "client" });
error_store.append(&caller_error).await.unwrap();
let buckets = health_timeline(&pool, start, start + Duration::minutes(15))
.await
.unwrap();
assert_eq!(
buckets,
vec![RequestHealthTimeBucket {
bucket_start: start,
success_requests: 2,
failed_requests: 1,
cancelled_requests: 1,
caller_error_requests: 1,
}]
);
}
fn success(id: &str, request_id: &str, created_at: chrono::DateTime<Utc>) -> UsageRecord {
let mut record = UsageRecord::new("v1.response", "completed", "acct_health", "gpt-5", 200);
record.id = id.to_string();
record.request_id = Some(request_id.to_string());
record.created_at = created_at;
record
}
fn service_error(id: &str, request_id: &str, created_at: chrono::DateTime<Utc>) -> OpsErrorLog {
let mut error = OpsErrorLog::new("v1.response", "request failed");
error.id = id.to_string();
error.request_id = Some(request_id.to_string());
error.status_code = Some(503);
error.failure_class = Some("upstream_unavailable".to_string());
error.created_at = created_at;
error
}
+1
View File
@@ -1 +1,2 @@
mod insights;
mod store;
@@ -0,0 +1,94 @@
use codex_proxy_rs::upstream::openai::desktop_release::{
APPCAST_POLL_INTERVAL, DesktopReleaseChecker, DesktopReleaseStatus,
parse_latest_desktop_release,
};
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{method, path},
};
const APPCAST: &str = r#"
<rss xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle">
<channel>
<item>
<title>26.707.72221</title>
<pubDate>Tue, 14 Jul 2026 07:56:40 +0000</pubDate>
<sparkle:version>5307</sparkle:version>
<sparkle:shortVersionString>26.707.72221</sparkle:shortVersionString>
<sparkle:minimumSystemVersion>12.0</sparkle:minimumSystemVersion>
<sparkle:hardwareRequirements>arm64</sparkle:hardwareRequirements>
<enclosure url="https://example.invalid/Codex.zip" length="565564803"
sparkle:edSignature="signature" />
<sparkle:deltas>
<enclosure url="https://example.invalid/Codex.delta" length="12" />
</sparkle:deltas>
</item>
<item>
<sparkle:version>5263</sparkle:version>
<sparkle:shortVersionString>26.707.71524</sparkle:shortVersionString>
</item>
</channel>
</rss>
"#;
#[test]
fn desktop_release_poll_interval_should_be_one_day() {
assert_eq!(APPCAST_POLL_INTERVAL, std::time::Duration::from_hours(24));
}
#[test]
fn appcast_parser_should_return_first_full_release_and_ignore_deltas() {
let release = parse_latest_desktop_release(APPCAST).expect("parse appcast");
assert_eq!(release.version, "26.707.72221");
assert_eq!(release.build, "5307");
assert_eq!(
release.published_at,
Some(crate::support::storage::timestamp("2026-07-14T07:56:40Z"))
);
assert_eq!(release.minimum_system_version.as_deref(), Some("12.0"));
assert_eq!(release.hardware_requirements.as_deref(), Some("arm64"));
assert_eq!(
release.download_url.as_deref(),
Some("https://example.invalid/Codex.zip")
);
assert_eq!(release.download_size, Some(565_564_803));
assert!(release.signature_present);
}
#[test]
fn appcast_parser_should_reject_release_without_codex_desktop_version() {
let error = parse_latest_desktop_release("<rss><channel><item /></channel></rss>")
.expect_err("missing version should fail");
assert!(error.to_string().contains("shortVersionString"));
}
#[tokio::test]
async fn desktop_release_checker_should_update_observation_only() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/appcast.xml"))
.respond_with(ResponseTemplate::new(200).set_body_raw(APPCAST, "application/xml"))
.mount(&server)
.await;
let status = DesktopReleaseStatus::default();
let checker = DesktopReleaseChecker::with_client(
reqwest::Client::new(),
format!("{}/appcast.xml", server.uri()),
status.clone(),
);
checker.check_and_record().await.expect("check appcast");
let snapshot = status.snapshot();
assert_eq!(
snapshot
.latest
.as_ref()
.map(|release| release.version.as_str()),
Some("26.707.72221")
);
assert!(snapshot.checked_at.is_some());
assert!(snapshot.last_error.is_none());
}
@@ -1,190 +0,0 @@
use std::{num::NonZeroU32, time::Duration};
use codex_proxy_rs::upstream::openai::fingerprint::{
APPCAST_POLL_INTERVAL, PgFingerprintStore, UpdateChecker,
};
use crate::support::storage::init_test_db;
#[test]
fn fingerprint_appcast_poll_interval_should_be_one_day() {
assert_eq!(APPCAST_POLL_INTERVAL, Duration::from_hours(24));
}
#[tokio::test]
async fn fingerprint_store_should_update_current_record() {
let (pool, _guard) = init_test_db("fingerprint-store").await;
let repo = PgFingerprintStore::new(pool.clone());
let default_fingerprint = crate::support::fingerprint::test_fingerprint();
repo.ensure_current_seed(&default_fingerprint)
.await
.expect("seed current fingerprint");
repo.update_current_version("26.800.1", "6001", Some("147"))
.await
.expect("first update");
repo.update_current_version("26.800.2", "6002", None)
.await
.expect("second update");
let stored = repo
.load_current()
.await
.expect("load current")
.expect("stored fingerprint");
let count: (i64,) = sqlx::query_as("select count(*) from fingerprints where id = 'current'")
.fetch_one(&pool)
.await
.expect("count row");
assert_eq!(count.0, 1);
assert_eq!(stored.app_version, "26.800.2");
assert_eq!(stored.build_number, "6002");
assert_eq!(stored.chromium_version, "147");
}
#[tokio::test]
async fn fingerprint_store_should_keep_latest_update_history_rows() {
let (pool, _guard) = init_test_db("fingerprint-history-limit").await;
let repo = PgFingerprintStore::new(pool.clone());
repo.ensure_current_seed(&crate::support::fingerprint::test_fingerprint())
.await
.expect("seed current fingerprint");
sqlx::query(
"insert into fingerprint_update_history (
id, current_fingerprint_id, app_version, build_number, source, created_at
)
select
'history-' || lpad(sequence::text, 3, '0'),
'current',
'26.900.' || sequence::text,
sequence::text,
'auto_update',
$1 + sequence * interval '1 second'
from generate_series(0, 101) as sequence",
)
.bind(chrono::Utc::now())
.execute(&pool)
.await
.expect("insert fingerprint history rows");
let deleted = repo
.trim_update_history_to_limit(NonZeroU32::new(100).unwrap())
.await
.expect("trim fingerprint history rows");
let (remaining, oldest_remaining): (i64, i64) = sqlx::query_as(
"select
count(*),
count(*) filter (where id in ('history-000', 'history-001'))
from fingerprint_update_history",
)
.fetch_one(&pool)
.await
.expect("count retained fingerprint history rows");
assert_eq!((deleted, remaining, oldest_remaining), (2, 100, 0));
}
#[tokio::test]
async fn update_checker_should_report_available_update_from_appcast() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.and(wiremock::matchers::path("/appcast.xml"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_raw(
r#"
<rss>
<channel>
<item>
<enclosure url="https://example.invalid/download" sparkle:shortVersionString="26.900.1" sparkle:version="7001" />
</item>
</channel>
</rss>
"#,
"application/xml",
))
.mount(&server)
.await;
let dir = tempfile::tempdir().expect("temp dir");
let (pool, _guard) = init_test_db("fingerprint-update-check").await;
let repo = PgFingerprintStore::new(pool);
repo.ensure_current_seed(&crate::support::fingerprint::test_fingerprint())
.await
.expect("seed current fingerprint");
let checker = UpdateChecker::with_client(
repo,
reqwest::Client::new(),
format!("{}/appcast.xml", server.uri()),
dir.path().join("extracted-fingerprint.json"),
"26.800.1",
"6001",
);
let state = checker.check_for_update().await.expect("update state");
assert!(state.update_available);
assert_eq!(state.latest_version.as_deref(), Some("26.900.1"));
assert_eq!(state.latest_build.as_deref(), Some("7001"));
}
#[tokio::test]
async fn update_checker_should_apply_available_update_to_store() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.and(wiremock::matchers::path("/appcast.xml"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_raw(
r#"
<rss>
<channel>
<item>
<enclosure url="https://example.invalid/download" sparkle:shortVersionString="26.900.1" sparkle:version="7001" />
</item>
</channel>
</rss>
"#,
"application/xml",
))
.mount(&server)
.await;
let dir = tempfile::tempdir().expect("temp dir");
let (pool, _guard) = init_test_db("fingerprint-update-apply").await;
let repo = PgFingerprintStore::new(pool);
repo.ensure_current_seed(&crate::support::fingerprint::test_fingerprint())
.await
.expect("seed current fingerprint");
let extracted_path = dir.path().join("extracted-fingerprint.json");
std::fs::write(
&extracted_path,
r#"{"app_version":"26.900.1","build_number":"7001","chromium_version":"147"}"#,
)
.expect("write extracted fingerprint");
let checker = UpdateChecker::with_client(
repo.clone(),
reqwest::Client::new(),
format!("{}/appcast.xml", server.uri()),
extracted_path,
"26.800.1",
"6001",
);
let updated = checker
.check_and_apply_update()
.await
.expect("apply update")
.expect("available update should return current fingerprint");
let stored = repo
.load_current()
.await
.expect("load current")
.expect("stored fingerprint");
assert_eq!(updated.app_version, "26.900.1");
assert_eq!(updated.build_number, "7001");
assert_eq!(updated.chromium_version, "147");
assert_eq!(stored.app_version, "26.900.1");
assert_eq!(stored.build_number, "7001");
assert_eq!(stored.chromium_version, "147");
}
+2 -1
View File
@@ -1,4 +1,5 @@
pub(crate) mod fingerprint;
mod desktop_release;
mod profile;
pub(crate) mod protocol;
pub(crate) mod token_client;
pub(crate) mod transport;
+9
View File
@@ -0,0 +1,9 @@
#[test]
fn wire_profile_should_generate_codex_core_user_agent() {
let profile = crate::support::wire_profile::test_wire_profile_value();
assert_eq!(
profile.user_agent(),
"Codex Desktop/0.144.2 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.707.72221)"
);
}

Some files were not shown because too many files have changed in this diff Show More