mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-10 08:06:03 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3328a4356a | ||
|
|
77b0c29354 | ||
|
|
c872bc7881 | ||
|
|
487b2e87e2 | ||
|
|
eb8a01541a | ||
|
|
0acf114fa7 | ||
|
|
5d941008fc | ||
|
|
b44a790777 | ||
|
|
06e94ef69b | ||
|
|
0f2e0b18a9 | ||
|
|
71e26f44b7 | ||
|
|
8232e19bc0 | ||
|
|
dcc61915e9 | ||
|
|
47d08560a7 | ||
|
|
590c4a163b | ||
|
|
9cfcd371d6 | ||
|
|
223021630e | ||
|
|
42c2b249c2 | ||
|
|
0b3ebc23f5 | ||
|
|
65bb9ab9cf | ||
|
|
50d6f7444d | ||
|
|
bcfea792e8 | ||
|
|
1e7ea8d46f | ||
|
|
bd61e0391a | ||
|
|
499b43043e | ||
|
|
ae2da31380 | ||
|
|
1e6d69a349 | ||
|
|
dac57a81c6 | ||
|
|
d162f9b7f6 | ||
|
|
866f71e3fb | ||
|
|
47ab48b4cf | ||
|
|
e3cf4ab8be | ||
|
|
4fb10f33f1 | ||
|
|
c9318c54cd | ||
|
|
8d9aaf0bbd | ||
|
|
52d24d29c7 |
@@ -33,6 +33,9 @@ description: 当前仓库开发指南。修改或排查本仓库的 Rust 网关
|
||||
优先依据适用的官方源码确定方案。结合上表读取所需文档,先找可复用的同类实现,再修改所属模块。
|
||||
4. 文档和运行结果不一致时,追查当前源码、配置与版本,区分实现缺陷和文档过期;
|
||||
按 [文档职责与更新条件](../../../CONTRIBUTING.md#文档职责与更新条件) 判断是否修订,不为每次改动追加文档或发布说明。
|
||||
5. 按 `CONTRIBUTING.md` 验证并报告结果。后端 Cargo 命令从 `backend/` 执行,或显式指定该目录的 manifest。
|
||||
5. 交付前按文档职责逐段复核最终文档 diff:新增内容是否回答当前读者的使用或维护问题,是否属于该文档,
|
||||
是否与其他位置重复,脱离本次变更背景能否独立理解。仅解释实现经过、前后对比或验证结果的内容移到
|
||||
任务报告或 PR;仍影响升级、恢复的说明保留适用条件与操作。没有文档差异时,只核对行为变化是否使现有说明失真。
|
||||
6. 按 `CONTRIBUTING.md` 验证并报告结果。后端 Cargo 命令从 `backend/` 执行,或显式指定该目录的 manifest。
|
||||
|
||||
发布任务按 `release` 技能读取当前版本、发布入口和工作流,发布说明语言遵循 `CONTRIBUTING.md`。
|
||||
|
||||
@@ -5,8 +5,9 @@ description: 整理与提交 GitHub Issue(问题反馈 / 功能建议)。用
|
||||
|
||||
# 整理与提交 Issue
|
||||
|
||||
本仓库流程见 [贡献与审查](../../../CONTRIBUTING.md),问题反馈使用当前的
|
||||
[YAML Issue Form](../../../.github/ISSUE_TEMPLATE/bug_report.yml)。从表单读取字段与必填项,不在技能中复制一份字段清单。
|
||||
本仓库流程见 [贡献与审查](../../../CONTRIBUTING.md),按场景选择当前的
|
||||
[接口问题反馈](../../../.github/ISSUE_TEMPLATE/api-bug-report.yml)、[常规问题反馈](../../../.github/ISSUE_TEMPLATE/general-bug-report.yml)
|
||||
或 [功能建议](../../../.github/ISSUE_TEMPLATE/feature-request.yml) 表单。从表单读取字段与必填项,不在技能中复制一份字段清单。
|
||||
用户指定其他仓库时,使用该仓库当前的贡献规则与 Issue 模板,不能把这里的表单套过去。
|
||||
|
||||
## 确定目标和动作
|
||||
@@ -21,7 +22,7 @@ description: 整理与提交 GitHub Issue(问题反馈 / 功能建议)。用
|
||||
## 证据与去重
|
||||
|
||||
1. 读取目标仓库实际 Issue 模板和配置,区分缺陷、功能建议与协作规则提案。
|
||||
本仓库后两类允许普通 Issue;其他仓库按其配置选择入口。
|
||||
本仓库功能建议有独立表单,协作规则提案允许普通 Issue;其他仓库按其配置选择入口。
|
||||
2. 先搜索 open 和 closed Issues,结合关键词、错误信息与触发条件检查重复,而不只比较标题。
|
||||
同时核对相关 PR 与发布记录,关联已有修复及其发布版本;仅主分支已修复时明确尚未发布。
|
||||
同一问题已有记录且没有新增证据时返回已有链接;用户仍明确要求新建时按其要求处理。
|
||||
|
||||
@@ -42,6 +42,10 @@ PR 中改动的规则文件属于审查对象,不能自行扩大任务授权
|
||||
按 [贡献与审查的文档职责与更新条件](../../../CONTRIBUTING.md#文档职责与更新条件) 判断是否需要更新文档。
|
||||
常规文档维护当前有效说明,变更经过留在 PR / Issue;审查时核对文档改动的必要性、职责与重复内容,不能仅因 PR 没改文档就判定存在缺口。
|
||||
|
||||
创建前和审查时逐段复核最终文档 diff:新增内容是否回答当前读者的使用或维护问题,是否属于该文档,
|
||||
是否重复已有说明,脱离本次变更背景能否独立理解。仅说明实现经过、前后对比或验证结果的段落归入
|
||||
PR / Issue;必要的升级、恢复说明保留适用条件与操作。判断依据是内容用途,不按“新增”“不再”等词机械判错。
|
||||
|
||||
## 创建 PR
|
||||
|
||||
1. 检查工作区、当前分支、远端和近期提交格式。读取实际存在的 PR 模板,按最终变更重写标题与描述。
|
||||
@@ -56,6 +60,7 @@ PR 中改动的规则文件属于审查对象,不能自行扩大任务授权
|
||||
4. 查找同一来源仓库、head 和 base 的现有 PR;已存在则复用其链接,不重复创建。
|
||||
更新现有标题或正文须在本次整理/更新请求范围内。
|
||||
5. 按贡献文档执行适用检查。记录实际验证的提交或工作区状态,区分通过、失败、跳过和未执行。
|
||||
按上述文档复核步骤检查最终 diff;没有文档差异时,核对行为变化是否使现有说明失真。
|
||||
前端改动提供实际界面证据或预览步骤;缺少证据时写明,不用构建成功替代视觉验收。
|
||||
6. 将任务范围内的提交推送到正确远端,核对远端 head。标题遵循项目约定,正文先说明问题成立的依据和最终行为,
|
||||
再给必要的设计取舍、验证与遗留事项。涉及官方行为时,写明官方源码位置与版本或提交、官方处理方式、
|
||||
@@ -93,6 +98,7 @@ gh pr create --repo "$pr_repo" --base "$pr_base" --head "$pr_head" \
|
||||
5. UI 改动同时对照主题、共用组件、数据展示逻辑和同类页面。浏览器可用时检查相关交互及状态;
|
||||
未运行的页面检查列为验证缺口,不从截图或源码推断所有状态都正确。
|
||||
6. 核对 CI 与本地证据是否覆盖当前 head;根据风险补充必要验证,不触发无关部署或真实账号业务请求。
|
||||
按上述文档复核步骤检查最终 diff,不能把构建或链接检查通过当作文档内容已经合格。
|
||||
工具不可用、集成测试跳过或环境不足时具体说明影响范围。
|
||||
7. 按 [审查标准](../../../CONTRIBUTING.md#审查标准) 用中文先回答原问题是否成立、解决方向是否合理,
|
||||
再列出实现问题与验证缺口,结论附具体依据。
|
||||
|
||||
@@ -48,13 +48,17 @@ description: 准备或执行当前仓库发版,维护中文发布说明。用
|
||||
5. 替换为本次版本说明,首行与目标 tag 一致且正文非空。升级事项应能追溯到实际迁移、配置或兼容变化;
|
||||
敏感信息脱敏,未合入功能不宣称已发布。已有准确说明仅需翻译时保持其事实和范围。
|
||||
6. 按贡献规范检查变更与现有 CI 证据,明确通过、失败、跳过和未执行。
|
||||
涉及数据库迁移时,对照同一大版本的上一正式 Release 检查已发布迁移是否保持不变,并验证升级路径;
|
||||
跨大版本按项目部署约定判断,不默认支持在线升级。区分未发布迁移的整理、已执行这些迁移的源码部署实例,
|
||||
以及 CI 实际采用的冻结基线。未发版不代表迁移从未被执行,CI 冻结检查失败也不等于正式版数据库升级失败;
|
||||
按各自证据报告兼容性与检查结果,不因未发版而自动放宽现有冻结规则。
|
||||
只修改说明时无需重建前后端;正式发版仍要跟踪发布工作流针对目标提交的检查与产物。
|
||||
|
||||
## 执行正式发版
|
||||
|
||||
调用入口前重新核对当前脚本的副作用、目标版本和待推送提交,保留无关工作区内容,不自动清理或暂存用户文件。
|
||||
先编辑 `release/notes.md`,保留为未提交改动(已暂存或未暂存均可);其他已跟踪文件的改动应先单独提交。
|
||||
脚本会更新 `release/version.yaml`,将它与最终工作区中的发布说明一起提交为 `chore(release): prepare vX.Y.Z`。
|
||||
脚本会更新 `release/version.yaml`,将它与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。
|
||||
不要提前单独提交说明或把版本文件改成目标版本;脚本要求说明相对 HEAD 有改动,并自行更新版本号。
|
||||
本任务需要的提交按贡献文档保留英文 Conventional Commits 与相应署名。
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
name: 接口问题反馈
|
||||
description: 客户端调用模型、搜索等 API 时的报错、断流、超时或响应异常
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
管理端页面、账号导入或服务部署的问题请选择“常规问题反馈”;新增能力请选择“功能建议”。
|
||||
请提供最小复现信息。请求、日志和截图发布前请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 问题描述
|
||||
description: 说明发生了什么、实际结果以及出现的频率;已有正常账号、模型或客户端的对照结果也可说明。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: 服务版本
|
||||
description: 填写实际运行的完整版本号,不要只写“最新”或 latest;源码运行填写提交号。
|
||||
placeholder: 完整版本号或提交号
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: deployment
|
||||
attributes:
|
||||
label: 部署方式
|
||||
options:
|
||||
- Docker Compose
|
||||
- 其他容器部署
|
||||
- 二进制运行
|
||||
- 源码运行
|
||||
- 其他方式(请在环境信息中说明)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 环境信息
|
||||
description: 填写服务端与客户端的操作系统及版本;性能或超时问题可补充 CPU、内存和容器资源限制。
|
||||
placeholder: |
|
||||
服务端操作系统与版本:
|
||||
客户端操作系统与版本:
|
||||
相关资源限制(如有):
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: client-name
|
||||
attributes:
|
||||
label: 客户端名称
|
||||
description: 区分 CLI、桌面端、编辑器扩展、SDK 或自编脚本。
|
||||
placeholder: 例如 Codex CLI、Codex Desktop 或某个编辑器扩展
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: client-version
|
||||
attributes:
|
||||
label: 客户端版本
|
||||
description: 填写完整版本号;Codex CLI 可填写 codex --version 的输出。无法取得时说明原因。
|
||||
placeholder: 完整版本号或相关 SDK 版本
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: client-source
|
||||
attributes:
|
||||
label: 客户端安装来源与启动方式
|
||||
description: 说明安装来源,以及是否通过包装脚本、启动器或第三方修改版运行;不确定时填写已知部分。
|
||||
placeholder: 例如通过 npm 安装 @openai/codex,直接执行 codex
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: endpoint
|
||||
attributes:
|
||||
label: 请求方法与端点
|
||||
description: 填写发生异常的请求方法与路径,无需真实域名;不清楚端点时说明客户端当时执行的具体操作。
|
||||
placeholder: 例如 POST /v1/responses、POST /v1/alpha/search 或 WebSocket /v1/responses
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: intermediary
|
||||
attributes:
|
||||
label: 是否经过中转或反向代理
|
||||
description: |
|
||||
指客户端到本服务之间的链路。API 中转包括 New API、One API、CCH 等;反向代理或 CDN 包括 Nginx、Caddy、Cloudflare Tunnel 等。
|
||||
如有中转,请在下方填写链路详情;本服务到上游的出站代理请在补充信息中说明。
|
||||
options:
|
||||
- 否,直接连接本服务
|
||||
- 是,仅经过 API 中转服务
|
||||
- 是,仅经过反向代理或 CDN
|
||||
- 是,同时经过 API 中转和反向代理或 CDN
|
||||
- 不确定
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-chain
|
||||
attributes:
|
||||
label: 请求链路
|
||||
description: |
|
||||
按调用顺序填写中间服务的名称与版本,例如:Codex → New API → Nginx → 本服务。不确定时填写已知部分。
|
||||
如果已测试过直连本服务,请说明是否仍能复现;未测试可注明,不要求先完成对照测试。
|
||||
placeholder: |
|
||||
调用顺序与各中间服务版本:
|
||||
是否启用请求参数或请求头改写(如已知):
|
||||
直连本服务的对照结果(如已测试):
|
||||
|
||||
- type: textarea
|
||||
id: account-model
|
||||
attributes:
|
||||
label: 平台、账号类型与模型
|
||||
description: 填写与失败请求相关的信息,不要提供账号凭据;不涉及模型或无法确认的信息请注明。
|
||||
placeholder: |
|
||||
平台与认证类型:
|
||||
账号套餐(如相关):
|
||||
请求模型与映射后的上游模型(如已知):
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: 复现步骤
|
||||
description: 描述触发问题的操作;无法稳定复现时,说明已知触发条件。
|
||||
placeholder: |
|
||||
1. 使用什么配置或输入
|
||||
2. 执行什么操作
|
||||
3. 出现什么结果
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-response
|
||||
attributes:
|
||||
label: 请求、响应与诊断附件
|
||||
description: |
|
||||
提供脱敏后的最小请求、网关实际状态/响应片段和客户端显示,区分上游、中转、网关与客户端的错误来源。
|
||||
可在此直接上传管理端默认诊断包并检查其缺失说明;原始错误与日志片段需另行审阅脱敏,粘贴时请使用 Markdown 代码块。
|
||||
按部署文档的“请求错误排查”检查关联 ID、错误详情与 application 日志,不默认开启完整 dump;无法取得的信息请注明。
|
||||
placeholder: |
|
||||
发生时间与时区:
|
||||
传输(HTTP JSON / SSE / WebSocket):
|
||||
脱敏后的最小请求:
|
||||
网关关联 ID(x-gateway-request-id;配置的入口头请单独注明):
|
||||
上游关联 ID(x-request-id / x-oai-request-id,注明来源):
|
||||
网关实际状态、Content-Type 与响应片段:
|
||||
中转服务的状态与错误(如有):
|
||||
客户端显示的错误:
|
||||
诊断包或无法取得的信息:
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 最近有升级时填写升级前后版本及前后端是否同步更新;网络问题补充出站代理、系统代理、TUN 和自定义证书等相关环境。
|
||||
placeholder: |
|
||||
服务或客户端升级前版本 → 当前版本(如有):
|
||||
服务前后端是否同步更新(如有升级):
|
||||
最近的配置变更:
|
||||
其他相关环境:
|
||||
@@ -1,120 +0,0 @@
|
||||
name: 问题反馈
|
||||
description: 提交可复现的问题,帮助定位和修复
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
请提供最小复现信息。请求、日志和截图发布前请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: dropdown
|
||||
id: issue-type
|
||||
attributes:
|
||||
label: 问题类型
|
||||
options:
|
||||
- 接口请求或响应异常
|
||||
- 页面或交互异常
|
||||
- 配置或部署异常
|
||||
- 其他问题
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 问题描述
|
||||
description: 说明发生了什么、实际结果以及出现的频率。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: 服务版本
|
||||
description: 填写具体版本号;从源码运行时填写提交号。
|
||||
placeholder: 版本号或提交号
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: deployment
|
||||
attributes:
|
||||
label: 部署方式
|
||||
options:
|
||||
- Docker Compose
|
||||
- 其他容器部署
|
||||
- 二进制运行
|
||||
- 源码运行
|
||||
- 其他方式(请在环境信息中说明)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 环境信息
|
||||
description: 填写操作系统与版本;页面问题同时填写浏览器与版本。
|
||||
placeholder: |
|
||||
操作系统与版本:
|
||||
浏览器与版本(页面问题):
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: account-client
|
||||
attributes:
|
||||
label: 账号与客户端
|
||||
description: 接口问题请填写平台、认证类型、模型和客户端信息,不要提供账号凭据。
|
||||
placeholder: |
|
||||
平台与认证类型:
|
||||
模型名称:
|
||||
客户端名称与版本:
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: 复现步骤
|
||||
description: 描述触发问题的操作;无法稳定复现时,说明已知触发条件。
|
||||
placeholder: |
|
||||
1. 使用什么配置或输入
|
||||
2. 执行什么操作
|
||||
3. 出现什么结果
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-response
|
||||
attributes:
|
||||
label: 请求与响应
|
||||
description: |
|
||||
接口问题请提供脱敏后的最小请求、网关实际状态/响应片段和客户端终端显示,区分两者。
|
||||
优先附管理端默认诊断包并检查其缺失说明;原始错误与日志片段需另行审阅脱敏。
|
||||
先按部署文档的“请求错误排查”检查关联 ID、错误详情与 application 日志,不默认开启完整 dump。
|
||||
placeholder: |
|
||||
发生时间与时区:
|
||||
传输(HTTP JSON / SSE / WebSocket):
|
||||
网关关联 ID(x-gateway-request-id / 配置的入口头):
|
||||
上游关联 ID(x-request-id / x-oai-request-id,注明来源):
|
||||
网关实际状态与响应片段:
|
||||
客户端终端显示:
|
||||
诊断包或无法取得的信息:
|
||||
render: text
|
||||
|
||||
- type: textarea
|
||||
id: screenshots
|
||||
attributes:
|
||||
label: 截图或录屏
|
||||
description: 页面或交互问题可在此拖入相关截图、录屏,并说明对应的操作。
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 最近的升级或配置变更;网络问题补充系统代理、TUN 和自定义证书等相关环境。
|
||||
@@ -0,0 +1,39 @@
|
||||
name: 功能建议
|
||||
description: 说明使用场景、当前不便和期望行为,提出新能力或体验改进
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
如果已有功能无法正常使用,请选择对应的故障模板。提交前可搜索已有 Issues,相关讨论请附链接。
|
||||
每个 Issue 尽量围绕一个使用场景;截图和示例请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: textarea
|
||||
id: use-case
|
||||
attributes:
|
||||
label: 使用场景与当前不便
|
||||
description: 说明你希望完成什么任务、目前如何操作,以及现有能力有什么不足。
|
||||
placeholder: |
|
||||
希望完成的任务:
|
||||
当前做法与不便:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 期望行为
|
||||
description: 描述改进后希望如何使用,最好给出一个具体例子;不要求先设计技术方案。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: 已尝试的替代方案
|
||||
description: 如果已有配置、操作方式或其他工具能部分满足需求,说明仍缺少什么;未尝试可留空。
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 可附当前使用的具体版本、相关 Issue、截图或参考链接;引用其他项目时说明希望借鉴的具体行为。
|
||||
@@ -0,0 +1,143 @@
|
||||
name: 常规问题反馈
|
||||
description: 管理端页面、账号导入、安装部署、升级或其他非客户端 API 问题
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
客户端调用模型、搜索等 API 的问题请选择“接口问题反馈”;新增能力或体验改进请选择“功能建议”。
|
||||
请提供最小复现信息。请求、日志和截图发布前请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: dropdown
|
||||
id: issue-type
|
||||
attributes:
|
||||
label: 问题类型
|
||||
options:
|
||||
- 页面或交互异常
|
||||
- 配置或部署异常
|
||||
- 升级异常
|
||||
- 其他问题
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 问题描述
|
||||
description: 说明发生了什么、实际结果以及出现的频率。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: 服务版本
|
||||
description: 填写实际运行的完整版本号,不要只写“最新”或 latest;源码运行填写提交号。服务无法启动时填写准备运行的版本。
|
||||
placeholder: 完整版本号或提交号
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: deployment
|
||||
attributes:
|
||||
label: 部署方式
|
||||
options:
|
||||
- Docker Compose
|
||||
- 其他容器部署
|
||||
- 二进制运行
|
||||
- 源码运行
|
||||
- 其他方式(请在环境信息中说明)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 环境信息
|
||||
description: 填写操作系统与版本,并补充与问题相关的浏览器、访问方式或资源信息。
|
||||
placeholder: |
|
||||
操作系统与版本:
|
||||
浏览器与版本(页面问题):
|
||||
页面访问方式(HTTP / HTTPS,localhost / IP / 域名,无需真实地址):
|
||||
CPU 核数与内存、容器资源限制(部署或性能问题):
|
||||
Docker / Compose 版本(容器部署问题):
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: intermediary
|
||||
attributes:
|
||||
label: 是否经过中转或反向代理
|
||||
description: |
|
||||
指客户端或浏览器到本服务之间的链路。API 中转包括 New API、One API、CCH 等;反向代理或 CDN 包括 Nginx、Caddy、Cloudflare Tunnel 等。
|
||||
如有中转,请在下方填写链路详情;本服务到上游的出站代理请在补充信息中说明。
|
||||
options:
|
||||
- 否,直接连接本服务
|
||||
- 是,仅经过 API 中转服务
|
||||
- 是,仅经过反向代理或 CDN
|
||||
- 是,同时经过 API 中转和反向代理或 CDN
|
||||
- 不确定
|
||||
- 不适用(例如服务尚未启动)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-chain
|
||||
attributes:
|
||||
label: 请求链路
|
||||
description: |
|
||||
按调用顺序填写中间服务的名称与版本,例如:浏览器 → Cloudflare → Nginx → 本服务。不确定时填写已知部分。
|
||||
如果已测试过直连本服务,请说明是否仍能复现;未测试可注明,不要求先完成对照测试。
|
||||
placeholder: |
|
||||
调用顺序与各中间服务版本:
|
||||
是否启用请求参数或请求头改写(如已知):
|
||||
直连本服务的对照结果(如已测试):
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: 复现步骤
|
||||
description: 描述触发问题的操作;批量操作补充数量,升级问题说明升级方式。无法稳定复现时,说明已知触发条件。
|
||||
placeholder: |
|
||||
1. 使用什么配置或输入
|
||||
2. 执行什么操作
|
||||
3. 出现什么结果
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: 错误信息与日志
|
||||
description: |
|
||||
提供脱敏后的页面提示、浏览器控制台错误或相关启动、升级日志;如有失败请求,补充状态码与响应片段。
|
||||
可以在此上传诊断包或日志附件;粘贴日志时请使用 Markdown 代码块。不要上传完整配置或请求转储。
|
||||
placeholder: |
|
||||
发生时间与时区:
|
||||
页面提示或终端错误:
|
||||
浏览器控制台 / 服务日志:
|
||||
失败请求的状态码与响应片段(如有):
|
||||
|
||||
- type: textarea
|
||||
id: screenshots
|
||||
attributes:
|
||||
label: 截图或录屏
|
||||
description: 页面或交互问题可在此拖入相关截图、录屏,并说明对应的操作。
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 最近有升级时填写升级前后版本及前后端是否同步更新;网络问题补充出站代理、系统代理、TUN 和自定义证书等相关环境。
|
||||
placeholder: |
|
||||
升级前版本 → 当前版本(如有):
|
||||
前后端是否同步更新(如有升级):
|
||||
最近的配置变更:
|
||||
其他相关环境:
|
||||
@@ -669,9 +669,6 @@ jobs:
|
||||
shell: bash
|
||||
env:
|
||||
INSTALLATION_NOTES: |
|
||||
> [!WARNING]
|
||||
> **跨大版本升级:** 不支持跨大版本在线升级。请使用全新的 `.runtime/` 数据目录重新部署,再重新导入或重新授权服务商账号,并重新导入客户端密钥。详见[升级说明](https://github.com/${{ github.repository }}/blob/${{ needs.release-info.outputs.tag }}/deploy/README.md#镜像升级与源码构建)。
|
||||
|
||||
## 📥 安装
|
||||
|
||||
**Docker 部署:**
|
||||
|
||||
+2
-2
@@ -6,9 +6,9 @@
|
||||
|
||||
## 提交 Issue
|
||||
|
||||
问题反馈使用仓库的 [Issue 表单](.github/ISSUE_TEMPLATE/bug_report.yml),填写问题类型、描述、版本与环境、复现步骤和预期结果。接口问题补充平台与认证类型、模型、客户端,以及脱敏后的请求、响应和请求 ID;页面问题附相关截图或录屏。
|
||||
按场景选择 [接口问题反馈](.github/ISSUE_TEMPLATE/api-bug-report.yml)、[常规问题反馈](.github/ISSUE_TEMPLATE/general-bug-report.yml) 或 [功能建议](.github/ISSUE_TEMPLATE/feature-request.yml) 表单,按对应提示提供信息。故障反馈填写实际运行版本和中转链路;接口问题区分客户端显示与网关实际响应,页面问题附相关截图或录屏。
|
||||
|
||||
日志和配置只提供定位所需片段,不上传真实凭据或完整请求转储。功能建议与协作规则调整可直接提交 Issue,说明场景、原因和建议方案。
|
||||
日志和配置只提供定位所需片段,不上传真实凭据或完整请求转储。功能建议说明使用场景、当前不便和期望行为;协作规则调整可直接提交普通 Issue,说明原因和建议方案。
|
||||
|
||||
## 问题与方案依据
|
||||
|
||||
|
||||
@@ -25,64 +25,56 @@
|
||||
以下命令适用于 Linux amd64/arm64,需要 Docker Engine、Docker Compose Plugin、curl 和 OpenSSL。已有部署请先看
|
||||
[升级说明](deploy/README.md#镜像升级与源码构建),不要覆盖原配置。
|
||||
|
||||
### 1. 下载部署文件并配置
|
||||
### 一键安装
|
||||
|
||||
请确保当前用户能访问 Docker,并可通过 `sudo` 或 root 设置目录权限。
|
||||
|
||||
```bash
|
||||
mkdir -p codex-proxy-rs/deploy && cd codex-proxy-rs
|
||||
|
||||
# 只解析一次最新正式版本,确保两个文件来自同一 Release。
|
||||
CPR_RELEASE_URL="$(curl -fsSL -o /dev/null -w '%{url_effective}' https://github.com/zyycn/codex-proxy-rs/releases/latest)"
|
||||
CPR_RELEASE_TAG="${CPR_RELEASE_URL##*/}"
|
||||
curl -fsSL "https://github.com/zyycn/codex-proxy-rs/releases/download/${CPR_RELEASE_TAG}/compose.yaml" \
|
||||
-o deploy/compose.yaml
|
||||
curl -fsSL "https://github.com/zyycn/codex-proxy-rs/releases/download/${CPR_RELEASE_TAG}/config.example.yaml" \
|
||||
-o deploy/config.example.yaml
|
||||
|
||||
install -d -m 0750 .runtime/postgres .runtime/redis
|
||||
sudo install -d -m 0770 -o "$(id -u)" -g 10001 .runtime/data .runtime/logs
|
||||
sudo install -m 0640 -o "$(id -u)" -g 10001 deploy/config.example.yaml deploy/config.yaml
|
||||
curl -fsSL https://raw.githubusercontent.com/zyycn/codex-proxy-rs/main/deploy/install.sh -o install.sh && bash install.sh
|
||||
```
|
||||
|
||||
也可将 `CPR_RELEASE_TAG` 设置为指定的发布标签。Release 附带的 `compose.yaml` 默认使用该版本镜像,
|
||||
配置模板和部署文件均包含在 `checksums.txt` 中;不要混用 `main` 分支模板与已发布镜像。
|
||||
[安装脚本](deploy/install.sh) 默认安装到当前目录下的 `codex-proxy-rs/`,下载同一正式 Release 的部署文件,
|
||||
自动生成密码、设置目录权限并启动服务。完成后会显示访问地址和管理员密码,请保存密码,再按下方步骤
|
||||
[添加账号与客户端密钥](#添加账号与客户端密钥)。
|
||||
|
||||
分别生成数据库和 Redis 密码:
|
||||
可在执行 `bash install.sh` 时传入环境变量:
|
||||
|
||||
| 变量 | 用途 | 默认值 |
|
||||
| --- | --- | --- |
|
||||
| `INSTALL_DIR` | 安装目录,建议使用绝对路径 | 当前目录下的 `codex-proxy-rs/` |
|
||||
| `CPR_RELEASE_TAG` | 指定发布标签 | 最新正式版本 |
|
||||
| `ADMIN_PASSWORD` | 管理员初始密码,至少 12 位,不能包含 `$`,不能使用常见弱口令 | 随机生成 |
|
||||
|
||||
例如,自定义安装目录:
|
||||
|
||||
```bash
|
||||
openssl rand -hex 24
|
||||
openssl rand -hex 24
|
||||
INSTALL_DIR="$HOME/services/codex-proxy-rs" bash install.sh
|
||||
```
|
||||
|
||||
编辑 `deploy/config.yaml`,填好以下三项:
|
||||
重复运行时请使用同一安装目录。检测到 `deploy/config.yaml` 后,脚本保留现有配置和部署文件,
|
||||
忽略传入的管理员密码,也不执行版本升级。
|
||||
|
||||
| 配置项 | 填写内容 |
|
||||
| --- | --- |
|
||||
| `store.database.password` | 第一个生成的 48 位十六进制密码 |
|
||||
| `store.redis.password` | 第二个生成的 48 位十六进制密码 |
|
||||
| `admin.default_password` | 管理员初始密码,至少 12 位,不能包含 `$` |
|
||||
### 手动安装
|
||||
|
||||
### 2. 启动服务
|
||||
自行下载部署文件、配置密码和启动服务的完整步骤见 [部署文档](deploy/README.md#手动安装)。
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/compose.yaml config --quiet
|
||||
docker compose -f deploy/compose.yaml pull
|
||||
docker compose -f deploy/compose.yaml up -d --no-build --wait
|
||||
curl -i http://127.0.0.1:8080/healthz
|
||||
```
|
||||
### 登录管理端
|
||||
|
||||
健康检查返回 `204 No Content` 后,打开 `http://127.0.0.1:8080`,
|
||||
使用 `admin@cpr.local` 和刚设置的管理员密码登录。
|
||||
部署完成后,打开 `http://127.0.0.1:8080`,使用 `admin@cpr.local` 和管理员密码登录。
|
||||
API Key 持有者可在同一登录页切换登录身份,进入 `/key-usage` 查看自己的用量、趋势、请求日志、额度与健康时间线;不能访问管理员页面。
|
||||
|
||||
默认地址只能在服务器本机访问。从其他设备使用时,需要配置
|
||||
[HTTPS 反向代理](deploy/README.md#公网访问)。
|
||||
|
||||
### 3. 添加账号与客户端密钥
|
||||
### 添加账号与客户端密钥
|
||||
|
||||
1. 在「账号」中添加账号,完成授权或导入。
|
||||
2. 按需建立账号分组,再创建客户端密钥并选择可用分组。**不选分组表示可使用全部账号**。
|
||||
3. 打开密钥的「使用密钥」,复制客户端配置。
|
||||
|
||||
账号自动冻结默认关闭,可在「系统设置 → 运行参数」中启用。启用后,账号频繁命中上游容量错误时
|
||||
会暂停调度并显示为限流中;若开启恢复前探测,冷却到期后需探测成功才恢复。
|
||||
|
||||
## 客户端接入
|
||||
|
||||
**Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,或通过 CCSwitch 导入。
|
||||
|
||||
Generated
+13
@@ -1491,6 +1491,7 @@ dependencies = [
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.29.0",
|
||||
@@ -1509,6 +1510,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono-tz",
|
||||
"futures",
|
||||
"futures-timer",
|
||||
"semver",
|
||||
@@ -1539,6 +1541,7 @@ dependencies = [
|
||||
"scraper",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_ignored",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"tar",
|
||||
@@ -3344,6 +3347,16 @@ dependencies = [
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_ignored"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "115dffd5f3853e06e746965a20dcbae6ee747ae30b543d91b0e089668bb07798"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
|
||||
@@ -7,8 +7,8 @@ use serde::Deserialize;
|
||||
const CONFIG_SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
/// 顶层配置只组合各包拥有的配置段,不解释任何业务字段。
|
||||
/// 各启动配置忽略未知字段,允许升级时保留旧配置;已知字段仍按类型和业务约束校验。
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GatewayConfig {
|
||||
schema_version: u32,
|
||||
host: HostConfig,
|
||||
@@ -19,16 +19,20 @@ pub struct GatewayConfig {
|
||||
api: gateway_api::ApiConfig,
|
||||
openai: provider_openai::OpenAiConfig,
|
||||
xai: provider_xai::XaiConfig,
|
||||
#[serde(default, rename = "services")]
|
||||
_compose_services: Option<serde::de::IgnoredAny>,
|
||||
}
|
||||
|
||||
impl LoadableConfig for GatewayConfig {
|
||||
const EXTERNAL_SECTIONS: &'static [&'static str] = &["services"];
|
||||
|
||||
fn resolve_and_validate(&mut self, source_dir: &std::path::Path) -> Result<(), ConfigError> {
|
||||
if self.schema_version != CONFIG_SCHEMA_VERSION {
|
||||
return Err(ConfigError::InvalidField("schema_version"));
|
||||
}
|
||||
self.host.resolve_and_validate(source_dir)?;
|
||||
self.api
|
||||
.resolve_and_validate(source_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("api"))?;
|
||||
self.host
|
||||
.resolve_and_validate(source_dir, &self.api.asset_directory)?;
|
||||
let runtime_data_dir = self.host.runtime_data_dir().to_path_buf();
|
||||
self.store
|
||||
.resolve_and_validate(&runtime_data_dir)
|
||||
@@ -39,9 +43,6 @@ impl LoadableConfig for GatewayConfig {
|
||||
self.client
|
||||
.resolve_and_validate(source_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("client"))?;
|
||||
self.api
|
||||
.resolve_and_validate(source_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("api"))?;
|
||||
self.openai
|
||||
.resolve_and_validate(&runtime_data_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("openai"))?;
|
||||
@@ -64,7 +65,6 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
api,
|
||||
openai,
|
||||
xai,
|
||||
_compose_services: _,
|
||||
} = config;
|
||||
|
||||
let host = gateway_host::initialize(host).await?;
|
||||
|
||||
@@ -168,6 +168,50 @@ fn config_loader_should_resolve_paths_relative_to_config_file() {
|
||||
assert!(debug.contains("frontend/dist"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_share_resolved_assets_with_system_update() {
|
||||
const CHILD_ENV: &str = "CPR_TEST_UPDATE_ASSETS_CHILD";
|
||||
let Ok(case) = std::env::var(CHILD_ENV) else {
|
||||
// 使用子进程覆盖无环境变量、Docker 路径和相对路径,避免污染并行测试。
|
||||
for (case, web_dist) in [
|
||||
("binary", None),
|
||||
("docker", Some("/app/web/dist")),
|
||||
("relative", Some("../custom/dist")),
|
||||
] {
|
||||
let mut child = Command::new(std::env::current_exe().expect("test executable"));
|
||||
child
|
||||
.args([
|
||||
"--exact",
|
||||
"bootstrap::config_loader_should_share_resolved_assets_with_system_update",
|
||||
])
|
||||
.env(CHILD_ENV, case)
|
||||
.env_remove("CPR_WEB_DIST_DIR");
|
||||
if let Some(web_dist) = web_dist {
|
||||
child.env("CPR_WEB_DIST_DIR", web_dist);
|
||||
}
|
||||
let output = child.output().expect("isolated configuration test");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{case}: {}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
return;
|
||||
};
|
||||
let mut document = valid_config_document();
|
||||
document["api"]["asset_directory"] = serde_json::json!("../web/dist");
|
||||
let (config, directory) = parse_config(&document.to_string()).expect("binary configuration");
|
||||
let assets = match case.as_str() {
|
||||
"docker" => std::path::PathBuf::from("/app/web/dist"),
|
||||
"relative" => directory.path().join("deploy/../custom/dist"),
|
||||
_ => directory.path().join("deploy/../web/dist"),
|
||||
};
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains(&format!("asset_directory: {assets:?}")));
|
||||
assert!(debug.contains(&format!("web_dist_dir: Some({assets:?})")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_missing_runtime_data_dir() {
|
||||
let config = valid_config().replace(" runtime_data_dir: '../.runtime/data'\n", "");
|
||||
@@ -228,16 +272,107 @@ fn bootstrap_config_debug_should_redact_all_passwords() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_unknown_fields() {
|
||||
assert_rejected(format!(
|
||||
"{}\nunknown_terminal_field: true\n",
|
||||
valid_config()
|
||||
));
|
||||
fn config_loader_should_ignore_unknown_fields_in_configuration_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["host"]["system_update"] = serde_json::json!({});
|
||||
document["store"]["pool"] = serde_json::json!({});
|
||||
for path in [
|
||||
"",
|
||||
"/host",
|
||||
"/host/listen",
|
||||
"/host/logging",
|
||||
"/host/logging/file",
|
||||
"/host/system_update",
|
||||
"/store",
|
||||
"/store/database",
|
||||
"/store/redis",
|
||||
"/store/pool",
|
||||
"/admin",
|
||||
"/client",
|
||||
"/api",
|
||||
"/openai",
|
||||
"/openai/wire_profile",
|
||||
"/xai",
|
||||
"/xai/wire_profile",
|
||||
] {
|
||||
let mut extended = document.clone();
|
||||
extended.pointer_mut(path).expect("configuration section")["unused_setting"] =
|
||||
serde_json::json!({"nested": [true, null, "ignored"]});
|
||||
parse_config(&extended.to_string())
|
||||
.unwrap_or_else(|error| panic!("unknown field in {path}: {error}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_removed_tls_section() {
|
||||
assert_rejected(valid_config().replace("openai:\n", "openai:\n tls: {}\n"));
|
||||
fn config_loader_should_ignore_removed_tls_and_fingerprint_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["tls"] = serde_json::json!({});
|
||||
document["openai"]["fingerprint"] = serde_json::json!({"browser": "removed"});
|
||||
parse_config(&document.to_string()).expect("removed provider settings are ignored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_known_fields_alongside_unknown_fields() {
|
||||
for port in [serde_json::json!(0), serde_json::json!("not-a-port")] {
|
||||
let mut document = valid_config_document();
|
||||
document["host"]["listen"]["unused_setting"] = serde_json::json!(true);
|
||||
document["host"]["listen"]["port"] = port;
|
||||
assert_rejected(document.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_report_startup_configuration_diagnostics() {
|
||||
const CHILD_ENV: &str = "CPR_TEST_CONFIG_DIAGNOSTICS_CHILD";
|
||||
const UNUSED_SECRET: &str = "unused-value-must-not-appear-in-diagnostics";
|
||||
if std::env::var_os(CHILD_ENV).is_some() {
|
||||
gateway_host::load_config::<GatewayConfig>().expect("startup configuration");
|
||||
return;
|
||||
}
|
||||
for case in ["normal", "unused", "missing"] {
|
||||
let mut document = valid_config_document();
|
||||
if case == "unused" {
|
||||
document["openai"]["wire_profile"]["location"] = serde_json::json!(UNUSED_SECRET);
|
||||
} else if case == "missing" {
|
||||
document["host"]["listen"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("port");
|
||||
}
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
fs::create_dir(directory.path().join("deploy")).unwrap();
|
||||
fs::write(
|
||||
directory.path().join("deploy/config.yaml"),
|
||||
document.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let output = Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"bootstrap::config_loader_should_report_startup_configuration_diagnostics",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_ENV, "1")
|
||||
.current_dir(directory.path())
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert_eq!(
|
||||
output.status.success(),
|
||||
case != "missing",
|
||||
"{case}: {stderr}"
|
||||
);
|
||||
match case {
|
||||
"unused" => assert!(
|
||||
stderr.contains("openai.wire_profile.location") && stderr.contains("已忽略"),
|
||||
"{stderr}"
|
||||
),
|
||||
"missing" => assert!(stderr.contains("host.listen.port"), "{stderr}"),
|
||||
_ => assert!(!stderr.contains("警告"), "{stderr}"),
|
||||
}
|
||||
assert!(!stderr.contains(UNUSED_SECRET), "{stderr}");
|
||||
assert!(!stderr.contains("services"), "{stderr}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -309,14 +444,6 @@ fn config_loader_should_reject_missing_client_session_ttl() {
|
||||
assert_rejected(config.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
assert_rejected(valid_config().replace(
|
||||
"openai:\n",
|
||||
"openai:\n fingerprint:\n browser: removed\n",
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.153.4'", "codex_version: 'latest'"));
|
||||
@@ -342,36 +469,22 @@ fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_accept_empty_and_custom_request_locations() {
|
||||
fn config_loader_should_ignore_removed_request_location_setting() {
|
||||
let original = valid_config();
|
||||
let omitted = original
|
||||
.lines()
|
||||
.filter(|line| !line.trim_start().starts_with("location:"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
assert_ne!(original, omitted);
|
||||
parse_config(&omitted).expect("location passthrough when omitted");
|
||||
let location_line = original
|
||||
.lines()
|
||||
.find(|line| line.trim_start().starts_with("location:"))
|
||||
.expect("example location");
|
||||
for empty in [" location:", " location: null", " location: ~"] {
|
||||
parse_config(&original.replace(location_line, empty)).expect("empty YAML location");
|
||||
parse_config(&original).expect("location belongs to runtime settings");
|
||||
for value in [
|
||||
"null",
|
||||
"{}",
|
||||
"{ country: 'US', region: 'Ohio', city: 'Piketon', timezone: 'America/New_York' }",
|
||||
] {
|
||||
let legacy = original.replacen(
|
||||
" residency: us",
|
||||
&format!(" residency: us\n location: {value}"),
|
||||
1,
|
||||
);
|
||||
assert_ne!(legacy, original);
|
||||
parse_config(&legacy).expect("removed request location does not prevent startup");
|
||||
}
|
||||
let custom = original.replace(
|
||||
"location: { country: 'US', region: 'Ohio', city: 'Piketon', timezone: 'America/New_York' }",
|
||||
"location: { country: 'NZ', region: 'Auckland', city: 'Auckland', timezone: 'Pacific/Auckland' }",
|
||||
);
|
||||
assert_ne!(original, custom);
|
||||
parse_config(&custom).expect("custom location from YAML");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_request_location_timezones() {
|
||||
let original = valid_config();
|
||||
let invalid = original.replace("timezone: 'America/New_York'", "timezone: 'Not/A_Timezone'");
|
||||
assert_ne!(original, invalid);
|
||||
assert_rejected(invalid);
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -36,7 +36,7 @@ tokio = { version = "1.52", features = ["rt", "time", "macros", "sync"] }
|
||||
[dev-dependencies]
|
||||
sha2 = "0.11"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt", "test-util"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
//! 容量熔断冻结的恢复编排:到期探测、失败顺延与自适应并发下调。
|
||||
//!
|
||||
//! 冻结事实保存在可丢失的 Redis 冷却中,本服务只组合 Admin 端口:
|
||||
//! 观测(`AccountRuntimeStore`)、探测(`AccountsService`)与写回
|
||||
//! (原子降低并发 / 按冻结代次结束探测)。旧观测不会覆盖管理员操作。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::account::{AccountConcurrencyLimit, ProviderAccountId};
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::model::accounts::{
|
||||
AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot,
|
||||
};
|
||||
use crate::model::{MutationActor, MutationContext};
|
||||
use crate::ports::store::{AccountRuntimeStore, AccountStore, SettingsStore};
|
||||
use crate::use_case::accounts::AccountsService;
|
||||
|
||||
/// 自适应并发下调保留系数:下调到观测在途峰值的 80%。
|
||||
const ADAPTIVE_CONCURRENCY_FACTOR: f64 = 0.8;
|
||||
/// 自适应并发下限:过低的并发让账号几乎不可用,宁可保持冻结。
|
||||
const ADAPTIVE_CONCURRENCY_FLOOR: u32 = 2;
|
||||
/// worker 周期;冷却观测与探测都依赖该节奏。
|
||||
pub const FREEZE_RECOVERY_INTERVAL: Duration = Duration::from_secs(30);
|
||||
pub const FREEZE_RECOVERY_WORKER_OWNER: &str = "account-freeze-recovery";
|
||||
pub const WORKER_INITIAL_BACKOFF: Duration = Duration::from_secs(1);
|
||||
pub const WORKER_MAXIMUM_BACKOFF: Duration = Duration::from_secs(60);
|
||||
pub const WORKER_LEASE_TTL: Duration = Duration::from_secs(15 * 60);
|
||||
pub const WORKER_LEASE_RENEWAL: Duration = Duration::from_secs(5 * 60);
|
||||
/// 系统审计上下文的固定请求 ID;账号并发更新审计由此溯源到自动冻结编排。
|
||||
const SYSTEM_REQUEST_ID: &str = "account-freeze-recovery";
|
||||
|
||||
/// 冻结恢复所需的管理端口组合。
|
||||
#[derive(Clone)]
|
||||
pub struct FreezeRecoveryDeps {
|
||||
pub accounts: Arc<dyn AccountsService>,
|
||||
pub store: Arc<dyn AccountStore>,
|
||||
pub runtime: Arc<dyn AccountRuntimeStore>,
|
||||
pub settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
pub struct FreezeRecoveryTask {
|
||||
deps: FreezeRecoveryDeps,
|
||||
}
|
||||
|
||||
impl FreezeRecoveryTask {
|
||||
pub fn new(deps: FreezeRecoveryDeps) -> Self {
|
||||
Self { deps }
|
||||
}
|
||||
|
||||
fn system_context() -> MutationContext {
|
||||
MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: SYSTEM_REQUEST_ID.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 读取冻结策略;读取或校验失败跳过本轮,不能把未知配置当作允许解冻。
|
||||
async fn freeze_policy(&self) -> Option<gateway_core::provider_ports::ProviderFreezePolicy> {
|
||||
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
|
||||
gateway_core::provider_ports::ProviderFreezePolicy::try_new(
|
||||
settings.account_auto_freeze_enabled,
|
||||
settings.account_auto_freeze_threshold,
|
||||
settings.account_auto_freeze_window_seconds,
|
||||
settings.account_auto_freeze_duration_seconds,
|
||||
settings.account_auto_freeze_probe_enabled,
|
||||
settings.account_auto_freeze_probe_model,
|
||||
settings.account_auto_freeze_adaptive_concurrency,
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
async fn active_freezes(&self) -> BTreeMap<String, AccountFreeze> {
|
||||
self.deps.runtime.active_freezes().await.unwrap_or_default()
|
||||
}
|
||||
|
||||
async fn run_cycle_inner(&self) {
|
||||
let Some(policy) = self.freeze_policy().await else {
|
||||
return;
|
||||
};
|
||||
let freezes = self.active_freezes().await;
|
||||
if freezes.is_empty() {
|
||||
return;
|
||||
}
|
||||
if policy.enabled() && policy.adaptive_concurrency() {
|
||||
self.adapt_concurrency_limits(&freezes, &policy).await;
|
||||
}
|
||||
self.recover_due_freezes(&freezes, &policy).await;
|
||||
}
|
||||
|
||||
/// 对冻结中的账号执行自适应并发下调:目标为观测在途峰值的 80%(下限 2),
|
||||
/// 只降不升;未观测到峰值证据的账号保持现状。
|
||||
async fn adapt_concurrency_limits(
|
||||
&self,
|
||||
freezes: &BTreeMap<String, AccountFreeze>,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let account_ids = freezes.keys().cloned().collect::<Vec<_>>();
|
||||
let Ok(peaks) = self.deps.runtime.capacity_peaks(&account_ids).await else {
|
||||
return;
|
||||
};
|
||||
for (account_id, peak) in peaks {
|
||||
let Some(target) = adaptive_target(peak) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(account) = ProviderAccountId::new(account_id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(limit) = AccountConcurrencyLimit::new(target) else {
|
||||
continue;
|
||||
};
|
||||
match self
|
||||
.deps
|
||||
.accounts
|
||||
.lower_concurrency_limit(&Self::system_context(), account, limit)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {
|
||||
tracing::info!(
|
||||
account_id,
|
||||
adapted_limit = target,
|
||||
freeze_seconds = policy.freeze_duration().as_secs(),
|
||||
"容量熔断:按观测在途峰值下调账号并发上限",
|
||||
);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
warn!(account_id, error = %error, "自适应并发下调失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> Option<Option<AccountPageItem>> {
|
||||
let account_id = ProviderAccountId::new(account_id.to_owned()).ok()?;
|
||||
self.deps
|
||||
.store
|
||||
.load_account(account_id.as_str(), AccountRuntimeSnapshot::default())
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// 到达探测时间后才恢复;关闭自动冻结或探测时,已有冻结仍等待原冷却结束。
|
||||
async fn recover_due_freezes(
|
||||
&self,
|
||||
freezes: &BTreeMap<String, AccountFreeze>,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
for (account_id, freeze) in freezes {
|
||||
if freeze.until > Utc::now() {
|
||||
continue;
|
||||
}
|
||||
if policy.enabled() && policy.probe_enabled() && freeze.requires_probe {
|
||||
self.probe_and_recover(account_id, freeze, policy).await;
|
||||
} else if let Err(error) = self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account_id, freeze, None)
|
||||
.await
|
||||
{
|
||||
warn!(account_id, error = %error, "解除到期冻结失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_and_recover(
|
||||
&self,
|
||||
account_id: &str,
|
||||
freeze: &AccountFreeze,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let Some(Some(item)) = self.load_account(account_id).await else {
|
||||
return;
|
||||
};
|
||||
if !item.account.enabled {
|
||||
// 停用账号没有自动恢复意义;解冻交给管理员手动恢复。
|
||||
return;
|
||||
}
|
||||
let Ok(account) = ProviderAccountId::new(account_id.to_owned()) else {
|
||||
return;
|
||||
};
|
||||
let Some(model) = self.resolve_probe_model(&account, policy).await else {
|
||||
// 没有可用探测模型时按失败处理,顺延冻结等待下一轮。
|
||||
self.postpone(&account, freeze, policy).await;
|
||||
return;
|
||||
};
|
||||
let probe_succeeded = match self
|
||||
.deps
|
||||
.accounts
|
||||
.test_connection(account.clone(), model)
|
||||
.await
|
||||
{
|
||||
Ok(events) => drain_probe(events).await,
|
||||
Err(error) => {
|
||||
tracing::debug!(account_id, error = %error, "冻结恢复探测发起失败");
|
||||
false
|
||||
}
|
||||
};
|
||||
if probe_succeeded {
|
||||
match self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account_id, freeze, None)
|
||||
.await
|
||||
{
|
||||
Ok(true) => tracing::info!(account_id, "冻结恢复探测成功:账号已解冻"),
|
||||
Ok(false) => {}
|
||||
Err(error) => warn!(account_id, error = %error, "解除冻结失败"),
|
||||
}
|
||||
} else {
|
||||
self.postpone(&account, freeze, policy).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// 探测失败只顺延本次仍存在的冻结,旧探测不能重新冻结已手动恢复的账号。
|
||||
async fn postpone(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
freeze: &AccountFreeze,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let Some(until) = SystemTime::now().checked_add(policy.freeze_duration()) else {
|
||||
return;
|
||||
};
|
||||
let until = DateTime::<Utc>::from(until);
|
||||
match self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account.as_str(), freeze, Some(until))
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
tracing::info!(
|
||||
account_id = account.as_str(),
|
||||
postpone_seconds = policy.freeze_duration().as_secs(),
|
||||
"冻结恢复探测失败:账号冻结顺延",
|
||||
);
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
account_id = account.as_str(),
|
||||
error = %error,
|
||||
"冻结顺延失败"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn resolve_probe_model(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) -> Option<UpstreamModelId> {
|
||||
if let Some(model) = policy.probe_model() {
|
||||
return UpstreamModelId::new(model.to_owned()).ok();
|
||||
}
|
||||
let models = self.deps.accounts.models(account, false).await.ok()?;
|
||||
models.models.into_iter().next().map(|model| model.id)
|
||||
}
|
||||
}
|
||||
|
||||
impl ScheduledTask for FreezeRecoveryTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.run_cycle_inner().await;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 自适应目标并发:观测峰值 × 0.8 向下取整,下限 2;无证据(峰值为 0)不调整。
|
||||
fn adaptive_target(peak_in_flight: u32) -> Option<u32> {
|
||||
if peak_in_flight == 0 {
|
||||
return None;
|
||||
}
|
||||
let scaled = (f64::from(peak_in_flight) * ADAPTIVE_CONCURRENCY_FACTOR).floor() as u32;
|
||||
Some(scaled.max(ADAPTIVE_CONCURRENCY_FLOOR))
|
||||
}
|
||||
|
||||
/// 消耗连接测试事件流并返回是否以 `Completed` 终止。
|
||||
async fn drain_probe(mut events: crate::model::accounts::AccountConnectionTestEventStream) -> bool {
|
||||
let mut completed = false;
|
||||
while let Some(event) = events.next().await {
|
||||
match event {
|
||||
AccountConnectionTestEvent::Completed => completed = true,
|
||||
AccountConnectionTestEvent::Failed { .. } => return false,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
completed
|
||||
}
|
||||
@@ -10,14 +10,15 @@ use gateway_core::{
|
||||
routing::ProviderKind,
|
||||
runtime::SnapshotControl,
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration,
|
||||
WorkerRunnable,
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest,
|
||||
WorkerRegistration, WorkerRunnable, WorkerSchedule,
|
||||
},
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
mod use_case;
|
||||
@@ -27,8 +28,9 @@ pub use use_case::key_usage::KeyUsageService;
|
||||
pub use use_case::{
|
||||
account_groups::AccountGroupService, accounts::AccountsService, auth::AuthService,
|
||||
backup::BackupService, client_distribution::ClientDistributionService,
|
||||
client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService,
|
||||
proxies::ProxiesService, settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
client_keys::ClientKeyService, import_tasks::ImportTasksService,
|
||||
observability::ObservabilityService, openai::OpenAiService, proxies::ProxiesService,
|
||||
settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
};
|
||||
|
||||
use model::{AdminError, AdminErrorKind};
|
||||
@@ -95,7 +97,6 @@ impl fmt::Debug for InitialAdminPassword {
|
||||
|
||||
/// 管理控制面的启动配置。
|
||||
#[derive(Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AdminConfig {
|
||||
pub session_ttl_minutes: u64,
|
||||
pub default_username: String,
|
||||
@@ -104,7 +105,6 @@ pub struct AdminConfig {
|
||||
|
||||
/// Client 登录域的通用启动配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ClientConfig {
|
||||
pub session_ttl_minutes: u64,
|
||||
}
|
||||
@@ -195,14 +195,26 @@ pub struct AdminServices {
|
||||
openai: Arc<dyn OpenAiService>,
|
||||
xai: Arc<dyn XaiService>,
|
||||
backups: Arc<dyn BackupService>,
|
||||
import_tasks: Arc<dyn ImportTasksService>,
|
||||
}
|
||||
|
||||
impl AdminServices {
|
||||
#[must_use]
|
||||
pub fn import_tasks(&self) -> &dyn ImportTasksService {
|
||||
self.import_tasks.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn key_usage(&self) -> &dyn KeyUsageService {
|
||||
self.key_usage.as_ref()
|
||||
}
|
||||
|
||||
/// 取得账号服务的共享句柄;后台编排(冻结恢复 worker)需要持有 Arc。
|
||||
#[must_use]
|
||||
pub fn accounts_handle(&self) -> Arc<dyn AccountsService> {
|
||||
Arc::clone(&self.accounts)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn proxies(&self) -> &dyn ProxiesService {
|
||||
self.proxies.as_ref()
|
||||
@@ -276,7 +288,7 @@ impl AdminBundle {
|
||||
self.services.clone()
|
||||
}
|
||||
|
||||
/// 取出 Backup Worker 贡献;只能调用一次,与其它 Bundle 的贡献一并交给 Host。
|
||||
/// 取出 Admin Worker 贡献;只能调用一次,与其它 Bundle 的贡献一并交给 Host。
|
||||
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
|
||||
std::mem::take(&mut self.worker_contributions)
|
||||
}
|
||||
@@ -361,6 +373,21 @@ pub async fn initialize(
|
||||
store.client_keys(),
|
||||
store.observability(),
|
||||
));
|
||||
let openai = Arc::new(DefaultOpenAiService::new(
|
||||
openai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
));
|
||||
let xai = Arc::new(DefaultXaiService::new(
|
||||
xai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
));
|
||||
let import_tasks =
|
||||
use_case::import_tasks::DefaultImportTasksService::new(openai.clone(), xai.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let services = AdminServices {
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
@@ -370,7 +397,7 @@ pub async fn initialize(
|
||||
registry.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
@@ -392,21 +419,33 @@ pub async fn initialize(
|
||||
snapshot.clone(),
|
||||
)),
|
||||
system: Arc::new(DefaultSystemService::new(system)),
|
||||
openai: Arc::new(DefaultOpenAiService::new(
|
||||
openai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
xai: Arc::new(DefaultXaiService::new(
|
||||
xai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
openai,
|
||||
xai,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let mut worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(import_task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(registration));
|
||||
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
@@ -432,6 +471,37 @@ fn backup_worker_contribution(
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定。
|
||||
fn freeze_recovery_worker_contribution(
|
||||
task: freeze_recovery::FreezeRecoveryTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
freeze_recovery::FREEZE_RECOVERY_INTERVAL,
|
||||
freeze_recovery::WORKER_INITIAL_BACKOFF,
|
||||
freeze_recovery::WORKER_MAXIMUM_BACKOFF,
|
||||
freeze_recovery::WORKER_LEASE_TTL,
|
||||
freeze_recovery::WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
fn provider_kind(value: &'static str) -> Result<ProviderKind, AdminError> {
|
||||
ProviderKind::new(value).map_err(|_| AdminError::internal("内置 Provider 类型不合法"))
|
||||
}
|
||||
|
||||
@@ -79,6 +79,7 @@ pub struct AccountGroupListQuery {
|
||||
/// Complete account group summary.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupRecord {
|
||||
pub disable_fast: bool,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
@@ -107,6 +108,7 @@ pub struct AccountGroupPage {
|
||||
/// Create an account group.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreateAccountGroup {
|
||||
pub disable_fast: bool,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
pub color: AccountGroupColor,
|
||||
@@ -115,6 +117,7 @@ pub struct CreateAccountGroup {
|
||||
/// Store-ready create command with a generated stable ID.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct NewAccountGroup {
|
||||
pub disable_fast: bool,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
@@ -124,6 +127,7 @@ pub struct NewAccountGroup {
|
||||
/// Update an account group's descriptive fields.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateAccountGroup {
|
||||
pub disable_fast: Option<bool>,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
|
||||
@@ -69,11 +69,20 @@ pub struct AccountListQuery {
|
||||
/// Admin query service 从运行态存储取得的当前账号冷却快照。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountRuntimeSnapshot {
|
||||
pub rate_limited_until: BTreeMap<String, DateTime<Utc>>,
|
||||
pub cooldown: BTreeMap<String, gateway_core::account::AccountCooldown>,
|
||||
/// `None` 表示实时 lease 存储不可用;`Some` 中未出现的账号当前使用量为零。
|
||||
pub in_flight: Option<BTreeMap<String, u64>>,
|
||||
}
|
||||
|
||||
/// 恢复任务读取的冻结快照;generation 将异步结果绑定到本次冻结。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountFreeze {
|
||||
pub credential_revision: Revision,
|
||||
pub until: DateTime<Utc>,
|
||||
pub generation: String,
|
||||
pub requires_probe: bool,
|
||||
}
|
||||
|
||||
/// Optional account membership filter.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AccountGroupFilter {
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
//! 进程内账号导入任务;只保存执行所需输入与可公开的结果。
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{MutationContext, provider_credentials::ImportCredentials};
|
||||
|
||||
pub const MAX_IMPORT_TASK_ITEMS: usize = 200;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ImportTaskInput {
|
||||
pub provider: ProviderKind,
|
||||
pub command: ImportCredentials,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SubmitImportTask {
|
||||
pub submission_id: Uuid,
|
||||
/// API 对完整输入计算摘要;用于区分丢失响应后的重试与修改后的新提交。
|
||||
pub fingerprint: [u8; 32],
|
||||
pub context: MutationContext,
|
||||
pub items: Vec<ImportTaskInput>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ImportItemStatus {
|
||||
Pending,
|
||||
Running,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Unknown,
|
||||
Skipped,
|
||||
}
|
||||
|
||||
impl ImportItemStatus {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Pending => "pending",
|
||||
Self::Running => "running",
|
||||
Self::Succeeded => "succeeded",
|
||||
Self::Failed => "failed",
|
||||
Self::Unknown => "unknown",
|
||||
Self::Skipped => "skipped",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportTaskItem {
|
||||
pub index: usize,
|
||||
pub provider: ProviderKind,
|
||||
pub status: ImportItemStatus,
|
||||
pub account_ids: Vec<ProviderAccountId>,
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ImportTaskCounts {
|
||||
pub pending: usize,
|
||||
pub running: usize,
|
||||
pub succeeded: usize,
|
||||
pub failed: usize,
|
||||
pub unknown: usize,
|
||||
pub skipped: usize,
|
||||
pub imported_accounts: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportTaskSummary {
|
||||
pub task_id: Uuid,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub finished_at: Option<DateTime<Utc>>,
|
||||
pub stop_requested: bool,
|
||||
pub total: usize,
|
||||
pub counts: ImportTaskCounts,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportTaskDetail {
|
||||
pub summary: ImportTaskSummary,
|
||||
pub items: Vec<ImportTaskItem>,
|
||||
}
|
||||
@@ -8,6 +8,7 @@ pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_distribution;
|
||||
pub mod client_keys;
|
||||
pub mod import_tasks;
|
||||
pub mod key_usage;
|
||||
pub mod observability;
|
||||
pub mod provider_credentials;
|
||||
|
||||
@@ -544,6 +544,7 @@ pub struct UsageListRecord {
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub upstream_response_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
@@ -602,6 +603,7 @@ pub struct UsageRecord {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub upstream_response_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;管理领域保持不透明。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
|
||||
@@ -585,6 +585,7 @@ pub struct CredentialDeletionResult {
|
||||
pub struct RotateCredential {
|
||||
pub mutation: CredentialMutation,
|
||||
pub provider_material: ProviderDocument,
|
||||
pub settings: Option<super::accounts::UpdateAccount>,
|
||||
}
|
||||
|
||||
/// Provider 校验手工轮换材料时所需的非事务输入。
|
||||
@@ -613,6 +614,8 @@ pub struct PreparedCredentialRotationFacts {
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
/// Token 刷新保留提交时的资料,不以准备阶段的副本覆盖新套餐。
|
||||
pub preserve_profile: bool,
|
||||
pub provider_material: ProviderDocument,
|
||||
pub has_refresh_token: bool,
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
@@ -664,6 +667,7 @@ impl fmt::Debug for PreparedCredentialRotation {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CredentialRotationCommit {
|
||||
pub prepared: PreparedCredentialRotationFacts,
|
||||
pub settings: Option<super::accounts::UpdateAccount>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RotateCredential {
|
||||
@@ -672,6 +676,7 @@ impl fmt::Debug for RotateCredential {
|
||||
.debug_struct("RotateCredential")
|
||||
.field("mutation", &self.mutation)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.field("settings", &self.settings)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,11 +60,14 @@ pub struct ProxyTestResult {
|
||||
pub success: bool,
|
||||
pub latency_ms: u64,
|
||||
pub exit_ip: Option<std::net::IpAddr>,
|
||||
pub exit_ipv4: Option<std::net::Ipv4Addr>,
|
||||
pub exit_ipv6: Option<std::net::Ipv6Addr>,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProxyRecord {
|
||||
pub location: Option<gateway_core::account::RequestLocation>,
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub proxy: OutboundProxy,
|
||||
@@ -86,12 +89,15 @@ pub struct ProxyPage {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NewProxy {
|
||||
pub location: Option<gateway_core::account::RequestLocation>,
|
||||
pub name: String,
|
||||
pub proxy: OutboundProxy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UpdateProxy {
|
||||
/// 外层为空保留配置,内层为空恢复全局继承。
|
||||
pub location: Option<Option<gateway_core::account::RequestLocation>>,
|
||||
pub id: String,
|
||||
pub revision: Revision,
|
||||
pub name: String,
|
||||
|
||||
@@ -17,7 +17,10 @@ pub use gateway_core::account::RotationStrategy;
|
||||
/// 完整运行设置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RuntimeSettings {
|
||||
pub disable_fast: bool,
|
||||
pub config_revision: Revision,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
@@ -26,18 +29,29 @@ pub struct RuntimeSettings {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub rotation_strategy: RotationStrategy,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 原子替换运行设置的命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReplaceRuntimeSettings {
|
||||
pub disable_fast: Option<bool>,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
@@ -46,12 +60,20 @@ pub struct ReplaceRuntimeSettings {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub rotation_strategy: RotationStrategy,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
/// 明文管理员 API Key;按产品约束明文落库,但禁止 Debug 泄漏。
|
||||
|
||||
@@ -170,6 +170,14 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
command: PrepareCredentialRefresh,
|
||||
) -> Result<PreparedCredentialRotation, ProviderAdminError>;
|
||||
|
||||
/// 可公开的账号连接设置;实现只能显式投影非敏感字段,不能返回凭据原文。
|
||||
async fn account_configuration(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderDocument>, ProviderAdminError> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
request: ProviderQuotaRequest,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! 端口按业务资源拆分,方法使用领域模型,不暴露连接池、事务或 Redis client。
|
||||
|
||||
use std::{net::IpAddr, sync::Arc, time::Duration};
|
||||
use std::{collections::BTreeMap, net::IpAddr, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -159,6 +159,14 @@ pub trait AccountStore: Send + Sync {
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountUpdateResult>;
|
||||
|
||||
/// 在事务内按最新启用状态、账号上限和全局默认值判断,只降低并发上限。
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &gateway_core::account::ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>>;
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &gateway_core::account::ProviderAccountId,
|
||||
@@ -193,6 +201,25 @@ pub trait AccountRuntimeStore: Send + Sync {
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot>;
|
||||
|
||||
/// 容量熔断自动冻结中的账号与其冻结截止时间;429 临时限流不包含在内。
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, crate::model::accounts::AccountFreeze>>;
|
||||
|
||||
/// 读取容量失败窗口内观测到的在途并发峰值(自适应并发下调的证据)。
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>>;
|
||||
|
||||
/// 仅当冻结快照仍匹配时解除或顺延;旧探测不得覆盖手动恢复或新一轮冻结。
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &crate::model::accounts::AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool>;
|
||||
}
|
||||
|
||||
/// 控制面凭据、统一会话、登录限流与管理员安全审计。
|
||||
|
||||
@@ -144,6 +144,7 @@ impl AccountGroupService for DefaultAccountGroupService {
|
||||
id,
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
disable_fast: command.disable_fast,
|
||||
color: command.color,
|
||||
},
|
||||
context,
|
||||
@@ -192,11 +193,7 @@ fn project_group_runtime(
|
||||
let mut accounts_by_group = BTreeMap::<&str, Vec<&str>>::new();
|
||||
for member in members {
|
||||
let mut status = member.status.clone();
|
||||
status.rate_limited_until = runtime
|
||||
.rate_limited_until
|
||||
.get(&member.account_id)
|
||||
.copied()
|
||||
.map(Into::into);
|
||||
status.cooldown = runtime.cooldown.get(&member.account_id).copied();
|
||||
status_by_account
|
||||
.entry(member.account_id.as_str())
|
||||
.or_insert_with(|| resolve_account_status(&status, now).status);
|
||||
|
||||
@@ -17,8 +17,8 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountPageItem, AccountUpdateResult, AccountUsageWindowQuery, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, UpdateAccount,
|
||||
AccountPageItem, AccountUpdateResult, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountsUpdateResult, BatchUpdateAccounts, UpdateAccount,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
@@ -73,12 +73,26 @@ pub trait AccountsService: Send + Sync {
|
||||
command: UpdateAccount,
|
||||
) -> Result<AccountUpdateResult, AdminError>;
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
) -> Result<Option<AccountUpdateResult>, AdminError>;
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: BatchUpdateAccounts,
|
||||
) -> Result<AccountsUpdateResult, AdminError>;
|
||||
|
||||
async fn account_configuration(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<Option<crate::model::provider_credentials::ProviderDocument>, AdminError> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -250,6 +264,37 @@ impl DefaultAccountsService {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_api_key_usage(
|
||||
&self,
|
||||
accounts: &[AccountPageItem],
|
||||
) -> Result<BTreeMap<String, AccountUsage>, AdminError> {
|
||||
let now = Utc::now();
|
||||
// API Key 没有套餐周期;本地累计直接查询账号创建后仍保留的请求记录。
|
||||
let windows = accounts
|
||||
.iter()
|
||||
.filter(|item| item.account.authentication_kind == "api_key")
|
||||
.map(|item| AccountUsageWindowQuery {
|
||||
account_id: item.account.id.clone(),
|
||||
key: "account-lifetime".to_owned(),
|
||||
range: TimeRange {
|
||||
start: item.account.created_at,
|
||||
end: now,
|
||||
},
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if windows.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
Ok(self
|
||||
.accounts
|
||||
.load_account_usage_by_windows(&windows)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "API Key account usage"))?
|
||||
.into_iter()
|
||||
.map(|result| (result.account_id, result.usage))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn load_directory_item(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -287,9 +332,15 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let usage = quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone());
|
||||
let usage = self
|
||||
.load_api_key_usage(std::slice::from_ref(&stored))
|
||||
.await?
|
||||
.remove(&stored.account.id)
|
||||
.or_else(|| {
|
||||
quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone())
|
||||
});
|
||||
Ok(AccountDirectoryItem {
|
||||
plan_type_display: self.providers.resolve_account_plan(
|
||||
stored.account.provider_kind.as_str(),
|
||||
@@ -377,14 +428,17 @@ impl AccountsService for DefaultAccountsService {
|
||||
.collect::<Result<Vec<_>, AdminError>>()?;
|
||||
self.attach_quota_local_usage(&page.items, &mut quotas)
|
||||
.await?;
|
||||
let mut api_key_usage = self.load_api_key_usage(&page.items).await?;
|
||||
let items = page
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(mut item, quota)| {
|
||||
let usage = quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone());
|
||||
let usage = api_key_usage.remove(&item.account.id).or_else(|| {
|
||||
quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone())
|
||||
});
|
||||
AccountDirectoryItem {
|
||||
plan_type_display: self.providers.resolve_account_plan(
|
||||
item.account.provider_kind.as_str(),
|
||||
@@ -535,6 +589,27 @@ impl AccountsService for DefaultAccountsService {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
) -> Result<Option<AccountUpdateResult>, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
let result = self
|
||||
.accounts
|
||||
.lower_concurrency_limit(&account_id, limit, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "provider account"))?;
|
||||
if let Some(result) = &result {
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
@@ -581,6 +656,17 @@ impl AccountsService for DefaultAccountsService {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn account_configuration(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<Option<crate::model::provider_credentials::ProviderDocument>, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(account_id).await?;
|
||||
provider
|
||||
.account_configuration(account_id)
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider account configuration"))
|
||||
}
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
//! 导入队列由 Host 管理生命周期,HTTP 只负责接受输入和读取快照。
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
panic::AssertUnwindSafe,
|
||||
sync::{Arc, Mutex, MutexGuard},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::{FutureExt as _, StreamExt as _, future::BoxFuture, stream::FuturesUnordered};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
task::{DaemonTask, WorkerTaskError},
|
||||
};
|
||||
use tokio::{sync::Notify, time::Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{openai::OpenAiService, xai::XaiService};
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind, MutationActor, MutationContext,
|
||||
import_tasks::{
|
||||
ImportItemStatus, ImportTaskCounts, ImportTaskDetail, ImportTaskInput, ImportTaskItem,
|
||||
ImportTaskSummary, MAX_IMPORT_TASK_ITEMS, SubmitImportTask,
|
||||
},
|
||||
};
|
||||
|
||||
const CONCURRENCY: usize = 3;
|
||||
const MAX_ACTIVE_TASKS: usize = 8;
|
||||
const MAX_RETAINED_TASKS: usize = 100;
|
||||
const RETENTION: Duration = Duration::from_secs(3600);
|
||||
|
||||
pub trait ImportTasksService: Send + Sync {
|
||||
fn submit(&self, command: SubmitImportTask) -> Result<ImportTaskSummary, AdminError>;
|
||||
fn list(&self, context: &MutationContext) -> Vec<ImportTaskSummary>;
|
||||
fn detail(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError>;
|
||||
fn stop(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError>;
|
||||
}
|
||||
|
||||
struct Entry {
|
||||
task_id: Uuid,
|
||||
submission_id: Uuid,
|
||||
fingerprint: [u8; 32],
|
||||
owner: MutationActor,
|
||||
created_at: chrono::DateTime<Utc>,
|
||||
finished_at: Option<chrono::DateTime<Utc>>,
|
||||
finished_instant: Option<Instant>,
|
||||
stop_requested: bool,
|
||||
items: Vec<ImportTaskItem>,
|
||||
inputs: Vec<Option<ImportTaskInput>>,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
fn summary(&self) -> ImportTaskSummary {
|
||||
let mut counts = ImportTaskCounts::default();
|
||||
for item in &self.items {
|
||||
match item.status {
|
||||
ImportItemStatus::Pending => counts.pending += 1,
|
||||
ImportItemStatus::Running => counts.running += 1,
|
||||
ImportItemStatus::Succeeded => counts.succeeded += 1,
|
||||
ImportItemStatus::Failed => counts.failed += 1,
|
||||
ImportItemStatus::Unknown => counts.unknown += 1,
|
||||
ImportItemStatus::Skipped => counts.skipped += 1,
|
||||
}
|
||||
counts.imported_accounts += item.account_ids.len();
|
||||
}
|
||||
ImportTaskSummary {
|
||||
task_id: self.task_id,
|
||||
created_at: self.created_at,
|
||||
finished_at: self.finished_at,
|
||||
stop_requested: self.stop_requested,
|
||||
total: self.items.len(),
|
||||
counts,
|
||||
}
|
||||
}
|
||||
|
||||
fn detail(&self) -> ImportTaskDetail {
|
||||
ImportTaskDetail {
|
||||
summary: self.summary(),
|
||||
items: self.items.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_if_done(&mut self) {
|
||||
if self.finished_at.is_none()
|
||||
&& self.items.iter().all(|item| {
|
||||
!matches!(
|
||||
item.status,
|
||||
ImportItemStatus::Pending | ImportItemStatus::Running
|
||||
)
|
||||
})
|
||||
{
|
||||
self.finished_at = Some(Utc::now());
|
||||
self.finished_instant = Some(Instant::now());
|
||||
// 终态仅保留结果,连已清空的输入槽位分配也一并释放。
|
||||
self.inputs = Vec::new();
|
||||
}
|
||||
}
|
||||
|
||||
fn stop(&mut self) {
|
||||
if self.finished_at.is_some() {
|
||||
return;
|
||||
}
|
||||
self.stop_requested = true;
|
||||
for (item, input) in self.items.iter_mut().zip(&mut self.inputs) {
|
||||
if item.status == ImportItemStatus::Pending {
|
||||
item.status = ImportItemStatus::Skipped;
|
||||
*input = None;
|
||||
}
|
||||
}
|
||||
self.finish_if_done();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Registry {
|
||||
tasks: VecDeque<Entry>,
|
||||
shutting_down: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultImportTasksService {
|
||||
registry: Mutex<Registry>,
|
||||
notify: Notify,
|
||||
openai: Arc<dyn OpenAiService>,
|
||||
xai: Arc<dyn XaiService>,
|
||||
}
|
||||
|
||||
impl DefaultImportTasksService {
|
||||
pub(crate) fn new(openai: Arc<dyn OpenAiService>, xai: Arc<dyn XaiService>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
registry: Mutex::default(),
|
||||
notify: Notify::new(),
|
||||
openai,
|
||||
xai,
|
||||
})
|
||||
}
|
||||
|
||||
fn registry(&self) -> MutexGuard<'_, Registry> {
|
||||
let mut registry = self
|
||||
.registry
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
registry.tasks.retain(|entry| {
|
||||
entry
|
||||
.finished_instant
|
||||
.is_none_or(|at| at.elapsed() < RETENTION)
|
||||
});
|
||||
registry
|
||||
}
|
||||
|
||||
fn take_next(&self) -> Option<(Uuid, usize, ImportTaskInput)> {
|
||||
let mut registry = self.registry();
|
||||
if registry.shutting_down {
|
||||
return None;
|
||||
}
|
||||
let task_index = registry
|
||||
.tasks
|
||||
.iter()
|
||||
.position(|entry| entry.inputs.iter().any(Option::is_some))?;
|
||||
// 轮转任务,避免大批次独占后续全部执行机会。
|
||||
let mut entry = registry.tasks.remove(task_index)?;
|
||||
let index = entry.inputs.iter().position(Option::is_some)?;
|
||||
let input = entry.inputs[index].take()?;
|
||||
entry.items[index].status = ImportItemStatus::Running;
|
||||
let id = entry.task_id;
|
||||
registry.tasks.push_back(entry);
|
||||
Some((id, index, input))
|
||||
}
|
||||
|
||||
async fn execute(&self, id: Uuid, index: usize, input: ImportTaskInput) {
|
||||
let operation = async {
|
||||
match input.provider.as_str() {
|
||||
"openai" => self.openai.import_document(input.command).await,
|
||||
"xai" => self.xai.import_document(input.command).await,
|
||||
_ => Err(AdminError::invalid("不支持的导入平台")),
|
||||
}
|
||||
};
|
||||
let result = AssertUnwindSafe(operation).catch_unwind().await;
|
||||
let mut registry = self.registry();
|
||||
let Some(entry) = registry.tasks.iter_mut().find(|entry| entry.task_id == id) else {
|
||||
return;
|
||||
};
|
||||
let item = &mut entry.items[index];
|
||||
match result {
|
||||
Ok(Ok(result)) => {
|
||||
item.status = ImportItemStatus::Succeeded;
|
||||
item.account_ids = result.credential_ids;
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
// 存储或发布异常也可能发生在提交之后;不自动重放可能轮换 RT 的操作。
|
||||
item.status = if matches!(
|
||||
error.kind(),
|
||||
AdminErrorKind::UpstreamResultUnknown
|
||||
| AdminErrorKind::Internal
|
||||
| AdminErrorKind::Unavailable
|
||||
) {
|
||||
ImportItemStatus::Unknown
|
||||
} else {
|
||||
ImportItemStatus::Failed
|
||||
};
|
||||
item.message = Some(error.message().to_owned());
|
||||
}
|
||||
Err(_) => {
|
||||
item.status = ImportItemStatus::Unknown;
|
||||
item.message = Some("执行中断,请先核对账号列表再决定是否重新导入".to_owned());
|
||||
}
|
||||
}
|
||||
entry.finish_if_done();
|
||||
}
|
||||
}
|
||||
|
||||
impl ImportTasksService for DefaultImportTasksService {
|
||||
fn submit(&self, command: SubmitImportTask) -> Result<ImportTaskSummary, AdminError> {
|
||||
if command.items.is_empty()
|
||||
|| command.items.len() > MAX_IMPORT_TASK_ITEMS
|
||||
|| command.items.iter().any(|item| {
|
||||
!matches!(item.provider.as_str(), "openai" | "xai")
|
||||
|| item.command.context.actor != command.context.actor
|
||||
})
|
||||
{
|
||||
return Err(AdminError::invalid("导入任务需包含 1–200 条有效输入"));
|
||||
}
|
||||
let mut registry = self.registry();
|
||||
if let Some(existing) = registry.tasks.iter().find(|entry| {
|
||||
entry.owner == command.context.actor && entry.submission_id == command.submission_id
|
||||
}) {
|
||||
return if existing.fingerprint == command.fingerprint {
|
||||
Ok(existing.summary())
|
||||
} else {
|
||||
Err(AdminError::new(
|
||||
AdminErrorKind::Conflict,
|
||||
"提交标识已用于其他导入内容",
|
||||
))
|
||||
};
|
||||
}
|
||||
if registry.shutting_down {
|
||||
return Err(AdminError::new(
|
||||
AdminErrorKind::Unavailable,
|
||||
"服务正在关闭,暂不接受导入任务",
|
||||
));
|
||||
}
|
||||
if registry
|
||||
.tasks
|
||||
.iter()
|
||||
.filter(|entry| entry.finished_at.is_none())
|
||||
.count()
|
||||
>= MAX_ACTIVE_TASKS
|
||||
|| registry.tasks.len() >= MAX_RETAINED_TASKS
|
||||
{
|
||||
return Err(AdminError::new(
|
||||
AdminErrorKind::RateLimited,
|
||||
"导入任务已满,请稍后再试",
|
||||
));
|
||||
}
|
||||
let items = command
|
||||
.items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, input)| ImportTaskItem {
|
||||
index: index + 1,
|
||||
provider: input.provider.clone(),
|
||||
status: ImportItemStatus::Pending,
|
||||
account_ids: Vec::new(),
|
||||
message: None,
|
||||
})
|
||||
.collect();
|
||||
let entry = Entry {
|
||||
task_id: Uuid::now_v7(),
|
||||
submission_id: command.submission_id,
|
||||
fingerprint: command.fingerprint,
|
||||
owner: command.context.actor,
|
||||
created_at: Utc::now(),
|
||||
finished_at: None,
|
||||
finished_instant: None,
|
||||
stop_requested: false,
|
||||
items,
|
||||
inputs: command.items.into_iter().map(Some).collect(),
|
||||
};
|
||||
let summary = entry.summary();
|
||||
registry.tasks.push_back(entry);
|
||||
drop(registry);
|
||||
self.notify.notify_one();
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
fn list(&self, context: &MutationContext) -> Vec<ImportTaskSummary> {
|
||||
let mut tasks: Vec<_> = self
|
||||
.registry()
|
||||
.tasks
|
||||
.iter()
|
||||
.filter(|entry| entry.owner == context.actor)
|
||||
.map(Entry::summary)
|
||||
.collect();
|
||||
tasks.sort_by_key(|entry| std::cmp::Reverse(entry.task_id));
|
||||
tasks
|
||||
}
|
||||
|
||||
fn detail(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError> {
|
||||
self.registry()
|
||||
.tasks
|
||||
.iter()
|
||||
.find(|entry| entry.task_id == task_id && entry.owner == context.actor)
|
||||
.map(Entry::detail)
|
||||
.ok_or_else(|| AdminError::not_found("导入任务不存在或已过期"))
|
||||
}
|
||||
|
||||
fn stop(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError> {
|
||||
let mut registry = self.registry();
|
||||
let entry = registry
|
||||
.tasks
|
||||
.iter_mut()
|
||||
.find(|entry| entry.task_id == task_id && entry.owner == context.actor)
|
||||
.ok_or_else(|| AdminError::not_found("导入任务不存在或已过期"))?;
|
||||
entry.stop();
|
||||
Ok(entry.detail())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ImportTaskWorker(pub Arc<DefaultImportTasksService>);
|
||||
|
||||
impl DaemonTask for ImportTaskWorker {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut active = FuturesUnordered::new();
|
||||
// 固定一个取消 future,避免循环反复注册中立 CancellationToken 的 waiter。
|
||||
let cancelled = cancellation.cancelled();
|
||||
tokio::pin!(cancelled);
|
||||
let mut cleanup = tokio::time::interval(Duration::from_secs(60));
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
let mut registry = self.0.registry();
|
||||
registry.shutting_down = true;
|
||||
for entry in &mut registry.tasks {
|
||||
entry.stop();
|
||||
}
|
||||
break;
|
||||
}
|
||||
while active.len() < CONCURRENCY {
|
||||
let Some((id, index, input)) = self.0.take_next() else {
|
||||
break;
|
||||
};
|
||||
active.push(self.0.execute(id, index, input));
|
||||
}
|
||||
tokio::select! {
|
||||
() = &mut cancelled => {},
|
||||
_ = active.next(), if !active.is_empty() => {},
|
||||
() = self.0.notify.notified() => {},
|
||||
_ = cleanup.tick() => { drop(self.0.registry()); },
|
||||
}
|
||||
}
|
||||
// Host 提供总关闭预算;此处让已发出的交换与入库尽量完成。
|
||||
while active.next().await.is_some() {}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_distribution;
|
||||
pub mod client_keys;
|
||||
pub mod import_tasks;
|
||||
pub mod key_usage;
|
||||
pub mod observability;
|
||||
pub mod openai;
|
||||
@@ -190,9 +191,6 @@ async fn pending_authorization(
|
||||
.get(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
if !record.last_test.is_some_and(|test| test.success) {
|
||||
return Err(AdminError::conflict("请先测试代理连接"));
|
||||
}
|
||||
proxy_id = Some(record.id);
|
||||
Some(record.proxy)
|
||||
}
|
||||
@@ -345,12 +343,19 @@ async fn commit_authorization(
|
||||
async fn commit_credential_rotation(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedCredentialRotation,
|
||||
settings: Option<crate::model::accounts::UpdateAccount>,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_rotation(CredentialRotationCommit { prepared: facts }, context)
|
||||
.commit_credential_rotation(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
@@ -372,7 +377,13 @@ async fn commit_credential_refresh(
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_refresh(CredentialRotationCommit { prepared: facts }, context)
|
||||
.commit_credential_refresh(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
|
||||
@@ -189,6 +189,17 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let context = command.mutation.context;
|
||||
let account_id = command.mutation.account_id;
|
||||
if command
|
||||
.settings
|
||||
.as_ref()
|
||||
.is_some_and(|settings| settings.account_id != account_id.as_str())
|
||||
{
|
||||
return Err(AdminError::invalid("凭据和账号设置的目标不一致"));
|
||||
}
|
||||
let disable_account = command
|
||||
.settings
|
||||
.as_ref()
|
||||
.is_some_and(|settings| !settings.enabled);
|
||||
let details = required_credential(
|
||||
self.accounts.as_ref(),
|
||||
self.provider.provider_kind(),
|
||||
@@ -209,10 +220,14 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
let result = commit_credential_rotation(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
command.settings,
|
||||
&context,
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
if disable_account {
|
||||
self.provider.account_unavailable(&account_id).await;
|
||||
}
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
|
||||
@@ -181,6 +181,11 @@ impl ProxiesService for DefaultProxiesService {
|
||||
context: &MutationContext,
|
||||
) -> Result<ProxyMutation, AdminError> {
|
||||
command.name = validate_name(&command.name)?;
|
||||
command.location = command
|
||||
.location
|
||||
.map(|location| location.normalized())
|
||||
.transpose()
|
||||
.map_err(|_| AdminError::invalid("代理位置不合法"))?;
|
||||
let result = self
|
||||
.store
|
||||
.create(command, context)
|
||||
@@ -196,6 +201,11 @@ impl ProxiesService for DefaultProxiesService {
|
||||
context: &MutationContext,
|
||||
) -> Result<ProxyMutation, AdminError> {
|
||||
command.name = validate_name(&command.name)?;
|
||||
command.location = command
|
||||
.location
|
||||
.map(|location| location.map(|value| value.normalized()).transpose())
|
||||
.transpose()
|
||||
.map_err(|_| AdminError::invalid("代理位置不合法"))?;
|
||||
let result = self
|
||||
.store
|
||||
.update(command, context)
|
||||
|
||||
@@ -114,7 +114,10 @@ impl SettingsService for DefaultSettingsService {
|
||||
}
|
||||
|
||||
fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError> {
|
||||
let valid = command.refresh_margin_seconds > 0
|
||||
let valid = command.request_location.validate().is_ok()
|
||||
&& command.responses_max_decompressed_body_bytes > 0
|
||||
&& isize::try_from(command.responses_max_decompressed_body_bytes).is_ok()
|
||||
&& command.refresh_margin_seconds > 0
|
||||
&& command.refresh_concurrency > 0
|
||||
&& command.max_concurrent_per_account > 0
|
||||
&& command.max_waiting_per_key <= 1_000
|
||||
@@ -125,7 +128,11 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
|
||||
&& command.audit_retention_days > 0
|
||||
&& valid_client_version(command.min_codex_desktop_version.as_deref())
|
||||
&& valid_client_version(command.min_codex_cli_version.as_deref())
|
||||
&& i64::try_from(command.request_interval_ms).is_ok();
|
||||
&& valid_probe_model(command.account_auto_freeze_probe_model.as_deref())
|
||||
&& i64::try_from(command.request_interval_ms).is_ok()
|
||||
&& (2..=1_000).contains(&command.account_auto_freeze_threshold)
|
||||
&& (60..=3_600).contains(&command.account_auto_freeze_window_seconds)
|
||||
&& (300..=604_800).contains(&command.account_auto_freeze_duration_seconds);
|
||||
if valid {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -136,3 +143,12 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
|
||||
fn valid_client_version(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| CodexClientVersion::parse(value).is_ok())
|
||||
}
|
||||
|
||||
fn valid_probe_model(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 128
|
||||
&& value == value.trim()
|
||||
&& !value.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, PageSize, Revision,
|
||||
@@ -168,18 +168,41 @@ impl AccountRuntimeStore for FakeRuntimeStore {
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
*self.requested_accounts.lock().expect("requested accounts") = account_ids.to_vec();
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::from([(
|
||||
cooldown: BTreeMap::from([(
|
||||
"acct_limited".to_owned(),
|
||||
Utc::now() + Duration::minutes(5),
|
||||
std::time::SystemTime::from(Utc::now() + Duration::minutes(5)).into(),
|
||||
)]),
|
||||
in_flight: Some(BTreeMap::from([("acct_available".to_owned(), 2)])),
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
fn group_record() -> AccountGroupRecord {
|
||||
let now = Utc::now();
|
||||
AccountGroupRecord {
|
||||
disable_fast: false,
|
||||
id: group_id(),
|
||||
name: "Primary".to_owned(),
|
||||
description: None,
|
||||
@@ -215,7 +238,7 @@ fn member(account_id: &str, total_slots: u64) -> AccountGroupMemberFact {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
|
||||
@@ -67,6 +67,7 @@ pub(super) struct FakeProviderAdmin {
|
||||
kind: ProviderKind,
|
||||
events: EventLog,
|
||||
failure: Mutex<Option<ProviderAdminError>>,
|
||||
import_gate: Mutex<Option<Arc<tokio::sync::Semaphore>>>,
|
||||
quota_failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
retry_authorization_after_abort: Mutex<bool>,
|
||||
@@ -90,6 +91,7 @@ impl FakeProviderAdmin {
|
||||
kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
events,
|
||||
failure: Mutex::new(None),
|
||||
import_gate: Mutex::new(None),
|
||||
quota_failure: Mutex::new(None),
|
||||
pending: Arc::new(Mutex::new(None)),
|
||||
retry_authorization_after_abort: Mutex::new(false),
|
||||
@@ -108,6 +110,12 @@ impl FakeProviderAdmin {
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn block_imports(&self) -> Arc<tokio::sync::Semaphore> {
|
||||
let gate = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
*self.import_gate.lock().expect("import gate") = Some(gate.clone());
|
||||
gate
|
||||
}
|
||||
|
||||
pub(super) fn fail_next(&self, kind: ProviderAdminErrorKind) {
|
||||
*self.failure.lock().expect("provider failure") = Some(ProviderAdminError::new(kind));
|
||||
}
|
||||
@@ -235,6 +243,7 @@ impl FakeProviderAdmin {
|
||||
name: account.name.clone(),
|
||||
email: account.email.clone(),
|
||||
plan_type: account.plan_type.clone(),
|
||||
preserve_profile: false,
|
||||
provider_material: document(),
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
access_token_expires_at: account
|
||||
@@ -332,6 +341,10 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
_command: PrepareCredentialImport,
|
||||
) -> Result<PreparedCredentialImport, ProviderAdminError> {
|
||||
self.record("provider.prepare_import");
|
||||
let gate = self.import_gate.lock().expect("import gate").clone();
|
||||
if let Some(gate) = gate {
|
||||
gate.acquire().await.expect("import permit").forget();
|
||||
}
|
||||
self.require_available()?;
|
||||
let account_ids = self
|
||||
.import_account_ids
|
||||
@@ -531,6 +544,8 @@ pub(super) struct FakeAccountStore {
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
|
||||
quota_forecast_history: Mutex<QuotaForecastHistory>,
|
||||
update_commands: Mutex<Vec<UpdateAccount>>,
|
||||
pub(super) lowered_limits: Mutex<Vec<(String, u32)>>,
|
||||
}
|
||||
|
||||
impl FakeAccountStore {
|
||||
@@ -538,7 +553,7 @@ impl FakeAccountStore {
|
||||
Self::with_account(account_record(kind), events)
|
||||
}
|
||||
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
pub(super) fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
accounts: Mutex::new(vec![account]),
|
||||
@@ -549,9 +564,18 @@ impl FakeAccountStore {
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
quota_window_queries: Mutex::new(Vec::new()),
|
||||
quota_forecast_history: Mutex::new(QuotaForecastHistory::default()),
|
||||
update_commands: Mutex::new(Vec::new()),
|
||||
lowered_limits: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn update_commands(&self) -> Vec<UpdateAccount> {
|
||||
self.update_commands
|
||||
.lock()
|
||||
.expect("update commands")
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub(super) fn fail_next_commit(&self) {
|
||||
*self.fail_commit.lock().expect("store failure") = true;
|
||||
}
|
||||
@@ -617,7 +641,7 @@ impl FakeAccountStore {
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message.clone(),
|
||||
};
|
||||
@@ -839,12 +863,45 @@ impl AccountStore for FakeAccountStore {
|
||||
self.record("store.update_account");
|
||||
self.record_context(context);
|
||||
self.require_commit()?;
|
||||
self.update_commands
|
||||
.lock()
|
||||
.expect("update commands")
|
||||
.push(command.clone());
|
||||
Ok(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: ProviderAccountId::new(command.account_id).expect("account ID"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>> {
|
||||
let mut accounts = self.accounts.lock().expect("accounts");
|
||||
let Some(account) = accounts
|
||||
.iter_mut()
|
||||
.find(|account| account.id == account_id.as_str())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !account.enabled
|
||||
|| account.concurrency_limit.map_or(5, |value| value.get()) <= limit.get()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
account.concurrency_limit = Some(limit);
|
||||
self.lowered_limits
|
||||
.lock()
|
||||
.expect("lowered limits")
|
||||
.push((account.id.clone(), limit.get()));
|
||||
Ok(Some(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: account_id.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -916,6 +973,9 @@ struct StaticSettingsStore;
|
||||
impl SettingsStore for StaticSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(RuntimeSettings {
|
||||
disable_fast: false,
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: revision(1),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
@@ -925,12 +985,20 @@ impl SettingsStore for StaticSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 30,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
})
|
||||
}
|
||||
@@ -1956,6 +2024,52 @@ async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_key_list_and_detail_should_accumulate_local_usage_without_subscription_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.authentication_kind = "api_key".to_owned();
|
||||
account.created_at = Utc::now() - TimeDelta::days(60);
|
||||
let added_at = account.created_at;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
store.set_quota_window_usage(vec![AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "account-lifetime".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 4_330_000),
|
||||
}]);
|
||||
let services = accounts_service(provider, store.clone()).await;
|
||||
let page = services
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).unwrap(),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(page.items[0].quota.windows.is_empty());
|
||||
assert_eq!(
|
||||
page.items[0].usage.as_ref().unwrap().total_tokens,
|
||||
Some(4_330_000)
|
||||
);
|
||||
let queries = store.quota_window_queries();
|
||||
assert_eq!(queries.len(), 1);
|
||||
assert_eq!(queries[0].range.start, added_at);
|
||||
assert!(queries[0].range.end > added_at + TimeDelta::days(59));
|
||||
let detail = services
|
||||
.accounts()
|
||||
.quota(&ProviderAccountId::new("acct_test").unwrap(), false)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(detail.usage, page.items[0].usage);
|
||||
assert!(detail.quota.windows.is_empty());
|
||||
assert_eq!(store.quota_window_queries()[0].range.start, added_at);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
//! 容量熔断冻结恢复编排回归:到期探测、失败顺延与自适应并发下调。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration as TimeDelta, Utc};
|
||||
use gateway_admin::freeze_recovery::{FreezeRecoveryDeps, FreezeRecoveryTask};
|
||||
use gateway_admin::model::MutationContext;
|
||||
use gateway_admin::model::accounts::{AccountFreeze, AccountRuntimeSnapshot};
|
||||
use gateway_admin::model::settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RuntimeSettings,
|
||||
};
|
||||
use gateway_admin::ports::store::{
|
||||
AccountRuntimeStore, AccountStore, AdminStoreError, AdminStoreResult, SettingsStore,
|
||||
};
|
||||
use gateway_core::engine::probe::{
|
||||
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
|
||||
};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::task::{ScheduledTask as _, WorkerCycleContext, WorkerId, WorkerKind};
|
||||
|
||||
use super::AdminHarness;
|
||||
use super::accounts::{FakeAccountStore, FakeProviderAdmin, account_record, events, revision};
|
||||
|
||||
fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> RuntimeSettings {
|
||||
RuntimeSettings {
|
||||
disable_fast: false,
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: revision(1),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: enabled,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: probe_enabled,
|
||||
account_auto_freeze_probe_model: Some("gpt-5.5".to_owned()),
|
||||
account_auto_freeze_adaptive_concurrency: adaptive,
|
||||
updated_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
struct FreezeSettingsStore {
|
||||
settings: RuntimeSettings,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SettingsStore for FreezeSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(self.settings.clone())
|
||||
}
|
||||
|
||||
async fn admin_api_key_exists(&self) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn replace_runtime_settings(
|
||||
&self,
|
||||
_: ReplaceRuntimeSettings,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<RuntimeSettings> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable() -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"freeze recovery test",
|
||||
"unused",
|
||||
)
|
||||
}
|
||||
|
||||
/// 记录解冻/顺延调用的运行态 fake;冻结与峰值证据由测试预置。
|
||||
struct FreezeRuntimeStore {
|
||||
freezes: BTreeMap<String, AccountFreeze>,
|
||||
peaks: BTreeMap<String, u32>,
|
||||
cleared: Mutex<Vec<String>>,
|
||||
extended: Mutex<Vec<(String, DateTime<Utc>)>>,
|
||||
}
|
||||
|
||||
impl FreezeRuntimeStore {
|
||||
fn new(freezes: BTreeMap<String, AccountFreeze>, peaks: BTreeMap<String, u32>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
freezes,
|
||||
peaks,
|
||||
cleared: Mutex::new(Vec::new()),
|
||||
extended: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
fn cleared(&self) -> Vec<String> {
|
||||
self.cleared.lock().expect("cleared lock").clone()
|
||||
}
|
||||
|
||||
fn extended(&self) -> Vec<(String, DateTime<Utc>)> {
|
||||
self.extended.lock().expect("extended lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AccountRuntimeStore for FreezeRuntimeStore {
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn account_runtime(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn active_freezes(&self) -> AdminStoreResult<BTreeMap<String, AccountFreeze>> {
|
||||
Ok(self.freezes.clone())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(account_ids
|
||||
.iter()
|
||||
.filter_map(|id| self.peaks.get(id).map(|peak| (id.clone(), *peak)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
assert_eq!(self.freezes.get(account_id), Some(expected));
|
||||
if let Some(until) = postpone_until {
|
||||
self.extended
|
||||
.lock()
|
||||
.expect("extended")
|
||||
.push((account_id.to_owned(), until));
|
||||
} else {
|
||||
self.cleared
|
||||
.lock()
|
||||
.expect("cleared")
|
||||
.push(account_id.to_owned());
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
struct SuccessfulProbe;
|
||||
|
||||
impl AccountProbe for SuccessfulProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Ok(AccountProbeResult {
|
||||
text: vec!["OK".to_owned()],
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingProbe;
|
||||
|
||||
impl AccountProbe for FailingProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Err(AccountProbeError::new(
|
||||
gateway_core::error::GatewayError::new(
|
||||
gateway_core::error::GatewayErrorKind::RateLimited,
|
||||
"still overloaded",
|
||||
),
|
||||
gateway_core::engine::probe::AccountProbeErrorSource::Upstream,
|
||||
Some(gateway_core::upstream::UpstreamSendState::NotSent),
|
||||
None,
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn recovery_task(
|
||||
settings: RuntimeSettings,
|
||||
runtime: Arc<FreezeRuntimeStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
|
||||
recovery_task_with_store(settings, runtime, probe).await
|
||||
}
|
||||
|
||||
async fn recovery_task_with_store(
|
||||
settings: RuntimeSettings,
|
||||
runtime: Arc<FreezeRuntimeStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
let services = AdminHarness::new()
|
||||
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
|
||||
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
|
||||
.settings(Arc::new(FreezeSettingsStore {
|
||||
settings: settings.clone(),
|
||||
}))
|
||||
.provider(FakeProviderAdmin::new("openai", events()))
|
||||
.probe(probe)
|
||||
.build()
|
||||
.await;
|
||||
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
|
||||
accounts: services.accounts_handle(),
|
||||
store: store.clone(),
|
||||
runtime,
|
||||
settings: Arc::new(FreezeSettingsStore {
|
||||
settings: settings.clone(),
|
||||
}),
|
||||
});
|
||||
(task, store)
|
||||
}
|
||||
|
||||
async fn run_cycle(task: &FreezeRecoveryTask) {
|
||||
let worker = WorkerId::try_new(WorkerKind::AccountFreezeRecovery, "test").expect("worker id");
|
||||
let context = WorkerCycleContext::new(worker, None, CancellationToken::new());
|
||||
task.run_cycle(context)
|
||||
.await
|
||||
.expect("freeze recovery cycle");
|
||||
}
|
||||
|
||||
fn freeze_until(until: DateTime<Utc>) -> BTreeMap<String, AccountFreeze> {
|
||||
BTreeMap::from([(
|
||||
"acct_test".to_owned(),
|
||||
AccountFreeze {
|
||||
credential_revision: revision(1),
|
||||
until,
|
||||
generation: "freeze-generation".to_owned(),
|
||||
requires_probe: true,
|
||||
},
|
||||
)])
|
||||
}
|
||||
|
||||
fn freeze_due() -> BTreeMap<String, AccountFreeze> {
|
||||
freeze_until(Utc::now() - TimeDelta::seconds(1))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_success_clears_freeze() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_failure_postpones_freeze() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(FailingProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
let extended = runtime.extended();
|
||||
assert_eq!(extended.len(), 1);
|
||||
let (account_id, until) = &extended[0];
|
||||
assert_eq!(account_id, "acct_test");
|
||||
let remaining = until.timestamp_millis() - Utc::now().timestamp_millis();
|
||||
assert!(
|
||||
remaining > 7_000_000 && remaining <= 7_200_000,
|
||||
"postponed freeze should be about 2 hours, got {remaining}ms"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_policy_releases_due_freeze_without_probing() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(false, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_concurrency_lowers_limit_to_observed_peak() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
|
||||
);
|
||||
let (task, store) = recovery_task(
|
||||
runtime_settings(true, false, true),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
// 峰值 4 × 0.8 = 3(不低于下限 2),低于全局默认 5,应下调到 3。
|
||||
assert_eq!(
|
||||
*store.lowered_limits.lock().expect("lowered limits"),
|
||||
vec![("acct_test".to_owned(), 3)]
|
||||
);
|
||||
assert!(store.update_commands().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
assert!(runtime.cleared().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_concurrency_never_raises_limit() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::from([("acct_test".to_owned(), 40_u32)]),
|
||||
);
|
||||
let (task, store) = recovery_task(
|
||||
runtime_settings(true, false, true),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
// 峰值 40 的目标 32 高于全局默认 5;只降不升,不下发任何更新。
|
||||
assert!(store.update_commands().is_empty());
|
||||
}
|
||||
|
||||
/// 到期时间还早(超过探测提前量)的冻结不发起探测。
|
||||
#[tokio::test]
|
||||
async fn probe_skips_freezes_far_from_expiry() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::new(),
|
||||
);
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
/// 停用账号不参与探测恢复;并发下调同样跳过。
|
||||
#[tokio::test]
|
||||
async fn disabled_accounts_are_skipped() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_due(),
|
||||
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
|
||||
);
|
||||
let mut record = account_record("openai");
|
||||
record.enabled = false;
|
||||
let store = FakeAccountStore::with_account(record, events());
|
||||
let services = AdminHarness::new()
|
||||
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
|
||||
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
|
||||
.settings(Arc::new(FreezeSettingsStore {
|
||||
settings: runtime_settings(true, true, true),
|
||||
}))
|
||||
.provider(FakeProviderAdmin::new("openai", events()))
|
||||
.probe(Arc::new(SuccessfulProbe))
|
||||
.build()
|
||||
.await;
|
||||
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
|
||||
accounts: services.accounts_handle(),
|
||||
store: store.clone(),
|
||||
runtime: Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>,
|
||||
settings: Arc::new(FreezeSettingsStore {
|
||||
settings: runtime_settings(true, true, true),
|
||||
}),
|
||||
});
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
assert!(store.update_commands().is_empty());
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use gateway_admin::{
|
||||
AdminBundle, AdminServices,
|
||||
model::{
|
||||
AdminErrorKind, MutationActor, MutationContext,
|
||||
import_tasks::{ImportTaskInput, SubmitImportTask},
|
||||
provider_credentials::ImportCredentials,
|
||||
},
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
routing::ProviderKind,
|
||||
task::{WorkerContribution, WorkerKind, WorkerRunnable},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
AdminHarness,
|
||||
accounts::{
|
||||
EventLog, FakeAccountStore, FakeProviderAdmin, context, document, events, recorded,
|
||||
},
|
||||
};
|
||||
|
||||
fn command(id: Uuid, count: usize) -> SubmitImportTask {
|
||||
let context = context("task-submit");
|
||||
SubmitImportTask {
|
||||
submission_id: id,
|
||||
fingerprint: [1; 32],
|
||||
context: context.clone(),
|
||||
items: (0..count)
|
||||
.map(|_| ImportTaskInput {
|
||||
provider: ProviderKind::new("openai").unwrap(),
|
||||
command: ImportCredentials {
|
||||
outbound_proxy_id: None,
|
||||
settings: None,
|
||||
context: context.clone(),
|
||||
document: document(),
|
||||
},
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn harness() -> (AdminBundle, Arc<FakeProviderAdmin>, EventLog) {
|
||||
let log = events();
|
||||
let provider = FakeProviderAdmin::new("openai", log.clone());
|
||||
let bundle = AdminHarness::new()
|
||||
.provider(provider.clone())
|
||||
.accounts(FakeAccountStore::new("openai", log.clone()))
|
||||
.build_bundle()
|
||||
.await;
|
||||
(bundle, provider, log)
|
||||
}
|
||||
|
||||
fn start(bundle: &mut AdminBundle) -> (CancellationToken, tokio::task::JoinHandle<()>) {
|
||||
let registration = bundle
|
||||
.take_worker_contributions()
|
||||
.into_iter()
|
||||
.find_map(|contribution| match contribution {
|
||||
WorkerContribution::Registration(registration)
|
||||
if registration.id.kind() == WorkerKind::AccountImport =>
|
||||
{
|
||||
Some(registration)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.expect("import registration");
|
||||
let WorkerRunnable::Daemon { task, .. } = registration.runnable else {
|
||||
panic!("import daemon")
|
||||
};
|
||||
let cancellation = CancellationToken::new();
|
||||
let token = cancellation.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
task.run(token).await.expect("import daemon exits cleanly");
|
||||
});
|
||||
(cancellation, handle)
|
||||
}
|
||||
|
||||
async fn wait_started(log: &EventLog, count: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while recorded(log)
|
||||
.iter()
|
||||
.filter(|event| **event == "provider.prepare_import")
|
||||
.count()
|
||||
< count
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("imports started");
|
||||
}
|
||||
|
||||
async fn wait_finished(services: &AdminServices, id: Uuid) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), id)
|
||||
.unwrap()
|
||||
.summary
|
||||
.finished_at
|
||||
.is_none()
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("imports finished");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_should_return_the_same_task_but_reject_changed_content() {
|
||||
let (bundle, _, _) = harness().await;
|
||||
let services = bundle.services();
|
||||
let id = Uuid::now_v7();
|
||||
let first = services.import_tasks().submit(command(id, 2)).unwrap();
|
||||
let retry = services.import_tasks().submit(command(id, 2)).unwrap();
|
||||
assert_eq!(first.task_id, retry.task_id);
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.list(&context("reopened-page"))
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
let mut changed = command(id, 2);
|
||||
changed.fingerprint = [2; 32];
|
||||
assert_eq!(
|
||||
services.import_tasks().submit(changed).unwrap_err().kind(),
|
||||
AdminErrorKind::Conflict
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_reads_and_stop_should_be_isolated_by_owner() {
|
||||
let (bundle, _, _) = harness().await;
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap();
|
||||
let other = MutationContext {
|
||||
actor: MutationActor::AdminSession {
|
||||
admin_user_id: "another-admin".to_owned(),
|
||||
},
|
||||
request_id: "other-session".to_owned(),
|
||||
};
|
||||
assert!(services.import_tasks().list(&other).is_empty());
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.detail(&other, task.task_id)
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.stop(&other, task.task_id)
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::NotFound
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopping_should_skip_pending_items_and_allow_three_in_flight_to_commit() {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
let gate = provider.block_imports();
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 8))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_started(&log, 3).await;
|
||||
let stopped = services
|
||||
.import_tasks()
|
||||
.stop(&context("stop"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(stopped.summary.counts.running, 3);
|
||||
assert_eq!(stopped.summary.counts.skipped, 5);
|
||||
gate.add_permits(3);
|
||||
wait_finished(&services, task.task_id).await;
|
||||
let finished = services
|
||||
.import_tasks()
|
||||
.detail(&context("new-page"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(finished.summary.counts.imported_accounts, 3);
|
||||
assert_eq!(
|
||||
recorded(&log)
|
||||
.iter()
|
||||
.filter(|event| **event == "provider.prepare_import")
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
cancel.cancel();
|
||||
worker.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrency_limit_should_be_shared_across_tasks() {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
let gate = provider.block_imports();
|
||||
let services = bundle.services();
|
||||
let first = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 6))
|
||||
.unwrap();
|
||||
let second = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 6))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_started(&log, 3).await;
|
||||
let summaries = services.import_tasks().list(&context("poll"));
|
||||
assert_eq!(
|
||||
summaries
|
||||
.iter()
|
||||
.map(|task| task.counts.running)
|
||||
.sum::<usize>(),
|
||||
3
|
||||
);
|
||||
assert!(summaries.iter().all(|task| task.counts.running > 0));
|
||||
gate.add_permits(12);
|
||||
wait_finished(&services, first.task_id).await;
|
||||
wait_finished(&services, second.task_id).await;
|
||||
cancel.cancel();
|
||||
worker.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_and_unknown_items_should_not_abort_the_batch_or_retry_credentials() {
|
||||
for (kind, failed, unknown) in [
|
||||
(ProviderAdminErrorKind::Invalid, 1, 0),
|
||||
(ProviderAdminErrorKind::Ambiguous, 0, 1),
|
||||
] {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
provider.fail_next(kind);
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 4))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_finished(&services, task.task_id).await;
|
||||
let result = services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
result.summary.counts.succeeded,
|
||||
result.summary.counts.failed,
|
||||
result.summary.counts.unknown
|
||||
),
|
||||
(3, failed, unknown)
|
||||
);
|
||||
assert_eq!(
|
||||
recorded(&log)
|
||||
.iter()
|
||||
.filter(|event| **event == "provider.prepare_import")
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
cancel.cancel();
|
||||
worker.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_should_drain_running_items_and_reject_new_work() {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
let gate = provider.block_imports();
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 7))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_started(&log, 3).await;
|
||||
cancel.cancel();
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Unavailable
|
||||
);
|
||||
assert!(!worker.is_finished());
|
||||
gate.add_permits(3);
|
||||
worker.await.unwrap();
|
||||
let result = services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
result.summary.counts.succeeded,
|
||||
result.summary.counts.skipped
|
||||
),
|
||||
(3, 4)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn completed_records_should_expire_and_new_process_should_start_empty() {
|
||||
let (bundle, _, _) = harness().await;
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 2))
|
||||
.unwrap();
|
||||
services
|
||||
.import_tasks()
|
||||
.stop(&context("stop"), task.task_id)
|
||||
.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(3601)).await;
|
||||
assert!(services.import_tasks().list(&context("poll")).is_empty());
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), task.task_id)
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::NotFound
|
||||
);
|
||||
let (fresh, _, _) = harness().await;
|
||||
assert!(
|
||||
fresh
|
||||
.services()
|
||||
.import_tasks()
|
||||
.list(&context("poll"))
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_should_reject_excess_work_before_starting_any_imports() {
|
||||
let (bundle, _, log) = harness().await;
|
||||
let services = bundle.services();
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 201))
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Invalid
|
||||
);
|
||||
for _ in 0..8 {
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::RateLimited
|
||||
);
|
||||
assert!(recorded(&log).is_empty());
|
||||
}
|
||||
@@ -4,6 +4,8 @@ mod auth;
|
||||
mod auth_key;
|
||||
mod backup;
|
||||
mod client_keys;
|
||||
mod freeze_recovery;
|
||||
mod import_tasks;
|
||||
mod observability;
|
||||
mod openai;
|
||||
mod proxies;
|
||||
@@ -81,6 +83,7 @@ use gateway_core::{
|
||||
routing::{ConfigRevision, ProviderKind},
|
||||
runtime::SnapshotControl,
|
||||
};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub(super) struct AdminHarness {
|
||||
default_password: String,
|
||||
@@ -213,6 +216,10 @@ impl AdminHarness {
|
||||
}
|
||||
|
||||
pub(super) async fn build(self) -> AdminServices {
|
||||
self.build_bundle().await.services()
|
||||
}
|
||||
|
||||
pub(super) async fn build_bundle(self) -> gateway_admin::AdminBundle {
|
||||
gateway_admin::initialize(
|
||||
AdminConfig {
|
||||
session_ttl_minutes: self.session_ttl_minutes,
|
||||
@@ -247,7 +254,6 @@ impl AdminHarness {
|
||||
)
|
||||
.await
|
||||
.expect("initialize admin test harness")
|
||||
.services()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -484,6 +490,15 @@ impl AccountStore for UnavailableStore {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
_: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
@@ -526,6 +541,28 @@ impl AccountRuntimeStore for UnavailableStore {
|
||||
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -753,6 +753,9 @@ struct FixtureSettingsStore;
|
||||
impl SettingsStore for FixtureSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(RuntimeSettings {
|
||||
disable_fast: false,
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(1).expect("revision"),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
@@ -762,6 +765,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
@@ -769,6 +773,13 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
updated_at: Utc::now(),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -851,6 +862,7 @@ fn total_record(
|
||||
provider_account_authentication_kind: None,
|
||||
upstream_model_id: Some("gpt-5.5".to_owned()),
|
||||
upstream_transport: None,
|
||||
upstream_response_model: None,
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
|
||||
@@ -12,6 +12,7 @@ use gateway_core::account::{OutboundProxy, ProviderAccountId};
|
||||
pub(super) struct TestProxies {
|
||||
pub events: Option<super::accounts::EventLog>,
|
||||
pub accounts: Option<Vec<ProxyAccountRef>>,
|
||||
pub record: Option<ProxyRecord>,
|
||||
}
|
||||
|
||||
struct ImportGuard(super::accounts::EventLog);
|
||||
@@ -70,8 +71,11 @@ impl ProxyStore for TestProxies {
|
||||
page_size: query.page_size.get(),
|
||||
})
|
||||
}
|
||||
async fn get(&self, _: &str) -> AdminStoreResult<ProxyRecord> {
|
||||
Err(super::unavailable("proxy"))
|
||||
async fn get(&self, id: &str) -> AdminStoreResult<ProxyRecord> {
|
||||
self.record
|
||||
.clone()
|
||||
.filter(|record| record.id == id)
|
||||
.ok_or_else(|| super::unavailable("proxy"))
|
||||
}
|
||||
async fn create(&self, _: NewProxy, _: &MutationContext) -> AdminStoreResult<ProxyMutation> {
|
||||
Err(super::unavailable("proxy"))
|
||||
@@ -105,6 +109,60 @@ impl ProxyProbe for TestProxies {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authorization_uses_selected_proxy_regardless_of_probe_status() {
|
||||
use super::accounts::{FakeAccountStore, FakeProviderAdmin, context, events, recorded};
|
||||
use gateway_admin::model::provider_credentials::StartAuthorization;
|
||||
use std::sync::Arc;
|
||||
|
||||
for kind in ["openai", "xai"] {
|
||||
for probe_success in [None, Some(false), Some(true)] {
|
||||
let events = events();
|
||||
let now = chrono::Utc::now();
|
||||
let services = super::AdminHarness::new()
|
||||
.provider(FakeProviderAdmin::new(kind, events.clone()))
|
||||
.accounts(FakeAccountStore::new(kind, events.clone()))
|
||||
.proxies(Arc::new(TestProxies {
|
||||
record: Some(ProxyRecord {
|
||||
location: None,
|
||||
id: "proxy_oauth".to_owned(),
|
||||
name: "授权出口".to_owned(),
|
||||
proxy: OutboundProxy::parse("http://127.0.0.1:8080").unwrap(),
|
||||
revision: Revision::new(1).unwrap(),
|
||||
account_count: 0,
|
||||
last_test_at: probe_success.map(|_| now),
|
||||
last_test: probe_success.map(|success| ProxyTestResult {
|
||||
success,
|
||||
latency_ms: 10,
|
||||
exit_ip: None,
|
||||
exit_ipv4: None,
|
||||
exit_ipv6: None,
|
||||
message: "出口探测结果".to_owned(),
|
||||
}),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}),
|
||||
..Default::default()
|
||||
}))
|
||||
.build()
|
||||
.await;
|
||||
let command = StartAuthorization {
|
||||
outbound_proxy: Some(AccountProxySelection::Saved("proxy_oauth".to_owned())),
|
||||
context: context("oauth-proxy-status"),
|
||||
name: "授权账号".to_owned(),
|
||||
reauthorization: None,
|
||||
};
|
||||
let result = if kind == "openai" {
|
||||
services.openai().start_authorization(command).await
|
||||
} else {
|
||||
services.xai().start_authorization(command).await
|
||||
};
|
||||
assert!(result.is_ok(), "{kind}, {probe_success:?}: {result:?}");
|
||||
assert_eq!(recorded(&events), ["provider.start_authorization"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn linked_accounts_share_plan_resolution_and_only_read_cached_quota() {
|
||||
use super::accounts::{FakeProviderAdmin, events};
|
||||
|
||||
@@ -61,6 +61,9 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
request_id: "request-settings".to_owned(),
|
||||
},
|
||||
ReplaceRuntimeSettings {
|
||||
disable_fast: None,
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 0,
|
||||
refresh_concurrency: 1,
|
||||
@@ -69,12 +72,20 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -24,6 +24,7 @@ gateway-protocol = { path = "../gateway-protocol" }
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt", "sync", "time"] }
|
||||
tower-http = { version = "0.7.0", features = ["cors", "fs", "request-id", "timeout", "trace"] }
|
||||
|
||||
@@ -72,6 +72,8 @@ impl ListAccountGroupsQuery {
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct CreateAccountGroupRequest {
|
||||
#[serde(default)]
|
||||
disable_fast: bool,
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
color: String,
|
||||
@@ -80,6 +82,7 @@ struct CreateAccountGroupRequest {
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct UpdateAccountGroupRequest {
|
||||
disable_fast: Option<bool>,
|
||||
id: String,
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
@@ -95,6 +98,7 @@ struct AccountGroupIdRequest {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupView {
|
||||
disable_fast: bool,
|
||||
id: String,
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
@@ -138,6 +142,7 @@ impl From<AccountGroupRecord> for AccountGroupView {
|
||||
id: record.id.to_string(),
|
||||
name: record.name,
|
||||
description: record.description,
|
||||
disable_fast: record.disable_fast,
|
||||
color: record.color.as_str().to_owned(),
|
||||
enabled: record.enabled,
|
||||
member_count: record.member_count,
|
||||
@@ -273,6 +278,7 @@ where
|
||||
CreateAccountGroup {
|
||||
name: request.name,
|
||||
description: request.description,
|
||||
disable_fast: request.disable_fast,
|
||||
color: group_color(&request.color)?,
|
||||
},
|
||||
)
|
||||
@@ -300,6 +306,7 @@ where
|
||||
id: group_id(request.id)?,
|
||||
name: request.name,
|
||||
description: request.description,
|
||||
disable_fast: request.disable_fast,
|
||||
color: group_color(&request.color)?,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -31,7 +31,7 @@ impl AccountProvider {
|
||||
}
|
||||
|
||||
/// 导入统一设置,复用编辑账号的备注、调度和分组约束。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountImportSettingsRequest {
|
||||
pub notes: Option<String>,
|
||||
@@ -67,7 +67,7 @@ impl AccountImportSettingsRequest {
|
||||
}
|
||||
|
||||
/// Provider-owned 账号导入请求;公共 API 不解释 `data` 内部字段。
|
||||
#[derive(Deserialize)]
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountImportRequest {
|
||||
pub outbound_proxy_id: Option<String>,
|
||||
@@ -319,9 +319,13 @@ impl AccountDeletionRequest {
|
||||
pub struct RotateAccountRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub access_token: String,
|
||||
pub access_token: Option<String>,
|
||||
pub refresh_token: Option<String>,
|
||||
pub id_token: Option<String>,
|
||||
pub base_url: Option<String>,
|
||||
pub api_key: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
pub settings: Option<UpdateAccountRequest>,
|
||||
}
|
||||
|
||||
impl RotateAccountRequest {
|
||||
@@ -330,11 +334,40 @@ impl RotateAccountRequest {
|
||||
return Err(WireValidationError::new("provider"));
|
||||
}
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
validate_oauth_material(
|
||||
&self.access_token,
|
||||
self.refresh_token.as_deref(),
|
||||
self.id_token.as_deref(),
|
||||
)
|
||||
if let Some(settings) = &self.settings {
|
||||
settings.validate()?;
|
||||
if settings.account_id != self.account_id {
|
||||
return Err(WireValidationError::new("settings.accountId"));
|
||||
}
|
||||
}
|
||||
if let Some(base_url) = &self.base_url {
|
||||
if self.access_token.is_some()
|
||||
|| self.refresh_token.is_some()
|
||||
|| self.id_token.is_some()
|
||||
|| base_url.is_empty()
|
||||
|| base_url.len() > 2048
|
||||
|| !matches!(self.transport.as_deref(), Some("http" | "prefer_websocket"))
|
||||
|| self.api_key.as_ref().is_some_and(|key| {
|
||||
key.is_empty()
|
||||
|| key.len() > 16 * 1024
|
||||
|| !key.bytes().all(|byte| byte.is_ascii_graphic())
|
||||
})
|
||||
{
|
||||
return Err(WireValidationError::new("credential"));
|
||||
}
|
||||
Ok(())
|
||||
} else {
|
||||
if self.api_key.is_some() || self.transport.is_some() {
|
||||
return Err(WireValidationError::new("credential"));
|
||||
}
|
||||
validate_oauth_material(
|
||||
self.access_token
|
||||
.as_deref()
|
||||
.ok_or_else(|| WireValidationError::new("accessToken"))?,
|
||||
self.refresh_token.as_deref(),
|
||||
self.id_token.as_deref(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_command(
|
||||
@@ -343,16 +376,34 @@ impl RotateAccountRequest {
|
||||
) -> Result<RotateCredential, WireValidationError> {
|
||||
self.validate()?;
|
||||
let mut material = Map::new();
|
||||
material.insert("access_token".to_owned(), Value::String(self.access_token));
|
||||
material.insert(
|
||||
"refresh_token".to_owned(),
|
||||
self.refresh_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
material.insert(
|
||||
"id_token".to_owned(),
|
||||
self.id_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
if let Some(base_url) = self.base_url {
|
||||
material.insert("base_url".to_owned(), Value::String(base_url));
|
||||
material.insert(
|
||||
"transport".to_owned(),
|
||||
self.transport.map_or(Value::Null, Value::String),
|
||||
);
|
||||
if let Some(key) = self.api_key {
|
||||
material.insert("api_key".to_owned(), Value::String(key));
|
||||
}
|
||||
} else {
|
||||
material.insert(
|
||||
"access_token".to_owned(),
|
||||
self.access_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
material.insert(
|
||||
"refresh_token".to_owned(),
|
||||
self.refresh_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
material.insert(
|
||||
"id_token".to_owned(),
|
||||
self.id_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
}
|
||||
Ok(RotateCredential {
|
||||
settings: self
|
||||
.settings
|
||||
.map(UpdateAccountRequest::into_command)
|
||||
.transpose()?,
|
||||
mutation: CredentialMutation {
|
||||
context,
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
|
||||
@@ -10,6 +10,7 @@ where
|
||||
S: SessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.merge(super::import_tasks::router::<S>())
|
||||
.route("/api/admin/accounts", get(list_accounts::<S>))
|
||||
.route("/api/admin/accounts/detail", get(account_detail::<S>))
|
||||
.route("/api/admin/accounts/export", get(export_accounts::<S>))
|
||||
@@ -120,8 +121,15 @@ where
|
||||
.quota(&account_id, false)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = AccountQuotaData {
|
||||
let configuration = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.account_configuration(&account_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = AccountDetailData {
|
||||
account: account_view(result, Utc::now()),
|
||||
credential_configuration: configuration.map(provider_document_value),
|
||||
};
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
//! 后台导入任务的有界请求与安全结果投影。
|
||||
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use gateway_admin::model::import_tasks::{
|
||||
ImportTaskDetail, ImportTaskInput, ImportTaskSummary, MAX_IMPORT_TASK_ITEMS, SubmitImportTask,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use super::*;
|
||||
use crate::auth::SessionState;
|
||||
|
||||
pub(super) fn router<S>() -> Router<S>
|
||||
where
|
||||
S: SessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route(
|
||||
"/api/admin/accounts/import-tasks",
|
||||
get(list::<S>).post(submit::<S>),
|
||||
)
|
||||
.route("/api/admin/accounts/import-tasks/detail", get(detail::<S>))
|
||||
.route("/api/admin/accounts/import-tasks/stop", post(stop::<S>))
|
||||
.layer(DefaultBodyLimit::max(4 * 1024 * 1024))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct SubmitRequest {
|
||||
submission_id: String,
|
||||
items: Vec<AccountImportRequest>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct TaskId {
|
||||
task_id: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct SummaryView {
|
||||
task_id: String,
|
||||
created_at: String,
|
||||
finished_at: Option<String>,
|
||||
stop_requested: bool,
|
||||
total: usize,
|
||||
counts: CountsView,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CountsView {
|
||||
pending: usize,
|
||||
running: usize,
|
||||
succeeded: usize,
|
||||
failed: usize,
|
||||
unknown: usize,
|
||||
skipped: usize,
|
||||
imported_accounts: usize,
|
||||
}
|
||||
|
||||
impl From<ImportTaskSummary> for SummaryView {
|
||||
fn from(task: ImportTaskSummary) -> Self {
|
||||
let counts = task.counts;
|
||||
Self {
|
||||
task_id: task.task_id.to_string(),
|
||||
created_at: task.created_at.to_rfc3339(),
|
||||
finished_at: task.finished_at.map(|at| at.to_rfc3339()),
|
||||
stop_requested: task.stop_requested,
|
||||
total: task.total,
|
||||
counts: CountsView {
|
||||
pending: counts.pending,
|
||||
running: counts.running,
|
||||
succeeded: counts.succeeded,
|
||||
failed: counts.failed,
|
||||
unknown: counts.unknown,
|
||||
skipped: counts.skipped,
|
||||
imported_accounts: counts.imported_accounts,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct DetailView {
|
||||
#[serde(flatten)]
|
||||
summary: SummaryView,
|
||||
items: Vec<ItemView>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ItemView {
|
||||
index: usize,
|
||||
provider: String,
|
||||
status: &'static str,
|
||||
account_ids: Vec<String>,
|
||||
message: Option<String>,
|
||||
}
|
||||
|
||||
impl From<ImportTaskDetail> for DetailView {
|
||||
fn from(task: ImportTaskDetail) -> Self {
|
||||
Self {
|
||||
summary: task.summary.into(),
|
||||
items: task
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| ItemView {
|
||||
index: item.index,
|
||||
provider: item.provider.to_string(),
|
||||
status: item.status.as_str(),
|
||||
account_ids: item
|
||||
.account_ids
|
||||
.into_iter()
|
||||
.map(|id| id.to_string())
|
||||
.collect(),
|
||||
message: item.message,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ListView {
|
||||
items: Vec<SummaryView>,
|
||||
}
|
||||
|
||||
async fn submit<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminJson(request): AdminJson<SubmitRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let submission_id = Uuid::parse_str(&request.submission_id)
|
||||
.map_err(|_| map_wire_error(WireValidationError::new("submissionId")))?;
|
||||
if submission_id.is_nil() {
|
||||
return Err(map_wire_error(WireValidationError::new("submissionId")));
|
||||
}
|
||||
if request.items.is_empty() || request.items.len() > MAX_IMPORT_TASK_ITEMS {
|
||||
return Err(map_wire_error(WireValidationError::new("items")));
|
||||
}
|
||||
let encoded = serde_json::to_vec(&request.items)
|
||||
.map_err(|_| map_wire_error(WireValidationError::new("items")))?;
|
||||
let fingerprint = Sha256::digest(&encoded).into();
|
||||
drop(encoded);
|
||||
let context = auth.context().mutation_context();
|
||||
let items = request
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let (provider, command) = item.into_command(context.clone())?;
|
||||
let provider = ProviderKind::new(match provider {
|
||||
AccountProvider::OpenAi => "openai",
|
||||
AccountProvider::Xai => "xai",
|
||||
})
|
||||
.map_err(|_| WireValidationError::new("provider"))?;
|
||||
Ok(ImportTaskInput { provider, command })
|
||||
})
|
||||
.collect::<Result<Vec<_>, WireValidationError>>()
|
||||
.map_err(map_wire_error)?;
|
||||
let task = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.submit(SubmitImportTask {
|
||||
submission_id,
|
||||
fingerprint,
|
||||
context,
|
||||
items,
|
||||
})
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::ACCEPTED,
|
||||
AdminEnvelope::ok(SummaryView::from(task)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list<S>(auth: AdminAuth, State(state): State<S>) -> impl IntoResponse
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let items = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.list(&auth.context().mutation_context())
|
||||
.into_iter()
|
||||
.map(SummaryView::from)
|
||||
.collect();
|
||||
AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(ListView { items }))
|
||||
}
|
||||
|
||||
async fn detail<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<TaskId>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let task = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.detail(
|
||||
&auth.context().mutation_context(),
|
||||
parse_task_id(&query.task_id)?,
|
||||
)
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(DetailView::from(task)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn stop<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminJson(request): AdminJson<TaskId>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let task = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.stop(
|
||||
&auth.context().mutation_context(),
|
||||
parse_task_id(&request.task_id)?,
|
||||
)
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(DetailView::from(task)),
|
||||
))
|
||||
}
|
||||
|
||||
fn parse_task_id(value: &str) -> Result<Uuid, AdminError> {
|
||||
Uuid::parse_str(value).map_err(|_| map_wire_error(WireValidationError::new("taskId")))
|
||||
}
|
||||
@@ -66,6 +66,7 @@ const MAX_AVATAR_VERSION_BYTES: usize = 32;
|
||||
|
||||
mod credentials;
|
||||
mod handlers;
|
||||
mod import_tasks;
|
||||
mod presenter;
|
||||
mod wire;
|
||||
|
||||
|
||||
@@ -66,12 +66,16 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
|
||||
quota,
|
||||
} = item;
|
||||
let status = projection.status.as_str().to_owned();
|
||||
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
|
||||
let cooldown = projection.cooldown;
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
let usage_period = quota.usage_window().map(|(_, period)| period);
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
|
||||
let mut usage = account_usage_view(usage, usage_period, now);
|
||||
if account.authentication_kind == "api_key" {
|
||||
usage.window_label_display = "通用额度".to_owned();
|
||||
}
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now);
|
||||
AccountView {
|
||||
id: account.id.clone(),
|
||||
name: account.name,
|
||||
@@ -121,7 +125,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
|
||||
updated_at,
|
||||
updated_at_display: china_datetime(&account.updated_at),
|
||||
quota,
|
||||
usage: account_usage_view(usage, usage_period, now),
|
||||
usage,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,7 +189,7 @@ fn forecast_usd_display(value: Option<f64>) -> String {
|
||||
|
||||
pub(super) fn account_quota_view(
|
||||
mut quota: ProviderQuota,
|
||||
rate_limited_until: Option<DateTime<Utc>>,
|
||||
cooldown: Option<gateway_core::account::AccountCooldown>,
|
||||
now: DateTime<Utc>,
|
||||
) -> (AccountQuotaView, Option<String>) {
|
||||
quota.apply_limit_reached_display();
|
||||
@@ -196,12 +200,22 @@ pub(super) fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
|
||||
let rate_limited_until = cooldown.map(|value| china_datetime(&value.until.into()));
|
||||
let rate_limit_reason = cooldown.map(|value| {
|
||||
if value.kind.is_capacity_freeze() {
|
||||
"capacity_freeze".to_owned()
|
||||
} else {
|
||||
"upstream_rate_limit".to_owned()
|
||||
}
|
||||
});
|
||||
let recovery_probe_required = cooldown.is_some_and(|value| value.kind.requires_probe());
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
rate_limit_reason,
|
||||
recovery_probe_required,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
|
||||
@@ -361,8 +361,10 @@ pub struct AccountGroupRefView {
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
/// 冷却到期或可开始恢复探测的时间;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub rate_limit_reason: Option<String>,
|
||||
pub recovery_probe_required: bool,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -684,6 +686,14 @@ pub struct AccountQuotaData {
|
||||
pub account: AccountView,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountDetailData {
|
||||
pub account: AccountView,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub credential_configuration: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// 个人资料、累计统计与订阅的统一响应。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
|
||||
@@ -267,6 +267,7 @@ pub(crate) fn usage_list_record_view(record: domain::UsageListRecord) -> UsageLi
|
||||
model,
|
||||
requested_model: record.requested_model_id,
|
||||
upstream_model: record.upstream_model_id,
|
||||
upstream_response_model: record.upstream_response_model,
|
||||
service_tier: record.service_tier,
|
||||
client_transport: record.client_transport,
|
||||
upstream_transport: record.upstream_transport,
|
||||
@@ -360,6 +361,7 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView
|
||||
model,
|
||||
requested_model: record.requested_model_id,
|
||||
upstream_model: record.upstream_model_id,
|
||||
upstream_response_model: record.upstream_response_model,
|
||||
service_tier: record.service_tier,
|
||||
status_code,
|
||||
client_transport: record.client_transport,
|
||||
|
||||
@@ -84,6 +84,7 @@ pub struct UsageListRecordView {
|
||||
pub model: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
pub upstream_model: Option<String>,
|
||||
pub upstream_response_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub client_transport: String,
|
||||
pub upstream_transport: Option<String>,
|
||||
@@ -122,6 +123,7 @@ pub struct UsageRecordView {
|
||||
pub model: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
pub upstream_model: Option<String>,
|
||||
pub upstream_response_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub client_transport: String,
|
||||
|
||||
@@ -48,6 +48,7 @@ struct RemoveAccountRequest {
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct CreateRequest {
|
||||
location: Option<gateway_core::account::RequestLocation>,
|
||||
name: String,
|
||||
proxy_url: AccountProxyUpdate,
|
||||
}
|
||||
@@ -55,12 +56,23 @@ struct CreateRequest {
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct UpdateRequest {
|
||||
#[serde(default, deserialize_with = "deserialize_location_update")]
|
||||
location: Option<Option<gateway_core::account::RequestLocation>>,
|
||||
id: String,
|
||||
revision: u64,
|
||||
name: String,
|
||||
proxy_url: Option<AccountProxyUpdate>,
|
||||
}
|
||||
|
||||
fn deserialize_location_update<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<Option<gateway_core::account::RequestLocation>>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
Option::deserialize(deserializer).map(Some)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct IdRequest {
|
||||
@@ -80,6 +92,8 @@ struct ProxyTestView {
|
||||
success: bool,
|
||||
latency_ms: u64,
|
||||
exit_ip: Option<String>,
|
||||
exit_ipv4: Option<String>,
|
||||
exit_ipv6: Option<String>,
|
||||
message: String,
|
||||
}
|
||||
|
||||
@@ -89,6 +103,8 @@ impl From<ProxyTestResult> for ProxyTestView {
|
||||
success: result.success,
|
||||
latency_ms: result.latency_ms,
|
||||
exit_ip: result.exit_ip.map(|ip| ip.to_string()),
|
||||
exit_ipv4: result.exit_ipv4.map(|ip| ip.to_string()),
|
||||
exit_ipv6: result.exit_ipv6.map(|ip| ip.to_string()),
|
||||
message: result.message,
|
||||
}
|
||||
}
|
||||
@@ -111,6 +127,7 @@ struct ProxyAccountView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ProxyView {
|
||||
location: Option<gateway_core::account::RequestLocation>,
|
||||
id: String,
|
||||
name: String,
|
||||
endpoint: String,
|
||||
@@ -127,6 +144,7 @@ impl From<ProxyRecord> for ProxyView {
|
||||
fn from(record: ProxyRecord) -> Self {
|
||||
let endpoint = record.proxy.endpoint();
|
||||
Self {
|
||||
location: record.location,
|
||||
id: record.id,
|
||||
name: record.name,
|
||||
has_authentication: record.proxy.expose_url() != endpoint,
|
||||
@@ -307,6 +325,7 @@ where
|
||||
.proxies()
|
||||
.create(
|
||||
NewProxy {
|
||||
location: request.location,
|
||||
name: request.name,
|
||||
proxy,
|
||||
},
|
||||
@@ -365,6 +384,7 @@ where
|
||||
.proxies()
|
||||
.update(
|
||||
UpdateProxy {
|
||||
location: request.location,
|
||||
id: request.id,
|
||||
revision: revision(request.revision)?,
|
||||
name: request.name,
|
||||
|
||||
@@ -34,6 +34,9 @@ pub type ModelMappings = BTreeMap<String, String>;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RuntimeSettingsView {
|
||||
pub disable_fast: bool,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -42,12 +45,20 @@ pub struct RuntimeSettingsView {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub rotation_strategy: String,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u64,
|
||||
pub ops_event_retention_days: u64,
|
||||
pub audit_retention_days: u64,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u64,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -55,6 +66,9 @@ pub struct RuntimeSettingsView {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateRuntimeSettingsRequest {
|
||||
pub disable_fast: Option<bool>,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -63,17 +77,28 @@ pub struct UpdateRuntimeSettingsRequest {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub rotation_strategy: String,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u64,
|
||||
pub ops_event_retention_days: u64,
|
||||
pub audit_retention_days: u64,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u64,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl UpdateRuntimeSettingsRequest {
|
||||
/// 校验公共运行参数。
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
self.request_location
|
||||
.validate()
|
||||
.map_err(|_| WireValidationError::new("requestLocation"))?;
|
||||
validate_model_mappings(&self.model_mappings)?;
|
||||
for (value, field) in [
|
||||
(self.max_waiting_per_key, "maxWaitingPerKey"),
|
||||
@@ -83,6 +108,13 @@ impl UpdateRuntimeSettingsRequest {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
}
|
||||
if self.responses_max_decompressed_body_bytes == 0
|
||||
|| isize::try_from(self.responses_max_decompressed_body_bytes).is_err()
|
||||
{
|
||||
return Err(WireValidationError::new(
|
||||
"responsesMaxDecompressedBodyBytes",
|
||||
));
|
||||
}
|
||||
if !(1..=120).contains(&self.concurrency_wait_timeout_seconds) {
|
||||
return Err(WireValidationError::new("concurrencyWaitTimeoutSeconds"));
|
||||
}
|
||||
@@ -110,12 +142,47 @@ impl UpdateRuntimeSettingsRequest {
|
||||
self.min_codex_cli_version.as_deref(),
|
||||
"minCodexCliVersion",
|
||||
)?;
|
||||
for (value, field) in [
|
||||
(
|
||||
self.account_auto_freeze_threshold,
|
||||
"accountAutoFreezeThreshold",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_window_seconds,
|
||||
"accountAutoFreezeWindowSeconds",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_duration_seconds,
|
||||
"accountAutoFreezeDurationSeconds",
|
||||
),
|
||||
] {
|
||||
require_positive_i64(value, field)?;
|
||||
}
|
||||
if !(2..=1_000).contains(&self.account_auto_freeze_threshold) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeThreshold"));
|
||||
}
|
||||
if !(60..=3_600).contains(&self.account_auto_freeze_window_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeWindowSeconds"));
|
||||
}
|
||||
if !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeDurationSeconds"));
|
||||
}
|
||||
validate_optional_probe_model(
|
||||
self.account_auto_freeze_probe_model.as_deref(),
|
||||
"accountAutoFreezeProbeModel",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(self) -> Result<ReplaceRuntimeSettings, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(ReplaceRuntimeSettings {
|
||||
disable_fast: self.disable_fast,
|
||||
request_location_enabled: self.request_location_enabled,
|
||||
request_location: self
|
||||
.request_location
|
||||
.normalized()
|
||||
.map_err(|_| WireValidationError::new("requestLocation"))?,
|
||||
model_mappings: domain_model_mappings(self.model_mappings)?,
|
||||
refresh_margin_seconds: self.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(self.refresh_concurrency)
|
||||
@@ -126,6 +193,7 @@ impl UpdateRuntimeSettingsRequest {
|
||||
max_waiting_per_key: self.max_waiting_per_key,
|
||||
max_waiting_per_account: self.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
|
||||
responses_max_decompressed_body_bytes: self.responses_max_decompressed_body_bytes,
|
||||
rotation_strategy: RotationStrategy::parse(&self.rotation_strategy)
|
||||
.ok_or_else(|| WireValidationError::new("rotationStrategy"))?,
|
||||
min_codex_desktop_version: self.min_codex_desktop_version,
|
||||
@@ -136,6 +204,14 @@ impl UpdateRuntimeSettingsRequest {
|
||||
.map_err(|_| WireValidationError::new("settingsOpsRetentionOverflow"))?,
|
||||
audit_retention_days: u32::try_from(self.audit_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?,
|
||||
account_auto_freeze_enabled: self.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u32::try_from(self.account_auto_freeze_threshold)
|
||||
.map_err(|_| WireValidationError::new("settingsFreezeThresholdOverflow"))?,
|
||||
account_auto_freeze_window_seconds: self.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: self.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: self.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: self.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: self.account_auto_freeze_adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -143,6 +219,9 @@ impl UpdateRuntimeSettingsRequest {
|
||||
impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
fn from(settings: RuntimeSettings) -> Self {
|
||||
Self {
|
||||
disable_fast: settings.disable_fast,
|
||||
request_location_enabled: settings.request_location_enabled,
|
||||
request_location: settings.request_location,
|
||||
model_mappings: wire_model_mappings(settings.model_mappings),
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
@@ -151,12 +230,21 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
responses_max_decompressed_body_bytes: settings.responses_max_decompressed_body_bytes,
|
||||
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
|
||||
min_codex_desktop_version: settings.min_codex_desktop_version,
|
||||
min_codex_cli_version: settings.min_codex_cli_version,
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u64::from(settings.account_auto_freeze_threshold),
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
updated_at: settings.updated_at,
|
||||
}
|
||||
}
|
||||
@@ -452,6 +540,22 @@ fn validate_optional_client_version(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 探测模型为可选自由文本:非空、去首尾空白后不变、无控制字符且不超过 128 字节。
|
||||
fn validate_optional_probe_model(
|
||||
value: Option<&str>,
|
||||
field: &'static str,
|
||||
) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| {
|
||||
value.is_empty()
|
||||
|| value.len() > 128
|
||||
|| value != value.trim()
|
||||
|| value.bytes().any(|byte| byte.is_ascii_control())
|
||||
}) {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
let message = match error.field() {
|
||||
"settingsRefreshConcurrencyOverflow" => "refreshConcurrency 不合法".to_owned(),
|
||||
@@ -459,6 +563,12 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
"settingsUsageRetentionOverflow" => "usageRetentionDays 不合法".to_owned(),
|
||||
"settingsOpsRetentionOverflow" => "opsEventRetentionDays 不合法".to_owned(),
|
||||
"settingsAuditRetentionOverflow" => "auditRetentionDays 不合法".to_owned(),
|
||||
"requestLocation" => "请求位置不合法,请检查国家代码、地区和城市".to_owned(),
|
||||
"settingsFreezeThresholdOverflow" => "accountAutoFreezeThreshold 不合法".to_owned(),
|
||||
"accountAutoFreezeThreshold" => "账号自动冻结阈值应为 2~1000 的整数".to_owned(),
|
||||
"accountAutoFreezeWindowSeconds" => "账号自动冻结统计窗口应为 60~3600 秒".to_owned(),
|
||||
"accountAutoFreezeDurationSeconds" => "账号自动冻结时长应为 300~604800 秒".to_owned(),
|
||||
"accountAutoFreezeProbeModel" => "探测模型格式不合法".to_owned(),
|
||||
"minCodexDesktopVersion" => "Codex Desktop 最低版本格式不合法".to_owned(),
|
||||
"minCodexCliVersion" => "Codex CLI 最低版本格式不合法".to_owned(),
|
||||
field => format!("{field} 字段不合法"),
|
||||
|
||||
@@ -37,7 +37,6 @@ mod session_cookie;
|
||||
|
||||
/// API-owned HTTP 与静态资源配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ApiConfig {
|
||||
pub asset_directory: PathBuf,
|
||||
pub cors_allowed_origins: Vec<String>,
|
||||
@@ -62,12 +61,17 @@ impl ApiConfig {
|
||||
return Err(ApiConfigError::InvalidAssetDirectory);
|
||||
}
|
||||
}
|
||||
if self.asset_directory.as_os_str().is_empty() {
|
||||
return Err(ApiConfigError::InvalidAssetDirectory);
|
||||
}
|
||||
self.validate()?;
|
||||
if self.asset_directory.is_relative() {
|
||||
self.asset_directory = source_dir.join(&self.asset_directory);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate(&mut self) -> Result<(), ApiConfigError> {
|
||||
if self.asset_directory.as_os_str().is_empty() {
|
||||
return Err(ApiConfigError::InvalidAssetDirectory);
|
||||
}
|
||||
if self.request_timeout_seconds == Some(0) {
|
||||
return Err(ApiConfigError::InvalidRequestTimeout);
|
||||
}
|
||||
@@ -124,9 +128,8 @@ pub fn initialize(
|
||||
worker_health: Arc<dyn WorkerHealthSource>,
|
||||
lifecycle: Arc<dyn ConnectionLifecycle>,
|
||||
) -> Result<ApiBundle, ApiError> {
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(ApiError::Config)?;
|
||||
// 配置加载已解析环境变量与相对路径,初始化只校验,避免覆盖最终目录。
|
||||
config.validate().map_err(ApiError::Config)?;
|
||||
let request_id_header = HeaderName::from_str(&config.request_id_header)
|
||||
.map_err(|_| ApiError::Config(ApiConfigError::InvalidRequestIdHeader))?;
|
||||
let state = ApiState {
|
||||
|
||||
@@ -143,7 +143,7 @@ pub(crate) fn client_access_error_response(error: ClientAccessError) -> Response
|
||||
}
|
||||
}
|
||||
|
||||
/// Desktop 优先于其内嵌的 CLI/Core 标记;未知客户端保持兼容并返回 `None`。
|
||||
/// Desktop 优先于内嵌 CLI/Core;未知客户端及未提供应用版本的手机远程客户端返回 `None`。
|
||||
#[must_use]
|
||||
pub fn identify_codex_client(headers: &HeaderMap) -> Option<IdentifiedCodexClient> {
|
||||
const MAXIMUM_HEADER_LENGTH: usize = 4096;
|
||||
@@ -154,11 +154,21 @@ pub fn identify_codex_client(headers: &HeaderMap) -> Option<IdentifiedCodexClien
|
||||
|| user_agent.is_some_and(|value| contains_ascii_case_insensitive(value, "Codex Desktop"));
|
||||
if is_desktop {
|
||||
let explicit_version = bounded_ascii_header(headers, "version", MAXIMUM_HEADER_LENGTH);
|
||||
let user_agent_version = user_agent.and_then(desktop_version_from_user_agent);
|
||||
// 手机远程初始化会覆盖进程级 UA 后缀,保留的 Desktop/Core 前缀不代表应用版本。
|
||||
// 仅缺失版本信息时免于门禁;已提供的版本即使非法,也继续按 Desktop 校验。
|
||||
if !headers.contains_key("version")
|
||||
&& user_agent_version.is_none()
|
||||
&& headers
|
||||
.get("user-agent")
|
||||
.is_some_and(|value| value.as_bytes().len() <= MAXIMUM_HEADER_LENGTH)
|
||||
&& user_agent.is_some_and(has_chatgpt_remote_user_agent_suffix)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let version = match explicit_version {
|
||||
Some(value) => CodexClientVersion::parse(value).ok(),
|
||||
None => user_agent
|
||||
.and_then(desktop_version_from_user_agent)
|
||||
.and_then(|value| CodexClientVersion::parse(value).ok()),
|
||||
None => user_agent_version.and_then(|value| CodexClientVersion::parse(value).ok()),
|
||||
};
|
||||
return Some(IdentifiedCodexClient {
|
||||
kind: CodexClientKind::Desktop,
|
||||
@@ -206,11 +216,34 @@ fn bounded_ascii_header<'a>(
|
||||
fn desktop_version_from_user_agent(user_agent: &str) -> Option<&str> {
|
||||
let marker = "Codex Desktop;";
|
||||
let start = find_ascii_case_insensitive(user_agent, marker)? + marker.len();
|
||||
let version = user_agent[start..]
|
||||
// 空候选也代表已提供应用版本,不能被手机远程的缺失版本规则放行。
|
||||
user_agent[start..]
|
||||
.trim_start()
|
||||
.split([')', ' ', ';', ','])
|
||||
.next()?;
|
||||
(!version.is_empty()).then_some(version)
|
||||
.next()
|
||||
}
|
||||
|
||||
fn has_chatgpt_remote_user_agent_suffix(user_agent: &str) -> bool {
|
||||
let Some((_, suffix)) = user_agent
|
||||
.strip_suffix(')')
|
||||
.and_then(|value| value.rsplit_once(" ("))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let Some((name, version)) = suffix.split_once("; ") else {
|
||||
return false;
|
||||
};
|
||||
let Some(platform) = name
|
||||
.strip_prefix("codex_chatgpt_")
|
||||
.and_then(|value| value.strip_suffix("_remote"))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
[platform, version].into_iter().all(|value| {
|
||||
!value.is_empty()
|
||||
&& !value.contains(['(', ')', ';'])
|
||||
&& !value.bytes().any(|byte| byte.is_ascii_whitespace())
|
||||
})
|
||||
}
|
||||
|
||||
fn contains_ascii_case_insensitive(haystack: &str, needle: &str) -> bool {
|
||||
|
||||
@@ -42,7 +42,10 @@ pub enum RequestDecodeError {
|
||||
MalformedJson,
|
||||
/// 请求体解压后超过允许上限。
|
||||
#[error("decompressed request body is too large")]
|
||||
DecompressedBodyTooLarge,
|
||||
DecompressedBodyTooLarge {
|
||||
/// 当前请求快照中的解压输出上限。
|
||||
limit_bytes: usize,
|
||||
},
|
||||
/// 请求使用了网关不支持的 `Content-Encoding`。
|
||||
#[error("request content-encoding `{encoding}` is not supported")]
|
||||
UnsupportedContentEncoding {
|
||||
@@ -108,9 +111,11 @@ impl RequestDecodeError {
|
||||
"Request body must be valid JSON.".to_owned(),
|
||||
None,
|
||||
),
|
||||
Self::DecompressedBodyTooLarge => (
|
||||
Self::DecompressedBodyTooLarge { limit_bytes } => (
|
||||
"request_too_large",
|
||||
"Decompressed request body exceeds the allowed size.".to_owned(),
|
||||
format!(
|
||||
"Decompressed request body exceeds the allowed size ({limit_bytes} bytes)."
|
||||
),
|
||||
None,
|
||||
),
|
||||
Self::UnsupportedContentEncoding { encoding } => (
|
||||
|
||||
@@ -52,7 +52,11 @@ pub(crate) async fn responses(
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let decoded = match decode_request_with_headers(&body, &headers) {
|
||||
let decoded = match decode_request_with_headers(
|
||||
&body,
|
||||
&headers,
|
||||
client.snapshot().responses_max_decompressed_body_bytes(),
|
||||
) {
|
||||
Ok(decoded) => decoded,
|
||||
Err(error) => {
|
||||
return protocol_error_response(StatusCode::BAD_REQUEST, error.protocol_body());
|
||||
|
||||
@@ -327,13 +327,14 @@ impl fmt::Debug for DecodedResponsesRequest {
|
||||
pub fn decode_request_with_headers(
|
||||
body: &[u8],
|
||||
headers: &HeaderMap,
|
||||
max_decompressed_bytes: usize,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
let body = decompress_request_body(body, headers)?;
|
||||
let body = decompress_request_body(body, headers, max_decompressed_bytes)?;
|
||||
decode_request_inner(&body, &OpenAiRequestHeaders::from_headers(headers))
|
||||
}
|
||||
|
||||
/// 下游请求体解压后的最大字节数;防御解压炸弹。
|
||||
const MAX_DECOMPRESSED_REQUEST_BYTES: usize = 64 * 1024 * 1024;
|
||||
/// zstd 回溯窗口独立于输出上限,调整设置不能放大解码器内部窗口分配。
|
||||
const MAX_ZSTD_WINDOW_LOG: u32 = 26;
|
||||
|
||||
/// 按 `Content-Encoding` 解压下游请求体。
|
||||
///
|
||||
@@ -342,6 +343,7 @@ const MAX_DECOMPRESSED_REQUEST_BYTES: usize = 64 * 1024 * 1024;
|
||||
fn decompress_request_body<'a>(
|
||||
body: &'a [u8],
|
||||
headers: &HeaderMap,
|
||||
max_decompressed_bytes: usize,
|
||||
) -> Result<Cow<'a, [u8]>, RequestDecodeError> {
|
||||
let encoding = headers
|
||||
.get_all(axum::http::header::CONTENT_ENCODING)
|
||||
@@ -357,20 +359,20 @@ fn decompress_request_body<'a>(
|
||||
"" | "identity" => Ok(Cow::Borrowed(body)),
|
||||
"gzip" => {
|
||||
let decoder = flate2::read::MultiGzDecoder::new(body);
|
||||
read_bounded(decoder).map(Cow::Owned)
|
||||
read_bounded(decoder, max_decompressed_bytes).map(Cow::Owned)
|
||||
}
|
||||
"deflate" => {
|
||||
let decoder = flate2::read::ZlibDecoder::new(body);
|
||||
read_bounded(decoder).map(Cow::Owned)
|
||||
read_bounded(decoder, max_decompressed_bytes).map(Cow::Owned)
|
||||
}
|
||||
"zstd" => {
|
||||
let mut decoder = zstd::stream::read::Decoder::with_buffer(body)
|
||||
.map_err(|_| RequestDecodeError::MalformedJson)?;
|
||||
// 输出有界之外,也限制压缩帧声明的回溯窗口,防止解码器内部过量分配。
|
||||
decoder
|
||||
.window_log_max(MAX_DECOMPRESSED_REQUEST_BYTES.ilog2())
|
||||
.window_log_max(MAX_ZSTD_WINDOW_LOG)
|
||||
.map_err(|_| RequestDecodeError::MalformedJson)?;
|
||||
read_bounded(decoder).map(Cow::Owned)
|
||||
read_bounded(decoder, max_decompressed_bytes).map(Cow::Owned)
|
||||
}
|
||||
other => Err(RequestDecodeError::UnsupportedContentEncoding {
|
||||
encoding: other.to_owned(),
|
||||
@@ -378,12 +380,15 @@ fn decompress_request_body<'a>(
|
||||
}
|
||||
}
|
||||
|
||||
fn read_bounded<R: std::io::Read>(mut reader: R) -> Result<Vec<u8>, RequestDecodeError> {
|
||||
fn read_bounded<R: std::io::Read>(
|
||||
mut reader: R,
|
||||
max_decompressed_bytes: usize,
|
||||
) -> Result<Vec<u8>, RequestDecodeError> {
|
||||
let mut decoded = Vec::new();
|
||||
let mut chunk = [0; 8192];
|
||||
loop {
|
||||
let remaining = MAX_DECOMPRESSED_REQUEST_BYTES - decoded.len();
|
||||
let read_limit = chunk.len().min(remaining + 1);
|
||||
let remaining = max_decompressed_bytes - decoded.len();
|
||||
let read_limit = chunk.len().min(remaining.saturating_add(1));
|
||||
let read = reader
|
||||
.read(&mut chunk[..read_limit])
|
||||
.map_err(|_| RequestDecodeError::MalformedJson)?;
|
||||
@@ -391,15 +396,19 @@ fn read_bounded<R: std::io::Read>(mut reader: R) -> Result<Vec<u8>, RequestDecod
|
||||
return Ok(decoded);
|
||||
}
|
||||
if read > remaining {
|
||||
return Err(RequestDecodeError::DecompressedBodyTooLarge);
|
||||
return Err(RequestDecodeError::DecompressedBodyTooLarge {
|
||||
limit_bytes: max_decompressed_bytes,
|
||||
});
|
||||
}
|
||||
// 默认 Vec 扩容和 read_to_end 的 EOF 探测可能突破输出上限;
|
||||
// 容量增长也受相同边界约束,越界探测只使用栈上分块缓冲区。
|
||||
let required = decoded.len() + read;
|
||||
if required > decoded.capacity() {
|
||||
let capacity = (decoded.capacity() * 2)
|
||||
let capacity = decoded
|
||||
.capacity()
|
||||
.saturating_mul(2)
|
||||
.max(required)
|
||||
.min(MAX_DECOMPRESSED_REQUEST_BYTES);
|
||||
.min(max_decompressed_bytes);
|
||||
decoded.reserve_exact(capacity - decoded.len());
|
||||
}
|
||||
decoded.extend_from_slice(&chunk[..read]);
|
||||
@@ -535,8 +544,6 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
.iter()
|
||||
.any(|connection_header| connection_header.eq_ignore_ascii_case(name))
|
||||
|| is_transport_managed_request_header(name)
|
||||
|| name.starts_with("x-grok-")
|
||||
|| name.starts_with("x-xai-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -551,15 +558,9 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent/version
|
||||
// 不能作为不透明头透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
| "originator"
|
||||
|
||||
@@ -272,3 +272,27 @@ async fn response_json(response: axum::response::Response) -> Value {
|
||||
.expect("account group response body");
|
||||
serde_json::from_slice(&body).expect("account group response JSON")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disable_fast_group_updates_preserve_omitted_values() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
for (value, expected) in [(Some(true), true), (None, true), (Some(false), false)] {
|
||||
let mut body = json!({"id": PRIMARY_GROUP_ID, "name":"Policy", "description":null,"color":"#F43F5ECC"});
|
||||
if let Some(value) = value {
|
||||
body["disableFast"] = json!(value);
|
||||
}
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/update",
|
||||
Some(body),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["record"]["disableFast"],
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
async fn send(
|
||||
fixture: &AdminTestFixture,
|
||||
method: &str,
|
||||
uri: &str,
|
||||
body: Value,
|
||||
authenticated: bool,
|
||||
) -> (StatusCode, Value) {
|
||||
let mut request = Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header("x-request-id", "import-task-contract")
|
||||
.header(header::CONTENT_TYPE, "application/json");
|
||||
if authenticated {
|
||||
request = request.header(header::COOKIE, "cpr_session=valid-session");
|
||||
}
|
||||
let response = admin::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(request.body(Body::from(body.to_string())).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||
let status = response.status();
|
||||
let bytes = to_bytes(response.into_body(), 8 * 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
let value: Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert!(!value.to_string().contains("synthetic-sensitive-token"));
|
||||
(status, value)
|
||||
}
|
||||
|
||||
fn input() -> Value {
|
||||
json!({"submissionId": "01994fcb-a333-7333-8000-000000000001", "items": [{"provider": "openai", "data": {"refreshToken": "synthetic-sensitive-token"}}]})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn import_tasks_should_accept_list_restore_stop_and_deduplicate_without_echoing_credentials()
|
||||
{
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let (status, accepted) = send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
input(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED);
|
||||
let id = accepted["data"]["taskId"].as_str().unwrap();
|
||||
let (_, retried) = send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
input(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(retried["data"]["taskId"], id);
|
||||
let (_, list) = send(
|
||||
&fixture,
|
||||
"GET",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
Value::Null,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(list["data"]["items"].as_array().unwrap().len(), 1);
|
||||
let (_, detail) = send(
|
||||
&fixture,
|
||||
"GET",
|
||||
&format!("/api/admin/accounts/import-tasks/detail?taskId={id}"),
|
||||
Value::Null,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(detail["data"]["items"][0]["status"], "pending");
|
||||
let (_, stopped) = send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks/stop",
|
||||
json!({"taskId": id}),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(stopped["data"]["counts"]["skipped"], 1);
|
||||
assert!(stopped["data"]["finishedAt"].is_string());
|
||||
let mut changed = input();
|
||||
changed["items"][0]["data"]["refreshToken"] = json!("changed-synthetic-input");
|
||||
assert_eq!(
|
||||
send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
changed,
|
||||
true
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::CONFLICT
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn import_tasks_should_require_admin_and_enforce_input_bounds() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (method, uri, body) in [
|
||||
("POST", "/api/admin/accounts/import-tasks", input()),
|
||||
("GET", "/api/admin/accounts/import-tasks", Value::Null),
|
||||
(
|
||||
"GET",
|
||||
"/api/admin/accounts/import-tasks/detail?taskId=01994fcb-a333-7333-8000-000000000001",
|
||||
Value::Null,
|
||||
),
|
||||
(
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks/stop",
|
||||
json!({"taskId": "01994fcb-a333-7333-8000-000000000001"}),
|
||||
),
|
||||
] {
|
||||
assert_eq!(
|
||||
send(&fixture, method, uri, body, false).await.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
}
|
||||
for count in [0, 201] {
|
||||
let mut body = input();
|
||||
body["items"] = json!(vec![body["items"][0].clone(); count]);
|
||||
assert_eq!(
|
||||
send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
body,
|
||||
true
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
}
|
||||
let mut body = input();
|
||||
body["items"][0]["data"]["refreshToken"] = json!("x".repeat(4 * 1024 * 1024));
|
||||
assert_eq!(
|
||||
send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
body,
|
||||
true
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::PAYLOAD_TOO_LARGE
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
mod handlers;
|
||||
mod import_tasks;
|
||||
mod presenter;
|
||||
|
||||
mod personal_info {
|
||||
@@ -599,6 +600,44 @@ mod actions {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_key_rotation_settings_must_target_the_same_account_and_remain_valid() {
|
||||
let mut request = json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_api",
|
||||
"baseUrl": "https://api.example.invalid/v1",
|
||||
"transport": "http",
|
||||
"settings": {
|
||||
"accountId": "acct_api",
|
||||
"enabled": true,
|
||||
"concurrencyLimit": null,
|
||||
"weight": 1,
|
||||
"groupIds": []
|
||||
}
|
||||
});
|
||||
serde_json::from_value::<RotateAccountRequest>(request.clone())
|
||||
.expect("decode combined save")
|
||||
.validate()
|
||||
.expect("blank replacement key preserves the existing key");
|
||||
request["settings"]["accountId"] = json!("acct_other");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<RotateAccountRequest>(request.clone())
|
||||
.unwrap()
|
||||
.validate()
|
||||
.unwrap_err()
|
||||
.field(),
|
||||
"settings.accountId"
|
||||
);
|
||||
request["settings"]["accountId"] = json!("acct_api");
|
||||
request["settings"]["concurrencyLimit"] = json!(0);
|
||||
assert!(
|
||||
serde_json::from_value::<RotateAccountRequest>(request)
|
||||
.unwrap()
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_recovery_requests_should_not_accept_client_revision_fences() {
|
||||
let authorization: StartAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -463,7 +463,7 @@ mod provider {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
};
|
||||
|
||||
@@ -402,6 +402,9 @@ impl SettingsStore for MemorySettingsStore {
|
||||
) -> AdminStoreResult<RuntimeSettings> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
let updated = RuntimeSettings {
|
||||
disable_fast: command.disable_fast.unwrap_or(settings.disable_fast),
|
||||
request_location_enabled: command.request_location_enabled,
|
||||
request_location: command.request_location,
|
||||
config_revision: next_revision(settings.config_revision),
|
||||
model_mappings: command.model_mappings,
|
||||
refresh_margin_seconds: command.refresh_margin_seconds,
|
||||
@@ -411,12 +414,20 @@ impl SettingsStore for MemorySettingsStore {
|
||||
max_waiting_per_key: command.max_waiting_per_key,
|
||||
max_waiting_per_account: command.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: command.concurrency_wait_timeout_seconds,
|
||||
responses_max_decompressed_body_bytes: command.responses_max_decompressed_body_bytes,
|
||||
rotation_strategy: command.rotation_strategy,
|
||||
min_codex_desktop_version: command.min_codex_desktop_version,
|
||||
min_codex_cli_version: command.min_codex_cli_version,
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
};
|
||||
*settings = updated.clone();
|
||||
@@ -482,6 +493,7 @@ impl MemoryAccountGroupStore {
|
||||
(
|
||||
primary_id.clone(),
|
||||
AccountGroupRecord {
|
||||
disable_fast: false,
|
||||
id: primary_id,
|
||||
name: "Alpha routing".to_owned(),
|
||||
description: Some("Primary traffic".to_owned()),
|
||||
@@ -503,6 +515,7 @@ impl MemoryAccountGroupStore {
|
||||
(
|
||||
secondary_id.clone(),
|
||||
AccountGroupRecord {
|
||||
disable_fast: false,
|
||||
id: secondary_id,
|
||||
name: "Beta routing".to_owned(),
|
||||
description: None,
|
||||
@@ -587,7 +600,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
@@ -601,7 +614,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
@@ -618,6 +631,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
let mut state = self.state.lock().expect("account groups");
|
||||
let now = Utc::now();
|
||||
let record = AccountGroupRecord {
|
||||
disable_fast: command.disable_fast,
|
||||
id: command.id.clone(),
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
@@ -649,6 +663,9 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
record.name = command.name;
|
||||
record.description = command.description;
|
||||
record.color = command.color;
|
||||
if let Some(disable_fast) = command.disable_fast {
|
||||
record.disable_fast = disable_fast;
|
||||
}
|
||||
record.updated_at = Utc::now();
|
||||
mutation(&mut state, command.id, true)
|
||||
}
|
||||
@@ -896,6 +913,15 @@ impl AccountStore for UnusedStore {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
_: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
@@ -933,17 +959,39 @@ impl AccountStore for UnusedStore {
|
||||
impl AccountRuntimeStore for UnusedStore {
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::new(),
|
||||
cooldown: BTreeMap::new(),
|
||||
in_flight: Some(BTreeMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::new(),
|
||||
cooldown: BTreeMap::new(),
|
||||
in_flight: Some(BTreeMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -1230,6 +1278,9 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
),
|
||||
]);
|
||||
RuntimeSettings {
|
||||
disable_fast: false,
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: mappings,
|
||||
refresh_margin_seconds: 3_600,
|
||||
@@ -1239,12 +1290,20 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -840,6 +840,7 @@ fn usage_record_with_account(
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: None,
|
||||
upstream_response_model: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
@@ -963,6 +964,7 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
provider_account_authentication_kind: Some("oauth".to_owned()),
|
||||
upstream_model_id: Some("grok-4.5".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
upstream_response_model: Some("grok-4.6-build".to_owned()),
|
||||
service_tier: Some("default".to_owned()),
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
@@ -1075,6 +1077,7 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
"requestedModel": value["data"]["items"][0]["requestedModel"],
|
||||
"upstreamModel": value["data"]["items"][0]["upstreamModel"],
|
||||
"upstreamResponseModel": value["data"]["items"][0]["upstreamResponseModel"],
|
||||
"reasoningEffort": value["data"]["items"][0]["reasoningEffort"],
|
||||
"reasoningPreset": value["data"]["items"][0]["reasoningPreset"],
|
||||
"subagentKind": value["data"]["items"][0]["subagentKind"],
|
||||
@@ -1098,6 +1101,7 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
"imageOutputTokens": 9,
|
||||
"requestedModel": "grok-4.5",
|
||||
"upstreamModel": "grok-4.5",
|
||||
"upstreamResponseModel": "grok-4.6-build",
|
||||
"reasoningEffort": "max",
|
||||
"reasoningPreset": "ultra",
|
||||
"subagentKind": "worker",
|
||||
|
||||
@@ -19,6 +19,71 @@ use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_location_round_trips_preserves_omitted_and_clears_null() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let location =
|
||||
json!({"country":"JP", "region":"Tokyo", "city":"Tokyo", "timezone":"Asia/Tokyo"});
|
||||
let (status, created) = request(
|
||||
&fixture,
|
||||
"/api/admin/proxies/create",
|
||||
Some(json!({
|
||||
"name":"Tokyo", "proxyUrl":"http://proxy.example:8080", "location":location
|
||||
})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CREATED);
|
||||
assert_eq!(created["data"]["record"]["location"], location);
|
||||
let (_, listed) = request(&fixture, "/api/admin/proxies", None, true).await;
|
||||
assert_eq!(listed["data"]["items"][0]["location"], location);
|
||||
for (revision, change, expected) in [
|
||||
(1, None, location.clone()),
|
||||
(2, Some(Value::Null), Value::Null),
|
||||
(3, Some(location.clone()), location.clone()),
|
||||
] {
|
||||
let mut body = json!({"id":"proxy_test", "revision":revision, "name":"Renamed"});
|
||||
if let Some(change) = change {
|
||||
body["location"] = change;
|
||||
}
|
||||
let (status, changed) =
|
||||
request(&fixture, "/api/admin/proxies/update", Some(body), true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(changed["data"]["record"]["location"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_location_rejects_invalid_and_incomplete_input() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (field, value) in [
|
||||
("country", json!("jp")),
|
||||
("city", json!(" ")),
|
||||
("region", json!("Tokyo\n")),
|
||||
("city", json!("x".repeat(129))),
|
||||
("timezone", json!("Asia/Typo")),
|
||||
] {
|
||||
let mut invalid =
|
||||
json!({"country":"JP", "region":"Tokyo", "city":"Tokyo", "timezone":"Asia/Tokyo"});
|
||||
invalid[field] = value;
|
||||
let (status, _) = request(&fixture, "/api/admin/proxies/create", Some(json!({"name":"Invalid", "proxyUrl":"http://proxy.example:8080", "location":invalid})), true).await;
|
||||
assert!(status.is_client_error(), "invalid {field} accepted");
|
||||
}
|
||||
let (status, _) = request(&fixture, "/api/admin/proxies/create", Some(json!({"name":"Incomplete", "proxyUrl":"http://proxy.example:8080", "location":{"country":"JP"}})), true).await;
|
||||
assert!(status.is_client_error());
|
||||
let (status, created) = request(
|
||||
&fixture,
|
||||
"/api/admin/proxies/create",
|
||||
Some(json!({"name":"Legacy", "proxyUrl":"http://proxy.example:8080"})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CREATED);
|
||||
assert!(created["data"]["record"]["location"].is_null());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct MemoryProxies(Mutex<Option<ProxyRecord>>);
|
||||
|
||||
@@ -98,6 +163,7 @@ impl ProxyStore for MemoryProxies {
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<ProxyMutation> {
|
||||
let record = ProxyRecord {
|
||||
location: command.location,
|
||||
id: "proxy_test".to_owned(),
|
||||
name: command.name,
|
||||
proxy: command.proxy,
|
||||
@@ -122,6 +188,9 @@ impl ProxyStore for MemoryProxies {
|
||||
let mut stored = self.0.lock().unwrap();
|
||||
let record = stored.as_mut().ok_or_else(missing)?;
|
||||
record.name = command.name;
|
||||
if let Some(location) = command.location {
|
||||
record.location = location;
|
||||
}
|
||||
if let Some(proxy) = command.proxy {
|
||||
record.proxy = proxy;
|
||||
}
|
||||
@@ -168,6 +237,8 @@ impl ProxyProbe for SuccessfulProbe {
|
||||
success: true,
|
||||
latency_ms: 15,
|
||||
exit_ip: Some("203.0.113.2".parse().unwrap()),
|
||||
exit_ipv4: Some("203.0.113.2".parse().unwrap()),
|
||||
exit_ipv6: None,
|
||||
message: "Connected".to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -321,7 +392,14 @@ async fn proxy_probe_checks_unsaved_address_without_creating_or_changing_records
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(
|
||||
probed["data"],
|
||||
json!({"success": true, "latencyMs": 15, "exitIp": "203.0.113.2", "message": "Connected"})
|
||||
json!({
|
||||
"success": true,
|
||||
"latencyMs": 15,
|
||||
"exitIp": "203.0.113.2",
|
||||
"exitIpv4": "203.0.113.2",
|
||||
"exitIpv6": null,
|
||||
"message": "Connected"
|
||||
})
|
||||
);
|
||||
let (_, listed) = request(&fixture, "/api/admin/proxies", None, true).await;
|
||||
assert_eq!(listed["data"]["page"]["total"], 0);
|
||||
|
||||
@@ -37,6 +37,9 @@ async fn response_json(response: axum::response::Response) -> Value {
|
||||
|
||||
fn update_body() -> Value {
|
||||
json!({
|
||||
"disableFast": false,
|
||||
"requestLocationEnabled": false,
|
||||
"requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"},
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
@@ -48,12 +51,20 @@ fn update_body() -> Value {
|
||||
"maxWaitingPerKey": 0,
|
||||
"maxWaitingPerAccount": 0,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
"accountAutoFreezeThreshold": 12,
|
||||
"accountAutoFreezeWindowSeconds": 600,
|
||||
"accountAutoFreezeDurationSeconds": 7200,
|
||||
"accountAutoFreezeProbeEnabled": true,
|
||||
"accountAutoFreezeProbeModel": null,
|
||||
"accountAutoFreezeAdaptiveConcurrency": true
|
||||
})
|
||||
}
|
||||
|
||||
@@ -92,6 +103,9 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let settings = RuntimeSettings {
|
||||
disable_fast: false,
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: BTreeMap::from_iter([
|
||||
(
|
||||
@@ -110,12 +124,20 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
rotation_strategy: RotationStrategy::RoundRobin,
|
||||
min_codex_desktop_version: Some("26.825.6671".to_owned()),
|
||||
min_codex_cli_version: Some("0.40.0".to_owned()),
|
||||
usage_retention_days: 32,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
@@ -126,6 +148,9 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"disableFast": false,
|
||||
"requestLocationEnabled": false,
|
||||
"requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"},
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
@@ -137,12 +162,20 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
"maxWaitingPerKey": 0,
|
||||
"maxWaitingPerAccount": 0,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
"accountAutoFreezeThreshold": 12,
|
||||
"accountAutoFreezeWindowSeconds": 600,
|
||||
"accountAutoFreezeDurationSeconds": 7200,
|
||||
"accountAutoFreezeProbeEnabled": true,
|
||||
"accountAutoFreezeProbeModel": null,
|
||||
"accountAutoFreezeAdaptiveConcurrency": true,
|
||||
"updatedAt": "2026-08-02T10:30:00Z"
|
||||
})
|
||||
);
|
||||
@@ -169,6 +202,9 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
.cloned()
|
||||
.collect();
|
||||
let settings = RuntimeSettings {
|
||||
disable_fast: false,
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: request
|
||||
.model_mappings
|
||||
@@ -188,6 +224,7 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
rotation_strategy: RotationStrategy::parse(&request.rotation_strategy)
|
||||
.expect("fixture rotation strategy"),
|
||||
min_codex_desktop_version: request.min_codex_desktop_version,
|
||||
@@ -195,6 +232,13 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
@@ -422,3 +466,175 @@ fn concurrency_queue_fields_validate_bounds_and_accept_disabled_queues() {
|
||||
.validate()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_location_should_normalize_toggle_and_round_trip() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
body["requestLocationEnabled"] = json!(true);
|
||||
body["requestLocation"] =
|
||||
json!({"country":"JP", "region":" Tokyo ", "city":" Tokyo ", "timezone":"Asia/Tokyo"});
|
||||
let expected =
|
||||
json!({"country":"JP", "region":"Tokyo", "city":"Tokyo", "timezone":"Asia/Tokyo"});
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["requestLocation"],
|
||||
expected
|
||||
);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["requestLocation"],
|
||||
expected
|
||||
);
|
||||
for enabled in [false, true] {
|
||||
let mut body = update_body();
|
||||
body["requestLocationEnabled"] = json!(enabled);
|
||||
body["requestLocation"] = expected.clone();
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
let data = response_json(response).await["data"].clone();
|
||||
assert_eq!(data["requestLocationEnabled"], json!(enabled));
|
||||
assert_eq!(data["requestLocation"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_location_should_reject_invalid_or_missing_fields_without_replacing_settings() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let original = update_body()["requestLocation"].clone();
|
||||
// JSON 结构或时区解析失败由 Axum 返回 422,业务校验失败返回 400。
|
||||
let mut invalid = vec![
|
||||
(Value::Null, StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!({}), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(
|
||||
json!({"timezone":"Asia/Tokyo"}),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
];
|
||||
for (field, value, status) in [
|
||||
("country", "USA", StatusCode::BAD_REQUEST),
|
||||
("country", "us", StatusCode::BAD_REQUEST),
|
||||
("city", "", StatusCode::BAD_REQUEST),
|
||||
("region", "\nTokyo", StatusCode::BAD_REQUEST),
|
||||
(
|
||||
"timezone",
|
||||
"Not/A_Timezone",
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
] {
|
||||
let mut location = original.clone();
|
||||
location[field] = json!(value);
|
||||
invalid.push((location, status));
|
||||
}
|
||||
for (location, expected_status) in invalid {
|
||||
let mut body = update_body();
|
||||
body["requestLocation"] = location;
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), expected_status);
|
||||
}
|
||||
let mut body = update_body();
|
||||
body.as_object_mut().unwrap().remove("requestLocation");
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["requestLocation"],
|
||||
original
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decompression_setting_should_reject_invalid_values() {
|
||||
let field = "responsesMaxDecompressedBodyBytes";
|
||||
for invalid in [
|
||||
json!(0),
|
||||
json!(-1),
|
||||
json!(1.5),
|
||||
json!(u64::MAX),
|
||||
Value::Null,
|
||||
] {
|
||||
let mut body = update_body();
|
||||
body[field] = invalid;
|
||||
if let Ok(request) = serde_json::from_value::<UpdateRuntimeSettingsRequest>(body) {
|
||||
assert_eq!(request.validate().unwrap_err().field(), field);
|
||||
}
|
||||
}
|
||||
let mut body = update_body();
|
||||
body.as_object_mut().unwrap().remove(field);
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(body).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disable_fast_settings_updates_preserve_omitted_values() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let app = app(fixture.state());
|
||||
for (value, expected) in [(Some(true), true), (None, true), (Some(false), false)] {
|
||||
let mut body = update_body();
|
||||
if let Some(value) = value {
|
||||
body["disableFast"] = json!(value);
|
||||
} else {
|
||||
body.as_object_mut().unwrap().remove("disableFast");
|
||||
}
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["disableFast"],
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/admin/account_groups.rs",
|
||||
"src/admin/accounts/credentials.rs",
|
||||
"src/admin/accounts/handlers.rs",
|
||||
"src/admin/accounts/import_tasks.rs",
|
||||
"src/admin/accounts/mod.rs",
|
||||
"src/admin/accounts/presenter.rs",
|
||||
"src/admin/accounts/wire.rs",
|
||||
@@ -95,6 +96,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
let mut expected = vec![
|
||||
"tests/admin/account_groups.rs",
|
||||
"tests/admin/accounts/handlers.rs",
|
||||
"tests/admin/accounts/import_tasks.rs",
|
||||
"tests/admin/accounts/mod.rs",
|
||||
"tests/admin/accounts/presenter.rs",
|
||||
"tests/admin/auth.rs",
|
||||
|
||||
@@ -100,6 +100,7 @@ fn usage_record() -> UsageListRecord {
|
||||
provider_account_authentication_kind: Some("private-sentinel".to_owned()),
|
||||
upstream_model_id: Some("private-sentinel".to_owned()),
|
||||
upstream_transport: Some("websocket".to_owned()),
|
||||
upstream_response_model: Some("private-sentinel".to_owned()),
|
||||
service_tier: Some("private-sentinel".to_owned()),
|
||||
input_tokens: Some(1000),
|
||||
output_tokens: Some(100),
|
||||
|
||||
@@ -5,3 +5,55 @@ mod health;
|
||||
mod key_usage;
|
||||
mod openai;
|
||||
mod support;
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_should_serve_resolved_assets_with_relative_environment_override() {
|
||||
use axum::{body::Body, http::Request};
|
||||
use std::{fs, process::Command};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
const CHILD_ENV: &str = "CPR_TEST_ASSET_DIRECTORY";
|
||||
let Ok(directory) = std::env::var(CHILD_ENV) else {
|
||||
let directory = std::env::temp_dir().join(format!("cpr-assets-{}", uuid::Uuid::now_v7()));
|
||||
fs::create_dir_all(directory.join("deploy")).expect("config directory");
|
||||
fs::create_dir_all(directory.join("web/dist")).expect("asset directory");
|
||||
fs::write(directory.join("web/dist/index.html"), "resolved-web-assets").expect("index");
|
||||
let output = Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"api_should_serve_resolved_assets_with_relative_environment_override",
|
||||
])
|
||||
.current_dir(&directory)
|
||||
.env(CHILD_ENV, &directory)
|
||||
.env("CPR_WEB_DIST_DIR", "../web/dist")
|
||||
.output()
|
||||
.expect("isolated asset directory test");
|
||||
fs::remove_dir_all(directory).expect("remove fixture");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
return;
|
||||
};
|
||||
let mut config = gateway_api::ApiConfig {
|
||||
asset_directory: "unused".into(),
|
||||
cors_allowed_origins: Vec::new(),
|
||||
request_timeout_seconds: None,
|
||||
request_id_header: "x-request-id".to_owned(),
|
||||
};
|
||||
config
|
||||
.resolve_and_validate(&std::path::Path::new(&directory).join("deploy"))
|
||||
.expect("resolved API config");
|
||||
let admin = admin::AdminTestFixture::new().await;
|
||||
let response = openai::api_router_with_config(admin.services, config)
|
||||
.oneshot(Request::builder().uri("/").body(Body::empty()).unwrap())
|
||||
.await
|
||||
.expect("static response");
|
||||
assert_eq!(response.status(), axum::http::StatusCode::OK);
|
||||
let body = axum::body::to_bytes(response.into_body(), 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(body.as_ref(), b"resolved-web-assets");
|
||||
}
|
||||
|
||||
@@ -180,3 +180,121 @@ fn cli_user_agent_should_expose_semver_or_recognized_missing_version() {
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chatgpt_remote_desktop_without_app_version_should_not_use_a_version_gate() {
|
||||
for name in [
|
||||
"codex_chatgpt_android_remote",
|
||||
"codex_chatgpt_ios_remote",
|
||||
"codex_chatgpt_future_os_remote",
|
||||
] {
|
||||
for remote_version in ["dev", "1.2.3"] {
|
||||
for (product, originator) in [
|
||||
("Codex Desktop", None),
|
||||
("codex_cli_rs", Some("Codex Desktop")),
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"user-agent",
|
||||
HeaderValue::from_str(&format!(
|
||||
"{product}/0.154.0-alpha.6.2 (Windows 10.0.26200; x86_64) unknown ({name}; {remote_version})"
|
||||
))
|
||||
.unwrap(),
|
||||
);
|
||||
if let Some(originator) = originator {
|
||||
headers.insert("originator", HeaderValue::from_static(originator));
|
||||
}
|
||||
assert_eq!(identify_codex_client(&headers), None, "{headers:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chatgpt_remote_marker_should_require_a_complete_exact_client_suffix() {
|
||||
for suffix in [
|
||||
"(codex_chatgpt_android_remote_extra; dev)",
|
||||
"(unofficial_codex_chatgpt_ios_remote; dev)",
|
||||
"(codex_chatgpt__remote; dev)",
|
||||
"(codex_chatgpt_remote; dev)",
|
||||
"(codex_chatgpt_future os_remote; dev)",
|
||||
"(codex_chatgpt_future;os_remote; dev)",
|
||||
"codex_chatgpt_android_remote; dev",
|
||||
"(codex_chatgpt_android_remote; dev",
|
||||
"(codex_chatgpt_android_remote; dev) trailing",
|
||||
"(codex_chatgpt_android_remote; dev) (another_client; dev)",
|
||||
"(codex_chatgpt_android_remote; )",
|
||||
"(another_client; codex_chatgpt_android_remote)",
|
||||
"(Codex Desktop; dev)",
|
||||
"(Codex Desktop; ) (codex_chatgpt_android_remote; dev)",
|
||||
"(Codex Desktop; invalid) (codex_chatgpt_ios_remote; dev)",
|
||||
"(Codex Desktop)",
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"user-agent",
|
||||
HeaderValue::from_str(&format!("Codex Desktop/0.154.0-alpha.6.2 {suffix}")).unwrap(),
|
||||
);
|
||||
let client = identify_codex_client(&headers).expect("Desktop still requires a version");
|
||||
assert_eq!(client.kind(), CodexClientKind::Desktop, "{suffix}");
|
||||
assert!(client.version().is_none(), "{suffix}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chatgpt_remote_desktop_should_preserve_explicit_version_validation() {
|
||||
for (value, expected) in [
|
||||
(
|
||||
HeaderValue::from_static("26.908.70816"),
|
||||
Some("26.908.70816"),
|
||||
),
|
||||
(HeaderValue::from_static("26.1.0"), Some("26.1.0")),
|
||||
(HeaderValue::from_static("dev"), None),
|
||||
(HeaderValue::from_static(""), None),
|
||||
(HeaderValue::from_bytes(&[0xff]).unwrap(), None),
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("version", value);
|
||||
headers.insert(
|
||||
"user-agent",
|
||||
HeaderValue::from_static(
|
||||
"Codex Desktop/0.154.0-alpha.6.2 (codex_chatgpt_android_remote; dev)",
|
||||
),
|
||||
);
|
||||
let client = identify_codex_client(&headers).expect("explicit Desktop version");
|
||||
assert_eq!(client.kind(), CodexClientKind::Desktop);
|
||||
assert_eq!(
|
||||
client.version().map(ToString::to_string).as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chatgpt_remote_marker_should_not_override_an_existing_desktop_version_suffix() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"user-agent",
|
||||
HeaderValue::from_static(
|
||||
"Codex Desktop/0.154.0-alpha.6.2 (Codex Desktop; 26.1.0) (codex_chatgpt_ios_remote; dev)",
|
||||
),
|
||||
);
|
||||
let client = identify_codex_client(&headers).expect("Desktop version remains available");
|
||||
assert_eq!(client.version().unwrap().to_string(), "26.1.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chatgpt_remote_suffix_created_by_header_truncation_should_not_skip_the_gate() {
|
||||
let suffix = " (codex_chatgpt_android_remote; dev)";
|
||||
let mut user_agent = "Codex Desktop/0.154.0-alpha.6.2 ".to_owned();
|
||||
user_agent.push_str(&"x".repeat(4096 - user_agent.len() - suffix.len()));
|
||||
user_agent.push_str(suffix);
|
||||
user_agent.push_str(" (another_client; dev)");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("user-agent", HeaderValue::from_str(&user_agent).unwrap());
|
||||
|
||||
let client =
|
||||
identify_codex_client(&headers).expect("incomplete headers still require a version");
|
||||
assert_eq!(client.kind(), CodexClientKind::Desktop);
|
||||
assert!(client.version().is_none());
|
||||
}
|
||||
|
||||
@@ -615,3 +615,33 @@ mod model_routing {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_recovery_http_response_uses_projection_without_replacing_upstream_facts() {
|
||||
use bytes::Bytes;
|
||||
use gateway_core::error::{ClientVisibleUpstreamResponse, OpaqueUpstreamValue};
|
||||
let body = Bytes::from_static(br#"{"error":{"code":"previous_response_not_found","type":"invalid_request_error","message":"Previous response was not found. Retrying the full request."}}"#);
|
||||
let provider = ProviderError::new(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent)
|
||||
.with_status(429)
|
||||
.with_upstream_code(OpaqueUpstreamValue::new("usage_limit_reached"))
|
||||
.with_retry_after(std::time::Duration::from_secs(129_600))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
400,
|
||||
Some(b"application/json".to_vec()),
|
||||
body.clone(),
|
||||
));
|
||||
let error = EngineError::Provider(provider);
|
||||
let response = gateway_api::openai::error::engine_error_response(&error);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert_eq!(response.headers()["content-type"], "application/json");
|
||||
assert!(!response.headers().contains_key("retry-after"));
|
||||
assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), body);
|
||||
let EngineError::Provider(provider) = error else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(provider.upstream_status(), Some(429));
|
||||
assert_eq!(
|
||||
provider.upstream_code().unwrap().as_str(),
|
||||
"usage_limit_reached"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -50,13 +50,23 @@ pub(super) async fn api_router(execution: Arc<dyn ExecutionService>) -> axum::Ro
|
||||
}
|
||||
|
||||
pub(super) fn api_router_with_admin(admin: gateway_admin::AdminServices) -> axum::Router {
|
||||
gateway_api::initialize(
|
||||
api_router_with_config(
|
||||
admin,
|
||||
gateway_api::ApiConfig {
|
||||
asset_directory: std::env::temp_dir(),
|
||||
cors_allowed_origins: Vec::new(),
|
||||
request_timeout_seconds: None,
|
||||
request_id_header: "x-request-id".to_owned(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn api_router_with_config(
|
||||
admin: gateway_admin::AdminServices,
|
||||
config: gateway_api::ApiConfig,
|
||||
) -> axum::Router {
|
||||
gateway_api::initialize(
|
||||
config,
|
||||
Arc::new(DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot("unused-client-route-key", "openai")),
|
||||
Arc::new(UnusedExecutionStore),
|
||||
@@ -125,8 +135,18 @@ pub(super) fn authenticated_client_for_provider(
|
||||
plaintext: &str,
|
||||
provider_name: &str,
|
||||
) -> AuthenticatedClient {
|
||||
authenticated_client_for_provider_with_limit(plaintext, provider_name, 64 * 1024 * 1024)
|
||||
}
|
||||
|
||||
pub(super) fn authenticated_client_for_provider_with_limit(
|
||||
plaintext: &str,
|
||||
provider_name: &str,
|
||||
bytes: usize,
|
||||
) -> AuthenticatedClient {
|
||||
let snapshot = snapshot(plaintext, provider_name)
|
||||
.with_responses_max_decompressed_body_bytes(std::num::NonZeroUsize::new(bytes).unwrap());
|
||||
let source = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot(plaintext, provider_name)),
|
||||
RuntimeSnapshotHandle::new(snapshot),
|
||||
Arc::new(UnusedExecutionStore),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
|
||||
@@ -15,6 +15,7 @@ use axum::{
|
||||
header::{AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE},
|
||||
},
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::future::{BoxFuture, pending};
|
||||
@@ -40,7 +41,9 @@ use serde_json::{Value, json};
|
||||
use gateway_api::openai::responses::{collect_execution_response, stream_execution_response};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::openai::{api_router, authenticated_client_for_provider};
|
||||
use crate::openai::{
|
||||
api_router, authenticated_client_for_provider, authenticated_client_with_min_versions,
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Trace {
|
||||
@@ -545,7 +548,7 @@ async fn invalid_compressed_http_requests_should_fail_before_execution() {
|
||||
super::request::encode_body("zstd", std::io::repeat(0).take(64 * 1024 * 1024 + 1))
|
||||
.into(),
|
||||
"request_too_large",
|
||||
"Decompressed request body exceeds the allowed size.",
|
||||
"Decompressed request body exceeds the allowed size (67108864 bytes).",
|
||||
),
|
||||
] {
|
||||
let headers = HeaderMap::from_iter([
|
||||
@@ -638,7 +641,11 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
input: Some(json!("hello")),
|
||||
client_metadata: None,
|
||||
// 客户端 User-Agent 仅用于本地展示,不再透传给上游指纹上下文。
|
||||
protocol_context: None,
|
||||
protocol_context: Some(json!({"opaque_request_headers": [
|
||||
["cf-connecting-ip", STANDARD.encode(b"198.51.100.1")],
|
||||
["x-real-ip", STANDARD.encode(b"198.51.100.2")],
|
||||
["x-forwarded-for", STANDARD.encode(b"10.0.0.2, 203.0.113.3")]
|
||||
]})),
|
||||
prompt_cache_key: None,
|
||||
previous_response_id: None,
|
||||
}
|
||||
@@ -662,6 +669,26 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opaque_client_headers_should_not_change_local_client_observation() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("user-agent", "pi/synthetic".parse().unwrap());
|
||||
headers.insert("x-stainless-runtime", "node".parse().unwrap());
|
||||
headers.insert("origin", "https://synthetic.invalid".parse().unwrap());
|
||||
let captured = captured_client_context(headers, "192.0.2.10:443".parse().unwrap()).await;
|
||||
assert_eq!(captured.user_agent.as_deref(), Some("pi/synthetic"));
|
||||
assert_eq!(captured.client_ip, Some("192.0.2.10".parse().unwrap()));
|
||||
assert_eq!(
|
||||
captured.protocol_context,
|
||||
Some(json!({
|
||||
"opaque_request_headers": [
|
||||
["x-stainless-runtime", STANDARD.encode(b"node")],
|
||||
["origin", STANDARD.encode(b"https://synthetic.invalid")]
|
||||
]
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_model_catalog_should_not_block_a_new_model_request() {
|
||||
let model = " gpt-future-codex ";
|
||||
@@ -768,7 +795,7 @@ async fn subagent_header_should_not_replace_non_object_client_metadata() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
async fn xai_private_headers_should_remain_opaque_without_projecting_request_facts() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-grok-turn-idx", "7".parse().expect("header"));
|
||||
@@ -776,7 +803,15 @@ async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
|
||||
let captured = captured_client_context(headers, peer).await;
|
||||
|
||||
assert!(captured.protocol_context.is_none());
|
||||
assert_eq!(
|
||||
captured.protocol_context,
|
||||
Some(json!({
|
||||
"opaque_request_headers": [
|
||||
["x-grok-turn-idx", STANDARD.encode(b"7")],
|
||||
["x-grok-conv-id", STANDARD.encode(b"private-session")]
|
||||
]
|
||||
}))
|
||||
);
|
||||
assert!(captured.prompt_cache_key.is_none());
|
||||
}
|
||||
|
||||
@@ -1935,6 +1970,112 @@ struct SessionExecution {
|
||||
session: Mutex<Option<Box<dyn ExecutionSession>>>,
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chatgpt_remote_responses_should_only_skip_missing_desktop_versions() {
|
||||
for name in [
|
||||
"codex_chatgpt_android_remote",
|
||||
"codex_chatgpt_ios_remote",
|
||||
"codex_chatgpt_future_os_remote",
|
||||
] {
|
||||
for (version, expected_code) in [
|
||||
(None, None),
|
||||
(Some("26.908.70816"), None),
|
||||
(Some("26.1.0"), Some("client_version_too_old")),
|
||||
(Some("dev"), Some("client_version_unavailable")),
|
||||
] {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let execution = Arc::new(SessionExecution {
|
||||
client: authenticated_client_with_min_versions(
|
||||
"sk_remote_test",
|
||||
Some("26.908.70816"),
|
||||
Some("0.154.0"),
|
||||
),
|
||||
session: Mutex::new(Some(Box::new(FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(delivery(
|
||||
started(),
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)),
|
||||
NextStep::Event(delivery(completed(), CommitRequirement::AlreadyCommitted)),
|
||||
NextStep::FinalizeSuccess,
|
||||
],
|
||||
)))),
|
||||
});
|
||||
let mut request = Request::post("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_remote_test")
|
||||
.header("user-agent", format!(
|
||||
"Codex Desktop/0.154.0-alpha.6.2 (Windows 10.0.26200; x86_64) unknown ({name}; dev)"
|
||||
));
|
||||
if let Some(version) = version {
|
||||
request = request.header("version", version);
|
||||
}
|
||||
let response = api_router(execution.clone())
|
||||
.await
|
||||
.oneshot(
|
||||
request
|
||||
.body(Body::from(
|
||||
json!({"model": "model-a", "input": "synthetic", "stream": true})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
if let Some(code) = expected_code {
|
||||
let value: Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::UPGRADE_REQUIRED, "{name} {version:?}");
|
||||
assert_eq!(value["error"]["code"], code);
|
||||
assert!(execution.session.lock().unwrap().is_some());
|
||||
} else {
|
||||
assert_eq!(status, StatusCode::OK, "{name} {version:?}");
|
||||
let body = String::from_utf8(body.to_vec()).unwrap();
|
||||
assert!(body.contains("event: response.completed\n"));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert!(trace.snapshot().contains(&"commit"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chatgpt_remote_responses_should_still_require_client_key_authentication() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client_with_min_versions(
|
||||
"sk_context_test",
|
||||
Some("26.908.70816"),
|
||||
None,
|
||||
),
|
||||
});
|
||||
let router = api_router(execution).await;
|
||||
for authorization in [None, Some("Bearer sk_invalid")] {
|
||||
let mut request = Request::post("/v1/responses").header(
|
||||
"user-agent",
|
||||
"Codex Desktop/0.154.0-alpha.6.2 (codex_chatgpt_android_remote; dev)",
|
||||
);
|
||||
if let Some(authorization) = authorization {
|
||||
request = request.header(AUTHORIZATION, authorization);
|
||||
}
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
request
|
||||
.body(Body::from(
|
||||
json!({"model": "model-a", "input": "synthetic"}).to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert!(observed.lock().unwrap().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionService for SessionExecution {
|
||||
fn authenticate(&self, _: &str) -> Result<AuthenticatedClient, ClientAuthenticationError> {
|
||||
Ok(self.client.clone())
|
||||
@@ -2147,3 +2288,50 @@ async fn local_execution_failure_uses_the_existing_model_id() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compressed_http_request_above_default_limit_should_reach_execution_after_increase() {
|
||||
let body = json!({"model": "model-a", "input": "x".repeat(64 * 1024 * 1024)}).to_string();
|
||||
let compressed = super::request::encode_body("zstd", body.as_bytes());
|
||||
for limit in [64 * 1024 * 1024, 128 * 1024 * 1024] {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: crate::openai::authenticated_client_for_provider_with_limit(
|
||||
"sk_context_test",
|
||||
"openai",
|
||||
limit,
|
||||
),
|
||||
});
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.header(CONTENT_ENCODING, "zstd")
|
||||
.body(Body::from(compressed.clone()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
if limit == 64 * 1024 * 1024 {
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert!(observed.lock().unwrap().is_none());
|
||||
let error: Value =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(error["error"]["code"], "request_too_large");
|
||||
} else {
|
||||
// 捕获执行服务刻意返回 500;断言完整输入到达执行层,而不是只验证路由状态码。
|
||||
let captured = observed
|
||||
.lock()
|
||||
.unwrap()
|
||||
.take()
|
||||
.expect("request reached execution");
|
||||
assert_eq!(
|
||||
captured.input.unwrap().as_str().unwrap().len(),
|
||||
64 * 1024 * 1024
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ use gateway_protocol::openai::sse::parse_sse_events;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn decode_request(body: &[u8]) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
decode_request_with_headers(body, &HeaderMap::new())
|
||||
decode_request_with_headers(body, &HeaderMap::new(), 64 * 1024 * 1024)
|
||||
}
|
||||
|
||||
fn decode_response_create(
|
||||
@@ -205,8 +205,9 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
}
|
||||
});
|
||||
|
||||
let decoded = decode_request_with_headers(body.to_string().as_bytes(), &headers)
|
||||
.expect("request should decode");
|
||||
let decoded =
|
||||
decode_request_with_headers(body.to_string().as_bytes(), &headers, 64 * 1024 * 1024)
|
||||
.expect("request should decode");
|
||||
|
||||
assert_eq!(Value::Object(openai_wire_body(&decoded).clone()), body);
|
||||
assert_eq!(
|
||||
@@ -233,6 +234,14 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
#[test]
|
||||
fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for name in [
|
||||
"chatgpt-organization-id",
|
||||
"chatgpt-org-id",
|
||||
"x-openai-organization",
|
||||
"x-openai-project",
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static("unclassified-extension"));
|
||||
}
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_static("future-ascii"),
|
||||
@@ -309,9 +318,12 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
HeaderValue::from_static("client-is-update"),
|
||||
);
|
||||
|
||||
let decoded =
|
||||
decode_request_with_headers(br#"{"model":"smart-code","input":"hello"}"#, &headers)
|
||||
.expect("opaque request headers should not affect body decoding");
|
||||
let decoded = decode_request_with_headers(
|
||||
br#"{"model":"smart-code","input":"hello"}"#,
|
||||
&headers,
|
||||
64 * 1024 * 1024,
|
||||
)
|
||||
.expect("opaque request headers should not affect body decoding");
|
||||
let entries = openai_protocol_context(&decoded)
|
||||
.get("opaque_request_headers")
|
||||
.and_then(Value::as_array)
|
||||
@@ -338,6 +350,18 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
values("openai-beta"),
|
||||
vec![b"future_responses=v2".to_vec(), b"future_tools=v3".to_vec()]
|
||||
);
|
||||
for name in [
|
||||
"chatgpt-organization-id",
|
||||
"chatgpt-org-id",
|
||||
"x-openai-organization",
|
||||
"x-openai-project",
|
||||
] {
|
||||
assert_eq!(values(name), vec![b"unclassified-extension".to_vec()]);
|
||||
}
|
||||
assert_eq!(
|
||||
values("x-codex-installation-id"),
|
||||
vec![b"client-installation".to_vec()]
|
||||
);
|
||||
for preserved in [
|
||||
"accept",
|
||||
"content-type",
|
||||
@@ -357,7 +381,6 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
"cookie",
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-project-id",
|
||||
"x-codex-installation-id",
|
||||
// 上游指纹由运行时画像统一生成,客户端不得覆盖。
|
||||
"user-agent",
|
||||
"originator",
|
||||
@@ -371,7 +394,7 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_exclude_downstream_transport_headers_from_opaque_context() {
|
||||
fn decoder_should_strip_http_transport_but_leave_source_headers_for_provider() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for name in [
|
||||
"cf-visitor",
|
||||
@@ -401,16 +424,134 @@ fn decoder_should_exclude_downstream_transport_headers_from_opaque_context() {
|
||||
headers.insert("content-encoding", HeaderValue::from_static("identity"));
|
||||
headers.insert("x-openai-future-mode", HeaderValue::from_static("keep"));
|
||||
|
||||
let decoded =
|
||||
decode_request_with_headers(br#"{"model":"smart-code","input":"hello"}"#, &headers)
|
||||
.expect("decode request behind a reverse proxy");
|
||||
let decoded = decode_request_with_headers(
|
||||
br#"{"model":"smart-code","input":"hello"}"#,
|
||||
&headers,
|
||||
64 * 1024 * 1024,
|
||||
)
|
||||
.expect("decode request behind a reverse proxy");
|
||||
|
||||
assert_eq!(
|
||||
openai_protocol_context(&decoded).get("opaque_request_headers"),
|
||||
Some(&json!([["x-openai-future-mode", STANDARD.encode(b"keep")]])),
|
||||
Some(&Value::Array(
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| !matches!(
|
||||
name.as_str(),
|
||||
"accept-encoding" | "content-encoding"
|
||||
))
|
||||
.map(|(name, value)| json!([name.as_str(), STANDARD.encode(value.as_bytes())]))
|
||||
.collect()
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downstream_client_headers_should_remain_opaque_without_losing_session_semantics() {
|
||||
for canonical in [None, Some("canonical-session")] {
|
||||
let mut headers = HeaderMap::new();
|
||||
for name in [
|
||||
"X-Stainless-Runtime",
|
||||
"x-stainless-future-field",
|
||||
"Origin",
|
||||
"Referer",
|
||||
"Sec-Ch-Ua",
|
||||
"sec-ch-ua-platform",
|
||||
"Sec-Fetch-Site",
|
||||
"session_id",
|
||||
] {
|
||||
headers.append(name, HeaderValue::from_static("alias-session"));
|
||||
headers.append(name, HeaderValue::from_static("duplicate"));
|
||||
}
|
||||
if let Some(session) = canonical {
|
||||
headers.insert("session-id", HeaderValue::from_static(session));
|
||||
}
|
||||
for name in [
|
||||
"thread-id",
|
||||
"x-client-request-id",
|
||||
"traceparent",
|
||||
"tracestate",
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static("keep"));
|
||||
}
|
||||
headers.append("x-future-business", HeaderValue::from_static("first"));
|
||||
headers.append(
|
||||
"x-future-business",
|
||||
HeaderValue::from_bytes(b"\x80\xff").unwrap(),
|
||||
);
|
||||
let original_headers = headers.clone();
|
||||
let body = json!({
|
||||
"model": "smart-code", "input": "中文 pi 内容不能被请求头过滤改写",
|
||||
"prompt_cache_key": "synthetic-cache",
|
||||
"client_metadata": {"session_id": "body-session"}
|
||||
});
|
||||
let mut frame = body.clone();
|
||||
frame["type"] = json!("response.create");
|
||||
let opening = OpenAiRequestHeaders::from_headers(&headers);
|
||||
for decoded in [
|
||||
decode_request_with_headers(body.to_string().as_bytes(), &headers, 64 * 1024 * 1024)
|
||||
.unwrap(),
|
||||
decode_response_create_with_context(&frame.to_string(), &opening).unwrap(),
|
||||
decode_response_create_with_context(&frame.to_string(), &opening).unwrap(),
|
||||
] {
|
||||
assert_eq!(openai_wire_body(&decoded), body.as_object().unwrap());
|
||||
let context = openai_protocol_context(&decoded);
|
||||
assert_eq!(context["session_id"], canonical.unwrap_or("alias-session"));
|
||||
let entries = context["opaque_request_headers"].as_array().unwrap();
|
||||
for name in [
|
||||
"x-stainless-runtime",
|
||||
"x-stainless-future-field",
|
||||
"origin",
|
||||
"referer",
|
||||
"sec-ch-ua",
|
||||
"sec-ch-ua-platform",
|
||||
"sec-fetch-site",
|
||||
"session_id",
|
||||
] {
|
||||
let values: Vec<_> = entries
|
||||
.iter()
|
||||
.filter(|entry| entry[0] == name)
|
||||
.cloned()
|
||||
.collect();
|
||||
assert_eq!(
|
||||
values,
|
||||
vec![
|
||||
json!([name, STANDARD.encode(b"alias-session")]),
|
||||
json!([name, STANDARD.encode(b"duplicate")]),
|
||||
],
|
||||
"source header {name} belongs to the Provider"
|
||||
);
|
||||
}
|
||||
for name in [
|
||||
"thread-id",
|
||||
"x-client-request-id",
|
||||
"traceparent",
|
||||
"tracestate",
|
||||
] {
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.any(|entry| entry == &json!([name, STANDARD.encode(b"keep")]))
|
||||
);
|
||||
}
|
||||
let future: Vec<_> = entries
|
||||
.iter()
|
||||
.filter(|entry| entry[0] == "x-future-business")
|
||||
.cloned()
|
||||
.collect();
|
||||
assert_eq!(
|
||||
future,
|
||||
vec![
|
||||
json!(["x-future-business", STANDARD.encode(b"first")]),
|
||||
json!(["x-future-business", STANDARD.encode(b"\x80\xff")]),
|
||||
]
|
||||
);
|
||||
}
|
||||
// 解码不修改原始请求,CORS、鉴权和本地观测仍可读取原值。
|
||||
assert_eq!(headers, original_headers);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
let secret = "nested-private-value";
|
||||
@@ -432,7 +573,7 @@ fn decoder_should_preserve_unknown_arbitrary_precision_numbers() {
|
||||
let number = "12345678901234567890123456789012345678901234567890";
|
||||
let body = format!(r#"{{"model":"smart-code","input":"hello","future_number":{number}}}"#);
|
||||
|
||||
let decoded = decode_request_with_headers(body.as_bytes(), &HeaderMap::new())
|
||||
let decoded = decode_request_with_headers(body.as_bytes(), &HeaderMap::new(), 64 * 1024 * 1024)
|
||||
.expect("opaque numeric field should decode");
|
||||
let encoded = serde_json::to_string(openai_wire_body(&decoded)).expect("encode request body");
|
||||
|
||||
|
||||
@@ -41,7 +41,9 @@ fn http_decode_should_preserve_compressed_request_fields() {
|
||||
let expected = serde_json::from_slice::<serde_json::Value>(REQUEST).expect("fixture JSON");
|
||||
for encoding in ENCODINGS {
|
||||
let compressed = encode_body(encoding, REQUEST);
|
||||
let decoded = decode_request_with_headers(&compressed, &headers(encoding)).expect(encoding);
|
||||
let decoded =
|
||||
decode_request_with_headers(&compressed, &headers(encoding), 64 * 1024 * 1024)
|
||||
.expect(encoding);
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded),
|
||||
expected.as_object().expect("object")
|
||||
@@ -52,10 +54,10 @@ fn http_decode_should_preserve_compressed_request_fields() {
|
||||
#[test]
|
||||
fn http_decode_should_accept_plain_identity_and_case_insensitive_encodings() {
|
||||
for request_headers in [HeaderMap::new(), headers(""), headers(" identity ")] {
|
||||
assert!(decode_request_with_headers(REQUEST, &request_headers).is_ok());
|
||||
assert!(decode_request_with_headers(REQUEST, &request_headers, 64 * 1024 * 1024).is_ok());
|
||||
}
|
||||
let compressed = encode_body("gzip", REQUEST);
|
||||
assert!(decode_request_with_headers(&compressed, &headers(" GZip ")).is_ok());
|
||||
assert!(decode_request_with_headers(&compressed, &headers(" GZip "), 64 * 1024 * 1024).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -63,7 +65,7 @@ fn http_decode_should_reject_unknown_and_stacked_encodings() {
|
||||
let mut repeated = headers("identity");
|
||||
repeated.append(CONTENT_ENCODING, HeaderValue::from_static("gzip"));
|
||||
for request_headers in [headers("br"), headers("gzip, zstd"), repeated] {
|
||||
let error = decode_request_with_headers(REQUEST, &request_headers)
|
||||
let error = decode_request_with_headers(REQUEST, &request_headers, 64 * 1024 * 1024)
|
||||
.expect_err("unsupported encoding");
|
||||
assert!(matches!(
|
||||
error,
|
||||
@@ -82,7 +84,8 @@ fn http_decode_should_reject_corrupted_and_truncated_compressed_bodies() {
|
||||
let mut truncated = encode_body(encoding, REQUEST);
|
||||
truncated.truncate(truncated.len() - 4);
|
||||
for body in [b"not-a-compressed-stream".as_slice(), truncated.as_slice()] {
|
||||
let error = decode_request_with_headers(body, &headers(encoding)).expect_err(encoding);
|
||||
let error = decode_request_with_headers(body, &headers(encoding), 64 * 1024 * 1024)
|
||||
.expect_err(encoding);
|
||||
assert_eq!(
|
||||
error.protocol_body().into_value(),
|
||||
json!({"error": {
|
||||
@@ -99,7 +102,9 @@ fn http_decode_should_read_all_gzip_members_and_zstd_frames() {
|
||||
for encoding in ["gzip", "zstd"] {
|
||||
let mut compressed = encode_body(encoding, &REQUEST[..12]);
|
||||
compressed.extend(encode_body(encoding, &REQUEST[12..]));
|
||||
let decoded = decode_request_with_headers(&compressed, &headers(encoding)).expect(encoding);
|
||||
let decoded =
|
||||
decode_request_with_headers(&compressed, &headers(encoding), 64 * 1024 * 1024)
|
||||
.expect(encoding);
|
||||
assert_eq!(openai_wire_body(&decoded)["input"], "hello");
|
||||
}
|
||||
}
|
||||
@@ -110,7 +115,8 @@ fn http_decode_should_not_ignore_invalid_trailing_members() {
|
||||
let mut compressed = encode_body(encoding, REQUEST);
|
||||
compressed.extend(encode_body(encoding, b"invalid trailing JSON".as_slice()));
|
||||
assert_eq!(
|
||||
decode_request_with_headers(&compressed, &headers(encoding)).expect_err(encoding),
|
||||
decode_request_with_headers(&compressed, &headers(encoding), 64 * 1024 * 1024)
|
||||
.expect_err(encoding),
|
||||
RequestDecodeError::MalformedJson
|
||||
);
|
||||
}
|
||||
@@ -129,7 +135,8 @@ fn http_decode_should_reject_zstd_frames_with_an_excessive_window() {
|
||||
REQUEST
|
||||
);
|
||||
assert_eq!(
|
||||
decode_request_with_headers(&compressed, &headers("zstd")).expect_err("oversized window"),
|
||||
decode_request_with_headers(&compressed, &headers("zstd"), 128 * 1024 * 1024)
|
||||
.expect_err("oversized window"),
|
||||
RequestDecodeError::MalformedJson
|
||||
);
|
||||
}
|
||||
@@ -138,7 +145,8 @@ fn http_decode_should_reject_zstd_frames_with_an_excessive_window() {
|
||||
fn http_decode_should_accept_the_exact_decompressed_limit() {
|
||||
let padding = repeat(b' ').take(DECOMPRESSED_LIMIT - REQUEST.len() as u64);
|
||||
let compressed = encode_body("zstd", REQUEST.chain(padding));
|
||||
let decoded = decode_request_with_headers(&compressed, &headers("zstd")).expect("exact limit");
|
||||
let decoded = decode_request_with_headers(&compressed, &headers("zstd"), 64 * 1024 * 1024)
|
||||
.expect("exact limit");
|
||||
assert_eq!(decoded.metadata().requested_model(), "gpt-test");
|
||||
}
|
||||
|
||||
@@ -146,11 +154,13 @@ fn http_decode_should_accept_the_exact_decompressed_limit() {
|
||||
fn http_decode_should_reject_bodies_above_the_decompressed_limit() {
|
||||
for encoding in ENCODINGS {
|
||||
let compressed = encode_body(encoding, repeat(0).take(DECOMPRESSED_LIMIT + 1));
|
||||
let error =
|
||||
decode_request_with_headers(&compressed, &headers(encoding)).expect_err(encoding);
|
||||
let error = decode_request_with_headers(&compressed, &headers(encoding), 64 * 1024 * 1024)
|
||||
.expect_err(encoding);
|
||||
assert_eq!(
|
||||
error,
|
||||
RequestDecodeError::DecompressedBodyTooLarge,
|
||||
RequestDecodeError::DecompressedBodyTooLarge {
|
||||
limit_bytes: 64 * 1024 * 1024
|
||||
},
|
||||
"{encoding}"
|
||||
);
|
||||
}
|
||||
@@ -163,12 +173,54 @@ fn http_decode_should_stop_at_limit_before_reading_invalid_trailing_frames() {
|
||||
// 这个断言验证越界时已经停止读取,而不只是最终返回了某个超限错误。
|
||||
let mut compressed = encode_body(encoding, repeat(0).take(DECOMPRESSED_LIMIT + 1));
|
||||
compressed.extend_from_slice(b"invalid trailing frame");
|
||||
let error =
|
||||
decode_request_with_headers(&compressed, &headers(encoding)).expect_err(encoding);
|
||||
let error = decode_request_with_headers(&compressed, &headers(encoding), 64 * 1024 * 1024)
|
||||
.expect_err(encoding);
|
||||
assert_eq!(
|
||||
error,
|
||||
RequestDecodeError::DecompressedBodyTooLarge,
|
||||
RequestDecodeError::DecompressedBodyTooLarge {
|
||||
limit_bytes: 64 * 1024 * 1024
|
||||
},
|
||||
"{encoding}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_decode_should_apply_configured_limits_to_every_encoding_and_frame() {
|
||||
for encoding in ENCODINGS {
|
||||
let compressed = encode_body(encoding, REQUEST);
|
||||
assert!(
|
||||
decode_request_with_headers(&compressed, &headers(encoding), REQUEST.len()).is_ok()
|
||||
);
|
||||
let error = decode_request_with_headers(&compressed, &headers(encoding), REQUEST.len() - 1)
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
error,
|
||||
RequestDecodeError::DecompressedBodyTooLarge {
|
||||
limit_bytes: REQUEST.len() - 1
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
error
|
||||
.protocol_body()
|
||||
.error
|
||||
.message
|
||||
.contains(&(REQUEST.len() - 1).to_string())
|
||||
);
|
||||
}
|
||||
for encoding in ["gzip", "zstd"] {
|
||||
let split = REQUEST.len() / 2;
|
||||
let mut compressed = encode_body(encoding, &REQUEST[..split]);
|
||||
compressed.extend(encode_body(encoding, &REQUEST[split..]));
|
||||
assert!(
|
||||
decode_request_with_headers(&compressed, &headers(encoding), REQUEST.len()).is_ok()
|
||||
);
|
||||
assert!(matches!(
|
||||
decode_request_with_headers(&compressed, &headers(encoding), REQUEST.len() - 1),
|
||||
Err(RequestDecodeError::DecompressedBodyTooLarge { .. })
|
||||
));
|
||||
}
|
||||
for request_headers in [HeaderMap::new(), headers("identity")] {
|
||||
assert!(decode_request_with_headers(REQUEST, &request_headers, 1).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -447,3 +447,29 @@ async fn deliverable_business_failures_are_not_rewritten_or_followed_by_a_second
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn initial_quota_recovery_delivers_client_projection_instead_of_upstream_status() {
|
||||
let detail = json!({"message":"Previous response was not found. Retrying the full request.","code":"previous_response_not_found","type":"invalid_request_error"});
|
||||
let provider = ProviderError::new(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent)
|
||||
.with_status(429)
|
||||
.with_retry_after(Duration::from_secs(129_600))
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
detail["message"].as_str().unwrap(),
|
||||
Some("previous_response_not_found".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
))
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
400,
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from(json!({"error":detail}).to_string()),
|
||||
)
|
||||
.with_headers(vec![header("x-request-id", b"req-quota-rejected")]),
|
||||
);
|
||||
let error = initial_error(EngineError::Provider(provider), Vec::new()).await;
|
||||
assert_eq!(error["status"], 400);
|
||||
assert_eq!(error["error"], detail);
|
||||
assert_eq!(error["headers"]["x-request-id"], "req-quota-rejected");
|
||||
assert!(error["headers"].get("retry-after").is_none());
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use serde_json::json;
|
||||
use super::decode_response_create;
|
||||
|
||||
#[test]
|
||||
fn response_create_should_exclude_cloudflare_headers_on_every_frame() {
|
||||
fn response_create_should_carry_source_headers_to_provider_on_every_frame() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in [
|
||||
("CF-Visitor", r#"{"scheme":"https"}"#),
|
||||
@@ -24,7 +24,7 @@ fn response_create_should_exclude_cloudflare_headers_on_every_frame() {
|
||||
headers.append("x-openai-future-mode", HeaderValue::from_static("second"));
|
||||
let request_headers = OpenAiRequestHeaders::from_headers(&headers);
|
||||
|
||||
// 连接级头会用于同一 WebSocket 的每一帧,后续帧也不能恢复下游链路元数据。
|
||||
// 连接级头随每一帧传给 Provider,由相同的出站边界处理来源适配。
|
||||
for input in ["hello", "continue"] {
|
||||
let decoded = decode_response_create_with_context(
|
||||
&json!({"type": "response.create", "model": "smart-code", "input": input}).to_string(),
|
||||
@@ -39,10 +39,12 @@ fn response_create_should_exclude_cloudflare_headers_on_every_frame() {
|
||||
.protocol_payload()
|
||||
.context()
|
||||
.get("opaque_request_headers"),
|
||||
Some(&json!([
|
||||
["x-openai-future-mode", STANDARD.encode(b"first")],
|
||||
["x-openai-future-mode", STANDARD.encode(b"second")],
|
||||
])),
|
||||
Some(&serde_json::Value::Array(
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| json!([name.as_str(), STANDARD.encode(value.as_bytes())]))
|
||||
.collect()
|
||||
)),
|
||||
);
|
||||
assert_eq!(request.protocol_payload().body()["input"], input);
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ path = "tests/main.rs"
|
||||
async-trait = "0.1.89"
|
||||
base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
chrono-tz = { version = "0.10", features = ["serde"] }
|
||||
futures = "0.3.32"
|
||||
futures-timer = "3.0.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//! 请求位置只描述出口的业务配置,不改变代理连接身份或服务系统时区。
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RequestLocation {
|
||||
pub country: String,
|
||||
pub region: String,
|
||||
pub city: String,
|
||||
pub timezone: chrono_tz::Tz,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum InvalidRequestLocation {
|
||||
#[error("country must contain two uppercase ASCII letters")]
|
||||
Country,
|
||||
#[error("region must contain 1–128 characters without control characters")]
|
||||
Region,
|
||||
#[error("city must contain 1–128 characters without control characters")]
|
||||
City,
|
||||
}
|
||||
|
||||
impl Default for RequestLocation {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
country: "US".to_owned(),
|
||||
region: "Ohio".to_owned(),
|
||||
city: "Piketon".to_owned(),
|
||||
timezone: chrono_tz::America::New_York,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestLocation {
|
||||
pub fn validate(&self) -> Result<(), InvalidRequestLocation> {
|
||||
if self.country.len() != 2 || !self.country.bytes().all(|byte| byte.is_ascii_uppercase()) {
|
||||
return Err(InvalidRequestLocation::Country);
|
||||
}
|
||||
for (value, error) in [
|
||||
(&self.region, InvalidRequestLocation::Region),
|
||||
(&self.city, InvalidRequestLocation::City),
|
||||
] {
|
||||
// 先检查原始输入,避免 trim 掩盖首尾的换行和其他控制字符。
|
||||
if value.chars().any(char::is_control)
|
||||
|| value.trim().is_empty()
|
||||
|| value.trim().chars().count() > 128
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn normalized(mut self) -> Result<Self, InvalidRequestLocation> {
|
||||
self.validate()?;
|
||||
self.region = self.region.trim().to_owned();
|
||||
self.city = self.city.trim().to_owned();
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
//! Provider 账号领域、持久化端口与同一 target 内的账号选择。
|
||||
|
||||
mod error;
|
||||
mod location;
|
||||
pub use location::{InvalidRequestLocation, RequestLocation};
|
||||
mod model;
|
||||
mod model_access;
|
||||
pub use model_access::{
|
||||
|
||||
@@ -583,6 +583,72 @@ impl AccountErrorReason {
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号冷却的原因与恢复方式。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AccountCooldownKind {
|
||||
/// 上游 429 临时限流;默认类别,兼容升级前未标记的存量 key。
|
||||
#[default]
|
||||
RateLimit,
|
||||
/// 容量类错误高频触发后由熔断策略写入的自动冻结。
|
||||
CapacityFreeze,
|
||||
/// 到期后仍阻止调度,必须由恢复探测成功或管理员恢复解除。
|
||||
CapacityFreezeProbe,
|
||||
}
|
||||
|
||||
impl AccountCooldownKind {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::RateLimit => "rate_limit",
|
||||
Self::CapacityFreeze => "capacity_freeze",
|
||||
Self::CapacityFreezeProbe => "capacity_freeze_probe",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_capacity_freeze(self) -> bool {
|
||||
matches!(self, Self::CapacityFreeze | Self::CapacityFreezeProbe)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn requires_probe(self) -> bool {
|
||||
matches!(self, Self::CapacityFreezeProbe)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"rate_limit" => Some(Self::RateLimit),
|
||||
"capacity_freeze" => Some(Self::CapacityFreeze),
|
||||
"capacity_freeze_probe" => Some(Self::CapacityFreezeProbe),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号级冷却投影;探测冻结的 until 是下次探测时间,不是自动放行时间。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountCooldown {
|
||||
pub until: SystemTime,
|
||||
pub kind: AccountCooldownKind,
|
||||
}
|
||||
|
||||
impl AccountCooldown {
|
||||
#[must_use]
|
||||
pub fn is_active(self, now: SystemTime) -> bool {
|
||||
self.kind.requires_probe() || self.until > now
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SystemTime> for AccountCooldown {
|
||||
fn from(until: SystemTime) -> Self {
|
||||
Self {
|
||||
until,
|
||||
kind: AccountCooldownKind::RateLimit,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 唯一状态解析器的完整输入事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusFacts {
|
||||
@@ -590,7 +656,7 @@ pub struct AccountStatusFacts {
|
||||
pub credential_state: CredentialState,
|
||||
pub access_token_expires_at: Option<SystemTime>,
|
||||
pub quota: QuotaState,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub cooldown: Option<AccountCooldown>,
|
||||
pub last_error_reason: Option<AccountErrorReason>,
|
||||
pub last_error_message: Option<String>,
|
||||
}
|
||||
@@ -601,8 +667,8 @@ pub struct AccountStatusProjection {
|
||||
pub status: AccountStatus,
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub error_message: Option<String>,
|
||||
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却事实。
|
||||
pub cooldown: Option<AccountCooldown>,
|
||||
}
|
||||
|
||||
/// 从独立事实派生唯一、互斥的对外状态。
|
||||
@@ -625,21 +691,21 @@ pub fn resolve_account_status(
|
||||
status: AccountStatus::Error,
|
||||
error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)),
|
||||
error_message: facts.last_error_message.clone(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
};
|
||||
}
|
||||
if facts.quota.is_exhausted() {
|
||||
return status_projection(AccountStatus::QuotaExhausted);
|
||||
}
|
||||
if facts
|
||||
.rate_limited_until
|
||||
.is_some_and(|rate_limited_until| rate_limited_until > now)
|
||||
.cooldown
|
||||
.is_some_and(|cooldown| cooldown.is_active(now))
|
||||
{
|
||||
return AccountStatusProjection {
|
||||
status: AccountStatus::RateLimited,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: facts.rate_limited_until,
|
||||
cooldown: facts.cooldown,
|
||||
};
|
||||
}
|
||||
status_projection(AccountStatus::Normal)
|
||||
@@ -650,7 +716,7 @@ const fn status_projection(status: AccountStatus) -> AccountStatusProjection {
|
||||
status,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -678,6 +744,7 @@ pub struct ProviderAccount {
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
outbound_proxy: Option<super::OutboundProxy>,
|
||||
request_location: Option<super::RequestLocation>,
|
||||
}
|
||||
|
||||
impl ProviderAccount {
|
||||
@@ -714,6 +781,7 @@ impl ProviderAccount {
|
||||
next_refresh_at: None,
|
||||
has_refresh_token: false,
|
||||
outbound_proxy: None,
|
||||
request_location: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -743,6 +811,8 @@ impl ProviderAccount {
|
||||
|
||||
#[must_use]
|
||||
pub fn with_outbound_proxy(mut self, proxy: Option<super::OutboundProxy>) -> Self {
|
||||
// 出口变化后不能沿用旧出口的位置;存储投影应在绑定出口后设置位置。
|
||||
self.request_location = None;
|
||||
self.outbound_proxy = proxy;
|
||||
self
|
||||
}
|
||||
@@ -752,6 +822,17 @@ impl ProviderAccount {
|
||||
self.outbound_proxy.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_location(mut self, location: Option<super::RequestLocation>) -> Self {
|
||||
self.request_location = self.outbound_proxy.as_ref().and(location);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_location(&self) -> Option<&super::RequestLocation> {
|
||||
self.request_location.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_account_facts(
|
||||
mut self,
|
||||
@@ -762,11 +843,8 @@ impl ProviderAccount {
|
||||
last_error_message: Option<String>,
|
||||
) -> Self {
|
||||
self.enabled = enabled;
|
||||
self.credential_state = if self.upstream_user_id.is_some() {
|
||||
credential_state
|
||||
} else {
|
||||
CredentialState::Unknown
|
||||
};
|
||||
// 凭据是否可用由 Provider 判断;API Key 等认证不要求上游用户身份。
|
||||
self.credential_state = credential_state;
|
||||
self.quota = quota;
|
||||
self.last_error_reason = last_error_reason;
|
||||
self.last_error_message = last_error_message;
|
||||
@@ -908,7 +986,7 @@ impl ProviderAccount {
|
||||
pub fn status_projection(
|
||||
&self,
|
||||
now: SystemTime,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
cooldown: Option<AccountCooldown>,
|
||||
) -> AccountStatusProjection {
|
||||
resolve_account_status(
|
||||
&AccountStatusFacts {
|
||||
@@ -916,7 +994,7 @@ impl ProviderAccount {
|
||||
credential_state: self.credential_state,
|
||||
access_token_expires_at: self.access_token_expires_at,
|
||||
quota: self.quota,
|
||||
rate_limited_until,
|
||||
cooldown,
|
||||
last_error_reason: self.last_error_reason,
|
||||
last_error_message: self.last_error_message.clone(),
|
||||
},
|
||||
@@ -1068,6 +1146,7 @@ pub struct CredentialCasUpdateParts {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub profile: ProviderAccountUpdate,
|
||||
pub preserve_profile: bool,
|
||||
pub credential: PlaintextCredential,
|
||||
pub has_refresh_token: bool,
|
||||
pub access_token_expires_at: Option<SystemTime>,
|
||||
@@ -1081,6 +1160,7 @@ pub struct CredentialCasUpdate {
|
||||
account_id: ProviderAccountId,
|
||||
expected_revision: CredentialRevision,
|
||||
profile: ProviderAccountUpdate,
|
||||
preserve_profile: bool,
|
||||
credential: PlaintextCredential,
|
||||
has_refresh_token: bool,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
@@ -1095,6 +1175,7 @@ impl fmt::Debug for CredentialCasUpdate {
|
||||
.field("account_id", &self.account_id)
|
||||
.field("expected_revision", &self.expected_revision)
|
||||
.field("profile", &self.profile)
|
||||
.field("preserve_profile", &self.preserve_profile)
|
||||
.field("credential", &self.credential)
|
||||
.field("has_refresh_token", &self.has_refresh_token)
|
||||
.field("access_token_expires_at", &self.access_token_expires_at)
|
||||
@@ -1129,6 +1210,7 @@ impl CredentialCasUpdate {
|
||||
account_id,
|
||||
expected_revision,
|
||||
profile,
|
||||
preserve_profile: false,
|
||||
credential,
|
||||
has_refresh_token,
|
||||
access_token_expires_at,
|
||||
@@ -1137,6 +1219,13 @@ impl CredentialCasUpdate {
|
||||
})
|
||||
}
|
||||
|
||||
/// 仅轮换凭据,保留提交时的账号资料,避免覆盖并发额度观测更新的套餐。
|
||||
#[must_use]
|
||||
pub const fn preserving_profile(mut self) -> Self {
|
||||
self.preserve_profile = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// 将刷新调度与账号错误事实放入同一个 revision CAS。
|
||||
#[must_use]
|
||||
pub fn with_account_state(
|
||||
@@ -1202,6 +1291,7 @@ impl CredentialCasUpdate {
|
||||
account_id: self.account_id,
|
||||
expected_revision: self.expected_revision,
|
||||
profile: self.profile,
|
||||
preserve_profile: self.preserve_profile,
|
||||
credential: self.credential,
|
||||
has_refresh_token: self.has_refresh_token,
|
||||
access_token_expires_at: self.access_token_expires_at,
|
||||
@@ -1224,6 +1314,8 @@ pub struct QuotaObservation {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub quota: OpaqueProviderData,
|
||||
/// Provider 确认的账号套餐,与额度原子写入;`None` 保留已有套餐。
|
||||
pub plan_type: Option<String>,
|
||||
/// Provider 原始 quota document 的观察时间;不代表访问结论发生变化。
|
||||
pub observed_at: SystemTime,
|
||||
/// Provider 已从私有 JSON 归一化出的额度访问事实。
|
||||
@@ -1237,6 +1329,7 @@ impl fmt::Debug for QuotaObservation {
|
||||
.field("account_id", &self.account_id)
|
||||
.field("expected_revision", &self.expected_revision)
|
||||
.field("quota", &self.quota)
|
||||
.field("plan_type", &self.plan_type)
|
||||
.field("observed_at", &self.observed_at)
|
||||
.field("state", &self.state)
|
||||
.finish()
|
||||
|
||||
@@ -247,17 +247,31 @@ impl ClientRoutingScope {
|
||||
/// 一次认证随 RuntimeSnapshot 冻结的账号目录与 Key 权限。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FrozenAccountScope {
|
||||
disable_fast: bool,
|
||||
directory: Arc<RuntimeAccountDirectory>,
|
||||
client_scope: ClientRoutingScope,
|
||||
}
|
||||
|
||||
impl FrozenAccountScope {
|
||||
/// Key 绑定分组的冻结 Fast 限制,与账号成员资格无关。
|
||||
#[must_use]
|
||||
pub const fn with_disable_fast(mut self, disable_fast: bool) -> Self {
|
||||
self.disable_fast = disable_fast;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn disable_fast(&self) -> bool {
|
||||
self.disable_fast
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
directory: Arc<RuntimeAccountDirectory>,
|
||||
client_scope: ClientRoutingScope,
|
||||
) -> Self {
|
||||
Self {
|
||||
disable_fast: false,
|
||||
directory,
|
||||
client_scope,
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
/// Provider 归一化的剩余额度基点:0 耗尽,10_000 全部可用。
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub cooldown: Option<super::AccountCooldown>,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -316,8 +316,8 @@ impl AccountRuntimeSignals {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_rate_limit(mut self, rate_limited_until: Option<SystemTime>) -> Self {
|
||||
self.rate_limited_until = rate_limited_until;
|
||||
pub const fn with_rate_limit(mut self, cooldown: Option<super::AccountCooldown>) -> Self {
|
||||
self.cooldown = cooldown;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -654,7 +654,7 @@ impl AccountSelector {
|
||||
if !context.eligibility.bypasses_local_eligibility() {
|
||||
let status = candidate
|
||||
.account
|
||||
.status_projection(context.now, candidate.signals.rate_limited_until)
|
||||
.status_projection(context.now, candidate.signals.cooldown)
|
||||
.status;
|
||||
if status != AccountStatus::Normal {
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
|
||||
@@ -788,6 +788,8 @@ where
|
||||
}));
|
||||
let context = AttemptContext::new(
|
||||
RequestAttemptContext::new(self.request_id.clone(), self.client_api_key_ref.clone())
|
||||
.with_disable_fast(self.plan.disable_fast())
|
||||
.with_request_location(self.plan.request_location().cloned())
|
||||
.with_concurrency_wait_budget(self.concurrency_wait_budget.clone())
|
||||
.with_timing_started_at(self.observation.timing_started_at)
|
||||
.with_trace(self.trace.clone()),
|
||||
@@ -1497,6 +1499,12 @@ where
|
||||
http_version,
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
upstream_response_model: self
|
||||
.current
|
||||
.as_ref()
|
||||
.and_then(|current| current.response_observation.as_ref())
|
||||
.and_then(ProviderResponseObservation::upstream_response_model)
|
||||
.map(str::to_owned),
|
||||
provider_metadata_json,
|
||||
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
|
||||
error: None,
|
||||
@@ -1658,6 +1666,12 @@ where
|
||||
http_version,
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
upstream_response_model: self
|
||||
.current
|
||||
.as_ref()
|
||||
.and_then(|current| current.response_observation.as_ref())
|
||||
.and_then(ProviderResponseObservation::upstream_response_model)
|
||||
.map(str::to_owned),
|
||||
provider_metadata_json,
|
||||
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
|
||||
error: Some(finalization.error),
|
||||
|
||||
@@ -1051,10 +1051,7 @@ impl ExecutionService for DefaultExecutionService {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(model.clone()) {
|
||||
result.push(PublicModelDescriptor::Native {
|
||||
model,
|
||||
payload: source.payload.clone(),
|
||||
});
|
||||
result.push(source.content.for_public_model(model));
|
||||
}
|
||||
}
|
||||
for model in client.snapshot.public_models_for_provider(kind) {
|
||||
@@ -1073,10 +1070,7 @@ impl ExecutionService for DefaultExecutionService {
|
||||
}
|
||||
if let Some(source) = by_id.get(target.as_str()) {
|
||||
seen.insert(model.clone());
|
||||
result.push(PublicModelDescriptor::Native {
|
||||
model,
|
||||
payload: source.payload.clone(),
|
||||
});
|
||||
result.push(source.content.for_public_model(model));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,6 +314,8 @@ impl ContinuationAttempt {
|
||||
/// Provider 每次执行可见的 request-local context。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RequestAttemptContext {
|
||||
disable_fast: bool,
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
request_id: ModelRequestId,
|
||||
client_api_key_ref: ClientApiKeyId,
|
||||
timing_started_at: Instant,
|
||||
@@ -322,11 +324,28 @@ pub struct RequestAttemptContext {
|
||||
}
|
||||
|
||||
impl RequestAttemptContext {
|
||||
#[must_use]
|
||||
pub const fn with_disable_fast(mut self, disable_fast: bool) -> Self {
|
||||
self.disable_fast = disable_fast;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_location(
|
||||
mut self,
|
||||
location: Option<crate::account::RequestLocation>,
|
||||
) -> Self {
|
||||
self.request_location = location;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn new(request_id: ModelRequestId, client_api_key_ref: ClientApiKeyId) -> Self {
|
||||
Self {
|
||||
request_id,
|
||||
client_api_key_ref,
|
||||
disable_fast: false,
|
||||
request_location: None,
|
||||
timing_started_at: Instant::now(),
|
||||
trace: crate::diagnostics::TraceContext::default(),
|
||||
concurrency_wait_budget: crate::concurrency::ConcurrencyWaitBudget::default(),
|
||||
@@ -388,6 +407,16 @@ pub struct AttemptContext {
|
||||
}
|
||||
|
||||
impl AttemptContext {
|
||||
#[must_use]
|
||||
pub const fn disable_fast(&self) -> bool {
|
||||
self.request.disable_fast
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_location(&self) -> Option<&crate::account::RequestLocation> {
|
||||
self.request.request_location.as_ref()
|
||||
}
|
||||
|
||||
/// 当前 attempt 的诊断关联;克隆后可传给后台 transport 任务。
|
||||
#[must_use]
|
||||
pub fn trace(&self) -> crate::diagnostics::TraceContext {
|
||||
@@ -643,6 +672,7 @@ pub struct ModelRequestFinalization {
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub upstream_response_model: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;Core 不解释字段。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
/// 请求全程的有界诊断快照,跨 Provider 与重试保留。
|
||||
|
||||
@@ -485,6 +485,11 @@ pub trait Provider: Send + Sync {
|
||||
/// 返回当前进程已经成功发布的目录代次。
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration;
|
||||
|
||||
/// 发现型目录不用于提前拒绝上游可能支持的新模型或别名。
|
||||
fn model_catalog_is_exhaustive(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// 解释 Provider 差异化观测字段;不参与路由和传输。
|
||||
fn request_observation(
|
||||
&self,
|
||||
@@ -652,6 +657,12 @@ impl ProviderRegistry {
|
||||
}
|
||||
|
||||
impl ProviderCatalogPort for ProviderRegistry {
|
||||
fn model_catalog_is_exhaustive(&self, provider: &ProviderKind) -> bool {
|
||||
self.providers
|
||||
.get(provider)
|
||||
.is_none_or(|provider| provider.model_catalog_is_exhaustive())
|
||||
}
|
||||
|
||||
fn catalog_generations(&self) -> BTreeMap<ProviderKind, ProviderCatalogGeneration> {
|
||||
self.providers
|
||||
.iter()
|
||||
|
||||
@@ -338,7 +338,10 @@ impl ProviderConnectionObservation {
|
||||
}
|
||||
}
|
||||
|
||||
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
|
||||
/// 只供当前客户端请求使用的 HTTP 失败响应。
|
||||
///
|
||||
/// 通常保留原始上游响应;Provider 也可为明确的恢复流程准备客户端投影。
|
||||
/// 投影不能覆盖 [`ProviderError`] 中用于诊断和记账的真实上游事实。
|
||||
///
|
||||
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
|
||||
/// [`ProviderError`] 的普通 clone 会丢弃它,只有最终失败的原对象才能把响应交给
|
||||
@@ -352,7 +355,7 @@ pub struct ClientVisibleUpstreamResponse {
|
||||
}
|
||||
|
||||
impl ClientVisibleUpstreamResponse {
|
||||
/// 保存 transport 实际收到的状态码、Content-Type 原值和正文。
|
||||
/// 保存客户端响应的状态码、Content-Type 和正文。
|
||||
#[must_use]
|
||||
pub fn new(status: u16, content_type: Option<Vec<u8>>, body: Bytes) -> Self {
|
||||
Self {
|
||||
|
||||
@@ -530,6 +530,7 @@ pub struct ProviderResponseObservation {
|
||||
status_code: Option<u16>,
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
service_tier: Option<String>,
|
||||
upstream_response_model: Option<String>,
|
||||
timings: ProviderResponseTimings,
|
||||
client_headers: Vec<ProviderResponseHeader>,
|
||||
provider_metadata: Option<ProviderResponseMetadata>,
|
||||
@@ -545,6 +546,7 @@ impl ProviderResponseObservation {
|
||||
status_code: None,
|
||||
request_id: None,
|
||||
service_tier: None,
|
||||
upstream_response_model: None,
|
||||
timings: ProviderResponseTimings::default(),
|
||||
client_headers: Vec::new(),
|
||||
provider_metadata: None,
|
||||
@@ -600,6 +602,21 @@ impl ProviderResponseObservation {
|
||||
self
|
||||
}
|
||||
|
||||
/// 记录 Provider 从上游响应明确取得的模型,不改写发送模型或计费合同。
|
||||
#[must_use]
|
||||
pub fn with_upstream_response_model_if_valid(mut self, model: &str) -> Self {
|
||||
let model = model.trim();
|
||||
if validate_text(model, 256, false, None).is_ok() {
|
||||
self.upstream_response_model = Some(model.to_owned());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_response_model(&self) -> Option<&str> {
|
||||
self.upstream_response_model.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_timings(mut self, timings: ProviderResponseTimings) -> Self {
|
||||
self.timings = timings;
|
||||
|
||||
@@ -181,6 +181,16 @@ pub trait ProviderLeasePort: Send + Sync {
|
||||
&self,
|
||||
request: ProviderLeaseRequest,
|
||||
) -> BoxFuture<'_, Result<ProviderLeaseAcquisition, ProviderStoreError>>;
|
||||
|
||||
/// 读取指定账号当前的在途请求数;只用于容量熔断的峰值证据,
|
||||
/// 支持租约信号的存储实现覆盖,否则视为不可观测(空映射)。
|
||||
fn account_in_flight<'a>(
|
||||
&'a self,
|
||||
account_ids: &'a [ProviderAccountId],
|
||||
) -> BoxFuture<'a, Result<BTreeMap<ProviderAccountId, u32>, ProviderStoreError>> {
|
||||
let _ = account_ids;
|
||||
Box::pin(async move { Ok(BTreeMap::new()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 从原始会话锚点派生的不可逆亲和键。
|
||||
@@ -596,12 +606,16 @@ pub trait ProviderCredentialStatePort: Send + Sync {
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// 临时 cooldown 只保存可丢失的调度截止时间,不进入账号持久状态。
|
||||
/// 账号级 cooldown 的来源类别;调度状态统一按 `rate_limited` 处理。
|
||||
pub use crate::account::AccountCooldownKind as ProviderCooldownKind;
|
||||
|
||||
/// 可丢失的账号冷却事实,不进入持久状态;探测冻结到期后仍需确认恢复。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
}
|
||||
|
||||
impl ProviderCooldown {
|
||||
@@ -610,11 +624,27 @@ impl ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self::new_with_kind(
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
ProviderCooldownKind::RateLimit,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn new_with_kind(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
kind,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -632,6 +662,19 @@ impl ProviderCooldown {
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scheduling_state(&self) -> crate::account::AccountCooldown {
|
||||
crate::account::AccountCooldown {
|
||||
until: self.until,
|
||||
kind: self.kind,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> ProviderCooldownKind {
|
||||
self.kind
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失 cooldown 的细粒度作用域。
|
||||
@@ -749,6 +792,29 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 记录一次容量类失败并返回滑动窗口内的累计次数,同时把观测到的账号
|
||||
/// 在途并发并入窗口峰值(`in_flight` 为 0 表示本次未观测,跳过峰值更新)。
|
||||
/// 只服务容量熔断触发器;调用频率受失败频率约束,不需要批量接口。
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
|
||||
|
||||
/// 普通请求成功后原子清除临时限流及失败证据;必须保留任何容量冻结。
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
|
||||
/// 读取窗口内观测到的在途并发峰值;无证据时返回 `None`。
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -864,6 +930,118 @@ pub trait ProviderRuntimePolicyPort: Send + Sync {
|
||||
fn load_refresh_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderRefreshPolicy, ProviderStoreError>>;
|
||||
|
||||
/// 读取账号容量熔断策略;默认关闭,只有实现运行时设置的存储需要覆盖。
|
||||
fn load_freeze_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(ProviderFreezePolicy::disabled()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号容量熔断(自动冻结)策略;来源于 `runtime_settings`,
|
||||
/// 由 Provider 触发路径与恢复 worker 共享同一份配置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderFreezePolicy {
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window: Duration,
|
||||
freeze_duration: Duration,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl ProviderFreezePolicy {
|
||||
/// 边界与迁移 `0010_account_auto_freeze.sql` 的 check 约束一致;
|
||||
/// store 层写入前已校验,这里兜底防御越界配置。
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window_seconds: u64,
|
||||
freeze_duration_seconds: u64,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if !(2..=1_000).contains(&threshold)
|
||||
|| !(60..=3_600).contains(&window_seconds)
|
||||
|| !(300..=604_800).contains(&freeze_duration_seconds)
|
||||
|| probe_model.as_deref().is_some_and(|model| {
|
||||
model.is_empty()
|
||||
|| model.len() > 128
|
||||
|| model != model.trim()
|
||||
|| model.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
{
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate freeze policy",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
enabled,
|
||||
threshold,
|
||||
window: Duration::from_secs(window_seconds),
|
||||
freeze_duration: Duration::from_secs(freeze_duration_seconds),
|
||||
probe_enabled,
|
||||
probe_model,
|
||||
adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
|
||||
/// 功能关闭时的全零策略;触发路径与 worker 都以此短路。
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
threshold: 2,
|
||||
window: Duration::from_secs(60),
|
||||
freeze_duration: Duration::from_secs(300),
|
||||
probe_enabled: false,
|
||||
probe_model: None,
|
||||
adaptive_concurrency: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
/// 窗口内触发冻结的请求级失败次数阈值。
|
||||
#[must_use]
|
||||
pub const fn threshold(&self) -> u32 {
|
||||
self.threshold
|
||||
}
|
||||
|
||||
/// 失败计数滑动窗口;每次失败都会顺延窗口。
|
||||
#[must_use]
|
||||
pub const fn window(&self) -> Duration {
|
||||
self.window
|
||||
}
|
||||
|
||||
/// 冻结时长;探测失败后的顺延也使用该值。
|
||||
#[must_use]
|
||||
pub const fn freeze_duration(&self) -> Duration {
|
||||
self.freeze_duration
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn probe_enabled(&self) -> bool {
|
||||
self.probe_enabled
|
||||
}
|
||||
|
||||
/// 探测模型;`None` 表示由 worker 选择账号可用的第一个模型。
|
||||
#[must_use]
|
||||
pub fn probe_model(&self) -> Option<&str> {
|
||||
self.probe_model.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn adaptive_concurrency(&self) -> bool {
|
||||
self.adaptive_concurrency
|
||||
}
|
||||
}
|
||||
|
||||
/// OAuth pending flow 的原始绑定只在 Provider 与 Store 边界内短暂存在。
|
||||
|
||||
@@ -11,14 +11,20 @@ use super::{
|
||||
ModelCapabilities, ModelPresentation, PublicModelId, PublicModelProfile, UpstreamModelId,
|
||||
};
|
||||
|
||||
/// Provider 原生客户端目录条目;Core 只解释模型标识,不解释协议正文。
|
||||
/// Provider 客户端目录条目;缺少原生协议正文时使用已编译的通用画像。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderModelDescriptor {
|
||||
pub model: UpstreamModelId,
|
||||
pub payload: RawJsonPayload,
|
||||
pub content: ProviderModelContent,
|
||||
}
|
||||
|
||||
/// 对外目录保留原生正文;只有没有原生目录的 Provider 才使用通用画像适配。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ProviderModelContent {
|
||||
Native(RawJsonPayload),
|
||||
Adapted(ModelPresentation),
|
||||
}
|
||||
|
||||
/// 对外目录按条目保留原生正文或使用通用画像适配。
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum PublicModelDescriptor {
|
||||
Native {
|
||||
@@ -28,6 +34,20 @@ pub enum PublicModelDescriptor {
|
||||
Adapted(PublicModelProfile),
|
||||
}
|
||||
|
||||
impl ProviderModelContent {
|
||||
pub(crate) fn for_public_model(&self, model: PublicModelId) -> PublicModelDescriptor {
|
||||
match self {
|
||||
Self::Native(payload) => PublicModelDescriptor::Native {
|
||||
model,
|
||||
payload: payload.clone(),
|
||||
},
|
||||
Self::Adapted(presentation) => {
|
||||
PublicModelDescriptor::Adapted(PublicModelProfile::new(model, presentation.clone()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 实时目录编译后的单模型能力。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderModelCapabilities {
|
||||
@@ -93,12 +113,17 @@ pub struct ProviderCatalogUnavailable;
|
||||
|
||||
/// 快照编译与对账使用的对象安全目录端口。
|
||||
pub trait ProviderCatalogPort: Send + Sync {
|
||||
/// 目录是否完整到足以根据缺项拒绝请求;发现型目录交由上游验证模型名。
|
||||
fn model_catalog_is_exhaustive(&self, _provider: &ProviderKind) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// 返回全部已注册 Provider 的目录代次;注册集合在初始化后保持不变。
|
||||
///
|
||||
/// 即使某个目录暂时不可读,也必须保留它的 Provider 与最近成功发布的代次。
|
||||
fn catalog_generations(&self) -> BTreeMap<ProviderKind, ProviderCatalogGeneration>;
|
||||
|
||||
/// 查询指定 Provider 的模型事实;成功的空列表表示已知无模型,失败表示未知。
|
||||
/// 查询模型发现事实;只有完整目录的空列表能证明无模型,失败表示未知。
|
||||
fn query_model_capabilities(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
|
||||
@@ -10,7 +10,8 @@ pub use crate::account::scope::{
|
||||
pub use crate::identity::ProviderKind;
|
||||
pub use catalog::{
|
||||
ProviderCatalogGeneration, ProviderCatalogPort, ProviderCatalogUnavailable,
|
||||
ProviderModelCapabilities, ProviderModelDescriptor, PublicModelDescriptor,
|
||||
ProviderModelCapabilities, ProviderModelContent, ProviderModelDescriptor,
|
||||
PublicModelDescriptor,
|
||||
};
|
||||
pub use snapshot::RuntimeSnapshot;
|
||||
|
||||
@@ -535,6 +536,8 @@ impl ProviderCandidate {
|
||||
/// 一次请求冻结的 Provider 尝试顺序。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RoutingPlan {
|
||||
disable_fast: bool,
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
config_revision: ConfigRevision,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
operation: OperationKind,
|
||||
@@ -544,6 +547,17 @@ pub struct RoutingPlan {
|
||||
}
|
||||
|
||||
impl RoutingPlan {
|
||||
#[must_use]
|
||||
pub const fn disable_fast(&self) -> bool {
|
||||
self.disable_fast
|
||||
}
|
||||
|
||||
/// 本次请求冻结的全局位置,重试时沿用同一份配置。
|
||||
#[must_use]
|
||||
pub const fn request_location(&self) -> Option<&crate::account::RequestLocation> {
|
||||
self.request_location.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn config_revision(&self) -> ConfigRevision {
|
||||
self.config_revision
|
||||
|
||||
@@ -28,10 +28,14 @@ const MAXIMUM_CATALOG_STABILITY_ATTEMPTS: usize = 4;
|
||||
/// Store 在一个一致性读取中提供的调度设置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotSettingsFacts {
|
||||
disable_fast: bool,
|
||||
request_location_enabled: bool,
|
||||
request_location: crate::account::RequestLocation,
|
||||
max_concurrent_per_account: u32,
|
||||
max_waiting_per_key: u32,
|
||||
max_waiting_per_account: u32,
|
||||
concurrency_wait_timeout_seconds: u32,
|
||||
responses_max_decompressed_body_bytes: u64,
|
||||
request_interval_ms: u64,
|
||||
rotation_strategy: String,
|
||||
model_mappings: BTreeMap<String, String>,
|
||||
@@ -40,6 +44,29 @@ pub struct SnapshotSettingsFacts {
|
||||
}
|
||||
|
||||
impl SnapshotSettingsFacts {
|
||||
#[must_use]
|
||||
pub const fn with_disable_fast(mut self, disable_fast: bool) -> Self {
|
||||
self.disable_fast = disable_fast;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_responses_max_decompressed_body_bytes(mut self, bytes: u64) -> Self {
|
||||
self.responses_max_decompressed_body_bytes = bytes;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_location(
|
||||
mut self,
|
||||
location: crate::account::RequestLocation,
|
||||
enabled: bool,
|
||||
) -> Self {
|
||||
self.request_location_enabled = enabled;
|
||||
self.request_location = location;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_concurrency_queues(
|
||||
mut self,
|
||||
@@ -63,10 +90,14 @@ impl SnapshotSettingsFacts {
|
||||
min_codex_cli_version: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
disable_fast: false,
|
||||
request_location_enabled: false,
|
||||
request_location: crate::account::RequestLocation::default(),
|
||||
max_concurrent_per_account,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
request_interval_ms,
|
||||
rotation_strategy: rotation_strategy.into(),
|
||||
model_mappings,
|
||||
@@ -105,15 +136,27 @@ impl SnapshotClientPolicyFacts {
|
||||
/// Store 读取到的账号分组事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotAccountGroupFacts {
|
||||
disable_fast: bool,
|
||||
id: AccountGroupId,
|
||||
name: String,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl SnapshotAccountGroupFacts {
|
||||
#[must_use]
|
||||
pub const fn with_disable_fast(mut self, disable_fast: bool) -> Self {
|
||||
self.disable_fast = disable_fast;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn new(id: AccountGroupId, name: String, enabled: bool) -> Self {
|
||||
Self { id, name, enabled }
|
||||
Self {
|
||||
id,
|
||||
name,
|
||||
enabled,
|
||||
disable_fast: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,14 +338,16 @@ async fn compile_runtime_snapshot(
|
||||
) -> Result<RuntimeSnapshot, RuntimeSnapshotCompileError> {
|
||||
let registered_providers = provider_kinds.iter().cloned().collect::<BTreeSet<_>>();
|
||||
|
||||
// 目录查询失败表示未知;查询成功后,即使为空,也必须与“已知缺少模型”区分。
|
||||
// 只有成功取得的完整目录能证明模型缺项;发现型目录和查询失败均交由上游验证。
|
||||
let mut provider_models = Vec::new();
|
||||
let mut known_provider_catalogs = BTreeSet::new();
|
||||
let mut exhaustive_provider_catalogs = BTreeSet::new();
|
||||
for provider in &provider_kinds {
|
||||
let Ok(models) = catalogs.query_model_capabilities(provider).await else {
|
||||
continue;
|
||||
};
|
||||
known_provider_catalogs.insert(provider.clone());
|
||||
if catalogs.model_catalog_is_exhaustive(provider) {
|
||||
exhaustive_provider_catalogs.insert(provider.clone());
|
||||
}
|
||||
provider_models.extend(models.into_iter().map(|model| {
|
||||
let compiled = ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -374,6 +419,12 @@ async fn compile_runtime_snapshot(
|
||||
{
|
||||
return Err(RuntimeSnapshotCompileError::InvalidData);
|
||||
}
|
||||
let decompressed_body_limit =
|
||||
isize::try_from(facts.settings.responses_max_decompressed_body_bytes)
|
||||
.ok()
|
||||
.and_then(|bytes| usize::try_from(bytes).ok())
|
||||
.and_then(std::num::NonZeroUsize::new)
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
let queue_timeout =
|
||||
Duration::from_secs(u64::from(facts.settings.concurrency_wait_timeout_seconds));
|
||||
let client_queue_policy = ConcurrencyQueuePolicy {
|
||||
@@ -411,6 +462,7 @@ async fn compile_runtime_snapshot(
|
||||
});
|
||||
let mut client_policies = Vec::with_capacity(facts.client_policies.len());
|
||||
for policy in facts.client_policies {
|
||||
let mut disable_fast = false;
|
||||
let account_scope = if policy.group_ids.is_empty() {
|
||||
FrozenAccountScope::new(
|
||||
Arc::clone(&account_directory),
|
||||
@@ -427,6 +479,8 @@ async fn compile_runtime_snapshot(
|
||||
let group = groups
|
||||
.get(&group_id)
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
// 禁用分组仅影响选号;Key 仍绑定其 Fast 限制。
|
||||
disable_fast |= group.disable_fast;
|
||||
bound_groups.push(RoutingGroupSnapshot::new(
|
||||
group.id.clone(),
|
||||
group.name.clone(),
|
||||
@@ -446,12 +500,24 @@ async fn compile_runtime_snapshot(
|
||||
client_policies.push(ClientPolicy::new(
|
||||
policy.key_id,
|
||||
policy.plaintext_key,
|
||||
Arc::new(account_scope),
|
||||
Arc::new(account_scope.with_disable_fast(disable_fast)),
|
||||
true,
|
||||
policy.limits,
|
||||
));
|
||||
}
|
||||
|
||||
// 关闭自定义时保留持久化值,但不生成全局覆盖;请求继续使用客户端字段。
|
||||
let request_location = if facts.settings.request_location_enabled {
|
||||
Some(
|
||||
facts
|
||||
.settings
|
||||
.request_location
|
||||
.normalized()
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
RuntimeSnapshot::new(
|
||||
facts.config_revision,
|
||||
selection_policy,
|
||||
@@ -462,10 +528,13 @@ async fn compile_runtime_snapshot(
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)
|
||||
.map(|snapshot| {
|
||||
snapshot
|
||||
.with_disable_fast(facts.settings.disable_fast)
|
||||
.with_request_location(request_location)
|
||||
.with_responses_max_decompressed_body_bytes(decompressed_body_limit)
|
||||
.with_client_queue_policy(client_queue_policy)
|
||||
.with_model_mappings(model_mappings)
|
||||
.with_account_directory(account_directory)
|
||||
.with_known_provider_catalogs(known_provider_catalogs)
|
||||
.with_exhaustive_provider_catalogs(exhaustive_provider_catalogs)
|
||||
.with_min_codex_client_versions(min_client_versions)
|
||||
})
|
||||
}
|
||||
@@ -473,6 +542,9 @@ async fn compile_runtime_snapshot(
|
||||
/// 数据面使用的不可变配置快照。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RuntimeSnapshot {
|
||||
disable_fast: bool,
|
||||
responses_max_decompressed_body_bytes: std::num::NonZeroUsize,
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
revision: ConfigRevision,
|
||||
client_queue_policy: ConcurrencyQueuePolicy,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
@@ -482,13 +554,42 @@ pub struct RuntimeSnapshot {
|
||||
Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, super::ModelPresentation>>>,
|
||||
model_mappings: Arc<BTreeMap<String, String>>,
|
||||
provider_catalog_generations: Arc<BTreeMap<ProviderKind, ProviderCatalogGeneration>>,
|
||||
known_provider_catalogs: Arc<BTreeSet<ProviderKind>>,
|
||||
exhaustive_provider_catalogs: Arc<BTreeSet<ProviderKind>>,
|
||||
account_directory: Arc<RuntimeAccountDirectory>,
|
||||
client_policies: Arc<BTreeMap<ClientApiKeyId, ClientPolicy>>,
|
||||
min_codex_client_versions: CodexClientMinVersions,
|
||||
}
|
||||
|
||||
impl RuntimeSnapshot {
|
||||
#[must_use]
|
||||
pub const fn with_disable_fast(mut self, disable_fast: bool) -> Self {
|
||||
self.disable_fast = disable_fast;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn responses_max_decompressed_body_bytes(&self) -> usize {
|
||||
self.responses_max_decompressed_body_bytes.get()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_responses_max_decompressed_body_bytes(
|
||||
mut self,
|
||||
bytes: std::num::NonZeroUsize,
|
||||
) -> Self {
|
||||
self.responses_max_decompressed_body_bytes = bytes;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_location(
|
||||
mut self,
|
||||
location: Option<crate::account::RequestLocation>,
|
||||
) -> Self {
|
||||
self.request_location = location;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn client_queue_policy(&self) -> ConcurrencyQueuePolicy {
|
||||
self.client_queue_policy
|
||||
@@ -518,7 +619,7 @@ impl RuntimeSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
let mut known_provider_catalogs = BTreeSet::new();
|
||||
let mut exhaustive_provider_catalogs = BTreeSet::new();
|
||||
let mut model_map =
|
||||
BTreeMap::<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>::new();
|
||||
let mut presentation_map =
|
||||
@@ -530,7 +631,7 @@ impl RuntimeSnapshot {
|
||||
capabilities,
|
||||
presentation,
|
||||
} = model;
|
||||
known_provider_catalogs.insert(provider.clone());
|
||||
exhaustive_provider_catalogs.insert(provider.clone());
|
||||
if !provider_set.contains(&provider) {
|
||||
return Err(RoutingError::NotFound {
|
||||
entity: "provider",
|
||||
@@ -568,6 +669,10 @@ impl RuntimeSnapshot {
|
||||
client_policy_map.retain(|_, policy| policy.enabled());
|
||||
|
||||
Ok(Self {
|
||||
responses_max_decompressed_body_bytes: std::num::NonZeroUsize::new(64 * 1024 * 1024)
|
||||
.expect("positive default limit"),
|
||||
disable_fast: false,
|
||||
request_location: None,
|
||||
revision,
|
||||
account_selection_policy,
|
||||
client_queue_policy: ConcurrencyQueuePolicy::default(),
|
||||
@@ -576,7 +681,7 @@ impl RuntimeSnapshot {
|
||||
provider_model_presentations: Arc::new(presentation_map),
|
||||
model_mappings: Arc::new(BTreeMap::new()),
|
||||
provider_catalog_generations: Arc::new(BTreeMap::new()),
|
||||
known_provider_catalogs: Arc::new(known_provider_catalogs),
|
||||
exhaustive_provider_catalogs: Arc::new(exhaustive_provider_catalogs),
|
||||
account_directory: Arc::new(RuntimeAccountDirectory::default()),
|
||||
client_policies: Arc::new(client_policy_map),
|
||||
min_codex_client_versions: CodexClientMinVersions::default(),
|
||||
@@ -602,8 +707,8 @@ impl RuntimeSnapshot {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
fn with_known_provider_catalogs(mut self, providers: BTreeSet<ProviderKind>) -> Self {
|
||||
self.known_provider_catalogs = Arc::new(providers);
|
||||
fn with_exhaustive_provider_catalogs(mut self, providers: BTreeSet<ProviderKind>) -> Self {
|
||||
self.exhaustive_provider_catalogs = Arc::new(providers);
|
||||
self
|
||||
}
|
||||
|
||||
@@ -740,7 +845,7 @@ impl RuntimeSnapshot {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 已取得目录时以映射后的上游模型为准;目录不可用时由 Provider 在发送前判定。
|
||||
/// 完整目录按映射后的模型判定;发现型或不可用目录不作为能力白名单。
|
||||
#[must_use]
|
||||
pub fn contains_public_model_for_provider(
|
||||
&self,
|
||||
@@ -750,11 +855,13 @@ impl RuntimeSnapshot {
|
||||
if !self.providers.contains(provider) {
|
||||
return false;
|
||||
}
|
||||
let upstream_model = self.mapped_model(public_model.as_str());
|
||||
match self.provider_models.get(provider) {
|
||||
Some(models) => models.keys().any(|model| model.as_str() == upstream_model),
|
||||
None => !self.known_provider_catalogs.contains(provider),
|
||||
if !self.exhaustive_provider_catalogs.contains(provider) {
|
||||
return true;
|
||||
}
|
||||
let upstream_model = self.mapped_model(public_model.as_str());
|
||||
self.provider_models
|
||||
.get(provider)
|
||||
.is_some_and(|models| models.keys().any(|model| model.as_str() == upstream_model))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -844,7 +951,7 @@ impl RuntimeSnapshot {
|
||||
};
|
||||
emulated
|
||||
}
|
||||
None if self.known_provider_catalogs.contains(provider) => continue,
|
||||
None if self.exhaustive_provider_catalogs.contains(provider) => continue,
|
||||
None => BTreeSet::new(),
|
||||
};
|
||||
candidates.push(ProviderCandidate {
|
||||
@@ -860,7 +967,7 @@ impl RuntimeSnapshot {
|
||||
let mut scoped_providers = providers.intersection(&self.providers).peekable();
|
||||
if scoped_providers.peek().is_some()
|
||||
&& scoped_providers.all(|provider| {
|
||||
self.known_provider_catalogs.contains(provider)
|
||||
self.exhaustive_provider_catalogs.contains(provider)
|
||||
&& !self.contains_public_model_for_provider(public_model, provider)
|
||||
})
|
||||
{
|
||||
@@ -876,6 +983,8 @@ impl RuntimeSnapshot {
|
||||
|
||||
Ok(RoutingPlan {
|
||||
config_revision: self.revision,
|
||||
disable_fast: self.disable_fast || account_scope.disable_fast(),
|
||||
request_location: self.request_location.clone(),
|
||||
account_selection_policy: self.account_selection_policy,
|
||||
operation: operation.kind(),
|
||||
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
|
||||
@@ -917,6 +1026,8 @@ impl RuntimeSnapshot {
|
||||
};
|
||||
Ok(RoutingPlan {
|
||||
config_revision: self.revision,
|
||||
disable_fast: self.disable_fast || account_scope.disable_fast(),
|
||||
request_location: self.request_location.clone(),
|
||||
account_selection_policy: self.account_selection_policy,
|
||||
operation: operation.kind(),
|
||||
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
|
||||
|
||||
@@ -22,10 +22,12 @@ pub enum WorkerKind {
|
||||
Retention,
|
||||
OpsFlush,
|
||||
Backup,
|
||||
AccountImport,
|
||||
AccountFreezeRecovery,
|
||||
}
|
||||
|
||||
impl WorkerKind {
|
||||
pub const ALL: [Self; 8] = [
|
||||
pub const ALL: [Self; 10] = [
|
||||
Self::OAuthRefresh,
|
||||
Self::QuotaCatalogHealth,
|
||||
Self::RuntimeSnapshotReconciliation,
|
||||
@@ -34,6 +36,8 @@ impl WorkerKind {
|
||||
Self::Retention,
|
||||
Self::OpsFlush,
|
||||
Self::Backup,
|
||||
Self::AccountImport,
|
||||
Self::AccountFreezeRecovery,
|
||||
];
|
||||
|
||||
#[must_use]
|
||||
@@ -47,6 +51,8 @@ impl WorkerKind {
|
||||
Self::Retention => "retention",
|
||||
Self::OpsFlush => "ops_flush",
|
||||
Self::Backup => "backup",
|
||||
Self::AccountImport => "account_import",
|
||||
Self::AccountFreezeRecovery => "account_freeze_recovery",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
use gateway_core::account::{InvalidRequestLocation, RequestLocation};
|
||||
use serde_json::json;
|
||||
|
||||
fn location() -> RequestLocation {
|
||||
RequestLocation {
|
||||
country: "JP".to_owned(),
|
||||
region: "Tokyo".to_owned(),
|
||||
city: "Tokyo".to_owned(),
|
||||
timezone: chrono_tz::Asia::Tokyo,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changing_or_removing_proxy_drops_its_location() {
|
||||
use gateway_core::account::OutboundProxy;
|
||||
let direct = super::account("acct_location").with_request_location(Some(location()));
|
||||
assert!(direct.request_location().is_none());
|
||||
let proxied = direct
|
||||
.with_outbound_proxy(Some(OutboundProxy::parse("http://localhost:8080").unwrap()))
|
||||
.with_request_location(Some(location()));
|
||||
assert_eq!(proxied.request_location(), Some(&location()));
|
||||
assert!(
|
||||
proxied
|
||||
.clone()
|
||||
.with_outbound_proxy(None)
|
||||
.request_location()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
proxied
|
||||
.with_outbound_proxy(Some(OutboundProxy::parse("http://localhost:8081").unwrap()))
|
||||
.request_location()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_normalizes_names_and_preserves_timezone() {
|
||||
let value = RequestLocation {
|
||||
region: " Tokyo ".to_owned(),
|
||||
city: " 東京 ".to_owned(),
|
||||
..location()
|
||||
}
|
||||
.normalized()
|
||||
.expect("valid location");
|
||||
assert_eq!(value.region, "Tokyo");
|
||||
assert_eq!(value.city, "東京");
|
||||
assert_eq!(value.timezone.name(), "Asia/Tokyo");
|
||||
let encoded = serde_json::to_value(&value).expect("serialize");
|
||||
assert_eq!(encoded["timezone"], "Asia/Tokyo");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<RequestLocation>(encoded).unwrap(),
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_rejects_invalid_country_codes() {
|
||||
for country in ["", "J", "JPN", "jp", "J1", "日", " JP"] {
|
||||
assert_eq!(
|
||||
RequestLocation {
|
||||
country: country.to_owned(),
|
||||
..location()
|
||||
}
|
||||
.validate(),
|
||||
Err(InvalidRequestLocation::Country)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_rejects_empty_control_and_long_names() {
|
||||
for name in [
|
||||
String::new(),
|
||||
" ".to_owned(),
|
||||
"Tokyo\n".to_owned(),
|
||||
"\tTokyo".to_owned(),
|
||||
"To\0kyo".to_owned(),
|
||||
"東".repeat(129),
|
||||
] {
|
||||
assert_eq!(
|
||||
RequestLocation {
|
||||
region: name.clone(),
|
||||
..location()
|
||||
}
|
||||
.normalized(),
|
||||
Err(InvalidRequestLocation::Region)
|
||||
);
|
||||
assert_eq!(
|
||||
RequestLocation {
|
||||
city: name,
|
||||
..location()
|
||||
}
|
||||
.normalized(),
|
||||
Err(InvalidRequestLocation::City)
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
RequestLocation {
|
||||
city: "東".repeat(128),
|
||||
..location()
|
||||
}
|
||||
.validate()
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_requires_complete_object_and_valid_iana_timezone() {
|
||||
for timezone in ["Asia/Tokyo", "America/New_York", "Europe/London"] {
|
||||
assert!(
|
||||
serde_json::from_value::<RequestLocation>(json!({
|
||||
"country": "JP", "region": "Tokyo", "city": "Tokyo", "timezone": timezone
|
||||
}))
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
for timezone in ["", "Not/A_Zone", "+09:00", "Asia/Typo"] {
|
||||
assert!(
|
||||
serde_json::from_value::<RequestLocation>(json!({
|
||||
"country": "JP", "region": "Tokyo", "city": "Tokyo", "timezone": timezone
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert!(serde_json::from_value::<RequestLocation>(json!({"country": "JP"})).is_err());
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
mod location;
|
||||
mod model_access;
|
||||
mod proxy;
|
||||
mod selection;
|
||||
@@ -49,7 +50,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -222,7 +223,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -240,7 +241,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -313,23 +314,23 @@ fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
|
||||
let active_until = now + Duration::from_secs(60);
|
||||
let account = account("acct_rate_limited");
|
||||
|
||||
let active = account.status_projection(now, Some(active_until));
|
||||
let active = account.status_projection(now, Some(active_until.into()));
|
||||
assert_eq!(active.status, AccountStatus::RateLimited);
|
||||
assert_eq!(active.rate_limited_until, Some(active_until));
|
||||
assert_eq!(active.cooldown, Some(active_until.into()));
|
||||
|
||||
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
|
||||
let elapsed = account.status_projection(now, Some((now - Duration::from_secs(1)).into()));
|
||||
assert_eq!(elapsed.status, AccountStatus::Normal);
|
||||
assert_eq!(elapsed.rate_limited_until, None);
|
||||
assert_eq!(elapsed.cooldown, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
|
||||
fn account_without_upstream_identity_preserves_provider_credential_state() {
|
||||
let account = ProviderAccount::new(
|
||||
ProviderAccountId::new("acct_pending_identity").expect("valid account"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
"pending identity".to_owned(),
|
||||
None,
|
||||
"oauth".to_owned(),
|
||||
"api_key".to_owned(),
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
@@ -346,7 +347,18 @@ fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
|
||||
account.credential_state(),
|
||||
account.status_projection(SystemTime::now(), None).status
|
||||
),
|
||||
(CredentialState::Unknown, AccountStatus::Error)
|
||||
(CredentialState::Ready, AccountStatus::Normal)
|
||||
);
|
||||
let pending = account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Unknown,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
assert_eq!(
|
||||
pending.status_projection(SystemTime::now(), None).status,
|
||||
AccountStatus::Error
|
||||
);
|
||||
}
|
||||
|
||||
@@ -481,8 +493,8 @@ fn highest_priority_affinity_should_still_obey_existing_scheduling_constraints()
|
||||
);
|
||||
}
|
||||
AccountSchedulingBlocker::LocalAvailability => {
|
||||
candidates[0].signals.rate_limited_until =
|
||||
Some(selection.now + Duration::from_secs(60))
|
||||
candidates[0].signals.cooldown =
|
||||
Some((selection.now + Duration::from_secs(60)).into())
|
||||
}
|
||||
_ => unreachable!("test cases"),
|
||||
}
|
||||
@@ -632,7 +644,7 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
@@ -1051,3 +1063,32 @@ fn elapsed_quota_reset_does_not_fabricate_recovery() {
|
||||
.expect("healthy candidate available");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn due_probe_freeze_blocks_scheduling_and_remains_visible_as_rate_limited() {
|
||||
use gateway_core::account::{AccountCooldown, AccountCooldownKind};
|
||||
let mut candidates = [
|
||||
weighted_candidate("acct_frozen", 100, 0),
|
||||
weighted_candidate("acct_ready", 1, 0),
|
||||
];
|
||||
let selection = context(RotationStrategy::Smart);
|
||||
candidates[0].signals.cooldown = Some(AccountCooldown {
|
||||
until: selection.now - Duration::from_secs(120),
|
||||
kind: AccountCooldownKind::CapacityFreezeProbe,
|
||||
});
|
||||
let projection = candidates[0]
|
||||
.account
|
||||
.status_projection(selection.now, candidates[0].signals.cooldown);
|
||||
assert_eq!(projection.status, AccountStatus::RateLimited);
|
||||
assert!(
|
||||
projection
|
||||
.cooldown
|
||||
.expect("freeze details")
|
||||
.kind
|
||||
.requires_probe()
|
||||
);
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("available account");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_ready");
|
||||
}
|
||||
|
||||
@@ -73,6 +73,7 @@ struct FinalState {
|
||||
http_version: Option<String>,
|
||||
websocket_pool: Option<String>,
|
||||
service_tier: Option<String>,
|
||||
upstream_response_model: Option<String>,
|
||||
upstream_request_id: Option<String>,
|
||||
upstream_status_code: Option<u16>,
|
||||
transport_decision_wait_ms: Option<u64>,
|
||||
@@ -239,6 +240,7 @@ impl ExecutionStore for FakeStore {
|
||||
http_version: finalization.http_version,
|
||||
websocket_pool: finalization.websocket_pool,
|
||||
service_tier: finalization.service_tier,
|
||||
upstream_response_model: finalization.upstream_response_model,
|
||||
upstream_request_id: finalization.upstream_request_id,
|
||||
upstream_status_code: finalization.upstream_status_code,
|
||||
transport_decision_wait_ms: finalization.timings.transport_decision_wait_ms,
|
||||
@@ -584,6 +586,23 @@ fn plan(operation: &Operation) -> RoutingPlan {
|
||||
fn plan_with_policy(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
) -> RoutingPlan {
|
||||
plan_with_location(operation, account_selection_policy, Default::default())
|
||||
}
|
||||
|
||||
fn plan_with_location(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
request_location: gateway_core::account::RequestLocation,
|
||||
) -> RoutingPlan {
|
||||
plan_with_location_and_fast_policy(operation, account_selection_policy, request_location, false)
|
||||
}
|
||||
|
||||
fn plan_with_location_and_fast_policy(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
request_location: gateway_core::account::RequestLocation,
|
||||
disable_fast: bool,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
@@ -626,6 +645,8 @@ fn plan_with_policy(
|
||||
Vec::new(),
|
||||
)
|
||||
.expect("snapshot")
|
||||
.with_disable_fast(disable_fast)
|
||||
.with_request_location(Some(request_location))
|
||||
.with_account_directory(Arc::clone(&directory));
|
||||
snapshot
|
||||
.plan(
|
||||
@@ -1084,6 +1105,7 @@ fn response_observation_is_persisted_but_never_delivered() {
|
||||
.with_websocket_pool(WebSocketPoolKind::New)
|
||||
.with_status_code(200)
|
||||
.with_request_id(OpaqueUpstreamValue::new("upstream-observed"))
|
||||
.with_upstream_response_model_if_valid("gpt-returned")
|
||||
.try_with_service_tier("priority")
|
||||
.expect("service tier")
|
||||
.with_timings(ProviderResponseTimings {
|
||||
@@ -1123,6 +1145,10 @@ fn response_observation_is_persisted_but_never_delivered() {
|
||||
assert_eq!(finalization.http_version.as_deref(), Some("HTTP/2"));
|
||||
assert_eq!(finalization.websocket_pool.as_deref(), Some("new"));
|
||||
assert_eq!(finalization.service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(
|
||||
finalization.upstream_response_model.as_deref(),
|
||||
Some("gpt-returned")
|
||||
);
|
||||
assert_eq!(
|
||||
finalization.upstream_request_id.as_deref(),
|
||||
Some("upstream-observed")
|
||||
@@ -1341,6 +1367,7 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
.with_http_version(UpstreamHttpVersion::Http11)
|
||||
.with_status_code(503)
|
||||
.with_request_id(OpaqueUpstreamValue::new("discarded-request"))
|
||||
.with_upstream_response_model_if_valid("discarded-model")
|
||||
.with_timings(ProviderResponseTimings {
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
@@ -1400,6 +1427,7 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
|
||||
let state = store.state.lock().expect("store lock");
|
||||
let finalization = &state.finalizations[0];
|
||||
assert_eq!(finalization.upstream_response_model, None);
|
||||
let trace: Value =
|
||||
serde_json::from_str(finalization.diagnostic_trace_json.as_deref().unwrap()).unwrap();
|
||||
let events = trace["events"].as_array().unwrap();
|
||||
@@ -1701,61 +1729,72 @@ fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery()
|
||||
|
||||
#[test]
|
||||
fn native_continuation_client_replay_required_is_terminal() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
|
||||
));
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
for (kind, send_state) in [
|
||||
(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
),
|
||||
(
|
||||
ProviderErrorKind::QuotaExhausted,
|
||||
UpstreamSendState::NotSent,
|
||||
),
|
||||
(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent),
|
||||
] {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
|
||||
));
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![Err(ProviderError::new(kind, send_state)
|
||||
.with_replay_safe()
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("client replay requirement must stop proxy recovery");
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("client replay requirement must stop proxy recovery");
|
||||
|
||||
assert!(matches!(error, EngineError::Provider(_)));
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 1);
|
||||
assert_eq!(
|
||||
contexts[0].continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 0);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert!(matches!(error, EngineError::Provider(_)));
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 1);
|
||||
assert_eq!(
|
||||
contexts[0].continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 0);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert_eq!(state.finalizations[0].send_state, send_state);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4497,3 +4536,51 @@ fn deadline_before_first_event_records_no_provider_circuit_failure() {
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert!(!state.finalizations[0].committed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn global_location_and_fast_policy_reach_every_account_retry() {
|
||||
let operation = generate_operation();
|
||||
let location = gateway_core::account::RequestLocation {
|
||||
timezone: "Asia/Tokyo".parse().unwrap(),
|
||||
..Default::default()
|
||||
};
|
||||
let route_plan = plan_with_location_and_fast_policy(
|
||||
&operation,
|
||||
plan(&operation).account_selection_policy(),
|
||||
location.clone(),
|
||||
true,
|
||||
);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert!(
|
||||
contexts
|
||||
.iter()
|
||||
.all(|context| context.request_location() == Some(&location) && context.disable_fast())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -483,7 +483,9 @@ fn detached_early_failure_resumes_cancelled_store_write_and_settles_once_for_all
|
||||
});
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ChargedProvider {
|
||||
policies: Mutex<Vec<bool>>,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
@@ -510,8 +512,9 @@ impl Provider for ChargedProvider {
|
||||
async fn execute(
|
||||
&self,
|
||||
request: ProviderRequest,
|
||||
_: AttemptContext,
|
||||
context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
self.policies.lock().unwrap().push(context.disable_fast());
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
candidate.provider().clone(),
|
||||
@@ -552,7 +555,11 @@ fn charged_service(
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::new([Arc::new(ChargedProvider { fail }) as Arc<dyn Provider>]).unwrap(),
|
||||
ProviderRegistry::new([Arc::new(ChargedProvider {
|
||||
fail,
|
||||
..Default::default()
|
||||
}) as Arc<dyn Provider>])
|
||||
.unwrap(),
|
||||
admissions,
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
@@ -1045,13 +1052,15 @@ impl Provider for NativeCatalogProvider {
|
||||
.into_iter()
|
||||
.map(|id| gateway_core::routing::ProviderModelDescriptor {
|
||||
model: UpstreamModelId::new(id).expect("id"),
|
||||
payload: RawJsonPayload::new(
|
||||
"codex",
|
||||
serde_json::to_vec(&json!({"slug":id,"future":null}))
|
||||
.expect("JSON")
|
||||
.into(),
|
||||
)
|
||||
.expect("payload"),
|
||||
content: gateway_core::routing::ProviderModelContent::Native(
|
||||
RawJsonPayload::new(
|
||||
"codex",
|
||||
serde_json::to_vec(&json!({"slug":id,"future":null}))
|
||||
.expect("JSON")
|
||||
.into(),
|
||||
)
|
||||
.expect("payload"),
|
||||
),
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
@@ -2238,3 +2247,32 @@ fn cancelling_pending_admission_cleans_up_a_slot_acquired_before_the_reply() {
|
||||
drop(pending);
|
||||
assert!(admissions.active.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reused_websocket_client_gets_fast_policy_from_each_new_request_snapshot() {
|
||||
let snapshots = RuntimeSnapshotHandle::new(start_snapshot());
|
||||
let admissions = Arc::new(Admissions::default());
|
||||
let provider = Arc::new(ChargedProvider::default());
|
||||
let service = DefaultExecutionService::new(
|
||||
snapshots.clone(),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::new([provider.clone() as Arc<dyn Provider>]).unwrap(),
|
||||
admissions,
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service.authenticate("sk_start_test").unwrap();
|
||||
for (revision, disable_fast) in [(2, true), (3, false)] {
|
||||
snapshots.publish(
|
||||
start_snapshot_with_policy(revision, true, RateLimits::unlimited())
|
||||
.with_disable_fast(disable_fast),
|
||||
);
|
||||
let mut next = request(&service, ClientTransport::WebSocket);
|
||||
next.client = client.clone();
|
||||
let mut started = block_on(service.start(next)).unwrap();
|
||||
block_on(started.session.collect_uncommitted()).unwrap();
|
||||
block_on(started.session.detach_finalize());
|
||||
}
|
||||
assert_eq!(*provider.policies.lock().unwrap(), vec![true, false]);
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user