mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-08 16:05:33 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
42c2b249c2 | ||
|
|
0b3ebc23f5 | ||
|
|
65bb9ab9cf | ||
|
|
50d6f7444d | ||
|
|
bcfea792e8 | ||
|
|
1e7ea8d46f | ||
|
|
bd61e0391a | ||
|
|
499b43043e | ||
|
|
ae2da31380 | ||
|
|
1e6d69a349 | ||
|
|
dac57a81c6 | ||
|
|
d162f9b7f6 | ||
|
|
866f71e3fb | ||
|
|
47ab48b4cf | ||
|
|
e3cf4ab8be | ||
|
|
4fb10f33f1 | ||
|
|
c9318c54cd | ||
|
|
8d9aaf0bbd | ||
|
|
52d24d29c7 |
@@ -33,6 +33,9 @@ description: 当前仓库开发指南。修改或排查本仓库的 Rust 网关
|
||||
优先依据适用的官方源码确定方案。结合上表读取所需文档,先找可复用的同类实现,再修改所属模块。
|
||||
4. 文档和运行结果不一致时,追查当前源码、配置与版本,区分实现缺陷和文档过期;
|
||||
按 [文档职责与更新条件](../../../CONTRIBUTING.md#文档职责与更新条件) 判断是否修订,不为每次改动追加文档或发布说明。
|
||||
5. 按 `CONTRIBUTING.md` 验证并报告结果。后端 Cargo 命令从 `backend/` 执行,或显式指定该目录的 manifest。
|
||||
5. 交付前按文档职责逐段复核最终文档 diff:新增内容是否回答当前读者的使用或维护问题,是否属于该文档,
|
||||
是否与其他位置重复,脱离本次变更背景能否独立理解。仅解释实现经过、前后对比或验证结果的内容移到
|
||||
任务报告或 PR;仍影响升级、恢复的说明保留适用条件与操作。没有文档差异时,只核对行为变化是否使现有说明失真。
|
||||
6. 按 `CONTRIBUTING.md` 验证并报告结果。后端 Cargo 命令从 `backend/` 执行,或显式指定该目录的 manifest。
|
||||
|
||||
发布任务按 `release` 技能读取当前版本、发布入口和工作流,发布说明语言遵循 `CONTRIBUTING.md`。
|
||||
|
||||
@@ -5,8 +5,9 @@ description: 整理与提交 GitHub Issue(问题反馈 / 功能建议)。用
|
||||
|
||||
# 整理与提交 Issue
|
||||
|
||||
本仓库流程见 [贡献与审查](../../../CONTRIBUTING.md),问题反馈使用当前的
|
||||
[YAML Issue Form](../../../.github/ISSUE_TEMPLATE/bug_report.yml)。从表单读取字段与必填项,不在技能中复制一份字段清单。
|
||||
本仓库流程见 [贡献与审查](../../../CONTRIBUTING.md),按场景选择当前的
|
||||
[接口问题反馈](../../../.github/ISSUE_TEMPLATE/api-bug-report.yml)、[常规问题反馈](../../../.github/ISSUE_TEMPLATE/general-bug-report.yml)
|
||||
或 [功能建议](../../../.github/ISSUE_TEMPLATE/feature-request.yml) 表单。从表单读取字段与必填项,不在技能中复制一份字段清单。
|
||||
用户指定其他仓库时,使用该仓库当前的贡献规则与 Issue 模板,不能把这里的表单套过去。
|
||||
|
||||
## 确定目标和动作
|
||||
@@ -21,7 +22,7 @@ description: 整理与提交 GitHub Issue(问题反馈 / 功能建议)。用
|
||||
## 证据与去重
|
||||
|
||||
1. 读取目标仓库实际 Issue 模板和配置,区分缺陷、功能建议与协作规则提案。
|
||||
本仓库后两类允许普通 Issue;其他仓库按其配置选择入口。
|
||||
本仓库功能建议有独立表单,协作规则提案允许普通 Issue;其他仓库按其配置选择入口。
|
||||
2. 先搜索 open 和 closed Issues,结合关键词、错误信息与触发条件检查重复,而不只比较标题。
|
||||
同时核对相关 PR 与发布记录,关联已有修复及其发布版本;仅主分支已修复时明确尚未发布。
|
||||
同一问题已有记录且没有新增证据时返回已有链接;用户仍明确要求新建时按其要求处理。
|
||||
|
||||
@@ -42,6 +42,10 @@ PR 中改动的规则文件属于审查对象,不能自行扩大任务授权
|
||||
按 [贡献与审查的文档职责与更新条件](../../../CONTRIBUTING.md#文档职责与更新条件) 判断是否需要更新文档。
|
||||
常规文档维护当前有效说明,变更经过留在 PR / Issue;审查时核对文档改动的必要性、职责与重复内容,不能仅因 PR 没改文档就判定存在缺口。
|
||||
|
||||
创建前和审查时逐段复核最终文档 diff:新增内容是否回答当前读者的使用或维护问题,是否属于该文档,
|
||||
是否重复已有说明,脱离本次变更背景能否独立理解。仅说明实现经过、前后对比或验证结果的段落归入
|
||||
PR / Issue;必要的升级、恢复说明保留适用条件与操作。判断依据是内容用途,不按“新增”“不再”等词机械判错。
|
||||
|
||||
## 创建 PR
|
||||
|
||||
1. 检查工作区、当前分支、远端和近期提交格式。读取实际存在的 PR 模板,按最终变更重写标题与描述。
|
||||
@@ -56,6 +60,7 @@ PR 中改动的规则文件属于审查对象,不能自行扩大任务授权
|
||||
4. 查找同一来源仓库、head 和 base 的现有 PR;已存在则复用其链接,不重复创建。
|
||||
更新现有标题或正文须在本次整理/更新请求范围内。
|
||||
5. 按贡献文档执行适用检查。记录实际验证的提交或工作区状态,区分通过、失败、跳过和未执行。
|
||||
按上述文档复核步骤检查最终 diff;没有文档差异时,核对行为变化是否使现有说明失真。
|
||||
前端改动提供实际界面证据或预览步骤;缺少证据时写明,不用构建成功替代视觉验收。
|
||||
6. 将任务范围内的提交推送到正确远端,核对远端 head。标题遵循项目约定,正文先说明问题成立的依据和最终行为,
|
||||
再给必要的设计取舍、验证与遗留事项。涉及官方行为时,写明官方源码位置与版本或提交、官方处理方式、
|
||||
@@ -93,6 +98,7 @@ gh pr create --repo "$pr_repo" --base "$pr_base" --head "$pr_head" \
|
||||
5. UI 改动同时对照主题、共用组件、数据展示逻辑和同类页面。浏览器可用时检查相关交互及状态;
|
||||
未运行的页面检查列为验证缺口,不从截图或源码推断所有状态都正确。
|
||||
6. 核对 CI 与本地证据是否覆盖当前 head;根据风险补充必要验证,不触发无关部署或真实账号业务请求。
|
||||
按上述文档复核步骤检查最终 diff,不能把构建或链接检查通过当作文档内容已经合格。
|
||||
工具不可用、集成测试跳过或环境不足时具体说明影响范围。
|
||||
7. 按 [审查标准](../../../CONTRIBUTING.md#审查标准) 用中文先回答原问题是否成立、解决方向是否合理,
|
||||
再列出实现问题与验证缺口,结论附具体依据。
|
||||
|
||||
@@ -48,13 +48,17 @@ description: 准备或执行当前仓库发版,维护中文发布说明。用
|
||||
5. 替换为本次版本说明,首行与目标 tag 一致且正文非空。升级事项应能追溯到实际迁移、配置或兼容变化;
|
||||
敏感信息脱敏,未合入功能不宣称已发布。已有准确说明仅需翻译时保持其事实和范围。
|
||||
6. 按贡献规范检查变更与现有 CI 证据,明确通过、失败、跳过和未执行。
|
||||
涉及数据库迁移时,对照同一大版本的上一正式 Release 检查已发布迁移是否保持不变,并验证升级路径;
|
||||
跨大版本按项目部署约定判断,不默认支持在线升级。区分未发布迁移的整理、已执行这些迁移的源码部署实例,
|
||||
以及 CI 实际采用的冻结基线。未发版不代表迁移从未被执行,CI 冻结检查失败也不等于正式版数据库升级失败;
|
||||
按各自证据报告兼容性与检查结果,不因未发版而自动放宽现有冻结规则。
|
||||
只修改说明时无需重建前后端;正式发版仍要跟踪发布工作流针对目标提交的检查与产物。
|
||||
|
||||
## 执行正式发版
|
||||
|
||||
调用入口前重新核对当前脚本的副作用、目标版本和待推送提交,保留无关工作区内容,不自动清理或暂存用户文件。
|
||||
先编辑 `release/notes.md`,保留为未提交改动(已暂存或未暂存均可);其他已跟踪文件的改动应先单独提交。
|
||||
脚本会更新 `release/version.yaml`,将它与最终工作区中的发布说明一起提交为 `chore(release): prepare vX.Y.Z`。
|
||||
脚本会更新 `release/version.yaml`,将它与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。
|
||||
不要提前单独提交说明或把版本文件改成目标版本;脚本要求说明相对 HEAD 有改动,并自行更新版本号。
|
||||
本任务需要的提交按贡献文档保留英文 Conventional Commits 与相应署名。
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
name: 接口问题反馈
|
||||
description: 客户端调用模型、搜索等 API 时的报错、断流、超时或响应异常
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
管理端页面、账号导入或服务部署的问题请选择“常规问题反馈”;新增能力请选择“功能建议”。
|
||||
请提供最小复现信息。请求、日志和截图发布前请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 问题描述
|
||||
description: 说明发生了什么、实际结果以及出现的频率;已有正常账号、模型或客户端的对照结果也可说明。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: 服务版本
|
||||
description: 填写实际运行的完整版本号,不要只写“最新”或 latest;源码运行填写提交号。
|
||||
placeholder: 完整版本号或提交号
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: deployment
|
||||
attributes:
|
||||
label: 部署方式
|
||||
options:
|
||||
- Docker Compose
|
||||
- 其他容器部署
|
||||
- 二进制运行
|
||||
- 源码运行
|
||||
- 其他方式(请在环境信息中说明)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 环境信息
|
||||
description: 填写服务端与客户端的操作系统及版本;性能或超时问题可补充 CPU、内存和容器资源限制。
|
||||
placeholder: |
|
||||
服务端操作系统与版本:
|
||||
客户端操作系统与版本:
|
||||
相关资源限制(如有):
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: client-name
|
||||
attributes:
|
||||
label: 客户端名称
|
||||
description: 区分 CLI、桌面端、编辑器扩展、SDK 或自编脚本。
|
||||
placeholder: 例如 Codex CLI、Codex Desktop 或某个编辑器扩展
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: client-version
|
||||
attributes:
|
||||
label: 客户端版本
|
||||
description: 填写完整版本号;Codex CLI 可填写 codex --version 的输出。无法取得时说明原因。
|
||||
placeholder: 完整版本号或相关 SDK 版本
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: client-source
|
||||
attributes:
|
||||
label: 客户端安装来源与启动方式
|
||||
description: 说明安装来源,以及是否通过包装脚本、启动器或第三方修改版运行;不确定时填写已知部分。
|
||||
placeholder: 例如通过 npm 安装 @openai/codex,直接执行 codex
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: endpoint
|
||||
attributes:
|
||||
label: 请求方法与端点
|
||||
description: 填写发生异常的请求方法与路径,无需真实域名;不清楚端点时说明客户端当时执行的具体操作。
|
||||
placeholder: 例如 POST /v1/responses、POST /v1/alpha/search 或 WebSocket /v1/responses
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: intermediary
|
||||
attributes:
|
||||
label: 是否经过中转或反向代理
|
||||
description: |
|
||||
指客户端到本服务之间的链路。API 中转包括 New API、One API、CCH 等;反向代理或 CDN 包括 Nginx、Caddy、Cloudflare Tunnel 等。
|
||||
如有中转,请在下方填写链路详情;本服务到上游的出站代理请在补充信息中说明。
|
||||
options:
|
||||
- 否,直接连接本服务
|
||||
- 是,仅经过 API 中转服务
|
||||
- 是,仅经过反向代理或 CDN
|
||||
- 是,同时经过 API 中转和反向代理或 CDN
|
||||
- 不确定
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-chain
|
||||
attributes:
|
||||
label: 请求链路
|
||||
description: |
|
||||
按调用顺序填写中间服务的名称与版本,例如:Codex → New API → Nginx → 本服务。不确定时填写已知部分。
|
||||
如果已测试过直连本服务,请说明是否仍能复现;未测试可注明,不要求先完成对照测试。
|
||||
placeholder: |
|
||||
调用顺序与各中间服务版本:
|
||||
是否启用请求参数或请求头改写(如已知):
|
||||
直连本服务的对照结果(如已测试):
|
||||
|
||||
- type: textarea
|
||||
id: account-model
|
||||
attributes:
|
||||
label: 平台、账号类型与模型
|
||||
description: 填写与失败请求相关的信息,不要提供账号凭据;不涉及模型或无法确认的信息请注明。
|
||||
placeholder: |
|
||||
平台与认证类型:
|
||||
账号套餐(如相关):
|
||||
请求模型与映射后的上游模型(如已知):
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: 复现步骤
|
||||
description: 描述触发问题的操作;无法稳定复现时,说明已知触发条件。
|
||||
placeholder: |
|
||||
1. 使用什么配置或输入
|
||||
2. 执行什么操作
|
||||
3. 出现什么结果
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-response
|
||||
attributes:
|
||||
label: 请求、响应与诊断附件
|
||||
description: |
|
||||
提供脱敏后的最小请求、网关实际状态/响应片段和客户端显示,区分上游、中转、网关与客户端的错误来源。
|
||||
可在此直接上传管理端默认诊断包并检查其缺失说明;原始错误与日志片段需另行审阅脱敏,粘贴时请使用 Markdown 代码块。
|
||||
按部署文档的“请求错误排查”检查关联 ID、错误详情与 application 日志,不默认开启完整 dump;无法取得的信息请注明。
|
||||
placeholder: |
|
||||
发生时间与时区:
|
||||
传输(HTTP JSON / SSE / WebSocket):
|
||||
脱敏后的最小请求:
|
||||
网关关联 ID(x-gateway-request-id;配置的入口头请单独注明):
|
||||
上游关联 ID(x-request-id / x-oai-request-id,注明来源):
|
||||
网关实际状态、Content-Type 与响应片段:
|
||||
中转服务的状态与错误(如有):
|
||||
客户端显示的错误:
|
||||
诊断包或无法取得的信息:
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 最近有升级时填写升级前后版本及前后端是否同步更新;网络问题补充出站代理、系统代理、TUN 和自定义证书等相关环境。
|
||||
placeholder: |
|
||||
服务或客户端升级前版本 → 当前版本(如有):
|
||||
服务前后端是否同步更新(如有升级):
|
||||
最近的配置变更:
|
||||
其他相关环境:
|
||||
@@ -1,120 +0,0 @@
|
||||
name: 问题反馈
|
||||
description: 提交可复现的问题,帮助定位和修复
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
请提供最小复现信息。请求、日志和截图发布前请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: dropdown
|
||||
id: issue-type
|
||||
attributes:
|
||||
label: 问题类型
|
||||
options:
|
||||
- 接口请求或响应异常
|
||||
- 页面或交互异常
|
||||
- 配置或部署异常
|
||||
- 其他问题
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 问题描述
|
||||
description: 说明发生了什么、实际结果以及出现的频率。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: 服务版本
|
||||
description: 填写具体版本号;从源码运行时填写提交号。
|
||||
placeholder: 版本号或提交号
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: deployment
|
||||
attributes:
|
||||
label: 部署方式
|
||||
options:
|
||||
- Docker Compose
|
||||
- 其他容器部署
|
||||
- 二进制运行
|
||||
- 源码运行
|
||||
- 其他方式(请在环境信息中说明)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 环境信息
|
||||
description: 填写操作系统与版本;页面问题同时填写浏览器与版本。
|
||||
placeholder: |
|
||||
操作系统与版本:
|
||||
浏览器与版本(页面问题):
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: account-client
|
||||
attributes:
|
||||
label: 账号与客户端
|
||||
description: 接口问题请填写平台、认证类型、模型和客户端信息,不要提供账号凭据。
|
||||
placeholder: |
|
||||
平台与认证类型:
|
||||
模型名称:
|
||||
客户端名称与版本:
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: 复现步骤
|
||||
description: 描述触发问题的操作;无法稳定复现时,说明已知触发条件。
|
||||
placeholder: |
|
||||
1. 使用什么配置或输入
|
||||
2. 执行什么操作
|
||||
3. 出现什么结果
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-response
|
||||
attributes:
|
||||
label: 请求与响应
|
||||
description: |
|
||||
接口问题请提供脱敏后的最小请求、网关实际状态/响应片段和客户端终端显示,区分两者。
|
||||
优先附管理端默认诊断包并检查其缺失说明;原始错误与日志片段需另行审阅脱敏。
|
||||
先按部署文档的“请求错误排查”检查关联 ID、错误详情与 application 日志,不默认开启完整 dump。
|
||||
placeholder: |
|
||||
发生时间与时区:
|
||||
传输(HTTP JSON / SSE / WebSocket):
|
||||
网关关联 ID(x-gateway-request-id / 配置的入口头):
|
||||
上游关联 ID(x-request-id / x-oai-request-id,注明来源):
|
||||
网关实际状态与响应片段:
|
||||
客户端终端显示:
|
||||
诊断包或无法取得的信息:
|
||||
render: text
|
||||
|
||||
- type: textarea
|
||||
id: screenshots
|
||||
attributes:
|
||||
label: 截图或录屏
|
||||
description: 页面或交互问题可在此拖入相关截图、录屏,并说明对应的操作。
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 最近的升级或配置变更;网络问题补充系统代理、TUN 和自定义证书等相关环境。
|
||||
@@ -0,0 +1,39 @@
|
||||
name: 功能建议
|
||||
description: 说明使用场景、当前不便和期望行为,提出新能力或体验改进
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
如果已有功能无法正常使用,请选择对应的故障模板。提交前可搜索已有 Issues,相关讨论请附链接。
|
||||
每个 Issue 尽量围绕一个使用场景;截图和示例请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: textarea
|
||||
id: use-case
|
||||
attributes:
|
||||
label: 使用场景与当前不便
|
||||
description: 说明你希望完成什么任务、目前如何操作,以及现有能力有什么不足。
|
||||
placeholder: |
|
||||
希望完成的任务:
|
||||
当前做法与不便:
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 期望行为
|
||||
description: 描述改进后希望如何使用,最好给出一个具体例子;不要求先设计技术方案。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: 已尝试的替代方案
|
||||
description: 如果已有配置、操作方式或其他工具能部分满足需求,说明仍缺少什么;未尝试可留空。
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 可附当前使用的具体版本、相关 Issue、截图或参考链接;引用其他项目时说明希望借鉴的具体行为。
|
||||
@@ -0,0 +1,143 @@
|
||||
name: 常规问题反馈
|
||||
description: 管理端页面、账号导入、安装部署、升级或其他非客户端 API 问题
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
客户端调用模型、搜索等 API 的问题请选择“接口问题反馈”;新增能力或体验改进请选择“功能建议”。
|
||||
请提供最小复现信息。请求、日志和截图发布前请移除密钥、令牌、Cookie、代理认证及私人内容。
|
||||
|
||||
- type: dropdown
|
||||
id: issue-type
|
||||
attributes:
|
||||
label: 问题类型
|
||||
options:
|
||||
- 页面或交互异常
|
||||
- 配置或部署异常
|
||||
- 升级异常
|
||||
- 其他问题
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 问题描述
|
||||
description: 说明发生了什么、实际结果以及出现的频率。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: 服务版本
|
||||
description: 填写实际运行的完整版本号,不要只写“最新”或 latest;源码运行填写提交号。服务无法启动时填写准备运行的版本。
|
||||
placeholder: 完整版本号或提交号
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: deployment
|
||||
attributes:
|
||||
label: 部署方式
|
||||
options:
|
||||
- Docker Compose
|
||||
- 其他容器部署
|
||||
- 二进制运行
|
||||
- 源码运行
|
||||
- 其他方式(请在环境信息中说明)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 环境信息
|
||||
description: 填写操作系统与版本,并补充与问题相关的浏览器、访问方式或资源信息。
|
||||
placeholder: |
|
||||
操作系统与版本:
|
||||
浏览器与版本(页面问题):
|
||||
页面访问方式(HTTP / HTTPS,localhost / IP / 域名,无需真实地址):
|
||||
CPU 核数与内存、容器资源限制(部署或性能问题):
|
||||
Docker / Compose 版本(容器部署问题):
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: intermediary
|
||||
attributes:
|
||||
label: 是否经过中转或反向代理
|
||||
description: |
|
||||
指客户端或浏览器到本服务之间的链路。API 中转包括 New API、One API、CCH 等;反向代理或 CDN 包括 Nginx、Caddy、Cloudflare Tunnel 等。
|
||||
如有中转,请在下方填写链路详情;本服务到上游的出站代理请在补充信息中说明。
|
||||
options:
|
||||
- 否,直接连接本服务
|
||||
- 是,仅经过 API 中转服务
|
||||
- 是,仅经过反向代理或 CDN
|
||||
- 是,同时经过 API 中转和反向代理或 CDN
|
||||
- 不确定
|
||||
- 不适用(例如服务尚未启动)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: request-chain
|
||||
attributes:
|
||||
label: 请求链路
|
||||
description: |
|
||||
按调用顺序填写中间服务的名称与版本,例如:浏览器 → Cloudflare → Nginx → 本服务。不确定时填写已知部分。
|
||||
如果已测试过直连本服务,请说明是否仍能复现;未测试可注明,不要求先完成对照测试。
|
||||
placeholder: |
|
||||
调用顺序与各中间服务版本:
|
||||
是否启用请求参数或请求头改写(如已知):
|
||||
直连本服务的对照结果(如已测试):
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: 复现步骤
|
||||
description: 描述触发问题的操作;批量操作补充数量,升级问题说明升级方式。无法稳定复现时,说明已知触发条件。
|
||||
placeholder: |
|
||||
1. 使用什么配置或输入
|
||||
2. 执行什么操作
|
||||
3. 出现什么结果
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: 错误信息与日志
|
||||
description: |
|
||||
提供脱敏后的页面提示、浏览器控制台错误或相关启动、升级日志;如有失败请求,补充状态码与响应片段。
|
||||
可以在此上传诊断包或日志附件;粘贴日志时请使用 Markdown 代码块。不要上传完整配置或请求转储。
|
||||
placeholder: |
|
||||
发生时间与时区:
|
||||
页面提示或终端错误:
|
||||
浏览器控制台 / 服务日志:
|
||||
失败请求的状态码与响应片段(如有):
|
||||
|
||||
- type: textarea
|
||||
id: screenshots
|
||||
attributes:
|
||||
label: 截图或录屏
|
||||
description: 页面或交互问题可在此拖入相关截图、录屏,并说明对应的操作。
|
||||
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
attributes:
|
||||
label: 补充信息
|
||||
description: 最近有升级时填写升级前后版本及前后端是否同步更新;网络问题补充出站代理、系统代理、TUN 和自定义证书等相关环境。
|
||||
placeholder: |
|
||||
升级前版本 → 当前版本(如有):
|
||||
前后端是否同步更新(如有升级):
|
||||
最近的配置变更:
|
||||
其他相关环境:
|
||||
+2
-2
@@ -6,9 +6,9 @@
|
||||
|
||||
## 提交 Issue
|
||||
|
||||
问题反馈使用仓库的 [Issue 表单](.github/ISSUE_TEMPLATE/bug_report.yml),填写问题类型、描述、版本与环境、复现步骤和预期结果。接口问题补充平台与认证类型、模型、客户端,以及脱敏后的请求、响应和请求 ID;页面问题附相关截图或录屏。
|
||||
按场景选择 [接口问题反馈](.github/ISSUE_TEMPLATE/api-bug-report.yml)、[常规问题反馈](.github/ISSUE_TEMPLATE/general-bug-report.yml) 或 [功能建议](.github/ISSUE_TEMPLATE/feature-request.yml) 表单,按对应提示提供信息。故障反馈填写实际运行版本和中转链路;接口问题区分客户端显示与网关实际响应,页面问题附相关截图或录屏。
|
||||
|
||||
日志和配置只提供定位所需片段,不上传真实凭据或完整请求转储。功能建议与协作规则调整可直接提交 Issue,说明场景、原因和建议方案。
|
||||
日志和配置只提供定位所需片段,不上传真实凭据或完整请求转储。功能建议说明使用场景、当前不便和期望行为;协作规则调整可直接提交普通 Issue,说明原因和建议方案。
|
||||
|
||||
## 问题与方案依据
|
||||
|
||||
|
||||
@@ -25,64 +25,56 @@
|
||||
以下命令适用于 Linux amd64/arm64,需要 Docker Engine、Docker Compose Plugin、curl 和 OpenSSL。已有部署请先看
|
||||
[升级说明](deploy/README.md#镜像升级与源码构建),不要覆盖原配置。
|
||||
|
||||
### 1. 下载部署文件并配置
|
||||
### 一键安装
|
||||
|
||||
请确保当前用户能访问 Docker,并可通过 `sudo` 或 root 设置目录权限。
|
||||
|
||||
```bash
|
||||
mkdir -p codex-proxy-rs/deploy && cd codex-proxy-rs
|
||||
|
||||
# 只解析一次最新正式版本,确保两个文件来自同一 Release。
|
||||
CPR_RELEASE_URL="$(curl -fsSL -o /dev/null -w '%{url_effective}' https://github.com/zyycn/codex-proxy-rs/releases/latest)"
|
||||
CPR_RELEASE_TAG="${CPR_RELEASE_URL##*/}"
|
||||
curl -fsSL "https://github.com/zyycn/codex-proxy-rs/releases/download/${CPR_RELEASE_TAG}/compose.yaml" \
|
||||
-o deploy/compose.yaml
|
||||
curl -fsSL "https://github.com/zyycn/codex-proxy-rs/releases/download/${CPR_RELEASE_TAG}/config.example.yaml" \
|
||||
-o deploy/config.example.yaml
|
||||
|
||||
install -d -m 0750 .runtime/postgres .runtime/redis
|
||||
sudo install -d -m 0770 -o "$(id -u)" -g 10001 .runtime/data .runtime/logs
|
||||
sudo install -m 0640 -o "$(id -u)" -g 10001 deploy/config.example.yaml deploy/config.yaml
|
||||
curl -fsSL https://raw.githubusercontent.com/zyycn/codex-proxy-rs/main/deploy/install.sh -o install.sh && bash install.sh
|
||||
```
|
||||
|
||||
也可将 `CPR_RELEASE_TAG` 设置为指定的发布标签。Release 附带的 `compose.yaml` 默认使用该版本镜像,
|
||||
配置模板和部署文件均包含在 `checksums.txt` 中;不要混用 `main` 分支模板与已发布镜像。
|
||||
[安装脚本](deploy/install.sh) 默认安装到当前目录下的 `codex-proxy-rs/`,下载同一正式 Release 的部署文件,
|
||||
自动生成密码、设置目录权限并启动服务。完成后会显示访问地址和管理员密码,请保存密码,再按下方步骤
|
||||
[添加账号与客户端密钥](#添加账号与客户端密钥)。
|
||||
|
||||
分别生成数据库和 Redis 密码:
|
||||
可在执行 `bash install.sh` 时传入环境变量:
|
||||
|
||||
| 变量 | 用途 | 默认值 |
|
||||
| --- | --- | --- |
|
||||
| `INSTALL_DIR` | 安装目录,建议使用绝对路径 | 当前目录下的 `codex-proxy-rs/` |
|
||||
| `CPR_RELEASE_TAG` | 指定发布标签 | 最新正式版本 |
|
||||
| `ADMIN_PASSWORD` | 管理员初始密码,至少 12 位,不能包含 `$`,不能使用常见弱口令 | 随机生成 |
|
||||
|
||||
例如,自定义安装目录:
|
||||
|
||||
```bash
|
||||
openssl rand -hex 24
|
||||
openssl rand -hex 24
|
||||
INSTALL_DIR="$HOME/services/codex-proxy-rs" bash install.sh
|
||||
```
|
||||
|
||||
编辑 `deploy/config.yaml`,填好以下三项:
|
||||
重复运行时请使用同一安装目录。检测到 `deploy/config.yaml` 后,脚本保留现有配置和部署文件,
|
||||
忽略传入的管理员密码,也不执行版本升级。
|
||||
|
||||
| 配置项 | 填写内容 |
|
||||
| --- | --- |
|
||||
| `store.database.password` | 第一个生成的 48 位十六进制密码 |
|
||||
| `store.redis.password` | 第二个生成的 48 位十六进制密码 |
|
||||
| `admin.default_password` | 管理员初始密码,至少 12 位,不能包含 `$` |
|
||||
### 手动安装
|
||||
|
||||
### 2. 启动服务
|
||||
自行下载部署文件、配置密码和启动服务的完整步骤见 [部署文档](deploy/README.md#手动安装)。
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/compose.yaml config --quiet
|
||||
docker compose -f deploy/compose.yaml pull
|
||||
docker compose -f deploy/compose.yaml up -d --no-build --wait
|
||||
curl -i http://127.0.0.1:8080/healthz
|
||||
```
|
||||
### 登录管理端
|
||||
|
||||
健康检查返回 `204 No Content` 后,打开 `http://127.0.0.1:8080`,
|
||||
使用 `admin@cpr.local` 和刚设置的管理员密码登录。
|
||||
部署完成后,打开 `http://127.0.0.1:8080`,使用 `admin@cpr.local` 和管理员密码登录。
|
||||
API Key 持有者可在同一登录页切换登录身份,进入 `/key-usage` 查看自己的用量、趋势、请求日志、额度与健康时间线;不能访问管理员页面。
|
||||
|
||||
默认地址只能在服务器本机访问。从其他设备使用时,需要配置
|
||||
[HTTPS 反向代理](deploy/README.md#公网访问)。
|
||||
|
||||
### 3. 添加账号与客户端密钥
|
||||
### 添加账号与客户端密钥
|
||||
|
||||
1. 在「账号」中添加账号,完成授权或导入。
|
||||
2. 按需建立账号分组,再创建客户端密钥并选择可用分组。**不选分组表示可使用全部账号**。
|
||||
3. 打开密钥的「使用密钥」,复制客户端配置。
|
||||
|
||||
账号自动冻结默认关闭,可在「系统设置 → 运行参数」中启用。启用后,账号频繁命中上游容量错误时
|
||||
会暂停调度并显示为限流中;若开启恢复前探测,冷却到期后需探测成功才恢复。
|
||||
|
||||
## 客户端接入
|
||||
|
||||
**Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,或通过 CCSwitch 导入。
|
||||
|
||||
Generated
+13
@@ -1491,6 +1491,7 @@ dependencies = [
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.29.0",
|
||||
@@ -1509,6 +1510,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono-tz",
|
||||
"futures",
|
||||
"futures-timer",
|
||||
"semver",
|
||||
@@ -1539,6 +1541,7 @@ dependencies = [
|
||||
"scraper",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_ignored",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"tar",
|
||||
@@ -3344,6 +3347,16 @@ dependencies = [
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_ignored"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "115dffd5f3853e06e746965a20dcbae6ee747ae30b543d91b0e089668bb07798"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
|
||||
@@ -7,8 +7,8 @@ use serde::Deserialize;
|
||||
const CONFIG_SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
/// 顶层配置只组合各包拥有的配置段,不解释任何业务字段。
|
||||
/// 各启动配置忽略未知字段,允许升级时保留旧配置;已知字段仍按类型和业务约束校验。
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GatewayConfig {
|
||||
schema_version: u32,
|
||||
host: HostConfig,
|
||||
@@ -19,16 +19,20 @@ pub struct GatewayConfig {
|
||||
api: gateway_api::ApiConfig,
|
||||
openai: provider_openai::OpenAiConfig,
|
||||
xai: provider_xai::XaiConfig,
|
||||
#[serde(default, rename = "services")]
|
||||
_compose_services: Option<serde::de::IgnoredAny>,
|
||||
}
|
||||
|
||||
impl LoadableConfig for GatewayConfig {
|
||||
const EXTERNAL_SECTIONS: &'static [&'static str] = &["services"];
|
||||
|
||||
fn resolve_and_validate(&mut self, source_dir: &std::path::Path) -> Result<(), ConfigError> {
|
||||
if self.schema_version != CONFIG_SCHEMA_VERSION {
|
||||
return Err(ConfigError::InvalidField("schema_version"));
|
||||
}
|
||||
self.host.resolve_and_validate(source_dir)?;
|
||||
self.api
|
||||
.resolve_and_validate(source_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("api"))?;
|
||||
self.host
|
||||
.resolve_and_validate(source_dir, &self.api.asset_directory)?;
|
||||
let runtime_data_dir = self.host.runtime_data_dir().to_path_buf();
|
||||
self.store
|
||||
.resolve_and_validate(&runtime_data_dir)
|
||||
@@ -39,9 +43,6 @@ impl LoadableConfig for GatewayConfig {
|
||||
self.client
|
||||
.resolve_and_validate(source_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("client"))?;
|
||||
self.api
|
||||
.resolve_and_validate(source_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("api"))?;
|
||||
self.openai
|
||||
.resolve_and_validate(&runtime_data_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("openai"))?;
|
||||
@@ -64,7 +65,6 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
api,
|
||||
openai,
|
||||
xai,
|
||||
_compose_services: _,
|
||||
} = config;
|
||||
|
||||
let host = gateway_host::initialize(host).await?;
|
||||
|
||||
@@ -168,6 +168,50 @@ fn config_loader_should_resolve_paths_relative_to_config_file() {
|
||||
assert!(debug.contains("frontend/dist"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_share_resolved_assets_with_system_update() {
|
||||
const CHILD_ENV: &str = "CPR_TEST_UPDATE_ASSETS_CHILD";
|
||||
let Ok(case) = std::env::var(CHILD_ENV) else {
|
||||
// 使用子进程覆盖无环境变量、Docker 路径和相对路径,避免污染并行测试。
|
||||
for (case, web_dist) in [
|
||||
("binary", None),
|
||||
("docker", Some("/app/web/dist")),
|
||||
("relative", Some("../custom/dist")),
|
||||
] {
|
||||
let mut child = Command::new(std::env::current_exe().expect("test executable"));
|
||||
child
|
||||
.args([
|
||||
"--exact",
|
||||
"bootstrap::config_loader_should_share_resolved_assets_with_system_update",
|
||||
])
|
||||
.env(CHILD_ENV, case)
|
||||
.env_remove("CPR_WEB_DIST_DIR");
|
||||
if let Some(web_dist) = web_dist {
|
||||
child.env("CPR_WEB_DIST_DIR", web_dist);
|
||||
}
|
||||
let output = child.output().expect("isolated configuration test");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{case}: {}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
return;
|
||||
};
|
||||
let mut document = valid_config_document();
|
||||
document["api"]["asset_directory"] = serde_json::json!("../web/dist");
|
||||
let (config, directory) = parse_config(&document.to_string()).expect("binary configuration");
|
||||
let assets = match case.as_str() {
|
||||
"docker" => std::path::PathBuf::from("/app/web/dist"),
|
||||
"relative" => directory.path().join("deploy/../custom/dist"),
|
||||
_ => directory.path().join("deploy/../web/dist"),
|
||||
};
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains(&format!("asset_directory: {assets:?}")));
|
||||
assert!(debug.contains(&format!("web_dist_dir: Some({assets:?})")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_missing_runtime_data_dir() {
|
||||
let config = valid_config().replace(" runtime_data_dir: '../.runtime/data'\n", "");
|
||||
@@ -228,16 +272,107 @@ fn bootstrap_config_debug_should_redact_all_passwords() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_unknown_fields() {
|
||||
assert_rejected(format!(
|
||||
"{}\nunknown_terminal_field: true\n",
|
||||
valid_config()
|
||||
));
|
||||
fn config_loader_should_ignore_unknown_fields_in_configuration_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["host"]["system_update"] = serde_json::json!({});
|
||||
document["store"]["pool"] = serde_json::json!({});
|
||||
for path in [
|
||||
"",
|
||||
"/host",
|
||||
"/host/listen",
|
||||
"/host/logging",
|
||||
"/host/logging/file",
|
||||
"/host/system_update",
|
||||
"/store",
|
||||
"/store/database",
|
||||
"/store/redis",
|
||||
"/store/pool",
|
||||
"/admin",
|
||||
"/client",
|
||||
"/api",
|
||||
"/openai",
|
||||
"/openai/wire_profile",
|
||||
"/xai",
|
||||
"/xai/wire_profile",
|
||||
] {
|
||||
let mut extended = document.clone();
|
||||
extended.pointer_mut(path).expect("configuration section")["unused_setting"] =
|
||||
serde_json::json!({"nested": [true, null, "ignored"]});
|
||||
parse_config(&extended.to_string())
|
||||
.unwrap_or_else(|error| panic!("unknown field in {path}: {error}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_removed_tls_section() {
|
||||
assert_rejected(valid_config().replace("openai:\n", "openai:\n tls: {}\n"));
|
||||
fn config_loader_should_ignore_removed_tls_and_fingerprint_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["tls"] = serde_json::json!({});
|
||||
document["openai"]["fingerprint"] = serde_json::json!({"browser": "removed"});
|
||||
parse_config(&document.to_string()).expect("removed provider settings are ignored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_known_fields_alongside_unknown_fields() {
|
||||
for port in [serde_json::json!(0), serde_json::json!("not-a-port")] {
|
||||
let mut document = valid_config_document();
|
||||
document["host"]["listen"]["unused_setting"] = serde_json::json!(true);
|
||||
document["host"]["listen"]["port"] = port;
|
||||
assert_rejected(document.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_report_startup_configuration_diagnostics() {
|
||||
const CHILD_ENV: &str = "CPR_TEST_CONFIG_DIAGNOSTICS_CHILD";
|
||||
const UNUSED_SECRET: &str = "unused-value-must-not-appear-in-diagnostics";
|
||||
if std::env::var_os(CHILD_ENV).is_some() {
|
||||
gateway_host::load_config::<GatewayConfig>().expect("startup configuration");
|
||||
return;
|
||||
}
|
||||
for case in ["normal", "unused", "missing"] {
|
||||
let mut document = valid_config_document();
|
||||
if case == "unused" {
|
||||
document["openai"]["wire_profile"]["location"] = serde_json::json!(UNUSED_SECRET);
|
||||
} else if case == "missing" {
|
||||
document["host"]["listen"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("port");
|
||||
}
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
fs::create_dir(directory.path().join("deploy")).unwrap();
|
||||
fs::write(
|
||||
directory.path().join("deploy/config.yaml"),
|
||||
document.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let output = Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"bootstrap::config_loader_should_report_startup_configuration_diagnostics",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_ENV, "1")
|
||||
.current_dir(directory.path())
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert_eq!(
|
||||
output.status.success(),
|
||||
case != "missing",
|
||||
"{case}: {stderr}"
|
||||
);
|
||||
match case {
|
||||
"unused" => assert!(
|
||||
stderr.contains("openai.wire_profile.location") && stderr.contains("已忽略"),
|
||||
"{stderr}"
|
||||
),
|
||||
"missing" => assert!(stderr.contains("host.listen.port"), "{stderr}"),
|
||||
_ => assert!(!stderr.contains("警告"), "{stderr}"),
|
||||
}
|
||||
assert!(!stderr.contains(UNUSED_SECRET), "{stderr}");
|
||||
assert!(!stderr.contains("services"), "{stderr}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -309,14 +444,6 @@ fn config_loader_should_reject_missing_client_session_ttl() {
|
||||
assert_rejected(config.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
assert_rejected(valid_config().replace(
|
||||
"openai:\n",
|
||||
"openai:\n fingerprint:\n browser: removed\n",
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.153.4'", "codex_version: 'latest'"));
|
||||
@@ -342,36 +469,22 @@ fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_accept_empty_and_custom_request_locations() {
|
||||
fn config_loader_should_ignore_removed_request_location_setting() {
|
||||
let original = valid_config();
|
||||
let omitted = original
|
||||
.lines()
|
||||
.filter(|line| !line.trim_start().starts_with("location:"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
assert_ne!(original, omitted);
|
||||
parse_config(&omitted).expect("location passthrough when omitted");
|
||||
let location_line = original
|
||||
.lines()
|
||||
.find(|line| line.trim_start().starts_with("location:"))
|
||||
.expect("example location");
|
||||
for empty in [" location:", " location: null", " location: ~"] {
|
||||
parse_config(&original.replace(location_line, empty)).expect("empty YAML location");
|
||||
parse_config(&original).expect("location belongs to runtime settings");
|
||||
for value in [
|
||||
"null",
|
||||
"{}",
|
||||
"{ country: 'US', region: 'Ohio', city: 'Piketon', timezone: 'America/New_York' }",
|
||||
] {
|
||||
let legacy = original.replacen(
|
||||
" residency: us",
|
||||
&format!(" residency: us\n location: {value}"),
|
||||
1,
|
||||
);
|
||||
assert_ne!(legacy, original);
|
||||
parse_config(&legacy).expect("removed request location does not prevent startup");
|
||||
}
|
||||
let custom = original.replace(
|
||||
"location: { country: 'US', region: 'Ohio', city: 'Piketon', timezone: 'America/New_York' }",
|
||||
"location: { country: 'NZ', region: 'Auckland', city: 'Auckland', timezone: 'Pacific/Auckland' }",
|
||||
);
|
||||
assert_ne!(original, custom);
|
||||
parse_config(&custom).expect("custom location from YAML");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_request_location_timezones() {
|
||||
let original = valid_config();
|
||||
let invalid = original.replace("timezone: 'America/New_York'", "timezone: 'Not/A_Timezone'");
|
||||
assert_ne!(original, invalid);
|
||||
assert_rejected(invalid);
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -36,7 +36,7 @@ tokio = { version = "1.52", features = ["rt", "time", "macros", "sync"] }
|
||||
[dev-dependencies]
|
||||
sha2 = "0.11"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt", "test-util"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
//! 容量熔断冻结的恢复编排:到期探测、失败顺延与自适应并发下调。
|
||||
//!
|
||||
//! 冻结事实保存在可丢失的 Redis 冷却中,本服务只组合 Admin 端口:
|
||||
//! 观测(`AccountRuntimeStore`)、探测(`AccountsService`)与写回
|
||||
//! (原子降低并发 / 按冻结代次结束探测)。旧观测不会覆盖管理员操作。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::account::{AccountConcurrencyLimit, ProviderAccountId};
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::model::accounts::{
|
||||
AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot,
|
||||
};
|
||||
use crate::model::{MutationActor, MutationContext};
|
||||
use crate::ports::store::{AccountRuntimeStore, AccountStore, SettingsStore};
|
||||
use crate::use_case::accounts::AccountsService;
|
||||
|
||||
/// 自适应并发下调保留系数:下调到观测在途峰值的 80%。
|
||||
const ADAPTIVE_CONCURRENCY_FACTOR: f64 = 0.8;
|
||||
/// 自适应并发下限:过低的并发让账号几乎不可用,宁可保持冻结。
|
||||
const ADAPTIVE_CONCURRENCY_FLOOR: u32 = 2;
|
||||
/// worker 周期;冷却观测与探测都依赖该节奏。
|
||||
pub const FREEZE_RECOVERY_INTERVAL: Duration = Duration::from_secs(30);
|
||||
pub const FREEZE_RECOVERY_WORKER_OWNER: &str = "account-freeze-recovery";
|
||||
pub const WORKER_INITIAL_BACKOFF: Duration = Duration::from_secs(1);
|
||||
pub const WORKER_MAXIMUM_BACKOFF: Duration = Duration::from_secs(60);
|
||||
pub const WORKER_LEASE_TTL: Duration = Duration::from_secs(15 * 60);
|
||||
pub const WORKER_LEASE_RENEWAL: Duration = Duration::from_secs(5 * 60);
|
||||
/// 系统审计上下文的固定请求 ID;账号并发更新审计由此溯源到自动冻结编排。
|
||||
const SYSTEM_REQUEST_ID: &str = "account-freeze-recovery";
|
||||
|
||||
/// 冻结恢复所需的管理端口组合。
|
||||
#[derive(Clone)]
|
||||
pub struct FreezeRecoveryDeps {
|
||||
pub accounts: Arc<dyn AccountsService>,
|
||||
pub store: Arc<dyn AccountStore>,
|
||||
pub runtime: Arc<dyn AccountRuntimeStore>,
|
||||
pub settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
pub struct FreezeRecoveryTask {
|
||||
deps: FreezeRecoveryDeps,
|
||||
}
|
||||
|
||||
impl FreezeRecoveryTask {
|
||||
pub fn new(deps: FreezeRecoveryDeps) -> Self {
|
||||
Self { deps }
|
||||
}
|
||||
|
||||
fn system_context() -> MutationContext {
|
||||
MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: SYSTEM_REQUEST_ID.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 读取冻结策略;读取或校验失败跳过本轮,不能把未知配置当作允许解冻。
|
||||
async fn freeze_policy(&self) -> Option<gateway_core::provider_ports::ProviderFreezePolicy> {
|
||||
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
|
||||
gateway_core::provider_ports::ProviderFreezePolicy::try_new(
|
||||
settings.account_auto_freeze_enabled,
|
||||
settings.account_auto_freeze_threshold,
|
||||
settings.account_auto_freeze_window_seconds,
|
||||
settings.account_auto_freeze_duration_seconds,
|
||||
settings.account_auto_freeze_probe_enabled,
|
||||
settings.account_auto_freeze_probe_model,
|
||||
settings.account_auto_freeze_adaptive_concurrency,
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
async fn active_freezes(&self) -> BTreeMap<String, AccountFreeze> {
|
||||
self.deps.runtime.active_freezes().await.unwrap_or_default()
|
||||
}
|
||||
|
||||
async fn run_cycle_inner(&self) {
|
||||
let Some(policy) = self.freeze_policy().await else {
|
||||
return;
|
||||
};
|
||||
let freezes = self.active_freezes().await;
|
||||
if freezes.is_empty() {
|
||||
return;
|
||||
}
|
||||
if policy.enabled() && policy.adaptive_concurrency() {
|
||||
self.adapt_concurrency_limits(&freezes, &policy).await;
|
||||
}
|
||||
self.recover_due_freezes(&freezes, &policy).await;
|
||||
}
|
||||
|
||||
/// 对冻结中的账号执行自适应并发下调:目标为观测在途峰值的 80%(下限 2),
|
||||
/// 只降不升;未观测到峰值证据的账号保持现状。
|
||||
async fn adapt_concurrency_limits(
|
||||
&self,
|
||||
freezes: &BTreeMap<String, AccountFreeze>,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let account_ids = freezes.keys().cloned().collect::<Vec<_>>();
|
||||
let Ok(peaks) = self.deps.runtime.capacity_peaks(&account_ids).await else {
|
||||
return;
|
||||
};
|
||||
for (account_id, peak) in peaks {
|
||||
let Some(target) = adaptive_target(peak) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(account) = ProviderAccountId::new(account_id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(limit) = AccountConcurrencyLimit::new(target) else {
|
||||
continue;
|
||||
};
|
||||
match self
|
||||
.deps
|
||||
.accounts
|
||||
.lower_concurrency_limit(&Self::system_context(), account, limit)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {
|
||||
tracing::info!(
|
||||
account_id,
|
||||
adapted_limit = target,
|
||||
freeze_seconds = policy.freeze_duration().as_secs(),
|
||||
"容量熔断:按观测在途峰值下调账号并发上限",
|
||||
);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
warn!(account_id, error = %error, "自适应并发下调失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> Option<Option<AccountPageItem>> {
|
||||
let account_id = ProviderAccountId::new(account_id.to_owned()).ok()?;
|
||||
self.deps
|
||||
.store
|
||||
.load_account(account_id.as_str(), AccountRuntimeSnapshot::default())
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// 到达探测时间后才恢复;关闭自动冻结或探测时,已有冻结仍等待原冷却结束。
|
||||
async fn recover_due_freezes(
|
||||
&self,
|
||||
freezes: &BTreeMap<String, AccountFreeze>,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
for (account_id, freeze) in freezes {
|
||||
if freeze.until > Utc::now() {
|
||||
continue;
|
||||
}
|
||||
if policy.enabled() && policy.probe_enabled() && freeze.requires_probe {
|
||||
self.probe_and_recover(account_id, freeze, policy).await;
|
||||
} else if let Err(error) = self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account_id, freeze, None)
|
||||
.await
|
||||
{
|
||||
warn!(account_id, error = %error, "解除到期冻结失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_and_recover(
|
||||
&self,
|
||||
account_id: &str,
|
||||
freeze: &AccountFreeze,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let Some(Some(item)) = self.load_account(account_id).await else {
|
||||
return;
|
||||
};
|
||||
if !item.account.enabled {
|
||||
// 停用账号没有自动恢复意义;解冻交给管理员手动恢复。
|
||||
return;
|
||||
}
|
||||
let Ok(account) = ProviderAccountId::new(account_id.to_owned()) else {
|
||||
return;
|
||||
};
|
||||
let Some(model) = self.resolve_probe_model(&account, policy).await else {
|
||||
// 没有可用探测模型时按失败处理,顺延冻结等待下一轮。
|
||||
self.postpone(&account, freeze, policy).await;
|
||||
return;
|
||||
};
|
||||
let probe_succeeded = match self
|
||||
.deps
|
||||
.accounts
|
||||
.test_connection(account.clone(), model)
|
||||
.await
|
||||
{
|
||||
Ok(events) => drain_probe(events).await,
|
||||
Err(error) => {
|
||||
tracing::debug!(account_id, error = %error, "冻结恢复探测发起失败");
|
||||
false
|
||||
}
|
||||
};
|
||||
if probe_succeeded {
|
||||
match self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account_id, freeze, None)
|
||||
.await
|
||||
{
|
||||
Ok(true) => tracing::info!(account_id, "冻结恢复探测成功:账号已解冻"),
|
||||
Ok(false) => {}
|
||||
Err(error) => warn!(account_id, error = %error, "解除冻结失败"),
|
||||
}
|
||||
} else {
|
||||
self.postpone(&account, freeze, policy).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// 探测失败只顺延本次仍存在的冻结,旧探测不能重新冻结已手动恢复的账号。
|
||||
async fn postpone(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
freeze: &AccountFreeze,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let Some(until) = SystemTime::now().checked_add(policy.freeze_duration()) else {
|
||||
return;
|
||||
};
|
||||
let until = DateTime::<Utc>::from(until);
|
||||
match self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account.as_str(), freeze, Some(until))
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
tracing::info!(
|
||||
account_id = account.as_str(),
|
||||
postpone_seconds = policy.freeze_duration().as_secs(),
|
||||
"冻结恢复探测失败:账号冻结顺延",
|
||||
);
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
account_id = account.as_str(),
|
||||
error = %error,
|
||||
"冻结顺延失败"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn resolve_probe_model(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) -> Option<UpstreamModelId> {
|
||||
if let Some(model) = policy.probe_model() {
|
||||
return UpstreamModelId::new(model.to_owned()).ok();
|
||||
}
|
||||
let models = self.deps.accounts.models(account, false).await.ok()?;
|
||||
models.models.into_iter().next().map(|model| model.id)
|
||||
}
|
||||
}
|
||||
|
||||
impl ScheduledTask for FreezeRecoveryTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.run_cycle_inner().await;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 自适应目标并发:观测峰值 × 0.8 向下取整,下限 2;无证据(峰值为 0)不调整。
|
||||
fn adaptive_target(peak_in_flight: u32) -> Option<u32> {
|
||||
if peak_in_flight == 0 {
|
||||
return None;
|
||||
}
|
||||
let scaled = (f64::from(peak_in_flight) * ADAPTIVE_CONCURRENCY_FACTOR).floor() as u32;
|
||||
Some(scaled.max(ADAPTIVE_CONCURRENCY_FLOOR))
|
||||
}
|
||||
|
||||
/// 消耗连接测试事件流并返回是否以 `Completed` 终止。
|
||||
async fn drain_probe(mut events: crate::model::accounts::AccountConnectionTestEventStream) -> bool {
|
||||
let mut completed = false;
|
||||
while let Some(event) = events.next().await {
|
||||
match event {
|
||||
AccountConnectionTestEvent::Completed => completed = true,
|
||||
AccountConnectionTestEvent::Failed { .. } => return false,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
completed
|
||||
}
|
||||
@@ -10,14 +10,15 @@ use gateway_core::{
|
||||
routing::ProviderKind,
|
||||
runtime::SnapshotControl,
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration,
|
||||
WorkerRunnable,
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest,
|
||||
WorkerRegistration, WorkerRunnable, WorkerSchedule,
|
||||
},
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
mod use_case;
|
||||
@@ -27,8 +28,9 @@ pub use use_case::key_usage::KeyUsageService;
|
||||
pub use use_case::{
|
||||
account_groups::AccountGroupService, accounts::AccountsService, auth::AuthService,
|
||||
backup::BackupService, client_distribution::ClientDistributionService,
|
||||
client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService,
|
||||
proxies::ProxiesService, settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
client_keys::ClientKeyService, import_tasks::ImportTasksService,
|
||||
observability::ObservabilityService, openai::OpenAiService, proxies::ProxiesService,
|
||||
settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
};
|
||||
|
||||
use model::{AdminError, AdminErrorKind};
|
||||
@@ -95,7 +97,6 @@ impl fmt::Debug for InitialAdminPassword {
|
||||
|
||||
/// 管理控制面的启动配置。
|
||||
#[derive(Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AdminConfig {
|
||||
pub session_ttl_minutes: u64,
|
||||
pub default_username: String,
|
||||
@@ -104,7 +105,6 @@ pub struct AdminConfig {
|
||||
|
||||
/// Client 登录域的通用启动配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ClientConfig {
|
||||
pub session_ttl_minutes: u64,
|
||||
}
|
||||
@@ -195,14 +195,26 @@ pub struct AdminServices {
|
||||
openai: Arc<dyn OpenAiService>,
|
||||
xai: Arc<dyn XaiService>,
|
||||
backups: Arc<dyn BackupService>,
|
||||
import_tasks: Arc<dyn ImportTasksService>,
|
||||
}
|
||||
|
||||
impl AdminServices {
|
||||
#[must_use]
|
||||
pub fn import_tasks(&self) -> &dyn ImportTasksService {
|
||||
self.import_tasks.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn key_usage(&self) -> &dyn KeyUsageService {
|
||||
self.key_usage.as_ref()
|
||||
}
|
||||
|
||||
/// 取得账号服务的共享句柄;后台编排(冻结恢复 worker)需要持有 Arc。
|
||||
#[must_use]
|
||||
pub fn accounts_handle(&self) -> Arc<dyn AccountsService> {
|
||||
Arc::clone(&self.accounts)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn proxies(&self) -> &dyn ProxiesService {
|
||||
self.proxies.as_ref()
|
||||
@@ -276,7 +288,7 @@ impl AdminBundle {
|
||||
self.services.clone()
|
||||
}
|
||||
|
||||
/// 取出 Backup Worker 贡献;只能调用一次,与其它 Bundle 的贡献一并交给 Host。
|
||||
/// 取出 Admin Worker 贡献;只能调用一次,与其它 Bundle 的贡献一并交给 Host。
|
||||
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
|
||||
std::mem::take(&mut self.worker_contributions)
|
||||
}
|
||||
@@ -361,6 +373,21 @@ pub async fn initialize(
|
||||
store.client_keys(),
|
||||
store.observability(),
|
||||
));
|
||||
let openai = Arc::new(DefaultOpenAiService::new(
|
||||
openai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
));
|
||||
let xai = Arc::new(DefaultXaiService::new(
|
||||
xai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
));
|
||||
let import_tasks =
|
||||
use_case::import_tasks::DefaultImportTasksService::new(openai.clone(), xai.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let services = AdminServices {
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
@@ -370,7 +397,7 @@ pub async fn initialize(
|
||||
registry.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
@@ -392,21 +419,33 @@ pub async fn initialize(
|
||||
snapshot.clone(),
|
||||
)),
|
||||
system: Arc::new(DefaultSystemService::new(system)),
|
||||
openai: Arc::new(DefaultOpenAiService::new(
|
||||
openai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
xai: Arc::new(DefaultXaiService::new(
|
||||
xai,
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
openai,
|
||||
xai,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let mut worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(import_task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(registration));
|
||||
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
@@ -432,6 +471,37 @@ fn backup_worker_contribution(
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定。
|
||||
fn freeze_recovery_worker_contribution(
|
||||
task: freeze_recovery::FreezeRecoveryTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
freeze_recovery::FREEZE_RECOVERY_INTERVAL,
|
||||
freeze_recovery::WORKER_INITIAL_BACKOFF,
|
||||
freeze_recovery::WORKER_MAXIMUM_BACKOFF,
|
||||
freeze_recovery::WORKER_LEASE_TTL,
|
||||
freeze_recovery::WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
fn provider_kind(value: &'static str) -> Result<ProviderKind, AdminError> {
|
||||
ProviderKind::new(value).map_err(|_| AdminError::internal("内置 Provider 类型不合法"))
|
||||
}
|
||||
|
||||
@@ -69,11 +69,20 @@ pub struct AccountListQuery {
|
||||
/// Admin query service 从运行态存储取得的当前账号冷却快照。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountRuntimeSnapshot {
|
||||
pub rate_limited_until: BTreeMap<String, DateTime<Utc>>,
|
||||
pub cooldown: BTreeMap<String, gateway_core::account::AccountCooldown>,
|
||||
/// `None` 表示实时 lease 存储不可用;`Some` 中未出现的账号当前使用量为零。
|
||||
pub in_flight: Option<BTreeMap<String, u64>>,
|
||||
}
|
||||
|
||||
/// 恢复任务读取的冻结快照;generation 将异步结果绑定到本次冻结。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountFreeze {
|
||||
pub credential_revision: Revision,
|
||||
pub until: DateTime<Utc>,
|
||||
pub generation: String,
|
||||
pub requires_probe: bool,
|
||||
}
|
||||
|
||||
/// Optional account membership filter.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AccountGroupFilter {
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
//! 进程内账号导入任务;只保存执行所需输入与可公开的结果。
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{MutationContext, provider_credentials::ImportCredentials};
|
||||
|
||||
pub const MAX_IMPORT_TASK_ITEMS: usize = 200;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ImportTaskInput {
|
||||
pub provider: ProviderKind,
|
||||
pub command: ImportCredentials,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SubmitImportTask {
|
||||
pub submission_id: Uuid,
|
||||
/// API 对完整输入计算摘要;用于区分丢失响应后的重试与修改后的新提交。
|
||||
pub fingerprint: [u8; 32],
|
||||
pub context: MutationContext,
|
||||
pub items: Vec<ImportTaskInput>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ImportItemStatus {
|
||||
Pending,
|
||||
Running,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Unknown,
|
||||
Skipped,
|
||||
}
|
||||
|
||||
impl ImportItemStatus {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Pending => "pending",
|
||||
Self::Running => "running",
|
||||
Self::Succeeded => "succeeded",
|
||||
Self::Failed => "failed",
|
||||
Self::Unknown => "unknown",
|
||||
Self::Skipped => "skipped",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportTaskItem {
|
||||
pub index: usize,
|
||||
pub provider: ProviderKind,
|
||||
pub status: ImportItemStatus,
|
||||
pub account_ids: Vec<ProviderAccountId>,
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ImportTaskCounts {
|
||||
pub pending: usize,
|
||||
pub running: usize,
|
||||
pub succeeded: usize,
|
||||
pub failed: usize,
|
||||
pub unknown: usize,
|
||||
pub skipped: usize,
|
||||
pub imported_accounts: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportTaskSummary {
|
||||
pub task_id: Uuid,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub finished_at: Option<DateTime<Utc>>,
|
||||
pub stop_requested: bool,
|
||||
pub total: usize,
|
||||
pub counts: ImportTaskCounts,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportTaskDetail {
|
||||
pub summary: ImportTaskSummary,
|
||||
pub items: Vec<ImportTaskItem>,
|
||||
}
|
||||
@@ -8,6 +8,7 @@ pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_distribution;
|
||||
pub mod client_keys;
|
||||
pub mod import_tasks;
|
||||
pub mod key_usage;
|
||||
pub mod observability;
|
||||
pub mod provider_credentials;
|
||||
|
||||
@@ -585,6 +585,7 @@ pub struct CredentialDeletionResult {
|
||||
pub struct RotateCredential {
|
||||
pub mutation: CredentialMutation,
|
||||
pub provider_material: ProviderDocument,
|
||||
pub settings: Option<super::accounts::UpdateAccount>,
|
||||
}
|
||||
|
||||
/// Provider 校验手工轮换材料时所需的非事务输入。
|
||||
@@ -613,6 +614,8 @@ pub struct PreparedCredentialRotationFacts {
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
/// Token 刷新保留提交时的资料,不以准备阶段的副本覆盖新套餐。
|
||||
pub preserve_profile: bool,
|
||||
pub provider_material: ProviderDocument,
|
||||
pub has_refresh_token: bool,
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
@@ -664,6 +667,7 @@ impl fmt::Debug for PreparedCredentialRotation {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CredentialRotationCommit {
|
||||
pub prepared: PreparedCredentialRotationFacts,
|
||||
pub settings: Option<super::accounts::UpdateAccount>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RotateCredential {
|
||||
@@ -672,6 +676,7 @@ impl fmt::Debug for RotateCredential {
|
||||
.debug_struct("RotateCredential")
|
||||
.field("mutation", &self.mutation)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.field("settings", &self.settings)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,7 @@ pub struct ProxyTestResult {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProxyRecord {
|
||||
pub location: Option<gateway_core::account::RequestLocation>,
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub proxy: OutboundProxy,
|
||||
@@ -86,12 +87,15 @@ pub struct ProxyPage {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NewProxy {
|
||||
pub location: Option<gateway_core::account::RequestLocation>,
|
||||
pub name: String,
|
||||
pub proxy: OutboundProxy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UpdateProxy {
|
||||
/// 外层为空保留配置,内层为空恢复全局继承。
|
||||
pub location: Option<Option<gateway_core::account::RequestLocation>>,
|
||||
pub id: String,
|
||||
pub revision: Revision,
|
||||
pub name: String,
|
||||
|
||||
@@ -18,6 +18,8 @@ pub use gateway_core::account::RotationStrategy;
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RuntimeSettings {
|
||||
pub config_revision: Revision,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
@@ -32,12 +34,21 @@ pub struct RuntimeSettings {
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 原子替换运行设置的命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReplaceRuntimeSettings {
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
@@ -52,6 +63,13 @@ pub struct ReplaceRuntimeSettings {
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
/// 明文管理员 API Key;按产品约束明文落库,但禁止 Debug 泄漏。
|
||||
|
||||
@@ -170,6 +170,14 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
command: PrepareCredentialRefresh,
|
||||
) -> Result<PreparedCredentialRotation, ProviderAdminError>;
|
||||
|
||||
/// 可公开的账号连接设置;实现只能显式投影非敏感字段,不能返回凭据原文。
|
||||
async fn account_configuration(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderDocument>, ProviderAdminError> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
request: ProviderQuotaRequest,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! 端口按业务资源拆分,方法使用领域模型,不暴露连接池、事务或 Redis client。
|
||||
|
||||
use std::{net::IpAddr, sync::Arc, time::Duration};
|
||||
use std::{collections::BTreeMap, net::IpAddr, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -159,6 +159,14 @@ pub trait AccountStore: Send + Sync {
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountUpdateResult>;
|
||||
|
||||
/// 在事务内按最新启用状态、账号上限和全局默认值判断,只降低并发上限。
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &gateway_core::account::ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>>;
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &gateway_core::account::ProviderAccountId,
|
||||
@@ -193,6 +201,25 @@ pub trait AccountRuntimeStore: Send + Sync {
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot>;
|
||||
|
||||
/// 容量熔断自动冻结中的账号与其冻结截止时间;429 临时限流不包含在内。
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, crate::model::accounts::AccountFreeze>>;
|
||||
|
||||
/// 读取容量失败窗口内观测到的在途并发峰值(自适应并发下调的证据)。
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>>;
|
||||
|
||||
/// 仅当冻结快照仍匹配时解除或顺延;旧探测不得覆盖手动恢复或新一轮冻结。
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &crate::model::accounts::AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool>;
|
||||
}
|
||||
|
||||
/// 控制面凭据、统一会话、登录限流与管理员安全审计。
|
||||
|
||||
@@ -192,11 +192,7 @@ fn project_group_runtime(
|
||||
let mut accounts_by_group = BTreeMap::<&str, Vec<&str>>::new();
|
||||
for member in members {
|
||||
let mut status = member.status.clone();
|
||||
status.rate_limited_until = runtime
|
||||
.rate_limited_until
|
||||
.get(&member.account_id)
|
||||
.copied()
|
||||
.map(Into::into);
|
||||
status.cooldown = runtime.cooldown.get(&member.account_id).copied();
|
||||
status_by_account
|
||||
.entry(member.account_id.as_str())
|
||||
.or_insert_with(|| resolve_account_status(&status, now).status);
|
||||
|
||||
@@ -17,8 +17,8 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountPageItem, AccountUpdateResult, AccountUsageWindowQuery, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, UpdateAccount,
|
||||
AccountPageItem, AccountUpdateResult, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountsUpdateResult, BatchUpdateAccounts, UpdateAccount,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
@@ -73,12 +73,26 @@ pub trait AccountsService: Send + Sync {
|
||||
command: UpdateAccount,
|
||||
) -> Result<AccountUpdateResult, AdminError>;
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
) -> Result<Option<AccountUpdateResult>, AdminError>;
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: BatchUpdateAccounts,
|
||||
) -> Result<AccountsUpdateResult, AdminError>;
|
||||
|
||||
async fn account_configuration(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<Option<crate::model::provider_credentials::ProviderDocument>, AdminError> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -250,6 +264,37 @@ impl DefaultAccountsService {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_api_key_usage(
|
||||
&self,
|
||||
accounts: &[AccountPageItem],
|
||||
) -> Result<BTreeMap<String, AccountUsage>, AdminError> {
|
||||
let now = Utc::now();
|
||||
// API Key 没有套餐周期;本地累计直接查询账号创建后仍保留的请求记录。
|
||||
let windows = accounts
|
||||
.iter()
|
||||
.filter(|item| item.account.authentication_kind == "api_key")
|
||||
.map(|item| AccountUsageWindowQuery {
|
||||
account_id: item.account.id.clone(),
|
||||
key: "account-lifetime".to_owned(),
|
||||
range: TimeRange {
|
||||
start: item.account.created_at,
|
||||
end: now,
|
||||
},
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if windows.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
Ok(self
|
||||
.accounts
|
||||
.load_account_usage_by_windows(&windows)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "API Key account usage"))?
|
||||
.into_iter()
|
||||
.map(|result| (result.account_id, result.usage))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn load_directory_item(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -287,9 +332,15 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let usage = quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone());
|
||||
let usage = self
|
||||
.load_api_key_usage(std::slice::from_ref(&stored))
|
||||
.await?
|
||||
.remove(&stored.account.id)
|
||||
.or_else(|| {
|
||||
quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone())
|
||||
});
|
||||
Ok(AccountDirectoryItem {
|
||||
plan_type_display: self.providers.resolve_account_plan(
|
||||
stored.account.provider_kind.as_str(),
|
||||
@@ -377,14 +428,17 @@ impl AccountsService for DefaultAccountsService {
|
||||
.collect::<Result<Vec<_>, AdminError>>()?;
|
||||
self.attach_quota_local_usage(&page.items, &mut quotas)
|
||||
.await?;
|
||||
let mut api_key_usage = self.load_api_key_usage(&page.items).await?;
|
||||
let items = page
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(mut item, quota)| {
|
||||
let usage = quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone());
|
||||
let usage = api_key_usage.remove(&item.account.id).or_else(|| {
|
||||
quota
|
||||
.usage_window()
|
||||
.and_then(|(window, _)| window.local_usage.clone())
|
||||
});
|
||||
AccountDirectoryItem {
|
||||
plan_type_display: self.providers.resolve_account_plan(
|
||||
item.account.provider_kind.as_str(),
|
||||
@@ -535,6 +589,27 @@ impl AccountsService for DefaultAccountsService {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
) -> Result<Option<AccountUpdateResult>, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
let result = self
|
||||
.accounts
|
||||
.lower_concurrency_limit(&account_id, limit, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "provider account"))?;
|
||||
if let Some(result) = &result {
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
@@ -581,6 +656,17 @@ impl AccountsService for DefaultAccountsService {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn account_configuration(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<Option<crate::model::provider_credentials::ProviderDocument>, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(account_id).await?;
|
||||
provider
|
||||
.account_configuration(account_id)
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider account configuration"))
|
||||
}
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
//! 导入队列由 Host 管理生命周期,HTTP 只负责接受输入和读取快照。
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
panic::AssertUnwindSafe,
|
||||
sync::{Arc, Mutex, MutexGuard},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::{FutureExt as _, StreamExt as _, future::BoxFuture, stream::FuturesUnordered};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
task::{DaemonTask, WorkerTaskError},
|
||||
};
|
||||
use tokio::{sync::Notify, time::Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{openai::OpenAiService, xai::XaiService};
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind, MutationActor, MutationContext,
|
||||
import_tasks::{
|
||||
ImportItemStatus, ImportTaskCounts, ImportTaskDetail, ImportTaskInput, ImportTaskItem,
|
||||
ImportTaskSummary, MAX_IMPORT_TASK_ITEMS, SubmitImportTask,
|
||||
},
|
||||
};
|
||||
|
||||
const CONCURRENCY: usize = 3;
|
||||
const MAX_ACTIVE_TASKS: usize = 8;
|
||||
const MAX_RETAINED_TASKS: usize = 100;
|
||||
const RETENTION: Duration = Duration::from_secs(3600);
|
||||
|
||||
pub trait ImportTasksService: Send + Sync {
|
||||
fn submit(&self, command: SubmitImportTask) -> Result<ImportTaskSummary, AdminError>;
|
||||
fn list(&self, context: &MutationContext) -> Vec<ImportTaskSummary>;
|
||||
fn detail(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError>;
|
||||
fn stop(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError>;
|
||||
}
|
||||
|
||||
struct Entry {
|
||||
task_id: Uuid,
|
||||
submission_id: Uuid,
|
||||
fingerprint: [u8; 32],
|
||||
owner: MutationActor,
|
||||
created_at: chrono::DateTime<Utc>,
|
||||
finished_at: Option<chrono::DateTime<Utc>>,
|
||||
finished_instant: Option<Instant>,
|
||||
stop_requested: bool,
|
||||
items: Vec<ImportTaskItem>,
|
||||
inputs: Vec<Option<ImportTaskInput>>,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
fn summary(&self) -> ImportTaskSummary {
|
||||
let mut counts = ImportTaskCounts::default();
|
||||
for item in &self.items {
|
||||
match item.status {
|
||||
ImportItemStatus::Pending => counts.pending += 1,
|
||||
ImportItemStatus::Running => counts.running += 1,
|
||||
ImportItemStatus::Succeeded => counts.succeeded += 1,
|
||||
ImportItemStatus::Failed => counts.failed += 1,
|
||||
ImportItemStatus::Unknown => counts.unknown += 1,
|
||||
ImportItemStatus::Skipped => counts.skipped += 1,
|
||||
}
|
||||
counts.imported_accounts += item.account_ids.len();
|
||||
}
|
||||
ImportTaskSummary {
|
||||
task_id: self.task_id,
|
||||
created_at: self.created_at,
|
||||
finished_at: self.finished_at,
|
||||
stop_requested: self.stop_requested,
|
||||
total: self.items.len(),
|
||||
counts,
|
||||
}
|
||||
}
|
||||
|
||||
fn detail(&self) -> ImportTaskDetail {
|
||||
ImportTaskDetail {
|
||||
summary: self.summary(),
|
||||
items: self.items.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_if_done(&mut self) {
|
||||
if self.finished_at.is_none()
|
||||
&& self.items.iter().all(|item| {
|
||||
!matches!(
|
||||
item.status,
|
||||
ImportItemStatus::Pending | ImportItemStatus::Running
|
||||
)
|
||||
})
|
||||
{
|
||||
self.finished_at = Some(Utc::now());
|
||||
self.finished_instant = Some(Instant::now());
|
||||
// 终态仅保留结果,连已清空的输入槽位分配也一并释放。
|
||||
self.inputs = Vec::new();
|
||||
}
|
||||
}
|
||||
|
||||
fn stop(&mut self) {
|
||||
if self.finished_at.is_some() {
|
||||
return;
|
||||
}
|
||||
self.stop_requested = true;
|
||||
for (item, input) in self.items.iter_mut().zip(&mut self.inputs) {
|
||||
if item.status == ImportItemStatus::Pending {
|
||||
item.status = ImportItemStatus::Skipped;
|
||||
*input = None;
|
||||
}
|
||||
}
|
||||
self.finish_if_done();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Registry {
|
||||
tasks: VecDeque<Entry>,
|
||||
shutting_down: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultImportTasksService {
|
||||
registry: Mutex<Registry>,
|
||||
notify: Notify,
|
||||
openai: Arc<dyn OpenAiService>,
|
||||
xai: Arc<dyn XaiService>,
|
||||
}
|
||||
|
||||
impl DefaultImportTasksService {
|
||||
pub(crate) fn new(openai: Arc<dyn OpenAiService>, xai: Arc<dyn XaiService>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
registry: Mutex::default(),
|
||||
notify: Notify::new(),
|
||||
openai,
|
||||
xai,
|
||||
})
|
||||
}
|
||||
|
||||
fn registry(&self) -> MutexGuard<'_, Registry> {
|
||||
let mut registry = self
|
||||
.registry
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
registry.tasks.retain(|entry| {
|
||||
entry
|
||||
.finished_instant
|
||||
.is_none_or(|at| at.elapsed() < RETENTION)
|
||||
});
|
||||
registry
|
||||
}
|
||||
|
||||
fn take_next(&self) -> Option<(Uuid, usize, ImportTaskInput)> {
|
||||
let mut registry = self.registry();
|
||||
if registry.shutting_down {
|
||||
return None;
|
||||
}
|
||||
let task_index = registry
|
||||
.tasks
|
||||
.iter()
|
||||
.position(|entry| entry.inputs.iter().any(Option::is_some))?;
|
||||
// 轮转任务,避免大批次独占后续全部执行机会。
|
||||
let mut entry = registry.tasks.remove(task_index)?;
|
||||
let index = entry.inputs.iter().position(Option::is_some)?;
|
||||
let input = entry.inputs[index].take()?;
|
||||
entry.items[index].status = ImportItemStatus::Running;
|
||||
let id = entry.task_id;
|
||||
registry.tasks.push_back(entry);
|
||||
Some((id, index, input))
|
||||
}
|
||||
|
||||
async fn execute(&self, id: Uuid, index: usize, input: ImportTaskInput) {
|
||||
let operation = async {
|
||||
match input.provider.as_str() {
|
||||
"openai" => self.openai.import_document(input.command).await,
|
||||
"xai" => self.xai.import_document(input.command).await,
|
||||
_ => Err(AdminError::invalid("不支持的导入平台")),
|
||||
}
|
||||
};
|
||||
let result = AssertUnwindSafe(operation).catch_unwind().await;
|
||||
let mut registry = self.registry();
|
||||
let Some(entry) = registry.tasks.iter_mut().find(|entry| entry.task_id == id) else {
|
||||
return;
|
||||
};
|
||||
let item = &mut entry.items[index];
|
||||
match result {
|
||||
Ok(Ok(result)) => {
|
||||
item.status = ImportItemStatus::Succeeded;
|
||||
item.account_ids = result.credential_ids;
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
// 存储或发布异常也可能发生在提交之后;不自动重放可能轮换 RT 的操作。
|
||||
item.status = if matches!(
|
||||
error.kind(),
|
||||
AdminErrorKind::UpstreamResultUnknown
|
||||
| AdminErrorKind::Internal
|
||||
| AdminErrorKind::Unavailable
|
||||
) {
|
||||
ImportItemStatus::Unknown
|
||||
} else {
|
||||
ImportItemStatus::Failed
|
||||
};
|
||||
item.message = Some(error.message().to_owned());
|
||||
}
|
||||
Err(_) => {
|
||||
item.status = ImportItemStatus::Unknown;
|
||||
item.message = Some("执行中断,请先核对账号列表再决定是否重新导入".to_owned());
|
||||
}
|
||||
}
|
||||
entry.finish_if_done();
|
||||
}
|
||||
}
|
||||
|
||||
impl ImportTasksService for DefaultImportTasksService {
|
||||
fn submit(&self, command: SubmitImportTask) -> Result<ImportTaskSummary, AdminError> {
|
||||
if command.items.is_empty()
|
||||
|| command.items.len() > MAX_IMPORT_TASK_ITEMS
|
||||
|| command.items.iter().any(|item| {
|
||||
!matches!(item.provider.as_str(), "openai" | "xai")
|
||||
|| item.command.context.actor != command.context.actor
|
||||
})
|
||||
{
|
||||
return Err(AdminError::invalid("导入任务需包含 1–200 条有效输入"));
|
||||
}
|
||||
let mut registry = self.registry();
|
||||
if let Some(existing) = registry.tasks.iter().find(|entry| {
|
||||
entry.owner == command.context.actor && entry.submission_id == command.submission_id
|
||||
}) {
|
||||
return if existing.fingerprint == command.fingerprint {
|
||||
Ok(existing.summary())
|
||||
} else {
|
||||
Err(AdminError::new(
|
||||
AdminErrorKind::Conflict,
|
||||
"提交标识已用于其他导入内容",
|
||||
))
|
||||
};
|
||||
}
|
||||
if registry.shutting_down {
|
||||
return Err(AdminError::new(
|
||||
AdminErrorKind::Unavailable,
|
||||
"服务正在关闭,暂不接受导入任务",
|
||||
));
|
||||
}
|
||||
if registry
|
||||
.tasks
|
||||
.iter()
|
||||
.filter(|entry| entry.finished_at.is_none())
|
||||
.count()
|
||||
>= MAX_ACTIVE_TASKS
|
||||
|| registry.tasks.len() >= MAX_RETAINED_TASKS
|
||||
{
|
||||
return Err(AdminError::new(
|
||||
AdminErrorKind::RateLimited,
|
||||
"导入任务已满,请稍后再试",
|
||||
));
|
||||
}
|
||||
let items = command
|
||||
.items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, input)| ImportTaskItem {
|
||||
index: index + 1,
|
||||
provider: input.provider.clone(),
|
||||
status: ImportItemStatus::Pending,
|
||||
account_ids: Vec::new(),
|
||||
message: None,
|
||||
})
|
||||
.collect();
|
||||
let entry = Entry {
|
||||
task_id: Uuid::now_v7(),
|
||||
submission_id: command.submission_id,
|
||||
fingerprint: command.fingerprint,
|
||||
owner: command.context.actor,
|
||||
created_at: Utc::now(),
|
||||
finished_at: None,
|
||||
finished_instant: None,
|
||||
stop_requested: false,
|
||||
items,
|
||||
inputs: command.items.into_iter().map(Some).collect(),
|
||||
};
|
||||
let summary = entry.summary();
|
||||
registry.tasks.push_back(entry);
|
||||
drop(registry);
|
||||
self.notify.notify_one();
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
fn list(&self, context: &MutationContext) -> Vec<ImportTaskSummary> {
|
||||
let mut tasks: Vec<_> = self
|
||||
.registry()
|
||||
.tasks
|
||||
.iter()
|
||||
.filter(|entry| entry.owner == context.actor)
|
||||
.map(Entry::summary)
|
||||
.collect();
|
||||
tasks.sort_by_key(|entry| std::cmp::Reverse(entry.task_id));
|
||||
tasks
|
||||
}
|
||||
|
||||
fn detail(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError> {
|
||||
self.registry()
|
||||
.tasks
|
||||
.iter()
|
||||
.find(|entry| entry.task_id == task_id && entry.owner == context.actor)
|
||||
.map(Entry::detail)
|
||||
.ok_or_else(|| AdminError::not_found("导入任务不存在或已过期"))
|
||||
}
|
||||
|
||||
fn stop(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
task_id: Uuid,
|
||||
) -> Result<ImportTaskDetail, AdminError> {
|
||||
let mut registry = self.registry();
|
||||
let entry = registry
|
||||
.tasks
|
||||
.iter_mut()
|
||||
.find(|entry| entry.task_id == task_id && entry.owner == context.actor)
|
||||
.ok_or_else(|| AdminError::not_found("导入任务不存在或已过期"))?;
|
||||
entry.stop();
|
||||
Ok(entry.detail())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ImportTaskWorker(pub Arc<DefaultImportTasksService>);
|
||||
|
||||
impl DaemonTask for ImportTaskWorker {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut active = FuturesUnordered::new();
|
||||
// 固定一个取消 future,避免循环反复注册中立 CancellationToken 的 waiter。
|
||||
let cancelled = cancellation.cancelled();
|
||||
tokio::pin!(cancelled);
|
||||
let mut cleanup = tokio::time::interval(Duration::from_secs(60));
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
let mut registry = self.0.registry();
|
||||
registry.shutting_down = true;
|
||||
for entry in &mut registry.tasks {
|
||||
entry.stop();
|
||||
}
|
||||
break;
|
||||
}
|
||||
while active.len() < CONCURRENCY {
|
||||
let Some((id, index, input)) = self.0.take_next() else {
|
||||
break;
|
||||
};
|
||||
active.push(self.0.execute(id, index, input));
|
||||
}
|
||||
tokio::select! {
|
||||
() = &mut cancelled => {},
|
||||
_ = active.next(), if !active.is_empty() => {},
|
||||
() = self.0.notify.notified() => {},
|
||||
_ = cleanup.tick() => { drop(self.0.registry()); },
|
||||
}
|
||||
}
|
||||
// Host 提供总关闭预算;此处让已发出的交换与入库尽量完成。
|
||||
while active.next().await.is_some() {}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_distribution;
|
||||
pub mod client_keys;
|
||||
pub mod import_tasks;
|
||||
pub mod key_usage;
|
||||
pub mod observability;
|
||||
pub mod openai;
|
||||
@@ -345,12 +346,19 @@ async fn commit_authorization(
|
||||
async fn commit_credential_rotation(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedCredentialRotation,
|
||||
settings: Option<crate::model::accounts::UpdateAccount>,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_rotation(CredentialRotationCommit { prepared: facts }, context)
|
||||
.commit_credential_rotation(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
@@ -372,7 +380,13 @@ async fn commit_credential_refresh(
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_refresh(CredentialRotationCommit { prepared: facts }, context)
|
||||
.commit_credential_refresh(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
|
||||
@@ -189,6 +189,17 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let context = command.mutation.context;
|
||||
let account_id = command.mutation.account_id;
|
||||
if command
|
||||
.settings
|
||||
.as_ref()
|
||||
.is_some_and(|settings| settings.account_id != account_id.as_str())
|
||||
{
|
||||
return Err(AdminError::invalid("凭据和账号设置的目标不一致"));
|
||||
}
|
||||
let disable_account = command
|
||||
.settings
|
||||
.as_ref()
|
||||
.is_some_and(|settings| !settings.enabled);
|
||||
let details = required_credential(
|
||||
self.accounts.as_ref(),
|
||||
self.provider.provider_kind(),
|
||||
@@ -209,10 +220,14 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
let result = commit_credential_rotation(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
command.settings,
|
||||
&context,
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
if disable_account {
|
||||
self.provider.account_unavailable(&account_id).await;
|
||||
}
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
|
||||
@@ -181,6 +181,11 @@ impl ProxiesService for DefaultProxiesService {
|
||||
context: &MutationContext,
|
||||
) -> Result<ProxyMutation, AdminError> {
|
||||
command.name = validate_name(&command.name)?;
|
||||
command.location = command
|
||||
.location
|
||||
.map(|location| location.normalized())
|
||||
.transpose()
|
||||
.map_err(|_| AdminError::invalid("代理位置不合法"))?;
|
||||
let result = self
|
||||
.store
|
||||
.create(command, context)
|
||||
@@ -196,6 +201,11 @@ impl ProxiesService for DefaultProxiesService {
|
||||
context: &MutationContext,
|
||||
) -> Result<ProxyMutation, AdminError> {
|
||||
command.name = validate_name(&command.name)?;
|
||||
command.location = command
|
||||
.location
|
||||
.map(|location| location.map(|value| value.normalized()).transpose())
|
||||
.transpose()
|
||||
.map_err(|_| AdminError::invalid("代理位置不合法"))?;
|
||||
let result = self
|
||||
.store
|
||||
.update(command, context)
|
||||
|
||||
@@ -114,7 +114,8 @@ impl SettingsService for DefaultSettingsService {
|
||||
}
|
||||
|
||||
fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError> {
|
||||
let valid = command.refresh_margin_seconds > 0
|
||||
let valid = command.request_location.validate().is_ok()
|
||||
&& command.refresh_margin_seconds > 0
|
||||
&& command.refresh_concurrency > 0
|
||||
&& command.max_concurrent_per_account > 0
|
||||
&& command.max_waiting_per_key <= 1_000
|
||||
@@ -125,7 +126,11 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
|
||||
&& command.audit_retention_days > 0
|
||||
&& valid_client_version(command.min_codex_desktop_version.as_deref())
|
||||
&& valid_client_version(command.min_codex_cli_version.as_deref())
|
||||
&& i64::try_from(command.request_interval_ms).is_ok();
|
||||
&& valid_probe_model(command.account_auto_freeze_probe_model.as_deref())
|
||||
&& i64::try_from(command.request_interval_ms).is_ok()
|
||||
&& (2..=1_000).contains(&command.account_auto_freeze_threshold)
|
||||
&& (60..=3_600).contains(&command.account_auto_freeze_window_seconds)
|
||||
&& (300..=604_800).contains(&command.account_auto_freeze_duration_seconds);
|
||||
if valid {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -136,3 +141,12 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
|
||||
fn valid_client_version(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| CodexClientVersion::parse(value).is_ok())
|
||||
}
|
||||
|
||||
fn valid_probe_model(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 128
|
||||
&& value == value.trim()
|
||||
&& !value.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, PageSize, Revision,
|
||||
@@ -168,13 +168,35 @@ impl AccountRuntimeStore for FakeRuntimeStore {
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
*self.requested_accounts.lock().expect("requested accounts") = account_ids.to_vec();
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::from([(
|
||||
cooldown: BTreeMap::from([(
|
||||
"acct_limited".to_owned(),
|
||||
Utc::now() + Duration::minutes(5),
|
||||
std::time::SystemTime::from(Utc::now() + Duration::minutes(5)).into(),
|
||||
)]),
|
||||
in_flight: Some(BTreeMap::from([("acct_available".to_owned(), 2)])),
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
fn group_record() -> AccountGroupRecord {
|
||||
@@ -215,7 +237,7 @@ fn member(account_id: &str, total_slots: u64) -> AccountGroupMemberFact {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
|
||||
@@ -67,6 +67,7 @@ pub(super) struct FakeProviderAdmin {
|
||||
kind: ProviderKind,
|
||||
events: EventLog,
|
||||
failure: Mutex<Option<ProviderAdminError>>,
|
||||
import_gate: Mutex<Option<Arc<tokio::sync::Semaphore>>>,
|
||||
quota_failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
retry_authorization_after_abort: Mutex<bool>,
|
||||
@@ -90,6 +91,7 @@ impl FakeProviderAdmin {
|
||||
kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
events,
|
||||
failure: Mutex::new(None),
|
||||
import_gate: Mutex::new(None),
|
||||
quota_failure: Mutex::new(None),
|
||||
pending: Arc::new(Mutex::new(None)),
|
||||
retry_authorization_after_abort: Mutex::new(false),
|
||||
@@ -108,6 +110,12 @@ impl FakeProviderAdmin {
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn block_imports(&self) -> Arc<tokio::sync::Semaphore> {
|
||||
let gate = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
*self.import_gate.lock().expect("import gate") = Some(gate.clone());
|
||||
gate
|
||||
}
|
||||
|
||||
pub(super) fn fail_next(&self, kind: ProviderAdminErrorKind) {
|
||||
*self.failure.lock().expect("provider failure") = Some(ProviderAdminError::new(kind));
|
||||
}
|
||||
@@ -235,6 +243,7 @@ impl FakeProviderAdmin {
|
||||
name: account.name.clone(),
|
||||
email: account.email.clone(),
|
||||
plan_type: account.plan_type.clone(),
|
||||
preserve_profile: false,
|
||||
provider_material: document(),
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
access_token_expires_at: account
|
||||
@@ -332,6 +341,10 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
_command: PrepareCredentialImport,
|
||||
) -> Result<PreparedCredentialImport, ProviderAdminError> {
|
||||
self.record("provider.prepare_import");
|
||||
let gate = self.import_gate.lock().expect("import gate").clone();
|
||||
if let Some(gate) = gate {
|
||||
gate.acquire().await.expect("import permit").forget();
|
||||
}
|
||||
self.require_available()?;
|
||||
let account_ids = self
|
||||
.import_account_ids
|
||||
@@ -531,6 +544,8 @@ pub(super) struct FakeAccountStore {
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
|
||||
quota_forecast_history: Mutex<QuotaForecastHistory>,
|
||||
update_commands: Mutex<Vec<UpdateAccount>>,
|
||||
pub(super) lowered_limits: Mutex<Vec<(String, u32)>>,
|
||||
}
|
||||
|
||||
impl FakeAccountStore {
|
||||
@@ -538,7 +553,7 @@ impl FakeAccountStore {
|
||||
Self::with_account(account_record(kind), events)
|
||||
}
|
||||
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
pub(super) fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
accounts: Mutex::new(vec![account]),
|
||||
@@ -549,9 +564,18 @@ impl FakeAccountStore {
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
quota_window_queries: Mutex::new(Vec::new()),
|
||||
quota_forecast_history: Mutex::new(QuotaForecastHistory::default()),
|
||||
update_commands: Mutex::new(Vec::new()),
|
||||
lowered_limits: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn update_commands(&self) -> Vec<UpdateAccount> {
|
||||
self.update_commands
|
||||
.lock()
|
||||
.expect("update commands")
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub(super) fn fail_next_commit(&self) {
|
||||
*self.fail_commit.lock().expect("store failure") = true;
|
||||
}
|
||||
@@ -617,7 +641,7 @@ impl FakeAccountStore {
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message.clone(),
|
||||
};
|
||||
@@ -839,12 +863,45 @@ impl AccountStore for FakeAccountStore {
|
||||
self.record("store.update_account");
|
||||
self.record_context(context);
|
||||
self.require_commit()?;
|
||||
self.update_commands
|
||||
.lock()
|
||||
.expect("update commands")
|
||||
.push(command.clone());
|
||||
Ok(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: ProviderAccountId::new(command.account_id).expect("account ID"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>> {
|
||||
let mut accounts = self.accounts.lock().expect("accounts");
|
||||
let Some(account) = accounts
|
||||
.iter_mut()
|
||||
.find(|account| account.id == account_id.as_str())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !account.enabled
|
||||
|| account.concurrency_limit.map_or(5, |value| value.get()) <= limit.get()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
account.concurrency_limit = Some(limit);
|
||||
self.lowered_limits
|
||||
.lock()
|
||||
.expect("lowered limits")
|
||||
.push((account.id.clone(), limit.get()));
|
||||
Ok(Some(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: account_id.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -916,6 +973,8 @@ struct StaticSettingsStore;
|
||||
impl SettingsStore for StaticSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(RuntimeSettings {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: revision(1),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
@@ -931,6 +990,13 @@ impl SettingsStore for StaticSettingsStore {
|
||||
usage_retention_days: 30,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
})
|
||||
}
|
||||
@@ -1956,6 +2022,52 @@ async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_key_list_and_detail_should_accumulate_local_usage_without_subscription_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.authentication_kind = "api_key".to_owned();
|
||||
account.created_at = Utc::now() - TimeDelta::days(60);
|
||||
let added_at = account.created_at;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
store.set_quota_window_usage(vec![AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "account-lifetime".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 4_330_000),
|
||||
}]);
|
||||
let services = accounts_service(provider, store.clone()).await;
|
||||
let page = services
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).unwrap(),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(page.items[0].quota.windows.is_empty());
|
||||
assert_eq!(
|
||||
page.items[0].usage.as_ref().unwrap().total_tokens,
|
||||
Some(4_330_000)
|
||||
);
|
||||
let queries = store.quota_window_queries();
|
||||
assert_eq!(queries.len(), 1);
|
||||
assert_eq!(queries[0].range.start, added_at);
|
||||
assert!(queries[0].range.end > added_at + TimeDelta::days(59));
|
||||
let detail = services
|
||||
.accounts()
|
||||
.quota(&ProviderAccountId::new("acct_test").unwrap(), false)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(detail.usage, page.items[0].usage);
|
||||
assert!(detail.quota.windows.is_empty());
|
||||
assert_eq!(store.quota_window_queries()[0].range.start, added_at);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
//! 容量熔断冻结恢复编排回归:到期探测、失败顺延与自适应并发下调。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration as TimeDelta, Utc};
|
||||
use gateway_admin::freeze_recovery::{FreezeRecoveryDeps, FreezeRecoveryTask};
|
||||
use gateway_admin::model::MutationContext;
|
||||
use gateway_admin::model::accounts::{AccountFreeze, AccountRuntimeSnapshot};
|
||||
use gateway_admin::model::settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RuntimeSettings,
|
||||
};
|
||||
use gateway_admin::ports::store::{
|
||||
AccountRuntimeStore, AccountStore, AdminStoreError, AdminStoreResult, SettingsStore,
|
||||
};
|
||||
use gateway_core::engine::probe::{
|
||||
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
|
||||
};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::task::{ScheduledTask as _, WorkerCycleContext, WorkerId, WorkerKind};
|
||||
|
||||
use super::AdminHarness;
|
||||
use super::accounts::{FakeAccountStore, FakeProviderAdmin, account_record, events, revision};
|
||||
|
||||
fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> RuntimeSettings {
|
||||
RuntimeSettings {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: revision(1),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: enabled,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: probe_enabled,
|
||||
account_auto_freeze_probe_model: Some("gpt-5.5".to_owned()),
|
||||
account_auto_freeze_adaptive_concurrency: adaptive,
|
||||
updated_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
struct FreezeSettingsStore {
|
||||
settings: RuntimeSettings,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SettingsStore for FreezeSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(self.settings.clone())
|
||||
}
|
||||
|
||||
async fn admin_api_key_exists(&self) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn replace_runtime_settings(
|
||||
&self,
|
||||
_: ReplaceRuntimeSettings,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<RuntimeSettings> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable() -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"freeze recovery test",
|
||||
"unused",
|
||||
)
|
||||
}
|
||||
|
||||
/// 记录解冻/顺延调用的运行态 fake;冻结与峰值证据由测试预置。
|
||||
struct FreezeRuntimeStore {
|
||||
freezes: BTreeMap<String, AccountFreeze>,
|
||||
peaks: BTreeMap<String, u32>,
|
||||
cleared: Mutex<Vec<String>>,
|
||||
extended: Mutex<Vec<(String, DateTime<Utc>)>>,
|
||||
}
|
||||
|
||||
impl FreezeRuntimeStore {
|
||||
fn new(freezes: BTreeMap<String, AccountFreeze>, peaks: BTreeMap<String, u32>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
freezes,
|
||||
peaks,
|
||||
cleared: Mutex::new(Vec::new()),
|
||||
extended: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
fn cleared(&self) -> Vec<String> {
|
||||
self.cleared.lock().expect("cleared lock").clone()
|
||||
}
|
||||
|
||||
fn extended(&self) -> Vec<(String, DateTime<Utc>)> {
|
||||
self.extended.lock().expect("extended lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AccountRuntimeStore for FreezeRuntimeStore {
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn account_runtime(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn active_freezes(&self) -> AdminStoreResult<BTreeMap<String, AccountFreeze>> {
|
||||
Ok(self.freezes.clone())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(account_ids
|
||||
.iter()
|
||||
.filter_map(|id| self.peaks.get(id).map(|peak| (id.clone(), *peak)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
assert_eq!(self.freezes.get(account_id), Some(expected));
|
||||
if let Some(until) = postpone_until {
|
||||
self.extended
|
||||
.lock()
|
||||
.expect("extended")
|
||||
.push((account_id.to_owned(), until));
|
||||
} else {
|
||||
self.cleared
|
||||
.lock()
|
||||
.expect("cleared")
|
||||
.push(account_id.to_owned());
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
struct SuccessfulProbe;
|
||||
|
||||
impl AccountProbe for SuccessfulProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Ok(AccountProbeResult {
|
||||
text: vec!["OK".to_owned()],
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingProbe;
|
||||
|
||||
impl AccountProbe for FailingProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Err(AccountProbeError::new(
|
||||
gateway_core::error::GatewayError::new(
|
||||
gateway_core::error::GatewayErrorKind::RateLimited,
|
||||
"still overloaded",
|
||||
),
|
||||
gateway_core::engine::probe::AccountProbeErrorSource::Upstream,
|
||||
Some(gateway_core::upstream::UpstreamSendState::NotSent),
|
||||
None,
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn recovery_task(
|
||||
settings: RuntimeSettings,
|
||||
runtime: Arc<FreezeRuntimeStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
|
||||
recovery_task_with_store(settings, runtime, probe).await
|
||||
}
|
||||
|
||||
async fn recovery_task_with_store(
|
||||
settings: RuntimeSettings,
|
||||
runtime: Arc<FreezeRuntimeStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
let services = AdminHarness::new()
|
||||
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
|
||||
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
|
||||
.settings(Arc::new(FreezeSettingsStore {
|
||||
settings: settings.clone(),
|
||||
}))
|
||||
.provider(FakeProviderAdmin::new("openai", events()))
|
||||
.probe(probe)
|
||||
.build()
|
||||
.await;
|
||||
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
|
||||
accounts: services.accounts_handle(),
|
||||
store: store.clone(),
|
||||
runtime,
|
||||
settings: Arc::new(FreezeSettingsStore {
|
||||
settings: settings.clone(),
|
||||
}),
|
||||
});
|
||||
(task, store)
|
||||
}
|
||||
|
||||
async fn run_cycle(task: &FreezeRecoveryTask) {
|
||||
let worker = WorkerId::try_new(WorkerKind::AccountFreezeRecovery, "test").expect("worker id");
|
||||
let context = WorkerCycleContext::new(worker, None, CancellationToken::new());
|
||||
task.run_cycle(context)
|
||||
.await
|
||||
.expect("freeze recovery cycle");
|
||||
}
|
||||
|
||||
fn freeze_until(until: DateTime<Utc>) -> BTreeMap<String, AccountFreeze> {
|
||||
BTreeMap::from([(
|
||||
"acct_test".to_owned(),
|
||||
AccountFreeze {
|
||||
credential_revision: revision(1),
|
||||
until,
|
||||
generation: "freeze-generation".to_owned(),
|
||||
requires_probe: true,
|
||||
},
|
||||
)])
|
||||
}
|
||||
|
||||
fn freeze_due() -> BTreeMap<String, AccountFreeze> {
|
||||
freeze_until(Utc::now() - TimeDelta::seconds(1))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_success_clears_freeze() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_failure_postpones_freeze() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(FailingProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
let extended = runtime.extended();
|
||||
assert_eq!(extended.len(), 1);
|
||||
let (account_id, until) = &extended[0];
|
||||
assert_eq!(account_id, "acct_test");
|
||||
let remaining = until.timestamp_millis() - Utc::now().timestamp_millis();
|
||||
assert!(
|
||||
remaining > 7_000_000 && remaining <= 7_200_000,
|
||||
"postponed freeze should be about 2 hours, got {remaining}ms"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_policy_releases_due_freeze_without_probing() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(false, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_concurrency_lowers_limit_to_observed_peak() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
|
||||
);
|
||||
let (task, store) = recovery_task(
|
||||
runtime_settings(true, false, true),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
// 峰值 4 × 0.8 = 3(不低于下限 2),低于全局默认 5,应下调到 3。
|
||||
assert_eq!(
|
||||
*store.lowered_limits.lock().expect("lowered limits"),
|
||||
vec![("acct_test".to_owned(), 3)]
|
||||
);
|
||||
assert!(store.update_commands().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
assert!(runtime.cleared().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_concurrency_never_raises_limit() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::from([("acct_test".to_owned(), 40_u32)]),
|
||||
);
|
||||
let (task, store) = recovery_task(
|
||||
runtime_settings(true, false, true),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
// 峰值 40 的目标 32 高于全局默认 5;只降不升,不下发任何更新。
|
||||
assert!(store.update_commands().is_empty());
|
||||
}
|
||||
|
||||
/// 到期时间还早(超过探测提前量)的冻结不发起探测。
|
||||
#[tokio::test]
|
||||
async fn probe_skips_freezes_far_from_expiry() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::new(),
|
||||
);
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
/// 停用账号不参与探测恢复;并发下调同样跳过。
|
||||
#[tokio::test]
|
||||
async fn disabled_accounts_are_skipped() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_due(),
|
||||
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
|
||||
);
|
||||
let mut record = account_record("openai");
|
||||
record.enabled = false;
|
||||
let store = FakeAccountStore::with_account(record, events());
|
||||
let services = AdminHarness::new()
|
||||
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
|
||||
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
|
||||
.settings(Arc::new(FreezeSettingsStore {
|
||||
settings: runtime_settings(true, true, true),
|
||||
}))
|
||||
.provider(FakeProviderAdmin::new("openai", events()))
|
||||
.probe(Arc::new(SuccessfulProbe))
|
||||
.build()
|
||||
.await;
|
||||
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
|
||||
accounts: services.accounts_handle(),
|
||||
store: store.clone(),
|
||||
runtime: Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>,
|
||||
settings: Arc::new(FreezeSettingsStore {
|
||||
settings: runtime_settings(true, true, true),
|
||||
}),
|
||||
});
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
assert!(store.update_commands().is_empty());
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use gateway_admin::{
|
||||
AdminBundle, AdminServices,
|
||||
model::{
|
||||
AdminErrorKind, MutationActor, MutationContext,
|
||||
import_tasks::{ImportTaskInput, SubmitImportTask},
|
||||
provider_credentials::ImportCredentials,
|
||||
},
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
routing::ProviderKind,
|
||||
task::{WorkerContribution, WorkerKind, WorkerRunnable},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
AdminHarness,
|
||||
accounts::{
|
||||
EventLog, FakeAccountStore, FakeProviderAdmin, context, document, events, recorded,
|
||||
},
|
||||
};
|
||||
|
||||
fn command(id: Uuid, count: usize) -> SubmitImportTask {
|
||||
let context = context("task-submit");
|
||||
SubmitImportTask {
|
||||
submission_id: id,
|
||||
fingerprint: [1; 32],
|
||||
context: context.clone(),
|
||||
items: (0..count)
|
||||
.map(|_| ImportTaskInput {
|
||||
provider: ProviderKind::new("openai").unwrap(),
|
||||
command: ImportCredentials {
|
||||
outbound_proxy_id: None,
|
||||
settings: None,
|
||||
context: context.clone(),
|
||||
document: document(),
|
||||
},
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn harness() -> (AdminBundle, Arc<FakeProviderAdmin>, EventLog) {
|
||||
let log = events();
|
||||
let provider = FakeProviderAdmin::new("openai", log.clone());
|
||||
let bundle = AdminHarness::new()
|
||||
.provider(provider.clone())
|
||||
.accounts(FakeAccountStore::new("openai", log.clone()))
|
||||
.build_bundle()
|
||||
.await;
|
||||
(bundle, provider, log)
|
||||
}
|
||||
|
||||
fn start(bundle: &mut AdminBundle) -> (CancellationToken, tokio::task::JoinHandle<()>) {
|
||||
let registration = bundle
|
||||
.take_worker_contributions()
|
||||
.into_iter()
|
||||
.find_map(|contribution| match contribution {
|
||||
WorkerContribution::Registration(registration)
|
||||
if registration.id.kind() == WorkerKind::AccountImport =>
|
||||
{
|
||||
Some(registration)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.expect("import registration");
|
||||
let WorkerRunnable::Daemon { task, .. } = registration.runnable else {
|
||||
panic!("import daemon")
|
||||
};
|
||||
let cancellation = CancellationToken::new();
|
||||
let token = cancellation.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
task.run(token).await.expect("import daemon exits cleanly");
|
||||
});
|
||||
(cancellation, handle)
|
||||
}
|
||||
|
||||
async fn wait_started(log: &EventLog, count: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while recorded(log)
|
||||
.iter()
|
||||
.filter(|event| **event == "provider.prepare_import")
|
||||
.count()
|
||||
< count
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("imports started");
|
||||
}
|
||||
|
||||
async fn wait_finished(services: &AdminServices, id: Uuid) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), id)
|
||||
.unwrap()
|
||||
.summary
|
||||
.finished_at
|
||||
.is_none()
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("imports finished");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_should_return_the_same_task_but_reject_changed_content() {
|
||||
let (bundle, _, _) = harness().await;
|
||||
let services = bundle.services();
|
||||
let id = Uuid::now_v7();
|
||||
let first = services.import_tasks().submit(command(id, 2)).unwrap();
|
||||
let retry = services.import_tasks().submit(command(id, 2)).unwrap();
|
||||
assert_eq!(first.task_id, retry.task_id);
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.list(&context("reopened-page"))
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
let mut changed = command(id, 2);
|
||||
changed.fingerprint = [2; 32];
|
||||
assert_eq!(
|
||||
services.import_tasks().submit(changed).unwrap_err().kind(),
|
||||
AdminErrorKind::Conflict
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_reads_and_stop_should_be_isolated_by_owner() {
|
||||
let (bundle, _, _) = harness().await;
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap();
|
||||
let other = MutationContext {
|
||||
actor: MutationActor::AdminSession {
|
||||
admin_user_id: "another-admin".to_owned(),
|
||||
},
|
||||
request_id: "other-session".to_owned(),
|
||||
};
|
||||
assert!(services.import_tasks().list(&other).is_empty());
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.detail(&other, task.task_id)
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.stop(&other, task.task_id)
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::NotFound
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopping_should_skip_pending_items_and_allow_three_in_flight_to_commit() {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
let gate = provider.block_imports();
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 8))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_started(&log, 3).await;
|
||||
let stopped = services
|
||||
.import_tasks()
|
||||
.stop(&context("stop"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(stopped.summary.counts.running, 3);
|
||||
assert_eq!(stopped.summary.counts.skipped, 5);
|
||||
gate.add_permits(3);
|
||||
wait_finished(&services, task.task_id).await;
|
||||
let finished = services
|
||||
.import_tasks()
|
||||
.detail(&context("new-page"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(finished.summary.counts.imported_accounts, 3);
|
||||
assert_eq!(
|
||||
recorded(&log)
|
||||
.iter()
|
||||
.filter(|event| **event == "provider.prepare_import")
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
cancel.cancel();
|
||||
worker.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrency_limit_should_be_shared_across_tasks() {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
let gate = provider.block_imports();
|
||||
let services = bundle.services();
|
||||
let first = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 6))
|
||||
.unwrap();
|
||||
let second = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 6))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_started(&log, 3).await;
|
||||
let summaries = services.import_tasks().list(&context("poll"));
|
||||
assert_eq!(
|
||||
summaries
|
||||
.iter()
|
||||
.map(|task| task.counts.running)
|
||||
.sum::<usize>(),
|
||||
3
|
||||
);
|
||||
assert!(summaries.iter().all(|task| task.counts.running > 0));
|
||||
gate.add_permits(12);
|
||||
wait_finished(&services, first.task_id).await;
|
||||
wait_finished(&services, second.task_id).await;
|
||||
cancel.cancel();
|
||||
worker.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_and_unknown_items_should_not_abort_the_batch_or_retry_credentials() {
|
||||
for (kind, failed, unknown) in [
|
||||
(ProviderAdminErrorKind::Invalid, 1, 0),
|
||||
(ProviderAdminErrorKind::Ambiguous, 0, 1),
|
||||
] {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
provider.fail_next(kind);
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 4))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_finished(&services, task.task_id).await;
|
||||
let result = services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
result.summary.counts.succeeded,
|
||||
result.summary.counts.failed,
|
||||
result.summary.counts.unknown
|
||||
),
|
||||
(3, failed, unknown)
|
||||
);
|
||||
assert_eq!(
|
||||
recorded(&log)
|
||||
.iter()
|
||||
.filter(|event| **event == "provider.prepare_import")
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
cancel.cancel();
|
||||
worker.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_should_drain_running_items_and_reject_new_work() {
|
||||
let (mut bundle, provider, log) = harness().await;
|
||||
let gate = provider.block_imports();
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 7))
|
||||
.unwrap();
|
||||
let (cancel, worker) = start(&mut bundle);
|
||||
wait_started(&log, 3).await;
|
||||
cancel.cancel();
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Unavailable
|
||||
);
|
||||
assert!(!worker.is_finished());
|
||||
gate.add_permits(3);
|
||||
worker.await.unwrap();
|
||||
let result = services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), task.task_id)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
result.summary.counts.succeeded,
|
||||
result.summary.counts.skipped
|
||||
),
|
||||
(3, 4)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn completed_records_should_expire_and_new_process_should_start_empty() {
|
||||
let (bundle, _, _) = harness().await;
|
||||
let services = bundle.services();
|
||||
let task = services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 2))
|
||||
.unwrap();
|
||||
services
|
||||
.import_tasks()
|
||||
.stop(&context("stop"), task.task_id)
|
||||
.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(3601)).await;
|
||||
assert!(services.import_tasks().list(&context("poll")).is_empty());
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.detail(&context("poll"), task.task_id)
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::NotFound
|
||||
);
|
||||
let (fresh, _, _) = harness().await;
|
||||
assert!(
|
||||
fresh
|
||||
.services()
|
||||
.import_tasks()
|
||||
.list(&context("poll"))
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_should_reject_excess_work_before_starting_any_imports() {
|
||||
let (bundle, _, log) = harness().await;
|
||||
let services = bundle.services();
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 201))
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Invalid
|
||||
);
|
||||
for _ in 0..8 {
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
services
|
||||
.import_tasks()
|
||||
.submit(command(Uuid::now_v7(), 1))
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::RateLimited
|
||||
);
|
||||
assert!(recorded(&log).is_empty());
|
||||
}
|
||||
@@ -4,6 +4,8 @@ mod auth;
|
||||
mod auth_key;
|
||||
mod backup;
|
||||
mod client_keys;
|
||||
mod freeze_recovery;
|
||||
mod import_tasks;
|
||||
mod observability;
|
||||
mod openai;
|
||||
mod proxies;
|
||||
@@ -81,6 +83,7 @@ use gateway_core::{
|
||||
routing::{ConfigRevision, ProviderKind},
|
||||
runtime::SnapshotControl,
|
||||
};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub(super) struct AdminHarness {
|
||||
default_password: String,
|
||||
@@ -213,6 +216,10 @@ impl AdminHarness {
|
||||
}
|
||||
|
||||
pub(super) async fn build(self) -> AdminServices {
|
||||
self.build_bundle().await.services()
|
||||
}
|
||||
|
||||
pub(super) async fn build_bundle(self) -> gateway_admin::AdminBundle {
|
||||
gateway_admin::initialize(
|
||||
AdminConfig {
|
||||
session_ttl_minutes: self.session_ttl_minutes,
|
||||
@@ -247,7 +254,6 @@ impl AdminHarness {
|
||||
)
|
||||
.await
|
||||
.expect("initialize admin test harness")
|
||||
.services()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -484,6 +490,15 @@ impl AccountStore for UnavailableStore {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
_: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
@@ -526,6 +541,28 @@ impl AccountRuntimeStore for UnavailableStore {
|
||||
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -753,6 +753,8 @@ struct FixtureSettingsStore;
|
||||
impl SettingsStore for FixtureSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(RuntimeSettings {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(1).expect("revision"),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
@@ -769,6 +771,13 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
updated_at: Utc::now(),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +61,8 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
request_id: "request-settings".to_owned(),
|
||||
},
|
||||
ReplaceRuntimeSettings {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 0,
|
||||
refresh_concurrency: 1,
|
||||
@@ -75,6 +77,13 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -24,6 +24,7 @@ gateway-protocol = { path = "../gateway-protocol" }
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt", "sync", "time"] }
|
||||
tower-http = { version = "0.7.0", features = ["cors", "fs", "request-id", "timeout", "trace"] }
|
||||
|
||||
@@ -31,7 +31,7 @@ impl AccountProvider {
|
||||
}
|
||||
|
||||
/// 导入统一设置,复用编辑账号的备注、调度和分组约束。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountImportSettingsRequest {
|
||||
pub notes: Option<String>,
|
||||
@@ -67,7 +67,7 @@ impl AccountImportSettingsRequest {
|
||||
}
|
||||
|
||||
/// Provider-owned 账号导入请求;公共 API 不解释 `data` 内部字段。
|
||||
#[derive(Deserialize)]
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountImportRequest {
|
||||
pub outbound_proxy_id: Option<String>,
|
||||
@@ -319,9 +319,13 @@ impl AccountDeletionRequest {
|
||||
pub struct RotateAccountRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub access_token: String,
|
||||
pub access_token: Option<String>,
|
||||
pub refresh_token: Option<String>,
|
||||
pub id_token: Option<String>,
|
||||
pub base_url: Option<String>,
|
||||
pub api_key: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
pub settings: Option<UpdateAccountRequest>,
|
||||
}
|
||||
|
||||
impl RotateAccountRequest {
|
||||
@@ -330,11 +334,40 @@ impl RotateAccountRequest {
|
||||
return Err(WireValidationError::new("provider"));
|
||||
}
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
validate_oauth_material(
|
||||
&self.access_token,
|
||||
self.refresh_token.as_deref(),
|
||||
self.id_token.as_deref(),
|
||||
)
|
||||
if let Some(settings) = &self.settings {
|
||||
settings.validate()?;
|
||||
if settings.account_id != self.account_id {
|
||||
return Err(WireValidationError::new("settings.accountId"));
|
||||
}
|
||||
}
|
||||
if let Some(base_url) = &self.base_url {
|
||||
if self.access_token.is_some()
|
||||
|| self.refresh_token.is_some()
|
||||
|| self.id_token.is_some()
|
||||
|| base_url.is_empty()
|
||||
|| base_url.len() > 2048
|
||||
|| !matches!(self.transport.as_deref(), Some("http" | "prefer_websocket"))
|
||||
|| self.api_key.as_ref().is_some_and(|key| {
|
||||
key.is_empty()
|
||||
|| key.len() > 16 * 1024
|
||||
|| !key.bytes().all(|byte| byte.is_ascii_graphic())
|
||||
})
|
||||
{
|
||||
return Err(WireValidationError::new("credential"));
|
||||
}
|
||||
Ok(())
|
||||
} else {
|
||||
if self.api_key.is_some() || self.transport.is_some() {
|
||||
return Err(WireValidationError::new("credential"));
|
||||
}
|
||||
validate_oauth_material(
|
||||
self.access_token
|
||||
.as_deref()
|
||||
.ok_or_else(|| WireValidationError::new("accessToken"))?,
|
||||
self.refresh_token.as_deref(),
|
||||
self.id_token.as_deref(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_command(
|
||||
@@ -343,16 +376,34 @@ impl RotateAccountRequest {
|
||||
) -> Result<RotateCredential, WireValidationError> {
|
||||
self.validate()?;
|
||||
let mut material = Map::new();
|
||||
material.insert("access_token".to_owned(), Value::String(self.access_token));
|
||||
material.insert(
|
||||
"refresh_token".to_owned(),
|
||||
self.refresh_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
material.insert(
|
||||
"id_token".to_owned(),
|
||||
self.id_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
if let Some(base_url) = self.base_url {
|
||||
material.insert("base_url".to_owned(), Value::String(base_url));
|
||||
material.insert(
|
||||
"transport".to_owned(),
|
||||
self.transport.map_or(Value::Null, Value::String),
|
||||
);
|
||||
if let Some(key) = self.api_key {
|
||||
material.insert("api_key".to_owned(), Value::String(key));
|
||||
}
|
||||
} else {
|
||||
material.insert(
|
||||
"access_token".to_owned(),
|
||||
self.access_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
material.insert(
|
||||
"refresh_token".to_owned(),
|
||||
self.refresh_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
material.insert(
|
||||
"id_token".to_owned(),
|
||||
self.id_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
}
|
||||
Ok(RotateCredential {
|
||||
settings: self
|
||||
.settings
|
||||
.map(UpdateAccountRequest::into_command)
|
||||
.transpose()?,
|
||||
mutation: CredentialMutation {
|
||||
context,
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
|
||||
@@ -10,6 +10,7 @@ where
|
||||
S: SessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.merge(super::import_tasks::router::<S>())
|
||||
.route("/api/admin/accounts", get(list_accounts::<S>))
|
||||
.route("/api/admin/accounts/detail", get(account_detail::<S>))
|
||||
.route("/api/admin/accounts/export", get(export_accounts::<S>))
|
||||
@@ -120,8 +121,15 @@ where
|
||||
.quota(&account_id, false)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = AccountQuotaData {
|
||||
let configuration = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.account_configuration(&account_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = AccountDetailData {
|
||||
account: account_view(result, Utc::now()),
|
||||
credential_configuration: configuration.map(provider_document_value),
|
||||
};
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
//! 后台导入任务的有界请求与安全结果投影。
|
||||
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use gateway_admin::model::import_tasks::{
|
||||
ImportTaskDetail, ImportTaskInput, ImportTaskSummary, MAX_IMPORT_TASK_ITEMS, SubmitImportTask,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use super::*;
|
||||
use crate::auth::SessionState;
|
||||
|
||||
pub(super) fn router<S>() -> Router<S>
|
||||
where
|
||||
S: SessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route(
|
||||
"/api/admin/accounts/import-tasks",
|
||||
get(list::<S>).post(submit::<S>),
|
||||
)
|
||||
.route("/api/admin/accounts/import-tasks/detail", get(detail::<S>))
|
||||
.route("/api/admin/accounts/import-tasks/stop", post(stop::<S>))
|
||||
.layer(DefaultBodyLimit::max(4 * 1024 * 1024))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct SubmitRequest {
|
||||
submission_id: String,
|
||||
items: Vec<AccountImportRequest>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct TaskId {
|
||||
task_id: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct SummaryView {
|
||||
task_id: String,
|
||||
created_at: String,
|
||||
finished_at: Option<String>,
|
||||
stop_requested: bool,
|
||||
total: usize,
|
||||
counts: CountsView,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CountsView {
|
||||
pending: usize,
|
||||
running: usize,
|
||||
succeeded: usize,
|
||||
failed: usize,
|
||||
unknown: usize,
|
||||
skipped: usize,
|
||||
imported_accounts: usize,
|
||||
}
|
||||
|
||||
impl From<ImportTaskSummary> for SummaryView {
|
||||
fn from(task: ImportTaskSummary) -> Self {
|
||||
let counts = task.counts;
|
||||
Self {
|
||||
task_id: task.task_id.to_string(),
|
||||
created_at: task.created_at.to_rfc3339(),
|
||||
finished_at: task.finished_at.map(|at| at.to_rfc3339()),
|
||||
stop_requested: task.stop_requested,
|
||||
total: task.total,
|
||||
counts: CountsView {
|
||||
pending: counts.pending,
|
||||
running: counts.running,
|
||||
succeeded: counts.succeeded,
|
||||
failed: counts.failed,
|
||||
unknown: counts.unknown,
|
||||
skipped: counts.skipped,
|
||||
imported_accounts: counts.imported_accounts,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct DetailView {
|
||||
#[serde(flatten)]
|
||||
summary: SummaryView,
|
||||
items: Vec<ItemView>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ItemView {
|
||||
index: usize,
|
||||
provider: String,
|
||||
status: &'static str,
|
||||
account_ids: Vec<String>,
|
||||
message: Option<String>,
|
||||
}
|
||||
|
||||
impl From<ImportTaskDetail> for DetailView {
|
||||
fn from(task: ImportTaskDetail) -> Self {
|
||||
Self {
|
||||
summary: task.summary.into(),
|
||||
items: task
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| ItemView {
|
||||
index: item.index,
|
||||
provider: item.provider.to_string(),
|
||||
status: item.status.as_str(),
|
||||
account_ids: item
|
||||
.account_ids
|
||||
.into_iter()
|
||||
.map(|id| id.to_string())
|
||||
.collect(),
|
||||
message: item.message,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ListView {
|
||||
items: Vec<SummaryView>,
|
||||
}
|
||||
|
||||
async fn submit<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminJson(request): AdminJson<SubmitRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let submission_id = Uuid::parse_str(&request.submission_id)
|
||||
.map_err(|_| map_wire_error(WireValidationError::new("submissionId")))?;
|
||||
if submission_id.is_nil() {
|
||||
return Err(map_wire_error(WireValidationError::new("submissionId")));
|
||||
}
|
||||
if request.items.is_empty() || request.items.len() > MAX_IMPORT_TASK_ITEMS {
|
||||
return Err(map_wire_error(WireValidationError::new("items")));
|
||||
}
|
||||
let encoded = serde_json::to_vec(&request.items)
|
||||
.map_err(|_| map_wire_error(WireValidationError::new("items")))?;
|
||||
let fingerprint = Sha256::digest(&encoded).into();
|
||||
drop(encoded);
|
||||
let context = auth.context().mutation_context();
|
||||
let items = request
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let (provider, command) = item.into_command(context.clone())?;
|
||||
let provider = ProviderKind::new(match provider {
|
||||
AccountProvider::OpenAi => "openai",
|
||||
AccountProvider::Xai => "xai",
|
||||
})
|
||||
.map_err(|_| WireValidationError::new("provider"))?;
|
||||
Ok(ImportTaskInput { provider, command })
|
||||
})
|
||||
.collect::<Result<Vec<_>, WireValidationError>>()
|
||||
.map_err(map_wire_error)?;
|
||||
let task = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.submit(SubmitImportTask {
|
||||
submission_id,
|
||||
fingerprint,
|
||||
context,
|
||||
items,
|
||||
})
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::ACCEPTED,
|
||||
AdminEnvelope::ok(SummaryView::from(task)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list<S>(auth: AdminAuth, State(state): State<S>) -> impl IntoResponse
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let items = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.list(&auth.context().mutation_context())
|
||||
.into_iter()
|
||||
.map(SummaryView::from)
|
||||
.collect();
|
||||
AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(ListView { items }))
|
||||
}
|
||||
|
||||
async fn detail<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<TaskId>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let task = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.detail(
|
||||
&auth.context().mutation_context(),
|
||||
parse_task_id(&query.task_id)?,
|
||||
)
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(DetailView::from(task)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn stop<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminJson(request): AdminJson<TaskId>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let task = state
|
||||
.admin_services()
|
||||
.import_tasks()
|
||||
.stop(
|
||||
&auth.context().mutation_context(),
|
||||
parse_task_id(&request.task_id)?,
|
||||
)
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(DetailView::from(task)),
|
||||
))
|
||||
}
|
||||
|
||||
fn parse_task_id(value: &str) -> Result<Uuid, AdminError> {
|
||||
Uuid::parse_str(value).map_err(|_| map_wire_error(WireValidationError::new("taskId")))
|
||||
}
|
||||
@@ -66,6 +66,7 @@ const MAX_AVATAR_VERSION_BYTES: usize = 32;
|
||||
|
||||
mod credentials;
|
||||
mod handlers;
|
||||
mod import_tasks;
|
||||
mod presenter;
|
||||
mod wire;
|
||||
|
||||
|
||||
@@ -66,12 +66,16 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
|
||||
quota,
|
||||
} = item;
|
||||
let status = projection.status.as_str().to_owned();
|
||||
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
|
||||
let cooldown = projection.cooldown;
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
let usage_period = quota.usage_window().map(|(_, period)| period);
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
|
||||
let mut usage = account_usage_view(usage, usage_period, now);
|
||||
if account.authentication_kind == "api_key" {
|
||||
usage.window_label_display = "通用额度".to_owned();
|
||||
}
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now);
|
||||
AccountView {
|
||||
id: account.id.clone(),
|
||||
name: account.name,
|
||||
@@ -121,7 +125,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
|
||||
updated_at,
|
||||
updated_at_display: china_datetime(&account.updated_at),
|
||||
quota,
|
||||
usage: account_usage_view(usage, usage_period, now),
|
||||
usage,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,7 +189,7 @@ fn forecast_usd_display(value: Option<f64>) -> String {
|
||||
|
||||
pub(super) fn account_quota_view(
|
||||
mut quota: ProviderQuota,
|
||||
rate_limited_until: Option<DateTime<Utc>>,
|
||||
cooldown: Option<gateway_core::account::AccountCooldown>,
|
||||
now: DateTime<Utc>,
|
||||
) -> (AccountQuotaView, Option<String>) {
|
||||
quota.apply_limit_reached_display();
|
||||
@@ -196,12 +200,22 @@ pub(super) fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
|
||||
let rate_limited_until = cooldown.map(|value| china_datetime(&value.until.into()));
|
||||
let rate_limit_reason = cooldown.map(|value| {
|
||||
if value.kind.is_capacity_freeze() {
|
||||
"capacity_freeze".to_owned()
|
||||
} else {
|
||||
"upstream_rate_limit".to_owned()
|
||||
}
|
||||
});
|
||||
let recovery_probe_required = cooldown.is_some_and(|value| value.kind.requires_probe());
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
rate_limit_reason,
|
||||
recovery_probe_required,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
|
||||
@@ -361,8 +361,10 @@ pub struct AccountGroupRefView {
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
/// 冷却到期或可开始恢复探测的时间;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub rate_limit_reason: Option<String>,
|
||||
pub recovery_probe_required: bool,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -684,6 +686,14 @@ pub struct AccountQuotaData {
|
||||
pub account: AccountView,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountDetailData {
|
||||
pub account: AccountView,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub credential_configuration: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// 个人资料、累计统计与订阅的统一响应。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
|
||||
@@ -48,6 +48,7 @@ struct RemoveAccountRequest {
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct CreateRequest {
|
||||
location: Option<gateway_core::account::RequestLocation>,
|
||||
name: String,
|
||||
proxy_url: AccountProxyUpdate,
|
||||
}
|
||||
@@ -55,12 +56,23 @@ struct CreateRequest {
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct UpdateRequest {
|
||||
#[serde(default, deserialize_with = "deserialize_location_update")]
|
||||
location: Option<Option<gateway_core::account::RequestLocation>>,
|
||||
id: String,
|
||||
revision: u64,
|
||||
name: String,
|
||||
proxy_url: Option<AccountProxyUpdate>,
|
||||
}
|
||||
|
||||
fn deserialize_location_update<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<Option<gateway_core::account::RequestLocation>>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
Option::deserialize(deserializer).map(Some)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct IdRequest {
|
||||
@@ -111,6 +123,7 @@ struct ProxyAccountView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ProxyView {
|
||||
location: Option<gateway_core::account::RequestLocation>,
|
||||
id: String,
|
||||
name: String,
|
||||
endpoint: String,
|
||||
@@ -127,6 +140,7 @@ impl From<ProxyRecord> for ProxyView {
|
||||
fn from(record: ProxyRecord) -> Self {
|
||||
let endpoint = record.proxy.endpoint();
|
||||
Self {
|
||||
location: record.location,
|
||||
id: record.id,
|
||||
name: record.name,
|
||||
has_authentication: record.proxy.expose_url() != endpoint,
|
||||
@@ -307,6 +321,7 @@ where
|
||||
.proxies()
|
||||
.create(
|
||||
NewProxy {
|
||||
location: request.location,
|
||||
name: request.name,
|
||||
proxy,
|
||||
},
|
||||
@@ -365,6 +380,7 @@ where
|
||||
.proxies()
|
||||
.update(
|
||||
UpdateProxy {
|
||||
location: request.location,
|
||||
id: request.id,
|
||||
revision: revision(request.revision)?,
|
||||
name: request.name,
|
||||
|
||||
@@ -34,6 +34,8 @@ pub type ModelMappings = BTreeMap<String, String>;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RuntimeSettingsView {
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -48,6 +50,13 @@ pub struct RuntimeSettingsView {
|
||||
pub usage_retention_days: u64,
|
||||
pub ops_event_retention_days: u64,
|
||||
pub audit_retention_days: u64,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u64,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -55,6 +64,8 @@ pub struct RuntimeSettingsView {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateRuntimeSettingsRequest {
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -69,11 +80,21 @@ pub struct UpdateRuntimeSettingsRequest {
|
||||
pub usage_retention_days: u64,
|
||||
pub ops_event_retention_days: u64,
|
||||
pub audit_retention_days: u64,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u64,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl UpdateRuntimeSettingsRequest {
|
||||
/// 校验公共运行参数。
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
self.request_location
|
||||
.validate()
|
||||
.map_err(|_| WireValidationError::new("requestLocation"))?;
|
||||
validate_model_mappings(&self.model_mappings)?;
|
||||
for (value, field) in [
|
||||
(self.max_waiting_per_key, "maxWaitingPerKey"),
|
||||
@@ -110,12 +131,46 @@ impl UpdateRuntimeSettingsRequest {
|
||||
self.min_codex_cli_version.as_deref(),
|
||||
"minCodexCliVersion",
|
||||
)?;
|
||||
for (value, field) in [
|
||||
(
|
||||
self.account_auto_freeze_threshold,
|
||||
"accountAutoFreezeThreshold",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_window_seconds,
|
||||
"accountAutoFreezeWindowSeconds",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_duration_seconds,
|
||||
"accountAutoFreezeDurationSeconds",
|
||||
),
|
||||
] {
|
||||
require_positive_i64(value, field)?;
|
||||
}
|
||||
if !(2..=1_000).contains(&self.account_auto_freeze_threshold) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeThreshold"));
|
||||
}
|
||||
if !(60..=3_600).contains(&self.account_auto_freeze_window_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeWindowSeconds"));
|
||||
}
|
||||
if !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeDurationSeconds"));
|
||||
}
|
||||
validate_optional_probe_model(
|
||||
self.account_auto_freeze_probe_model.as_deref(),
|
||||
"accountAutoFreezeProbeModel",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(self) -> Result<ReplaceRuntimeSettings, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(ReplaceRuntimeSettings {
|
||||
request_location_enabled: self.request_location_enabled,
|
||||
request_location: self
|
||||
.request_location
|
||||
.normalized()
|
||||
.map_err(|_| WireValidationError::new("requestLocation"))?,
|
||||
model_mappings: domain_model_mappings(self.model_mappings)?,
|
||||
refresh_margin_seconds: self.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(self.refresh_concurrency)
|
||||
@@ -136,6 +191,14 @@ impl UpdateRuntimeSettingsRequest {
|
||||
.map_err(|_| WireValidationError::new("settingsOpsRetentionOverflow"))?,
|
||||
audit_retention_days: u32::try_from(self.audit_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?,
|
||||
account_auto_freeze_enabled: self.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u32::try_from(self.account_auto_freeze_threshold)
|
||||
.map_err(|_| WireValidationError::new("settingsFreezeThresholdOverflow"))?,
|
||||
account_auto_freeze_window_seconds: self.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: self.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: self.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: self.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: self.account_auto_freeze_adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -143,6 +206,8 @@ impl UpdateRuntimeSettingsRequest {
|
||||
impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
fn from(settings: RuntimeSettings) -> Self {
|
||||
Self {
|
||||
request_location_enabled: settings.request_location_enabled,
|
||||
request_location: settings.request_location,
|
||||
model_mappings: wire_model_mappings(settings.model_mappings),
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
@@ -157,6 +222,14 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u64::from(settings.account_auto_freeze_threshold),
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
updated_at: settings.updated_at,
|
||||
}
|
||||
}
|
||||
@@ -452,6 +525,22 @@ fn validate_optional_client_version(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 探测模型为可选自由文本:非空、去首尾空白后不变、无控制字符且不超过 128 字节。
|
||||
fn validate_optional_probe_model(
|
||||
value: Option<&str>,
|
||||
field: &'static str,
|
||||
) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| {
|
||||
value.is_empty()
|
||||
|| value.len() > 128
|
||||
|| value != value.trim()
|
||||
|| value.bytes().any(|byte| byte.is_ascii_control())
|
||||
}) {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
let message = match error.field() {
|
||||
"settingsRefreshConcurrencyOverflow" => "refreshConcurrency 不合法".to_owned(),
|
||||
@@ -459,6 +548,12 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
"settingsUsageRetentionOverflow" => "usageRetentionDays 不合法".to_owned(),
|
||||
"settingsOpsRetentionOverflow" => "opsEventRetentionDays 不合法".to_owned(),
|
||||
"settingsAuditRetentionOverflow" => "auditRetentionDays 不合法".to_owned(),
|
||||
"requestLocation" => "请求位置不合法,请检查国家代码、地区和城市".to_owned(),
|
||||
"settingsFreezeThresholdOverflow" => "accountAutoFreezeThreshold 不合法".to_owned(),
|
||||
"accountAutoFreezeThreshold" => "账号自动冻结阈值应为 2~1000 的整数".to_owned(),
|
||||
"accountAutoFreezeWindowSeconds" => "账号自动冻结统计窗口应为 60~3600 秒".to_owned(),
|
||||
"accountAutoFreezeDurationSeconds" => "账号自动冻结时长应为 300~604800 秒".to_owned(),
|
||||
"accountAutoFreezeProbeModel" => "探测模型格式不合法".to_owned(),
|
||||
"minCodexDesktopVersion" => "Codex Desktop 最低版本格式不合法".to_owned(),
|
||||
"minCodexCliVersion" => "Codex CLI 最低版本格式不合法".to_owned(),
|
||||
field => format!("{field} 字段不合法"),
|
||||
|
||||
@@ -37,7 +37,6 @@ mod session_cookie;
|
||||
|
||||
/// API-owned HTTP 与静态资源配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ApiConfig {
|
||||
pub asset_directory: PathBuf,
|
||||
pub cors_allowed_origins: Vec<String>,
|
||||
@@ -62,12 +61,17 @@ impl ApiConfig {
|
||||
return Err(ApiConfigError::InvalidAssetDirectory);
|
||||
}
|
||||
}
|
||||
if self.asset_directory.as_os_str().is_empty() {
|
||||
return Err(ApiConfigError::InvalidAssetDirectory);
|
||||
}
|
||||
self.validate()?;
|
||||
if self.asset_directory.is_relative() {
|
||||
self.asset_directory = source_dir.join(&self.asset_directory);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate(&mut self) -> Result<(), ApiConfigError> {
|
||||
if self.asset_directory.as_os_str().is_empty() {
|
||||
return Err(ApiConfigError::InvalidAssetDirectory);
|
||||
}
|
||||
if self.request_timeout_seconds == Some(0) {
|
||||
return Err(ApiConfigError::InvalidRequestTimeout);
|
||||
}
|
||||
@@ -124,9 +128,8 @@ pub fn initialize(
|
||||
worker_health: Arc<dyn WorkerHealthSource>,
|
||||
lifecycle: Arc<dyn ConnectionLifecycle>,
|
||||
) -> Result<ApiBundle, ApiError> {
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(ApiError::Config)?;
|
||||
// 配置加载已解析环境变量与相对路径,初始化只校验,避免覆盖最终目录。
|
||||
config.validate().map_err(ApiError::Config)?;
|
||||
let request_id_header = HeaderName::from_str(&config.request_id_header)
|
||||
.map_err(|_| ApiError::Config(ApiConfigError::InvalidRequestIdHeader))?;
|
||||
let state = ApiState {
|
||||
|
||||
@@ -535,8 +535,6 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
.iter()
|
||||
.any(|connection_header| connection_header.eq_ignore_ascii_case(name))
|
||||
|| is_transport_managed_request_header(name)
|
||||
|| name.starts_with("x-grok-")
|
||||
|| name.starts_with("x-xai-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -551,15 +549,9 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent/version
|
||||
// 不能作为不透明头透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
| "originator"
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
async fn send(
|
||||
fixture: &AdminTestFixture,
|
||||
method: &str,
|
||||
uri: &str,
|
||||
body: Value,
|
||||
authenticated: bool,
|
||||
) -> (StatusCode, Value) {
|
||||
let mut request = Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header("x-request-id", "import-task-contract")
|
||||
.header(header::CONTENT_TYPE, "application/json");
|
||||
if authenticated {
|
||||
request = request.header(header::COOKIE, "cpr_session=valid-session");
|
||||
}
|
||||
let response = admin::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(request.body(Body::from(body.to_string())).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||
let status = response.status();
|
||||
let bytes = to_bytes(response.into_body(), 8 * 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
let value: Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert!(!value.to_string().contains("synthetic-sensitive-token"));
|
||||
(status, value)
|
||||
}
|
||||
|
||||
fn input() -> Value {
|
||||
json!({"submissionId": "01994fcb-a333-7333-8000-000000000001", "items": [{"provider": "openai", "data": {"refreshToken": "synthetic-sensitive-token"}}]})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn import_tasks_should_accept_list_restore_stop_and_deduplicate_without_echoing_credentials()
|
||||
{
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let (status, accepted) = send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
input(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED);
|
||||
let id = accepted["data"]["taskId"].as_str().unwrap();
|
||||
let (_, retried) = send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
input(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(retried["data"]["taskId"], id);
|
||||
let (_, list) = send(
|
||||
&fixture,
|
||||
"GET",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
Value::Null,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(list["data"]["items"].as_array().unwrap().len(), 1);
|
||||
let (_, detail) = send(
|
||||
&fixture,
|
||||
"GET",
|
||||
&format!("/api/admin/accounts/import-tasks/detail?taskId={id}"),
|
||||
Value::Null,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(detail["data"]["items"][0]["status"], "pending");
|
||||
let (_, stopped) = send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks/stop",
|
||||
json!({"taskId": id}),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(stopped["data"]["counts"]["skipped"], 1);
|
||||
assert!(stopped["data"]["finishedAt"].is_string());
|
||||
let mut changed = input();
|
||||
changed["items"][0]["data"]["refreshToken"] = json!("changed-synthetic-input");
|
||||
assert_eq!(
|
||||
send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
changed,
|
||||
true
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::CONFLICT
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn import_tasks_should_require_admin_and_enforce_input_bounds() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (method, uri, body) in [
|
||||
("POST", "/api/admin/accounts/import-tasks", input()),
|
||||
("GET", "/api/admin/accounts/import-tasks", Value::Null),
|
||||
(
|
||||
"GET",
|
||||
"/api/admin/accounts/import-tasks/detail?taskId=01994fcb-a333-7333-8000-000000000001",
|
||||
Value::Null,
|
||||
),
|
||||
(
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks/stop",
|
||||
json!({"taskId": "01994fcb-a333-7333-8000-000000000001"}),
|
||||
),
|
||||
] {
|
||||
assert_eq!(
|
||||
send(&fixture, method, uri, body, false).await.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
}
|
||||
for count in [0, 201] {
|
||||
let mut body = input();
|
||||
body["items"] = json!(vec![body["items"][0].clone(); count]);
|
||||
assert_eq!(
|
||||
send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
body,
|
||||
true
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
}
|
||||
let mut body = input();
|
||||
body["items"][0]["data"]["refreshToken"] = json!("x".repeat(4 * 1024 * 1024));
|
||||
assert_eq!(
|
||||
send(
|
||||
&fixture,
|
||||
"POST",
|
||||
"/api/admin/accounts/import-tasks",
|
||||
body,
|
||||
true
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::PAYLOAD_TOO_LARGE
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
mod handlers;
|
||||
mod import_tasks;
|
||||
mod presenter;
|
||||
|
||||
mod personal_info {
|
||||
@@ -599,6 +600,44 @@ mod actions {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_key_rotation_settings_must_target_the_same_account_and_remain_valid() {
|
||||
let mut request = json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_api",
|
||||
"baseUrl": "https://api.example.invalid/v1",
|
||||
"transport": "http",
|
||||
"settings": {
|
||||
"accountId": "acct_api",
|
||||
"enabled": true,
|
||||
"concurrencyLimit": null,
|
||||
"weight": 1,
|
||||
"groupIds": []
|
||||
}
|
||||
});
|
||||
serde_json::from_value::<RotateAccountRequest>(request.clone())
|
||||
.expect("decode combined save")
|
||||
.validate()
|
||||
.expect("blank replacement key preserves the existing key");
|
||||
request["settings"]["accountId"] = json!("acct_other");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<RotateAccountRequest>(request.clone())
|
||||
.unwrap()
|
||||
.validate()
|
||||
.unwrap_err()
|
||||
.field(),
|
||||
"settings.accountId"
|
||||
);
|
||||
request["settings"]["accountId"] = json!("acct_api");
|
||||
request["settings"]["concurrencyLimit"] = json!(0);
|
||||
assert!(
|
||||
serde_json::from_value::<RotateAccountRequest>(request)
|
||||
.unwrap()
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_recovery_requests_should_not_accept_client_revision_fences() {
|
||||
let authorization: StartAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -463,7 +463,7 @@ mod provider {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
};
|
||||
|
||||
@@ -402,6 +402,8 @@ impl SettingsStore for MemorySettingsStore {
|
||||
) -> AdminStoreResult<RuntimeSettings> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
let updated = RuntimeSettings {
|
||||
request_location_enabled: command.request_location_enabled,
|
||||
request_location: command.request_location,
|
||||
config_revision: next_revision(settings.config_revision),
|
||||
model_mappings: command.model_mappings,
|
||||
refresh_margin_seconds: command.refresh_margin_seconds,
|
||||
@@ -417,6 +419,13 @@ impl SettingsStore for MemorySettingsStore {
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
};
|
||||
*settings = updated.clone();
|
||||
@@ -587,7 +596,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
@@ -601,7 +610,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
@@ -896,6 +905,15 @@ impl AccountStore for UnusedStore {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
_: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
@@ -933,17 +951,39 @@ impl AccountStore for UnusedStore {
|
||||
impl AccountRuntimeStore for UnusedStore {
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::new(),
|
||||
cooldown: BTreeMap::new(),
|
||||
in_flight: Some(BTreeMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::new(),
|
||||
cooldown: BTreeMap::new(),
|
||||
in_flight: Some(BTreeMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -1230,6 +1270,8 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
),
|
||||
]);
|
||||
RuntimeSettings {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: mappings,
|
||||
refresh_margin_seconds: 3_600,
|
||||
@@ -1245,6 +1287,13 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,71 @@ use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_location_round_trips_preserves_omitted_and_clears_null() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let location =
|
||||
json!({"country":"JP", "region":"Tokyo", "city":"Tokyo", "timezone":"Asia/Tokyo"});
|
||||
let (status, created) = request(
|
||||
&fixture,
|
||||
"/api/admin/proxies/create",
|
||||
Some(json!({
|
||||
"name":"Tokyo", "proxyUrl":"http://proxy.example:8080", "location":location
|
||||
})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CREATED);
|
||||
assert_eq!(created["data"]["record"]["location"], location);
|
||||
let (_, listed) = request(&fixture, "/api/admin/proxies", None, true).await;
|
||||
assert_eq!(listed["data"]["items"][0]["location"], location);
|
||||
for (revision, change, expected) in [
|
||||
(1, None, location.clone()),
|
||||
(2, Some(Value::Null), Value::Null),
|
||||
(3, Some(location.clone()), location.clone()),
|
||||
] {
|
||||
let mut body = json!({"id":"proxy_test", "revision":revision, "name":"Renamed"});
|
||||
if let Some(change) = change {
|
||||
body["location"] = change;
|
||||
}
|
||||
let (status, changed) =
|
||||
request(&fixture, "/api/admin/proxies/update", Some(body), true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(changed["data"]["record"]["location"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_location_rejects_invalid_and_incomplete_input() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (field, value) in [
|
||||
("country", json!("jp")),
|
||||
("city", json!(" ")),
|
||||
("region", json!("Tokyo\n")),
|
||||
("city", json!("x".repeat(129))),
|
||||
("timezone", json!("Asia/Typo")),
|
||||
] {
|
||||
let mut invalid =
|
||||
json!({"country":"JP", "region":"Tokyo", "city":"Tokyo", "timezone":"Asia/Tokyo"});
|
||||
invalid[field] = value;
|
||||
let (status, _) = request(&fixture, "/api/admin/proxies/create", Some(json!({"name":"Invalid", "proxyUrl":"http://proxy.example:8080", "location":invalid})), true).await;
|
||||
assert!(status.is_client_error(), "invalid {field} accepted");
|
||||
}
|
||||
let (status, _) = request(&fixture, "/api/admin/proxies/create", Some(json!({"name":"Incomplete", "proxyUrl":"http://proxy.example:8080", "location":{"country":"JP"}})), true).await;
|
||||
assert!(status.is_client_error());
|
||||
let (status, created) = request(
|
||||
&fixture,
|
||||
"/api/admin/proxies/create",
|
||||
Some(json!({"name":"Legacy", "proxyUrl":"http://proxy.example:8080"})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CREATED);
|
||||
assert!(created["data"]["record"]["location"].is_null());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct MemoryProxies(Mutex<Option<ProxyRecord>>);
|
||||
|
||||
@@ -98,6 +163,7 @@ impl ProxyStore for MemoryProxies {
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<ProxyMutation> {
|
||||
let record = ProxyRecord {
|
||||
location: command.location,
|
||||
id: "proxy_test".to_owned(),
|
||||
name: command.name,
|
||||
proxy: command.proxy,
|
||||
@@ -122,6 +188,9 @@ impl ProxyStore for MemoryProxies {
|
||||
let mut stored = self.0.lock().unwrap();
|
||||
let record = stored.as_mut().ok_or_else(missing)?;
|
||||
record.name = command.name;
|
||||
if let Some(location) = command.location {
|
||||
record.location = location;
|
||||
}
|
||||
if let Some(proxy) = command.proxy {
|
||||
record.proxy = proxy;
|
||||
}
|
||||
|
||||
@@ -37,6 +37,8 @@ async fn response_json(response: axum::response::Response) -> Value {
|
||||
|
||||
fn update_body() -> Value {
|
||||
json!({
|
||||
"requestLocationEnabled": false,
|
||||
"requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"},
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
@@ -53,7 +55,14 @@ fn update_body() -> Value {
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
"accountAutoFreezeThreshold": 12,
|
||||
"accountAutoFreezeWindowSeconds": 600,
|
||||
"accountAutoFreezeDurationSeconds": 7200,
|
||||
"accountAutoFreezeProbeEnabled": true,
|
||||
"accountAutoFreezeProbeModel": null,
|
||||
"accountAutoFreezeAdaptiveConcurrency": true
|
||||
})
|
||||
}
|
||||
|
||||
@@ -92,6 +101,8 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let settings = RuntimeSettings {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: BTreeMap::from_iter([
|
||||
(
|
||||
@@ -116,6 +127,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
usage_retention_days: 32,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
@@ -126,6 +144,8 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"requestLocationEnabled": false,
|
||||
"requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"},
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
@@ -143,6 +163,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
"accountAutoFreezeThreshold": 12,
|
||||
"accountAutoFreezeWindowSeconds": 600,
|
||||
"accountAutoFreezeDurationSeconds": 7200,
|
||||
"accountAutoFreezeProbeEnabled": true,
|
||||
"accountAutoFreezeProbeModel": null,
|
||||
"accountAutoFreezeAdaptiveConcurrency": true,
|
||||
"updatedAt": "2026-08-02T10:30:00Z"
|
||||
})
|
||||
);
|
||||
@@ -169,6 +196,8 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
.cloned()
|
||||
.collect();
|
||||
let settings = RuntimeSettings {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: request
|
||||
.model_mappings
|
||||
@@ -195,6 +224,13 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
@@ -422,3 +458,120 @@ fn concurrency_queue_fields_validate_bounds_and_accept_disabled_queues() {
|
||||
.validate()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_location_should_normalize_toggle_and_round_trip() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
body["requestLocationEnabled"] = json!(true);
|
||||
body["requestLocation"] =
|
||||
json!({"country":"JP", "region":" Tokyo ", "city":" Tokyo ", "timezone":"Asia/Tokyo"});
|
||||
let expected =
|
||||
json!({"country":"JP", "region":"Tokyo", "city":"Tokyo", "timezone":"Asia/Tokyo"});
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["requestLocation"],
|
||||
expected
|
||||
);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["requestLocation"],
|
||||
expected
|
||||
);
|
||||
for enabled in [false, true] {
|
||||
let mut body = update_body();
|
||||
body["requestLocationEnabled"] = json!(enabled);
|
||||
body["requestLocation"] = expected.clone();
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
let data = response_json(response).await["data"].clone();
|
||||
assert_eq!(data["requestLocationEnabled"], json!(enabled));
|
||||
assert_eq!(data["requestLocation"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_location_should_reject_invalid_or_missing_fields_without_replacing_settings() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let original = update_body()["requestLocation"].clone();
|
||||
// JSON 结构或时区解析失败由 Axum 返回 422,业务校验失败返回 400。
|
||||
let mut invalid = vec![
|
||||
(Value::Null, StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!({}), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(
|
||||
json!({"timezone":"Asia/Tokyo"}),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
];
|
||||
for (field, value, status) in [
|
||||
("country", "USA", StatusCode::BAD_REQUEST),
|
||||
("country", "us", StatusCode::BAD_REQUEST),
|
||||
("city", "", StatusCode::BAD_REQUEST),
|
||||
("region", "\nTokyo", StatusCode::BAD_REQUEST),
|
||||
(
|
||||
"timezone",
|
||||
"Not/A_Timezone",
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
] {
|
||||
let mut location = original.clone();
|
||||
location[field] = json!(value);
|
||||
invalid.push((location, status));
|
||||
}
|
||||
for (location, expected_status) in invalid {
|
||||
let mut body = update_body();
|
||||
body["requestLocation"] = location;
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), expected_status);
|
||||
}
|
||||
let mut body = update_body();
|
||||
body.as_object_mut().unwrap().remove("requestLocation");
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["requestLocation"],
|
||||
original
|
||||
);
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/admin/account_groups.rs",
|
||||
"src/admin/accounts/credentials.rs",
|
||||
"src/admin/accounts/handlers.rs",
|
||||
"src/admin/accounts/import_tasks.rs",
|
||||
"src/admin/accounts/mod.rs",
|
||||
"src/admin/accounts/presenter.rs",
|
||||
"src/admin/accounts/wire.rs",
|
||||
@@ -95,6 +96,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
let mut expected = vec![
|
||||
"tests/admin/account_groups.rs",
|
||||
"tests/admin/accounts/handlers.rs",
|
||||
"tests/admin/accounts/import_tasks.rs",
|
||||
"tests/admin/accounts/mod.rs",
|
||||
"tests/admin/accounts/presenter.rs",
|
||||
"tests/admin/auth.rs",
|
||||
|
||||
@@ -5,3 +5,55 @@ mod health;
|
||||
mod key_usage;
|
||||
mod openai;
|
||||
mod support;
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_should_serve_resolved_assets_with_relative_environment_override() {
|
||||
use axum::{body::Body, http::Request};
|
||||
use std::{fs, process::Command};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
const CHILD_ENV: &str = "CPR_TEST_ASSET_DIRECTORY";
|
||||
let Ok(directory) = std::env::var(CHILD_ENV) else {
|
||||
let directory = std::env::temp_dir().join(format!("cpr-assets-{}", uuid::Uuid::now_v7()));
|
||||
fs::create_dir_all(directory.join("deploy")).expect("config directory");
|
||||
fs::create_dir_all(directory.join("web/dist")).expect("asset directory");
|
||||
fs::write(directory.join("web/dist/index.html"), "resolved-web-assets").expect("index");
|
||||
let output = Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"api_should_serve_resolved_assets_with_relative_environment_override",
|
||||
])
|
||||
.current_dir(&directory)
|
||||
.env(CHILD_ENV, &directory)
|
||||
.env("CPR_WEB_DIST_DIR", "../web/dist")
|
||||
.output()
|
||||
.expect("isolated asset directory test");
|
||||
fs::remove_dir_all(directory).expect("remove fixture");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
return;
|
||||
};
|
||||
let mut config = gateway_api::ApiConfig {
|
||||
asset_directory: "unused".into(),
|
||||
cors_allowed_origins: Vec::new(),
|
||||
request_timeout_seconds: None,
|
||||
request_id_header: "x-request-id".to_owned(),
|
||||
};
|
||||
config
|
||||
.resolve_and_validate(&std::path::Path::new(&directory).join("deploy"))
|
||||
.expect("resolved API config");
|
||||
let admin = admin::AdminTestFixture::new().await;
|
||||
let response = openai::api_router_with_config(admin.services, config)
|
||||
.oneshot(Request::builder().uri("/").body(Body::empty()).unwrap())
|
||||
.await
|
||||
.expect("static response");
|
||||
assert_eq!(response.status(), axum::http::StatusCode::OK);
|
||||
let body = axum::body::to_bytes(response.into_body(), 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(body.as_ref(), b"resolved-web-assets");
|
||||
}
|
||||
|
||||
@@ -50,13 +50,23 @@ pub(super) async fn api_router(execution: Arc<dyn ExecutionService>) -> axum::Ro
|
||||
}
|
||||
|
||||
pub(super) fn api_router_with_admin(admin: gateway_admin::AdminServices) -> axum::Router {
|
||||
gateway_api::initialize(
|
||||
api_router_with_config(
|
||||
admin,
|
||||
gateway_api::ApiConfig {
|
||||
asset_directory: std::env::temp_dir(),
|
||||
cors_allowed_origins: Vec::new(),
|
||||
request_timeout_seconds: None,
|
||||
request_id_header: "x-request-id".to_owned(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn api_router_with_config(
|
||||
admin: gateway_admin::AdminServices,
|
||||
config: gateway_api::ApiConfig,
|
||||
) -> axum::Router {
|
||||
gateway_api::initialize(
|
||||
config,
|
||||
Arc::new(DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot("unused-client-route-key", "openai")),
|
||||
Arc::new(UnusedExecutionStore),
|
||||
|
||||
@@ -15,6 +15,7 @@ use axum::{
|
||||
header::{AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE},
|
||||
},
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::future::{BoxFuture, pending};
|
||||
@@ -638,7 +639,11 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
input: Some(json!("hello")),
|
||||
client_metadata: None,
|
||||
// 客户端 User-Agent 仅用于本地展示,不再透传给上游指纹上下文。
|
||||
protocol_context: None,
|
||||
protocol_context: Some(json!({"opaque_request_headers": [
|
||||
["cf-connecting-ip", STANDARD.encode(b"198.51.100.1")],
|
||||
["x-real-ip", STANDARD.encode(b"198.51.100.2")],
|
||||
["x-forwarded-for", STANDARD.encode(b"10.0.0.2, 203.0.113.3")]
|
||||
]})),
|
||||
prompt_cache_key: None,
|
||||
previous_response_id: None,
|
||||
}
|
||||
@@ -662,6 +667,26 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opaque_client_headers_should_not_change_local_client_observation() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("user-agent", "pi/synthetic".parse().unwrap());
|
||||
headers.insert("x-stainless-runtime", "node".parse().unwrap());
|
||||
headers.insert("origin", "https://synthetic.invalid".parse().unwrap());
|
||||
let captured = captured_client_context(headers, "192.0.2.10:443".parse().unwrap()).await;
|
||||
assert_eq!(captured.user_agent.as_deref(), Some("pi/synthetic"));
|
||||
assert_eq!(captured.client_ip, Some("192.0.2.10".parse().unwrap()));
|
||||
assert_eq!(
|
||||
captured.protocol_context,
|
||||
Some(json!({
|
||||
"opaque_request_headers": [
|
||||
["x-stainless-runtime", STANDARD.encode(b"node")],
|
||||
["origin", STANDARD.encode(b"https://synthetic.invalid")]
|
||||
]
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_model_catalog_should_not_block_a_new_model_request() {
|
||||
let model = " gpt-future-codex ";
|
||||
@@ -768,7 +793,7 @@ async fn subagent_header_should_not_replace_non_object_client_metadata() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
async fn xai_private_headers_should_remain_opaque_without_projecting_request_facts() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-grok-turn-idx", "7".parse().expect("header"));
|
||||
@@ -776,7 +801,15 @@ async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
|
||||
let captured = captured_client_context(headers, peer).await;
|
||||
|
||||
assert!(captured.protocol_context.is_none());
|
||||
assert_eq!(
|
||||
captured.protocol_context,
|
||||
Some(json!({
|
||||
"opaque_request_headers": [
|
||||
["x-grok-turn-idx", STANDARD.encode(b"7")],
|
||||
["x-grok-conv-id", STANDARD.encode(b"private-session")]
|
||||
]
|
||||
}))
|
||||
);
|
||||
assert!(captured.prompt_cache_key.is_none());
|
||||
}
|
||||
|
||||
|
||||
@@ -233,6 +233,14 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
#[test]
|
||||
fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for name in [
|
||||
"chatgpt-organization-id",
|
||||
"chatgpt-org-id",
|
||||
"x-openai-organization",
|
||||
"x-openai-project",
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static("unclassified-extension"));
|
||||
}
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_static("future-ascii"),
|
||||
@@ -338,6 +346,18 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
values("openai-beta"),
|
||||
vec![b"future_responses=v2".to_vec(), b"future_tools=v3".to_vec()]
|
||||
);
|
||||
for name in [
|
||||
"chatgpt-organization-id",
|
||||
"chatgpt-org-id",
|
||||
"x-openai-organization",
|
||||
"x-openai-project",
|
||||
] {
|
||||
assert_eq!(values(name), vec![b"unclassified-extension".to_vec()]);
|
||||
}
|
||||
assert_eq!(
|
||||
values("x-codex-installation-id"),
|
||||
vec![b"client-installation".to_vec()]
|
||||
);
|
||||
for preserved in [
|
||||
"accept",
|
||||
"content-type",
|
||||
@@ -357,7 +377,6 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
"cookie",
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-project-id",
|
||||
"x-codex-installation-id",
|
||||
// 上游指纹由运行时画像统一生成,客户端不得覆盖。
|
||||
"user-agent",
|
||||
"originator",
|
||||
@@ -371,7 +390,7 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_exclude_downstream_transport_headers_from_opaque_context() {
|
||||
fn decoder_should_strip_http_transport_but_leave_source_headers_for_provider() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for name in [
|
||||
"cf-visitor",
|
||||
@@ -407,10 +426,124 @@ fn decoder_should_exclude_downstream_transport_headers_from_opaque_context() {
|
||||
|
||||
assert_eq!(
|
||||
openai_protocol_context(&decoded).get("opaque_request_headers"),
|
||||
Some(&json!([["x-openai-future-mode", STANDARD.encode(b"keep")]])),
|
||||
Some(&Value::Array(
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| !matches!(
|
||||
name.as_str(),
|
||||
"accept-encoding" | "content-encoding"
|
||||
))
|
||||
.map(|(name, value)| json!([name.as_str(), STANDARD.encode(value.as_bytes())]))
|
||||
.collect()
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downstream_client_headers_should_remain_opaque_without_losing_session_semantics() {
|
||||
for canonical in [None, Some("canonical-session")] {
|
||||
let mut headers = HeaderMap::new();
|
||||
for name in [
|
||||
"X-Stainless-Runtime",
|
||||
"x-stainless-future-field",
|
||||
"Origin",
|
||||
"Referer",
|
||||
"Sec-Ch-Ua",
|
||||
"sec-ch-ua-platform",
|
||||
"Sec-Fetch-Site",
|
||||
"session_id",
|
||||
] {
|
||||
headers.append(name, HeaderValue::from_static("alias-session"));
|
||||
headers.append(name, HeaderValue::from_static("duplicate"));
|
||||
}
|
||||
if let Some(session) = canonical {
|
||||
headers.insert("session-id", HeaderValue::from_static(session));
|
||||
}
|
||||
for name in [
|
||||
"thread-id",
|
||||
"x-client-request-id",
|
||||
"traceparent",
|
||||
"tracestate",
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static("keep"));
|
||||
}
|
||||
headers.append("x-future-business", HeaderValue::from_static("first"));
|
||||
headers.append(
|
||||
"x-future-business",
|
||||
HeaderValue::from_bytes(b"\x80\xff").unwrap(),
|
||||
);
|
||||
let original_headers = headers.clone();
|
||||
let body = json!({
|
||||
"model": "smart-code", "input": "中文 pi 内容不能被请求头过滤改写",
|
||||
"prompt_cache_key": "synthetic-cache",
|
||||
"client_metadata": {"session_id": "body-session"}
|
||||
});
|
||||
let mut frame = body.clone();
|
||||
frame["type"] = json!("response.create");
|
||||
let opening = OpenAiRequestHeaders::from_headers(&headers);
|
||||
for decoded in [
|
||||
decode_request_with_headers(body.to_string().as_bytes(), &headers).unwrap(),
|
||||
decode_response_create_with_context(&frame.to_string(), &opening).unwrap(),
|
||||
decode_response_create_with_context(&frame.to_string(), &opening).unwrap(),
|
||||
] {
|
||||
assert_eq!(openai_wire_body(&decoded), body.as_object().unwrap());
|
||||
let context = openai_protocol_context(&decoded);
|
||||
assert_eq!(context["session_id"], canonical.unwrap_or("alias-session"));
|
||||
let entries = context["opaque_request_headers"].as_array().unwrap();
|
||||
for name in [
|
||||
"x-stainless-runtime",
|
||||
"x-stainless-future-field",
|
||||
"origin",
|
||||
"referer",
|
||||
"sec-ch-ua",
|
||||
"sec-ch-ua-platform",
|
||||
"sec-fetch-site",
|
||||
"session_id",
|
||||
] {
|
||||
let values: Vec<_> = entries
|
||||
.iter()
|
||||
.filter(|entry| entry[0] == name)
|
||||
.cloned()
|
||||
.collect();
|
||||
assert_eq!(
|
||||
values,
|
||||
vec![
|
||||
json!([name, STANDARD.encode(b"alias-session")]),
|
||||
json!([name, STANDARD.encode(b"duplicate")]),
|
||||
],
|
||||
"source header {name} belongs to the Provider"
|
||||
);
|
||||
}
|
||||
for name in [
|
||||
"thread-id",
|
||||
"x-client-request-id",
|
||||
"traceparent",
|
||||
"tracestate",
|
||||
] {
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.any(|entry| entry == &json!([name, STANDARD.encode(b"keep")]))
|
||||
);
|
||||
}
|
||||
let future: Vec<_> = entries
|
||||
.iter()
|
||||
.filter(|entry| entry[0] == "x-future-business")
|
||||
.cloned()
|
||||
.collect();
|
||||
assert_eq!(
|
||||
future,
|
||||
vec![
|
||||
json!(["x-future-business", STANDARD.encode(b"first")]),
|
||||
json!(["x-future-business", STANDARD.encode(b"\x80\xff")]),
|
||||
]
|
||||
);
|
||||
}
|
||||
// 解码不修改原始请求,CORS、鉴权和本地观测仍可读取原值。
|
||||
assert_eq!(headers, original_headers);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
let secret = "nested-private-value";
|
||||
|
||||
@@ -7,7 +7,7 @@ use serde_json::json;
|
||||
use super::decode_response_create;
|
||||
|
||||
#[test]
|
||||
fn response_create_should_exclude_cloudflare_headers_on_every_frame() {
|
||||
fn response_create_should_carry_source_headers_to_provider_on_every_frame() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in [
|
||||
("CF-Visitor", r#"{"scheme":"https"}"#),
|
||||
@@ -24,7 +24,7 @@ fn response_create_should_exclude_cloudflare_headers_on_every_frame() {
|
||||
headers.append("x-openai-future-mode", HeaderValue::from_static("second"));
|
||||
let request_headers = OpenAiRequestHeaders::from_headers(&headers);
|
||||
|
||||
// 连接级头会用于同一 WebSocket 的每一帧,后续帧也不能恢复下游链路元数据。
|
||||
// 连接级头随每一帧传给 Provider,由相同的出站边界处理来源适配。
|
||||
for input in ["hello", "continue"] {
|
||||
let decoded = decode_response_create_with_context(
|
||||
&json!({"type": "response.create", "model": "smart-code", "input": input}).to_string(),
|
||||
@@ -39,10 +39,12 @@ fn response_create_should_exclude_cloudflare_headers_on_every_frame() {
|
||||
.protocol_payload()
|
||||
.context()
|
||||
.get("opaque_request_headers"),
|
||||
Some(&json!([
|
||||
["x-openai-future-mode", STANDARD.encode(b"first")],
|
||||
["x-openai-future-mode", STANDARD.encode(b"second")],
|
||||
])),
|
||||
Some(&serde_json::Value::Array(
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| json!([name.as_str(), STANDARD.encode(value.as_bytes())]))
|
||||
.collect()
|
||||
)),
|
||||
);
|
||||
assert_eq!(request.protocol_payload().body()["input"], input);
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ path = "tests/main.rs"
|
||||
async-trait = "0.1.89"
|
||||
base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
chrono-tz = { version = "0.10", features = ["serde"] }
|
||||
futures = "0.3.32"
|
||||
futures-timer = "3.0.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//! 请求位置只描述出口的业务配置,不改变代理连接身份或服务系统时区。
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RequestLocation {
|
||||
pub country: String,
|
||||
pub region: String,
|
||||
pub city: String,
|
||||
pub timezone: chrono_tz::Tz,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum InvalidRequestLocation {
|
||||
#[error("country must contain two uppercase ASCII letters")]
|
||||
Country,
|
||||
#[error("region must contain 1–128 characters without control characters")]
|
||||
Region,
|
||||
#[error("city must contain 1–128 characters without control characters")]
|
||||
City,
|
||||
}
|
||||
|
||||
impl Default for RequestLocation {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
country: "US".to_owned(),
|
||||
region: "Ohio".to_owned(),
|
||||
city: "Piketon".to_owned(),
|
||||
timezone: chrono_tz::America::New_York,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestLocation {
|
||||
pub fn validate(&self) -> Result<(), InvalidRequestLocation> {
|
||||
if self.country.len() != 2 || !self.country.bytes().all(|byte| byte.is_ascii_uppercase()) {
|
||||
return Err(InvalidRequestLocation::Country);
|
||||
}
|
||||
for (value, error) in [
|
||||
(&self.region, InvalidRequestLocation::Region),
|
||||
(&self.city, InvalidRequestLocation::City),
|
||||
] {
|
||||
// 先检查原始输入,避免 trim 掩盖首尾的换行和其他控制字符。
|
||||
if value.chars().any(char::is_control)
|
||||
|| value.trim().is_empty()
|
||||
|| value.trim().chars().count() > 128
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn normalized(mut self) -> Result<Self, InvalidRequestLocation> {
|
||||
self.validate()?;
|
||||
self.region = self.region.trim().to_owned();
|
||||
self.city = self.city.trim().to_owned();
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
//! Provider 账号领域、持久化端口与同一 target 内的账号选择。
|
||||
|
||||
mod error;
|
||||
mod location;
|
||||
pub use location::{InvalidRequestLocation, RequestLocation};
|
||||
mod model;
|
||||
mod model_access;
|
||||
pub use model_access::{
|
||||
|
||||
@@ -583,6 +583,72 @@ impl AccountErrorReason {
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号冷却的原因与恢复方式。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AccountCooldownKind {
|
||||
/// 上游 429 临时限流;默认类别,兼容升级前未标记的存量 key。
|
||||
#[default]
|
||||
RateLimit,
|
||||
/// 容量类错误高频触发后由熔断策略写入的自动冻结。
|
||||
CapacityFreeze,
|
||||
/// 到期后仍阻止调度,必须由恢复探测成功或管理员恢复解除。
|
||||
CapacityFreezeProbe,
|
||||
}
|
||||
|
||||
impl AccountCooldownKind {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::RateLimit => "rate_limit",
|
||||
Self::CapacityFreeze => "capacity_freeze",
|
||||
Self::CapacityFreezeProbe => "capacity_freeze_probe",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_capacity_freeze(self) -> bool {
|
||||
matches!(self, Self::CapacityFreeze | Self::CapacityFreezeProbe)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn requires_probe(self) -> bool {
|
||||
matches!(self, Self::CapacityFreezeProbe)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"rate_limit" => Some(Self::RateLimit),
|
||||
"capacity_freeze" => Some(Self::CapacityFreeze),
|
||||
"capacity_freeze_probe" => Some(Self::CapacityFreezeProbe),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号级冷却投影;探测冻结的 until 是下次探测时间,不是自动放行时间。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountCooldown {
|
||||
pub until: SystemTime,
|
||||
pub kind: AccountCooldownKind,
|
||||
}
|
||||
|
||||
impl AccountCooldown {
|
||||
#[must_use]
|
||||
pub fn is_active(self, now: SystemTime) -> bool {
|
||||
self.kind.requires_probe() || self.until > now
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SystemTime> for AccountCooldown {
|
||||
fn from(until: SystemTime) -> Self {
|
||||
Self {
|
||||
until,
|
||||
kind: AccountCooldownKind::RateLimit,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 唯一状态解析器的完整输入事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusFacts {
|
||||
@@ -590,7 +656,7 @@ pub struct AccountStatusFacts {
|
||||
pub credential_state: CredentialState,
|
||||
pub access_token_expires_at: Option<SystemTime>,
|
||||
pub quota: QuotaState,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub cooldown: Option<AccountCooldown>,
|
||||
pub last_error_reason: Option<AccountErrorReason>,
|
||||
pub last_error_message: Option<String>,
|
||||
}
|
||||
@@ -601,8 +667,8 @@ pub struct AccountStatusProjection {
|
||||
pub status: AccountStatus,
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub error_message: Option<String>,
|
||||
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却事实。
|
||||
pub cooldown: Option<AccountCooldown>,
|
||||
}
|
||||
|
||||
/// 从独立事实派生唯一、互斥的对外状态。
|
||||
@@ -625,21 +691,21 @@ pub fn resolve_account_status(
|
||||
status: AccountStatus::Error,
|
||||
error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)),
|
||||
error_message: facts.last_error_message.clone(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
};
|
||||
}
|
||||
if facts.quota.is_exhausted() {
|
||||
return status_projection(AccountStatus::QuotaExhausted);
|
||||
}
|
||||
if facts
|
||||
.rate_limited_until
|
||||
.is_some_and(|rate_limited_until| rate_limited_until > now)
|
||||
.cooldown
|
||||
.is_some_and(|cooldown| cooldown.is_active(now))
|
||||
{
|
||||
return AccountStatusProjection {
|
||||
status: AccountStatus::RateLimited,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: facts.rate_limited_until,
|
||||
cooldown: facts.cooldown,
|
||||
};
|
||||
}
|
||||
status_projection(AccountStatus::Normal)
|
||||
@@ -650,7 +716,7 @@ const fn status_projection(status: AccountStatus) -> AccountStatusProjection {
|
||||
status,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -678,6 +744,7 @@ pub struct ProviderAccount {
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
outbound_proxy: Option<super::OutboundProxy>,
|
||||
request_location: Option<super::RequestLocation>,
|
||||
}
|
||||
|
||||
impl ProviderAccount {
|
||||
@@ -714,6 +781,7 @@ impl ProviderAccount {
|
||||
next_refresh_at: None,
|
||||
has_refresh_token: false,
|
||||
outbound_proxy: None,
|
||||
request_location: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -743,6 +811,8 @@ impl ProviderAccount {
|
||||
|
||||
#[must_use]
|
||||
pub fn with_outbound_proxy(mut self, proxy: Option<super::OutboundProxy>) -> Self {
|
||||
// 出口变化后不能沿用旧出口的位置;存储投影应在绑定出口后设置位置。
|
||||
self.request_location = None;
|
||||
self.outbound_proxy = proxy;
|
||||
self
|
||||
}
|
||||
@@ -752,6 +822,17 @@ impl ProviderAccount {
|
||||
self.outbound_proxy.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_location(mut self, location: Option<super::RequestLocation>) -> Self {
|
||||
self.request_location = self.outbound_proxy.as_ref().and(location);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_location(&self) -> Option<&super::RequestLocation> {
|
||||
self.request_location.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_account_facts(
|
||||
mut self,
|
||||
@@ -762,11 +843,8 @@ impl ProviderAccount {
|
||||
last_error_message: Option<String>,
|
||||
) -> Self {
|
||||
self.enabled = enabled;
|
||||
self.credential_state = if self.upstream_user_id.is_some() {
|
||||
credential_state
|
||||
} else {
|
||||
CredentialState::Unknown
|
||||
};
|
||||
// 凭据是否可用由 Provider 判断;API Key 等认证不要求上游用户身份。
|
||||
self.credential_state = credential_state;
|
||||
self.quota = quota;
|
||||
self.last_error_reason = last_error_reason;
|
||||
self.last_error_message = last_error_message;
|
||||
@@ -908,7 +986,7 @@ impl ProviderAccount {
|
||||
pub fn status_projection(
|
||||
&self,
|
||||
now: SystemTime,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
cooldown: Option<AccountCooldown>,
|
||||
) -> AccountStatusProjection {
|
||||
resolve_account_status(
|
||||
&AccountStatusFacts {
|
||||
@@ -916,7 +994,7 @@ impl ProviderAccount {
|
||||
credential_state: self.credential_state,
|
||||
access_token_expires_at: self.access_token_expires_at,
|
||||
quota: self.quota,
|
||||
rate_limited_until,
|
||||
cooldown,
|
||||
last_error_reason: self.last_error_reason,
|
||||
last_error_message: self.last_error_message.clone(),
|
||||
},
|
||||
@@ -1068,6 +1146,7 @@ pub struct CredentialCasUpdateParts {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub profile: ProviderAccountUpdate,
|
||||
pub preserve_profile: bool,
|
||||
pub credential: PlaintextCredential,
|
||||
pub has_refresh_token: bool,
|
||||
pub access_token_expires_at: Option<SystemTime>,
|
||||
@@ -1081,6 +1160,7 @@ pub struct CredentialCasUpdate {
|
||||
account_id: ProviderAccountId,
|
||||
expected_revision: CredentialRevision,
|
||||
profile: ProviderAccountUpdate,
|
||||
preserve_profile: bool,
|
||||
credential: PlaintextCredential,
|
||||
has_refresh_token: bool,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
@@ -1095,6 +1175,7 @@ impl fmt::Debug for CredentialCasUpdate {
|
||||
.field("account_id", &self.account_id)
|
||||
.field("expected_revision", &self.expected_revision)
|
||||
.field("profile", &self.profile)
|
||||
.field("preserve_profile", &self.preserve_profile)
|
||||
.field("credential", &self.credential)
|
||||
.field("has_refresh_token", &self.has_refresh_token)
|
||||
.field("access_token_expires_at", &self.access_token_expires_at)
|
||||
@@ -1129,6 +1210,7 @@ impl CredentialCasUpdate {
|
||||
account_id,
|
||||
expected_revision,
|
||||
profile,
|
||||
preserve_profile: false,
|
||||
credential,
|
||||
has_refresh_token,
|
||||
access_token_expires_at,
|
||||
@@ -1137,6 +1219,13 @@ impl CredentialCasUpdate {
|
||||
})
|
||||
}
|
||||
|
||||
/// 仅轮换凭据,保留提交时的账号资料,避免覆盖并发额度观测更新的套餐。
|
||||
#[must_use]
|
||||
pub const fn preserving_profile(mut self) -> Self {
|
||||
self.preserve_profile = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// 将刷新调度与账号错误事实放入同一个 revision CAS。
|
||||
#[must_use]
|
||||
pub fn with_account_state(
|
||||
@@ -1202,6 +1291,7 @@ impl CredentialCasUpdate {
|
||||
account_id: self.account_id,
|
||||
expected_revision: self.expected_revision,
|
||||
profile: self.profile,
|
||||
preserve_profile: self.preserve_profile,
|
||||
credential: self.credential,
|
||||
has_refresh_token: self.has_refresh_token,
|
||||
access_token_expires_at: self.access_token_expires_at,
|
||||
@@ -1224,6 +1314,8 @@ pub struct QuotaObservation {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub quota: OpaqueProviderData,
|
||||
/// Provider 确认的账号套餐,与额度原子写入;`None` 保留已有套餐。
|
||||
pub plan_type: Option<String>,
|
||||
/// Provider 原始 quota document 的观察时间;不代表访问结论发生变化。
|
||||
pub observed_at: SystemTime,
|
||||
/// Provider 已从私有 JSON 归一化出的额度访问事实。
|
||||
@@ -1237,6 +1329,7 @@ impl fmt::Debug for QuotaObservation {
|
||||
.field("account_id", &self.account_id)
|
||||
.field("expected_revision", &self.expected_revision)
|
||||
.field("quota", &self.quota)
|
||||
.field("plan_type", &self.plan_type)
|
||||
.field("observed_at", &self.observed_at)
|
||||
.field("state", &self.state)
|
||||
.finish()
|
||||
|
||||
@@ -106,7 +106,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
/// Provider 归一化的剩余额度基点:0 耗尽,10_000 全部可用。
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub cooldown: Option<super::AccountCooldown>,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -316,8 +316,8 @@ impl AccountRuntimeSignals {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_rate_limit(mut self, rate_limited_until: Option<SystemTime>) -> Self {
|
||||
self.rate_limited_until = rate_limited_until;
|
||||
pub const fn with_rate_limit(mut self, cooldown: Option<super::AccountCooldown>) -> Self {
|
||||
self.cooldown = cooldown;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -654,7 +654,7 @@ impl AccountSelector {
|
||||
if !context.eligibility.bypasses_local_eligibility() {
|
||||
let status = candidate
|
||||
.account
|
||||
.status_projection(context.now, candidate.signals.rate_limited_until)
|
||||
.status_projection(context.now, candidate.signals.cooldown)
|
||||
.status;
|
||||
if status != AccountStatus::Normal {
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
|
||||
@@ -788,6 +788,7 @@ where
|
||||
}));
|
||||
let context = AttemptContext::new(
|
||||
RequestAttemptContext::new(self.request_id.clone(), self.client_api_key_ref.clone())
|
||||
.with_request_location(self.plan.request_location().cloned())
|
||||
.with_concurrency_wait_budget(self.concurrency_wait_budget.clone())
|
||||
.with_timing_started_at(self.observation.timing_started_at)
|
||||
.with_trace(self.trace.clone()),
|
||||
|
||||
@@ -1051,10 +1051,7 @@ impl ExecutionService for DefaultExecutionService {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(model.clone()) {
|
||||
result.push(PublicModelDescriptor::Native {
|
||||
model,
|
||||
payload: source.payload.clone(),
|
||||
});
|
||||
result.push(source.content.for_public_model(model));
|
||||
}
|
||||
}
|
||||
for model in client.snapshot.public_models_for_provider(kind) {
|
||||
@@ -1073,10 +1070,7 @@ impl ExecutionService for DefaultExecutionService {
|
||||
}
|
||||
if let Some(source) = by_id.get(target.as_str()) {
|
||||
seen.insert(model.clone());
|
||||
result.push(PublicModelDescriptor::Native {
|
||||
model,
|
||||
payload: source.payload.clone(),
|
||||
});
|
||||
result.push(source.content.for_public_model(model));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,6 +314,7 @@ impl ContinuationAttempt {
|
||||
/// Provider 每次执行可见的 request-local context。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RequestAttemptContext {
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
request_id: ModelRequestId,
|
||||
client_api_key_ref: ClientApiKeyId,
|
||||
timing_started_at: Instant,
|
||||
@@ -322,11 +323,21 @@ pub struct RequestAttemptContext {
|
||||
}
|
||||
|
||||
impl RequestAttemptContext {
|
||||
#[must_use]
|
||||
pub fn with_request_location(
|
||||
mut self,
|
||||
location: Option<crate::account::RequestLocation>,
|
||||
) -> Self {
|
||||
self.request_location = location;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn new(request_id: ModelRequestId, client_api_key_ref: ClientApiKeyId) -> Self {
|
||||
Self {
|
||||
request_id,
|
||||
client_api_key_ref,
|
||||
request_location: None,
|
||||
timing_started_at: Instant::now(),
|
||||
trace: crate::diagnostics::TraceContext::default(),
|
||||
concurrency_wait_budget: crate::concurrency::ConcurrencyWaitBudget::default(),
|
||||
@@ -388,6 +399,11 @@ pub struct AttemptContext {
|
||||
}
|
||||
|
||||
impl AttemptContext {
|
||||
#[must_use]
|
||||
pub const fn request_location(&self) -> Option<&crate::account::RequestLocation> {
|
||||
self.request.request_location.as_ref()
|
||||
}
|
||||
|
||||
/// 当前 attempt 的诊断关联;克隆后可传给后台 transport 任务。
|
||||
#[must_use]
|
||||
pub fn trace(&self) -> crate::diagnostics::TraceContext {
|
||||
|
||||
@@ -181,6 +181,16 @@ pub trait ProviderLeasePort: Send + Sync {
|
||||
&self,
|
||||
request: ProviderLeaseRequest,
|
||||
) -> BoxFuture<'_, Result<ProviderLeaseAcquisition, ProviderStoreError>>;
|
||||
|
||||
/// 读取指定账号当前的在途请求数;只用于容量熔断的峰值证据,
|
||||
/// 支持租约信号的存储实现覆盖,否则视为不可观测(空映射)。
|
||||
fn account_in_flight<'a>(
|
||||
&'a self,
|
||||
account_ids: &'a [ProviderAccountId],
|
||||
) -> BoxFuture<'a, Result<BTreeMap<ProviderAccountId, u32>, ProviderStoreError>> {
|
||||
let _ = account_ids;
|
||||
Box::pin(async move { Ok(BTreeMap::new()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 从原始会话锚点派生的不可逆亲和键。
|
||||
@@ -596,12 +606,16 @@ pub trait ProviderCredentialStatePort: Send + Sync {
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// 临时 cooldown 只保存可丢失的调度截止时间,不进入账号持久状态。
|
||||
/// 账号级 cooldown 的来源类别;调度状态统一按 `rate_limited` 处理。
|
||||
pub use crate::account::AccountCooldownKind as ProviderCooldownKind;
|
||||
|
||||
/// 可丢失的账号冷却事实,不进入持久状态;探测冻结到期后仍需确认恢复。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
}
|
||||
|
||||
impl ProviderCooldown {
|
||||
@@ -610,11 +624,27 @@ impl ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self::new_with_kind(
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
ProviderCooldownKind::RateLimit,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn new_with_kind(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
kind,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -632,6 +662,19 @@ impl ProviderCooldown {
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scheduling_state(&self) -> crate::account::AccountCooldown {
|
||||
crate::account::AccountCooldown {
|
||||
until: self.until,
|
||||
kind: self.kind,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> ProviderCooldownKind {
|
||||
self.kind
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失 cooldown 的细粒度作用域。
|
||||
@@ -749,6 +792,29 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 记录一次容量类失败并返回滑动窗口内的累计次数,同时把观测到的账号
|
||||
/// 在途并发并入窗口峰值(`in_flight` 为 0 表示本次未观测,跳过峰值更新)。
|
||||
/// 只服务容量熔断触发器;调用频率受失败频率约束,不需要批量接口。
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
|
||||
|
||||
/// 普通请求成功后原子清除临时限流及失败证据;必须保留任何容量冻结。
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
|
||||
/// 读取窗口内观测到的在途并发峰值;无证据时返回 `None`。
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -864,6 +930,118 @@ pub trait ProviderRuntimePolicyPort: Send + Sync {
|
||||
fn load_refresh_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderRefreshPolicy, ProviderStoreError>>;
|
||||
|
||||
/// 读取账号容量熔断策略;默认关闭,只有实现运行时设置的存储需要覆盖。
|
||||
fn load_freeze_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(ProviderFreezePolicy::disabled()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号容量熔断(自动冻结)策略;来源于 `runtime_settings`,
|
||||
/// 由 Provider 触发路径与恢复 worker 共享同一份配置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderFreezePolicy {
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window: Duration,
|
||||
freeze_duration: Duration,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl ProviderFreezePolicy {
|
||||
/// 边界与迁移 `0010_account_auto_freeze.sql` 的 check 约束一致;
|
||||
/// store 层写入前已校验,这里兜底防御越界配置。
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window_seconds: u64,
|
||||
freeze_duration_seconds: u64,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if !(2..=1_000).contains(&threshold)
|
||||
|| !(60..=3_600).contains(&window_seconds)
|
||||
|| !(300..=604_800).contains(&freeze_duration_seconds)
|
||||
|| probe_model.as_deref().is_some_and(|model| {
|
||||
model.is_empty()
|
||||
|| model.len() > 128
|
||||
|| model != model.trim()
|
||||
|| model.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
{
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate freeze policy",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
enabled,
|
||||
threshold,
|
||||
window: Duration::from_secs(window_seconds),
|
||||
freeze_duration: Duration::from_secs(freeze_duration_seconds),
|
||||
probe_enabled,
|
||||
probe_model,
|
||||
adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
|
||||
/// 功能关闭时的全零策略;触发路径与 worker 都以此短路。
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
threshold: 2,
|
||||
window: Duration::from_secs(60),
|
||||
freeze_duration: Duration::from_secs(300),
|
||||
probe_enabled: false,
|
||||
probe_model: None,
|
||||
adaptive_concurrency: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
/// 窗口内触发冻结的请求级失败次数阈值。
|
||||
#[must_use]
|
||||
pub const fn threshold(&self) -> u32 {
|
||||
self.threshold
|
||||
}
|
||||
|
||||
/// 失败计数滑动窗口;每次失败都会顺延窗口。
|
||||
#[must_use]
|
||||
pub const fn window(&self) -> Duration {
|
||||
self.window
|
||||
}
|
||||
|
||||
/// 冻结时长;探测失败后的顺延也使用该值。
|
||||
#[must_use]
|
||||
pub const fn freeze_duration(&self) -> Duration {
|
||||
self.freeze_duration
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn probe_enabled(&self) -> bool {
|
||||
self.probe_enabled
|
||||
}
|
||||
|
||||
/// 探测模型;`None` 表示由 worker 选择账号可用的第一个模型。
|
||||
#[must_use]
|
||||
pub fn probe_model(&self) -> Option<&str> {
|
||||
self.probe_model.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn adaptive_concurrency(&self) -> bool {
|
||||
self.adaptive_concurrency
|
||||
}
|
||||
}
|
||||
|
||||
/// OAuth pending flow 的原始绑定只在 Provider 与 Store 边界内短暂存在。
|
||||
|
||||
@@ -11,14 +11,20 @@ use super::{
|
||||
ModelCapabilities, ModelPresentation, PublicModelId, PublicModelProfile, UpstreamModelId,
|
||||
};
|
||||
|
||||
/// Provider 原生客户端目录条目;Core 只解释模型标识,不解释协议正文。
|
||||
/// Provider 客户端目录条目;缺少原生协议正文时使用已编译的通用画像。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderModelDescriptor {
|
||||
pub model: UpstreamModelId,
|
||||
pub payload: RawJsonPayload,
|
||||
pub content: ProviderModelContent,
|
||||
}
|
||||
|
||||
/// 对外目录保留原生正文;只有没有原生目录的 Provider 才使用通用画像适配。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ProviderModelContent {
|
||||
Native(RawJsonPayload),
|
||||
Adapted(ModelPresentation),
|
||||
}
|
||||
|
||||
/// 对外目录按条目保留原生正文或使用通用画像适配。
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum PublicModelDescriptor {
|
||||
Native {
|
||||
@@ -28,6 +34,20 @@ pub enum PublicModelDescriptor {
|
||||
Adapted(PublicModelProfile),
|
||||
}
|
||||
|
||||
impl ProviderModelContent {
|
||||
pub(crate) fn for_public_model(&self, model: PublicModelId) -> PublicModelDescriptor {
|
||||
match self {
|
||||
Self::Native(payload) => PublicModelDescriptor::Native {
|
||||
model,
|
||||
payload: payload.clone(),
|
||||
},
|
||||
Self::Adapted(presentation) => {
|
||||
PublicModelDescriptor::Adapted(PublicModelProfile::new(model, presentation.clone()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 实时目录编译后的单模型能力。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderModelCapabilities {
|
||||
|
||||
@@ -10,7 +10,8 @@ pub use crate::account::scope::{
|
||||
pub use crate::identity::ProviderKind;
|
||||
pub use catalog::{
|
||||
ProviderCatalogGeneration, ProviderCatalogPort, ProviderCatalogUnavailable,
|
||||
ProviderModelCapabilities, ProviderModelDescriptor, PublicModelDescriptor,
|
||||
ProviderModelCapabilities, ProviderModelContent, ProviderModelDescriptor,
|
||||
PublicModelDescriptor,
|
||||
};
|
||||
pub use snapshot::RuntimeSnapshot;
|
||||
|
||||
@@ -535,6 +536,7 @@ impl ProviderCandidate {
|
||||
/// 一次请求冻结的 Provider 尝试顺序。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RoutingPlan {
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
config_revision: ConfigRevision,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
operation: OperationKind,
|
||||
@@ -544,6 +546,12 @@ pub struct RoutingPlan {
|
||||
}
|
||||
|
||||
impl RoutingPlan {
|
||||
/// 本次请求冻结的全局位置,重试时沿用同一份配置。
|
||||
#[must_use]
|
||||
pub const fn request_location(&self) -> Option<&crate::account::RequestLocation> {
|
||||
self.request_location.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn config_revision(&self) -> ConfigRevision {
|
||||
self.config_revision
|
||||
|
||||
@@ -28,6 +28,8 @@ const MAXIMUM_CATALOG_STABILITY_ATTEMPTS: usize = 4;
|
||||
/// Store 在一个一致性读取中提供的调度设置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotSettingsFacts {
|
||||
request_location_enabled: bool,
|
||||
request_location: crate::account::RequestLocation,
|
||||
max_concurrent_per_account: u32,
|
||||
max_waiting_per_key: u32,
|
||||
max_waiting_per_account: u32,
|
||||
@@ -40,6 +42,17 @@ pub struct SnapshotSettingsFacts {
|
||||
}
|
||||
|
||||
impl SnapshotSettingsFacts {
|
||||
#[must_use]
|
||||
pub fn with_request_location(
|
||||
mut self,
|
||||
location: crate::account::RequestLocation,
|
||||
enabled: bool,
|
||||
) -> Self {
|
||||
self.request_location_enabled = enabled;
|
||||
self.request_location = location;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_concurrency_queues(
|
||||
mut self,
|
||||
@@ -63,6 +76,8 @@ impl SnapshotSettingsFacts {
|
||||
min_codex_cli_version: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
request_location_enabled: false,
|
||||
request_location: crate::account::RequestLocation::default(),
|
||||
max_concurrent_per_account,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
@@ -452,6 +467,18 @@ async fn compile_runtime_snapshot(
|
||||
));
|
||||
}
|
||||
|
||||
// 关闭自定义时保留持久化值,但不生成全局覆盖;请求继续使用客户端字段。
|
||||
let request_location = if facts.settings.request_location_enabled {
|
||||
Some(
|
||||
facts
|
||||
.settings
|
||||
.request_location
|
||||
.normalized()
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
RuntimeSnapshot::new(
|
||||
facts.config_revision,
|
||||
selection_policy,
|
||||
@@ -462,6 +489,7 @@ async fn compile_runtime_snapshot(
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)
|
||||
.map(|snapshot| {
|
||||
snapshot
|
||||
.with_request_location(request_location)
|
||||
.with_client_queue_policy(client_queue_policy)
|
||||
.with_model_mappings(model_mappings)
|
||||
.with_account_directory(account_directory)
|
||||
@@ -473,6 +501,7 @@ async fn compile_runtime_snapshot(
|
||||
/// 数据面使用的不可变配置快照。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RuntimeSnapshot {
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
revision: ConfigRevision,
|
||||
client_queue_policy: ConcurrencyQueuePolicy,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
@@ -489,6 +518,15 @@ pub struct RuntimeSnapshot {
|
||||
}
|
||||
|
||||
impl RuntimeSnapshot {
|
||||
#[must_use]
|
||||
pub fn with_request_location(
|
||||
mut self,
|
||||
location: Option<crate::account::RequestLocation>,
|
||||
) -> Self {
|
||||
self.request_location = location;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn client_queue_policy(&self) -> ConcurrencyQueuePolicy {
|
||||
self.client_queue_policy
|
||||
@@ -568,6 +606,7 @@ impl RuntimeSnapshot {
|
||||
client_policy_map.retain(|_, policy| policy.enabled());
|
||||
|
||||
Ok(Self {
|
||||
request_location: None,
|
||||
revision,
|
||||
account_selection_policy,
|
||||
client_queue_policy: ConcurrencyQueuePolicy::default(),
|
||||
@@ -876,6 +915,7 @@ impl RuntimeSnapshot {
|
||||
|
||||
Ok(RoutingPlan {
|
||||
config_revision: self.revision,
|
||||
request_location: self.request_location.clone(),
|
||||
account_selection_policy: self.account_selection_policy,
|
||||
operation: operation.kind(),
|
||||
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
|
||||
@@ -917,6 +957,7 @@ impl RuntimeSnapshot {
|
||||
};
|
||||
Ok(RoutingPlan {
|
||||
config_revision: self.revision,
|
||||
request_location: self.request_location.clone(),
|
||||
account_selection_policy: self.account_selection_policy,
|
||||
operation: operation.kind(),
|
||||
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
|
||||
|
||||
@@ -22,10 +22,12 @@ pub enum WorkerKind {
|
||||
Retention,
|
||||
OpsFlush,
|
||||
Backup,
|
||||
AccountImport,
|
||||
AccountFreezeRecovery,
|
||||
}
|
||||
|
||||
impl WorkerKind {
|
||||
pub const ALL: [Self; 8] = [
|
||||
pub const ALL: [Self; 10] = [
|
||||
Self::OAuthRefresh,
|
||||
Self::QuotaCatalogHealth,
|
||||
Self::RuntimeSnapshotReconciliation,
|
||||
@@ -34,6 +36,8 @@ impl WorkerKind {
|
||||
Self::Retention,
|
||||
Self::OpsFlush,
|
||||
Self::Backup,
|
||||
Self::AccountImport,
|
||||
Self::AccountFreezeRecovery,
|
||||
];
|
||||
|
||||
#[must_use]
|
||||
@@ -47,6 +51,8 @@ impl WorkerKind {
|
||||
Self::Retention => "retention",
|
||||
Self::OpsFlush => "ops_flush",
|
||||
Self::Backup => "backup",
|
||||
Self::AccountImport => "account_import",
|
||||
Self::AccountFreezeRecovery => "account_freeze_recovery",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
use gateway_core::account::{InvalidRequestLocation, RequestLocation};
|
||||
use serde_json::json;
|
||||
|
||||
fn location() -> RequestLocation {
|
||||
RequestLocation {
|
||||
country: "JP".to_owned(),
|
||||
region: "Tokyo".to_owned(),
|
||||
city: "Tokyo".to_owned(),
|
||||
timezone: chrono_tz::Asia::Tokyo,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changing_or_removing_proxy_drops_its_location() {
|
||||
use gateway_core::account::OutboundProxy;
|
||||
let direct = super::account("acct_location").with_request_location(Some(location()));
|
||||
assert!(direct.request_location().is_none());
|
||||
let proxied = direct
|
||||
.with_outbound_proxy(Some(OutboundProxy::parse("http://localhost:8080").unwrap()))
|
||||
.with_request_location(Some(location()));
|
||||
assert_eq!(proxied.request_location(), Some(&location()));
|
||||
assert!(
|
||||
proxied
|
||||
.clone()
|
||||
.with_outbound_proxy(None)
|
||||
.request_location()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
proxied
|
||||
.with_outbound_proxy(Some(OutboundProxy::parse("http://localhost:8081").unwrap()))
|
||||
.request_location()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_normalizes_names_and_preserves_timezone() {
|
||||
let value = RequestLocation {
|
||||
region: " Tokyo ".to_owned(),
|
||||
city: " 東京 ".to_owned(),
|
||||
..location()
|
||||
}
|
||||
.normalized()
|
||||
.expect("valid location");
|
||||
assert_eq!(value.region, "Tokyo");
|
||||
assert_eq!(value.city, "東京");
|
||||
assert_eq!(value.timezone.name(), "Asia/Tokyo");
|
||||
let encoded = serde_json::to_value(&value).expect("serialize");
|
||||
assert_eq!(encoded["timezone"], "Asia/Tokyo");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<RequestLocation>(encoded).unwrap(),
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_rejects_invalid_country_codes() {
|
||||
for country in ["", "J", "JPN", "jp", "J1", "日", " JP"] {
|
||||
assert_eq!(
|
||||
RequestLocation {
|
||||
country: country.to_owned(),
|
||||
..location()
|
||||
}
|
||||
.validate(),
|
||||
Err(InvalidRequestLocation::Country)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_rejects_empty_control_and_long_names() {
|
||||
for name in [
|
||||
String::new(),
|
||||
" ".to_owned(),
|
||||
"Tokyo\n".to_owned(),
|
||||
"\tTokyo".to_owned(),
|
||||
"To\0kyo".to_owned(),
|
||||
"東".repeat(129),
|
||||
] {
|
||||
assert_eq!(
|
||||
RequestLocation {
|
||||
region: name.clone(),
|
||||
..location()
|
||||
}
|
||||
.normalized(),
|
||||
Err(InvalidRequestLocation::Region)
|
||||
);
|
||||
assert_eq!(
|
||||
RequestLocation {
|
||||
city: name,
|
||||
..location()
|
||||
}
|
||||
.normalized(),
|
||||
Err(InvalidRequestLocation::City)
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
RequestLocation {
|
||||
city: "東".repeat(128),
|
||||
..location()
|
||||
}
|
||||
.validate()
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_requires_complete_object_and_valid_iana_timezone() {
|
||||
for timezone in ["Asia/Tokyo", "America/New_York", "Europe/London"] {
|
||||
assert!(
|
||||
serde_json::from_value::<RequestLocation>(json!({
|
||||
"country": "JP", "region": "Tokyo", "city": "Tokyo", "timezone": timezone
|
||||
}))
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
for timezone in ["", "Not/A_Zone", "+09:00", "Asia/Typo"] {
|
||||
assert!(
|
||||
serde_json::from_value::<RequestLocation>(json!({
|
||||
"country": "JP", "region": "Tokyo", "city": "Tokyo", "timezone": timezone
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert!(serde_json::from_value::<RequestLocation>(json!({"country": "JP"})).is_err());
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
mod location;
|
||||
mod model_access;
|
||||
mod proxy;
|
||||
mod selection;
|
||||
@@ -49,7 +50,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -222,7 +223,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -240,7 +241,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -313,23 +314,23 @@ fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
|
||||
let active_until = now + Duration::from_secs(60);
|
||||
let account = account("acct_rate_limited");
|
||||
|
||||
let active = account.status_projection(now, Some(active_until));
|
||||
let active = account.status_projection(now, Some(active_until.into()));
|
||||
assert_eq!(active.status, AccountStatus::RateLimited);
|
||||
assert_eq!(active.rate_limited_until, Some(active_until));
|
||||
assert_eq!(active.cooldown, Some(active_until.into()));
|
||||
|
||||
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
|
||||
let elapsed = account.status_projection(now, Some((now - Duration::from_secs(1)).into()));
|
||||
assert_eq!(elapsed.status, AccountStatus::Normal);
|
||||
assert_eq!(elapsed.rate_limited_until, None);
|
||||
assert_eq!(elapsed.cooldown, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
|
||||
fn account_without_upstream_identity_preserves_provider_credential_state() {
|
||||
let account = ProviderAccount::new(
|
||||
ProviderAccountId::new("acct_pending_identity").expect("valid account"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
"pending identity".to_owned(),
|
||||
None,
|
||||
"oauth".to_owned(),
|
||||
"api_key".to_owned(),
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
@@ -346,7 +347,18 @@ fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
|
||||
account.credential_state(),
|
||||
account.status_projection(SystemTime::now(), None).status
|
||||
),
|
||||
(CredentialState::Unknown, AccountStatus::Error)
|
||||
(CredentialState::Ready, AccountStatus::Normal)
|
||||
);
|
||||
let pending = account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Unknown,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
assert_eq!(
|
||||
pending.status_projection(SystemTime::now(), None).status,
|
||||
AccountStatus::Error
|
||||
);
|
||||
}
|
||||
|
||||
@@ -481,8 +493,8 @@ fn highest_priority_affinity_should_still_obey_existing_scheduling_constraints()
|
||||
);
|
||||
}
|
||||
AccountSchedulingBlocker::LocalAvailability => {
|
||||
candidates[0].signals.rate_limited_until =
|
||||
Some(selection.now + Duration::from_secs(60))
|
||||
candidates[0].signals.cooldown =
|
||||
Some((selection.now + Duration::from_secs(60)).into())
|
||||
}
|
||||
_ => unreachable!("test cases"),
|
||||
}
|
||||
@@ -632,7 +644,7 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
@@ -1051,3 +1063,32 @@ fn elapsed_quota_reset_does_not_fabricate_recovery() {
|
||||
.expect("healthy candidate available");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn due_probe_freeze_blocks_scheduling_and_remains_visible_as_rate_limited() {
|
||||
use gateway_core::account::{AccountCooldown, AccountCooldownKind};
|
||||
let mut candidates = [
|
||||
weighted_candidate("acct_frozen", 100, 0),
|
||||
weighted_candidate("acct_ready", 1, 0),
|
||||
];
|
||||
let selection = context(RotationStrategy::Smart);
|
||||
candidates[0].signals.cooldown = Some(AccountCooldown {
|
||||
until: selection.now - Duration::from_secs(120),
|
||||
kind: AccountCooldownKind::CapacityFreezeProbe,
|
||||
});
|
||||
let projection = candidates[0]
|
||||
.account
|
||||
.status_projection(selection.now, candidates[0].signals.cooldown);
|
||||
assert_eq!(projection.status, AccountStatus::RateLimited);
|
||||
assert!(
|
||||
projection
|
||||
.cooldown
|
||||
.expect("freeze details")
|
||||
.kind
|
||||
.requires_probe()
|
||||
);
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("available account");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_ready");
|
||||
}
|
||||
|
||||
@@ -584,6 +584,14 @@ fn plan(operation: &Operation) -> RoutingPlan {
|
||||
fn plan_with_policy(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
) -> RoutingPlan {
|
||||
plan_with_location(operation, account_selection_policy, Default::default())
|
||||
}
|
||||
|
||||
fn plan_with_location(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
request_location: gateway_core::account::RequestLocation,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
@@ -626,6 +634,7 @@ fn plan_with_policy(
|
||||
Vec::new(),
|
||||
)
|
||||
.expect("snapshot")
|
||||
.with_request_location(Some(request_location))
|
||||
.with_account_directory(Arc::clone(&directory));
|
||||
snapshot
|
||||
.plan(
|
||||
@@ -4497,3 +4506,50 @@ fn deadline_before_first_event_records_no_provider_circuit_failure() {
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert!(!state.finalizations[0].committed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn global_request_location_should_reach_every_account_retry() {
|
||||
let operation = generate_operation();
|
||||
let location = gateway_core::account::RequestLocation {
|
||||
timezone: "Asia/Tokyo".parse().unwrap(),
|
||||
..Default::default()
|
||||
};
|
||||
let route_plan = plan_with_location(
|
||||
&operation,
|
||||
plan(&operation).account_selection_policy(),
|
||||
location.clone(),
|
||||
);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert!(
|
||||
contexts
|
||||
.iter()
|
||||
.all(|context| context.request_location() == Some(&location))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1045,13 +1045,15 @@ impl Provider for NativeCatalogProvider {
|
||||
.into_iter()
|
||||
.map(|id| gateway_core::routing::ProviderModelDescriptor {
|
||||
model: UpstreamModelId::new(id).expect("id"),
|
||||
payload: RawJsonPayload::new(
|
||||
"codex",
|
||||
serde_json::to_vec(&json!({"slug":id,"future":null}))
|
||||
.expect("JSON")
|
||||
.into(),
|
||||
)
|
||||
.expect("payload"),
|
||||
content: gateway_core::routing::ProviderModelContent::Native(
|
||||
RawJsonPayload::new(
|
||||
"codex",
|
||||
serde_json::to_vec(&json!({"slug":id,"future":null}))
|
||||
.expect("JSON")
|
||||
.into(),
|
||||
)
|
||||
.expect("payload"),
|
||||
),
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
|
||||
@@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
@@ -290,3 +290,85 @@ fn compiler(store: Arc<dyn SnapshotStorePort>) -> RuntimeSnapshotCompiler {
|
||||
fn revision(value: u64) -> ConfigRevision {
|
||||
ConfigRevision::new(value).expect("positive revision")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn global_request_location_should_be_frozen_when_snapshot_is_published() {
|
||||
use gateway_core::account::{ProviderAccountId, RequestLocation};
|
||||
use gateway_core::runtime::RuntimeSnapshotHandle;
|
||||
let make_snapshot = |version, timezone: &str, enabled| {
|
||||
let location = RequestLocation {
|
||||
timezone: timezone.parse().unwrap(),
|
||||
..RequestLocation::default()
|
||||
};
|
||||
let facts = SnapshotFacts::new(
|
||||
revision(version),
|
||||
revision(version),
|
||||
SnapshotSettingsFacts::new(3, 0, "smart", BTreeMap::new(), None, None)
|
||||
.with_request_location(location, enabled),
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
vec![SnapshotProviderAccountFacts::new(
|
||||
ProviderAccountId::new("acct_location").unwrap(),
|
||||
"alpha",
|
||||
)],
|
||||
Vec::new(),
|
||||
);
|
||||
block_on(
|
||||
RuntimeSnapshotCompiler::new(
|
||||
Arc::new(TestSnapshotStore::new(Ok(facts))),
|
||||
Arc::new(TestCatalog::Unavailable),
|
||||
)
|
||||
.compile(),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let handle = RuntimeSnapshotHandle::new(make_snapshot(1, "Asia/Tokyo", true));
|
||||
let frozen = handle.acquire().unwrap();
|
||||
let plan = |snapshot: &gateway_core::routing::RuntimeSnapshot| {
|
||||
snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("public-model").unwrap(),
|
||||
&super::operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&gateway_core::routing::RoutingContext::default(),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let old_plan = plan(&frozen);
|
||||
handle.publish(make_snapshot(2, "America/New_York", true));
|
||||
assert_eq!(
|
||||
old_plan.request_location().unwrap().timezone.name(),
|
||||
"Asia/Tokyo"
|
||||
);
|
||||
assert_eq!(
|
||||
plan(&frozen).request_location().unwrap().timezone.name(),
|
||||
"Asia/Tokyo"
|
||||
);
|
||||
assert_eq!(
|
||||
plan(&handle.acquire().unwrap())
|
||||
.request_location()
|
||||
.unwrap()
|
||||
.timezone
|
||||
.name(),
|
||||
"America/New_York"
|
||||
);
|
||||
handle.publish(make_snapshot(3, "Asia/Tokyo", false));
|
||||
assert!(
|
||||
plan(&handle.acquire().unwrap())
|
||||
.request_location()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
old_plan.request_location().unwrap().timezone.name(),
|
||||
"Asia/Tokyo"
|
||||
);
|
||||
handle.publish(make_snapshot(4, "Asia/Tokyo", true));
|
||||
assert_eq!(
|
||||
plan(&handle.acquire().unwrap())
|
||||
.request_location()
|
||||
.unwrap()
|
||||
.timezone
|
||||
.name(),
|
||||
"Asia/Tokyo"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ reqwest = { version = "=0.12.28", default-features = false, features = ["json",
|
||||
scraper = "0.27.0"
|
||||
semver = "1.0.27"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_ignored = "0.1.14"
|
||||
serde_json = "1.0.150"
|
||||
sha2 = "0.11.0"
|
||||
tar = "0.4.44"
|
||||
|
||||
@@ -19,6 +19,9 @@ const SERVER_PORT_ENV: &str = "CPR_SERVER_PORT";
|
||||
/// Host 只负责找到和解析文件;每个包的字段解释与相对路径解析
|
||||
/// 由顶层配置委托给对应的包完成。
|
||||
pub trait LoadableConfig: DeserializeOwned {
|
||||
/// 同一配置文件中由其他工具消费的顶层配置段。
|
||||
const EXTERNAL_SECTIONS: &'static [&'static str] = &[];
|
||||
|
||||
fn resolve_and_validate(&mut self, source_dir: &Path) -> Result<(), ConfigError>;
|
||||
}
|
||||
|
||||
@@ -27,18 +30,44 @@ pub fn load_config<T: LoadableConfig>() -> Result<T, ConfigError> {
|
||||
let current = env::current_dir().map_err(|_| ConfigError::CurrentDirectory)?;
|
||||
let path = discover_config_path(¤t)?;
|
||||
let source_dir = path.parent().ok_or(ConfigError::InvalidConfigPath)?;
|
||||
let mut value: T = config::Config::builder()
|
||||
let document = config::Config::builder()
|
||||
.add_source(config::File::from(path.as_path()).required(true))
|
||||
.build()
|
||||
.and_then(config::Config::try_deserialize)
|
||||
.map_err(|_| ConfigError::InvalidDocument { path: path.clone() })?;
|
||||
let mut unused = std::collections::BTreeSet::new();
|
||||
let mut value: T = serde_ignored::deserialize(document, |field| {
|
||||
unused.insert(field.to_string());
|
||||
})
|
||||
.map_err(|error| match missing_config_field(&error) {
|
||||
Some(field) => ConfigError::MissingField {
|
||||
path: path.clone(),
|
||||
field,
|
||||
},
|
||||
None => ConfigError::InvalidDocument { path: path.clone() },
|
||||
})?;
|
||||
// 此时日志尚未初始化;只报告字段路径,不回显可能包含凭据的配置值。
|
||||
for field in unused {
|
||||
if !T::EXTERNAL_SECTIONS.contains(&field.as_str()) {
|
||||
eprintln!("[警告] 配置字段 {field:?} 未使用,已忽略");
|
||||
}
|
||||
}
|
||||
value.resolve_and_validate(source_dir)?;
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn missing_config_field(error: &config::ConfigError) -> Option<String> {
|
||||
match error {
|
||||
config::ConfigError::NotFound(field) => Some(field.clone()),
|
||||
config::ConfigError::At { error, key, .. } => missing_config_field(error).map(|field| {
|
||||
key.as_ref()
|
||||
.map_or_else(|| field.clone(), |key| format!("{key}.{field}"))
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Host 唯一拥有的进程配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HostConfig {
|
||||
pub listen: ListenConfig,
|
||||
pub runtime_data_dir: PathBuf,
|
||||
@@ -52,7 +81,12 @@ pub struct HostConfig {
|
||||
}
|
||||
|
||||
impl HostConfig {
|
||||
pub fn resolve_and_validate(&mut self, source_dir: &Path) -> Result<(), ConfigError> {
|
||||
/// 解析 Host 配置;在线更新默认沿用已解析的 API 静态资源目录。
|
||||
pub fn resolve_and_validate(
|
||||
&mut self,
|
||||
source_dir: &Path,
|
||||
asset_directory: &Path,
|
||||
) -> Result<(), ConfigError> {
|
||||
if let Some(host) = optional_environment_value(SERVER_HOST_ENV)? {
|
||||
self.listen.host = host;
|
||||
}
|
||||
@@ -82,8 +116,11 @@ impl HostConfig {
|
||||
}
|
||||
resolve_relative_path(source_dir, &mut self.runtime_data_dir);
|
||||
self.logging.resolve_and_validate(source_dir)?;
|
||||
self.system_update
|
||||
.resolve_and_validate(source_dir, &self.runtime_data_dir)?;
|
||||
self.system_update.resolve_and_validate(
|
||||
source_dir,
|
||||
&self.runtime_data_dir,
|
||||
asset_directory,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -105,7 +142,6 @@ impl HostConfig {
|
||||
|
||||
/// HTTP 监听地址。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListenConfig {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
@@ -113,7 +149,6 @@ pub struct ListenConfig {
|
||||
|
||||
/// Host 结构化日志配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct LoggingConfig {
|
||||
pub level: String,
|
||||
pub stdout: bool,
|
||||
@@ -170,7 +205,6 @@ impl LoggingConfig {
|
||||
|
||||
/// Host 文件日志配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct FileLoggingConfig {
|
||||
pub enabled: bool,
|
||||
pub directory: PathBuf,
|
||||
@@ -190,6 +224,8 @@ pub enum ConfigError {
|
||||
InvalidConfigPath,
|
||||
#[error("configuration document is invalid: {path}")]
|
||||
InvalidDocument { path: PathBuf },
|
||||
#[error("配置文件 {path} 缺少必填字段 {field:?}")]
|
||||
MissingField { path: PathBuf, field: String },
|
||||
#[error("configuration field is invalid: {0}")]
|
||||
InvalidField(&'static str),
|
||||
#[error("environment variable is invalid: {0}")]
|
||||
|
||||
@@ -21,7 +21,8 @@ type ProxyClientBuilder =
|
||||
|
||||
impl Default for HttpProxyProbe {
|
||||
fn default() -> Self {
|
||||
Self::new("https://api.ipify.org?format=json")
|
||||
// 使用双栈端点,避免仅有 IPv6 出口的代理被 IPv4 专用检测服务误判为不可用。
|
||||
Self::new("https://api64.ipify.org?format=json")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,6 @@ use self::swap::{replace_release_files, rollback_release};
|
||||
pub use self::release::validate_download_url;
|
||||
|
||||
const APP_BINARY_NAME: &str = "codex-proxy-rs";
|
||||
const DEFAULT_WEB_DIST_DIR: &str = "/app/web/dist";
|
||||
const DEFAULT_GITHUB_API_BASE: &str = "https://api.github.com/repos";
|
||||
const DEFAULT_UPDATE_REPOSITORY: &str = "zyycn/codex-proxy-rs";
|
||||
|
||||
@@ -51,7 +50,7 @@ type OperationError = SystemOperationError;
|
||||
|
||||
/// 系统更新与重启配置;所有字段只由 Host 解释。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
#[serde(default)]
|
||||
pub struct SystemUpdateConfig {
|
||||
pub version: String,
|
||||
pub git_sha: String,
|
||||
@@ -62,7 +61,8 @@ pub struct SystemUpdateConfig {
|
||||
pub update_repository: Option<String>,
|
||||
pub github_api_base: String,
|
||||
pub executable_path: Option<PathBuf>,
|
||||
pub web_dist_dir: PathBuf,
|
||||
/// 未显式指定时,由组合根传入 API 实际使用的静态资源目录。
|
||||
pub web_dist_dir: Option<PathBuf>,
|
||||
pub update_state_file: PathBuf,
|
||||
pub update_lock_file: PathBuf,
|
||||
pub update_temp_dir: PathBuf,
|
||||
@@ -107,9 +107,7 @@ impl Default for SystemUpdateConfig {
|
||||
github_api_base: environment_value("CPR_GITHUB_API_BASE")
|
||||
.unwrap_or_else(|| DEFAULT_GITHUB_API_BASE.to_owned()),
|
||||
executable_path,
|
||||
web_dist_dir: environment_value("CPR_WEB_DIST_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from(DEFAULT_WEB_DIST_DIR)),
|
||||
web_dist_dir: None,
|
||||
update_state_file,
|
||||
update_lock_file,
|
||||
update_temp_dir,
|
||||
@@ -124,8 +122,15 @@ impl SystemUpdateConfig {
|
||||
&mut self,
|
||||
source_dir: &Path,
|
||||
runtime_data_dir: &Path,
|
||||
asset_directory: &Path,
|
||||
) -> Result<(), ConfigError> {
|
||||
for path in [self.executable_path.as_mut(), Some(&mut self.web_dist_dir)]
|
||||
let web_dist_dir = self
|
||||
.web_dist_dir
|
||||
.get_or_insert_with(|| asset_directory.to_path_buf());
|
||||
if web_dist_dir.as_os_str().is_empty() {
|
||||
return Err(ConfigError::InvalidField("host.system_update.web_dist_dir"));
|
||||
}
|
||||
for path in [self.executable_path.as_mut(), self.web_dist_dir.as_mut()]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
@@ -174,6 +179,9 @@ impl SystemUpdateConfig {
|
||||
if let Err(error) = release::validate_api_base(&self.github_api_base) {
|
||||
return Some(error);
|
||||
}
|
||||
if let Err(error) = self.web_dist_dir() {
|
||||
return Some(error.to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -197,6 +205,13 @@ impl SystemUpdateConfig {
|
||||
.and_then(fs::canonicalize)
|
||||
.map_err(|error| internal(format!("failed to resolve executable: {error}")))
|
||||
}
|
||||
|
||||
pub(crate) fn web_dist_dir(&self) -> Result<&Path, OperationError> {
|
||||
self.web_dist_dir
|
||||
.as_deref()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.ok_or_else(|| conflict("web assets directory is not configured"))
|
||||
}
|
||||
}
|
||||
|
||||
/// gateway-admin 消费的真实进程/文件系统操作实现。
|
||||
@@ -395,7 +410,7 @@ impl ProcessSystemOperations {
|
||||
.info(Some(operation_id), Some("replace"), "正在替换应用文件");
|
||||
replace_release_files(
|
||||
&self.config.executable_path()?,
|
||||
&self.config.web_dist_dir,
|
||||
self.config.web_dist_dir()?,
|
||||
extracted,
|
||||
)?;
|
||||
self.events
|
||||
|
||||
@@ -81,6 +81,7 @@ pub(crate) fn replace_release_files(
|
||||
}
|
||||
|
||||
pub(crate) fn rollback_release(config: &SystemUpdateConfig) -> Result<(), OperationError> {
|
||||
let web_dist = config.web_dist_dir()?;
|
||||
let executable = config.executable_path()?;
|
||||
let binary_backup = backup_path_for(&executable);
|
||||
if !binary_backup.exists() {
|
||||
@@ -89,9 +90,9 @@ pub(crate) fn rollback_release(config: &SystemUpdateConfig) -> Result<(), Operat
|
||||
swap_file(&executable, &binary_backup)
|
||||
.map_err(|error| internal(format!("binary rollback failed: {error}")))?;
|
||||
|
||||
let web_backup = backup_path_for(&config.web_dist_dir);
|
||||
let web_backup = backup_path_for(web_dist);
|
||||
if web_backup.exists()
|
||||
&& let Err(error) = swap_dir(&config.web_dist_dir, &web_backup)
|
||||
&& let Err(error) = swap_dir(web_dist, &web_backup)
|
||||
{
|
||||
let mut rollback = Vec::new();
|
||||
collect_rollback_error(
|
||||
|
||||
@@ -21,11 +21,61 @@ fn system_update_defaults_should_use_host_build_metadata_and_official_repository
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_should_default_update_assets_to_the_resolved_api_directory() {
|
||||
let mut config = valid_config();
|
||||
config.system_update.web_dist_dir = None;
|
||||
let assets = std::path::Path::new("/srv/gateway/web/dist");
|
||||
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway/deploy"), assets)
|
||||
.expect("host config");
|
||||
|
||||
assert_eq!(config.system_update.web_dist_dir.as_deref(), Some(assets));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_should_preserve_explicit_update_asset_directory() {
|
||||
let mut config = valid_config();
|
||||
config.system_update.web_dist_dir = Some(PathBuf::from("../custom/dist"));
|
||||
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway/deploy"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("host config");
|
||||
|
||||
assert_eq!(
|
||||
config.system_update.web_dist_dir,
|
||||
Some(PathBuf::from("/srv/gateway/deploy/../custom/dist"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_should_reject_empty_update_asset_directory() {
|
||||
let mut config = valid_config();
|
||||
config.system_update.web_dist_dir = Some(PathBuf::new());
|
||||
|
||||
assert!(matches!(
|
||||
config.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway/deploy"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
),
|
||||
Err(gateway_host::ConfigError::InvalidField(
|
||||
"host.system_update.web_dist_dir"
|
||||
))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_resolves_only_host_owned_relative_paths() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
@@ -38,7 +88,10 @@ fn host_config_resolves_only_host_owned_relative_paths() {
|
||||
fn host_config_resolves_runtime_data_dir_relative_to_configuration() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
@@ -51,7 +104,10 @@ fn host_config_resolves_runtime_data_dir_relative_to_configuration() {
|
||||
fn host_config_derives_update_paths_from_runtime_data_dir() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
@@ -75,7 +131,10 @@ fn host_config_rejects_zero_drain_window() {
|
||||
|
||||
assert!(
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
@@ -91,14 +150,17 @@ fn logging_rejects_zero_retention_windows() {
|
||||
}
|
||||
assert!(
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_defaults_to_seven_days_and_one_day_for_payloads_and_rejects_count_retention() {
|
||||
fn logging_defaults_to_time_retention_and_ignores_removed_count_retention() {
|
||||
let mut value = serde_json::json!({
|
||||
"level": "info", "stdout": true,
|
||||
"file": { "enabled": true, "directory": "logs", "max_file_size_mb": 20 }
|
||||
@@ -108,7 +170,10 @@ fn logging_defaults_to_seven_days_and_one_day_for_payloads_and_rejects_count_ret
|
||||
assert_eq!(config.file.retention_days, 7);
|
||||
assert!(!config.oauth_recovery);
|
||||
value["file"]["max_files"] = serde_json::json!(20);
|
||||
assert!(serde_json::from_value::<LoggingConfig>(value).is_err());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<LoggingConfig>(value).unwrap(),
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -120,7 +185,10 @@ fn dedicated_file_channels_can_be_enabled_without_application_sinks() {
|
||||
config.logging.oauth_recovery = oauth_recovery;
|
||||
config.logging.request_dump = !oauth_recovery;
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,7 +53,10 @@ fn logging_requires_at_least_one_sink() {
|
||||
|
||||
assert!(
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/tmp"))
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/tmp"),
|
||||
std::path::Path::new("/tmp/web/dist"),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,20 +8,33 @@ use wiremock::{
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_probe_sends_authentication_through_explicit_proxy() {
|
||||
let proxy_server = MockServer::start().await;
|
||||
Mock::given(header("proxy-authorization", "Basic dXNlcjpwYXNzd29yZA=="))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"ip": "203.0.113.8"})))
|
||||
.expect(1)
|
||||
.mount(&proxy_server)
|
||||
.await;
|
||||
let proxy =
|
||||
OutboundProxy::parse(&format!("http://user:password@{}", proxy_server.address())).unwrap();
|
||||
let result = HttpProxyProbe::new("http://unresolvable.invalid/ip")
|
||||
.test(&proxy)
|
||||
.await;
|
||||
assert!(result.success);
|
||||
assert_eq!(result.exit_ip.unwrap().to_string(), "203.0.113.8");
|
||||
async fn proxy_probe_supports_ipv4_and_ipv6_proxies_and_exit_addresses() {
|
||||
for (listen_address, exit_ip) in [
|
||||
("127.0.0.1:0", "203.0.113.8"),
|
||||
("127.0.0.1:0", "2001:db8::8"),
|
||||
("[::1]:0", "203.0.113.8"),
|
||||
("[::1]:0", "2001:db8::8"),
|
||||
] {
|
||||
let listener = std::net::TcpListener::bind(listen_address).unwrap();
|
||||
let proxy_server = MockServer::builder().listener(listener).start().await;
|
||||
Mock::given(header("proxy-authorization", "Basic dXNlcjpwYXNzd29yZA=="))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"ip": exit_ip})))
|
||||
.expect(1)
|
||||
.mount(&proxy_server)
|
||||
.await;
|
||||
let proxy =
|
||||
OutboundProxy::parse(&format!("http://user:password@{}", proxy_server.address()))
|
||||
.unwrap();
|
||||
let result = HttpProxyProbe::new("http://unresolvable.invalid/ip")
|
||||
.test(&proxy)
|
||||
.await;
|
||||
assert!(
|
||||
result.success,
|
||||
"{listen_address} -> {exit_ip}: {}",
|
||||
result.message
|
||||
);
|
||||
assert_eq!(result.exit_ip.unwrap().to_string(), exit_ip);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -550,7 +550,7 @@ async fn update_should_remove_stale_file_lock_and_continue() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_replace_local_release_files_with_latest_asset() {
|
||||
async fn update_and_rollback_should_use_the_default_api_asset_directory() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
@@ -561,8 +561,21 @@ async fn update_should_replace_local_release_files_with_latest_asset() {
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
fixture
|
||||
.service(&server)
|
||||
let mut host: gateway_host::HostConfig = serde_json::from_value(serde_json::json!({
|
||||
"listen": { "host": "127.0.0.1", "port": 8080 },
|
||||
"runtime_data_dir": "../.runtime/data",
|
||||
"logging": {
|
||||
"level": "info", "stdout": true,
|
||||
"file": { "enabled": false, "directory": "../.runtime/logs", "max_file_size_mb": 20 }
|
||||
}
|
||||
}))
|
||||
.expect("binary host config");
|
||||
host.system_update = fixture.config(&format!("{}/repos", server.uri()));
|
||||
host.system_update.web_dist_dir = None;
|
||||
host.resolve_and_validate(&fixture.root.path().join("deploy"), &fixture.web())
|
||||
.expect("inherit API asset directory");
|
||||
let service = ProcessSystemOperations::new(CancellationToken::new(), host.system_update);
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
.expect("update");
|
||||
@@ -575,6 +588,16 @@ async fn update_should_replace_local_release_files_with_latest_asset() {
|
||||
fs::read(fixture.web().join("index.html")).expect("web"),
|
||||
b"new-web"
|
||||
);
|
||||
|
||||
service.rollback().await.expect("rollback");
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("binary"),
|
||||
b"old-binary"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.web().join("index.html")).expect("web"),
|
||||
b"old-web"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -593,17 +616,18 @@ async fn update_should_replace_web_assets_across_filesystems() {
|
||||
)
|
||||
.await;
|
||||
let mut config = fixture.config(&format!("{}/repos", server.uri()));
|
||||
config.web_dist_dir = external.path().join("dist");
|
||||
fs::create_dir_all(&config.web_dist_dir).expect("web dir");
|
||||
fs::write(config.web_dist_dir.join("index.html"), "old-web").expect("web");
|
||||
let service = ProcessSystemOperations::new(CancellationToken::new(), config.clone());
|
||||
let web_dist = external.path().join("dist");
|
||||
config.web_dist_dir = Some(web_dist.clone());
|
||||
fs::create_dir_all(&web_dist).expect("web dir");
|
||||
fs::write(web_dist.join("index.html"), "old-web").expect("web");
|
||||
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
.expect("update");
|
||||
|
||||
assert_eq!(
|
||||
fs::read(config.web_dist_dir.join("index.html")).expect("web"),
|
||||
fs::read(web_dist.join("index.html")).expect("web"),
|
||||
b"new-web"
|
||||
);
|
||||
}
|
||||
@@ -726,7 +750,7 @@ impl Fixture {
|
||||
update_repository: Some("owner/repository".to_owned()),
|
||||
github_api_base: api_base.to_owned(),
|
||||
executable_path: Some(self.executable()),
|
||||
web_dist_dir: self.web(),
|
||||
web_dist_dir: Some(self.web()),
|
||||
update_state_file: self.state(),
|
||||
update_lock_file: self.lock(),
|
||||
update_temp_dir: self.root.path().join("tmp"),
|
||||
|
||||
@@ -17,7 +17,7 @@ use gateway_core::task::{
|
||||
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
const ACTIVE_KINDS: [WorkerKind; 7] = [
|
||||
const ACTIVE_KINDS: [WorkerKind; 9] = [
|
||||
WorkerKind::OAuthRefresh,
|
||||
WorkerKind::QuotaCatalogHealth,
|
||||
WorkerKind::RuntimeSnapshotReconciliation,
|
||||
@@ -25,6 +25,8 @@ const ACTIVE_KINDS: [WorkerKind; 7] = [
|
||||
WorkerKind::StaleModelRequestRecovery,
|
||||
WorkerKind::Retention,
|
||||
WorkerKind::Backup,
|
||||
WorkerKind::AccountImport,
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
];
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! OpenAI 请求头的业务协议与传输边界。
|
||||
//! OpenAI 请求头的 HTTP 传输边界。
|
||||
|
||||
/// 判断小写请求头是否属于传输层管理的字段,不得作为业务扩展头透传。
|
||||
///
|
||||
@@ -6,10 +6,7 @@
|
||||
/// 只用于请求方向,不影响上游响应中的代理诊断信息。
|
||||
#[must_use]
|
||||
pub fn is_transport_managed_request_header(name: &str) -> bool {
|
||||
// 代理命名空间描述的是下游链路;包括未知扩展也不能冒充上游连接事实。
|
||||
name.starts_with("cf-")
|
||||
|| name.starts_with("x-forwarded-")
|
||||
|| name.starts_with("sec-websocket-")
|
||||
name.starts_with("sec-websocket-")
|
||||
|| matches!(
|
||||
name,
|
||||
"connection"
|
||||
@@ -26,11 +23,7 @@ pub fn is_transport_managed_request_header(name: &str) -> bool {
|
||||
// 请求实体与响应压缩能力属于各段 transport,不能继承下游协商。
|
||||
| "content-encoding"
|
||||
| "accept-encoding"
|
||||
| "forwarded"
|
||||
| "via"
|
||||
| "cdn-loop"
|
||||
| "x-real-ip"
|
||||
| "true-client-ip"
|
||||
// 网关中间件也会生成该链路诊断 ID,不作为业务上下文转发。
|
||||
| "x-request-id"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,31 @@
|
||||
use gateway_protocol::openai::is_transport_managed_request_header;
|
||||
|
||||
#[test]
|
||||
fn transport_headers_should_include_proxy_namespaces_and_compression() {
|
||||
fn transport_headers_should_include_hop_fields_and_compression() {
|
||||
for name in [
|
||||
"accept-encoding",
|
||||
"content-encoding",
|
||||
"content-length",
|
||||
"host",
|
||||
"x-request-id",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-connection",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
"sec-websocket-key",
|
||||
"sec-websocket-extensions",
|
||||
] {
|
||||
assert!(is_transport_managed_request_header(name), "missing {name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_headers_should_leave_business_extensions_to_the_protocol_owner() {
|
||||
for name in [
|
||||
"cf-visitor",
|
||||
"cf-connecting-ip",
|
||||
@@ -24,30 +48,6 @@ fn transport_headers_should_include_proxy_namespaces_and_compression() {
|
||||
"x-forwarded-future-field",
|
||||
"x-real-ip",
|
||||
"true-client-ip",
|
||||
"x-request-id",
|
||||
"accept-encoding",
|
||||
"content-encoding",
|
||||
"content-length",
|
||||
"host",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-connection",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
"sec-websocket-key",
|
||||
"sec-websocket-extensions",
|
||||
] {
|
||||
assert!(is_transport_managed_request_header(name), "missing {name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_headers_should_leave_business_extensions_to_the_protocol_owner() {
|
||||
for name in [
|
||||
"x-openai-future-mode",
|
||||
"x-custom-extension",
|
||||
"x-client-request-id",
|
||||
|
||||
@@ -43,6 +43,8 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
refresh_margin_seconds: command.refresh_margin_seconds,
|
||||
refresh_concurrency: command.refresh_concurrency,
|
||||
max_concurrent_per_account: command.max_concurrent_per_account,
|
||||
request_location_enabled: command.request_location_enabled,
|
||||
request_location: command.request_location,
|
||||
request_interval_ms: command.request_interval_ms,
|
||||
max_waiting_per_key: command.max_waiting_per_key,
|
||||
max_waiting_per_account: command.max_waiting_per_account,
|
||||
@@ -54,6 +56,14 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
account_auto_freeze_enabled: command.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: command.account_auto_freeze_threshold,
|
||||
account_auto_freeze_window_seconds: command.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: command.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: command.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: command.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: command
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
},
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
@@ -61,6 +71,8 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
"runtime_settings",
|
||||
"1",
|
||||
vec![
|
||||
"request_location_enabled".to_owned(),
|
||||
"request_location_json".to_owned(),
|
||||
"model_mappings_json".to_owned(),
|
||||
"refresh_margin_seconds".to_owned(),
|
||||
"refresh_concurrency".to_owned(),
|
||||
@@ -73,6 +85,7 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
"min_codex_desktop_version".to_owned(),
|
||||
"min_codex_cli_version".to_owned(),
|
||||
"retention".to_owned(),
|
||||
"account_auto_freeze".to_owned(),
|
||||
],
|
||||
),
|
||||
};
|
||||
@@ -167,6 +180,8 @@ pub(crate) fn admin_runtime_settings(
|
||||
.collect::<AdminStoreResult<ModelMappings>>()?;
|
||||
Ok(AdminRuntimeSettings {
|
||||
config_revision: admin_revision(settings.config_revision)?,
|
||||
request_location_enabled: settings.request_location_enabled,
|
||||
request_location: settings.request_location,
|
||||
model_mappings,
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: settings.refresh_concurrency,
|
||||
@@ -181,6 +196,13 @@ pub(crate) fn admin_runtime_settings(
|
||||
usage_retention_days: settings.usage_retention_days,
|
||||
ops_event_retention_days: settings.ops_event_retention_days,
|
||||
audit_retention_days: settings.audit_retention_days,
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: settings.account_auto_freeze_threshold,
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings.account_auto_freeze_adaptive_concurrency,
|
||||
updated_at: settings.updated_at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ pub(crate) const POSTGRES_HEALTH_RETRY_DELAY: Duration = Duration::from_millis(5
|
||||
|
||||
/// Store 自己拥有并校验的启动配置。
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StoreConfig {
|
||||
pub(crate) database: StoreConnectionConfig,
|
||||
pub(crate) redis: StoreConnectionConfig,
|
||||
@@ -31,7 +30,7 @@ pub struct StoreConfig {
|
||||
|
||||
/// PostgreSQL 连接池预算;acquire 超时决定池耗尽时快速失败而非排队积压。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
#[serde(default)]
|
||||
pub struct StorePoolConfig {
|
||||
pub max_connections: u32,
|
||||
pub acquire_timeout_seconds: u64,
|
||||
@@ -149,7 +148,6 @@ impl fmt::Debug for StoreConfig {
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct StoreConnectionConfig {
|
||||
pub(crate) url: String,
|
||||
pub(crate) password: String,
|
||||
|
||||
@@ -164,7 +164,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
|
||||
.map(|group_id| group_id.as_str().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let rows = sqlx::query(
|
||||
"select membership.account_group_id,
|
||||
"select proxy.location_country, proxy.location_region, proxy.location_city, proxy.location_timezone, membership.account_group_id,
|
||||
account.id, account.provider_kind, account.name, account.notes, account.email,
|
||||
account.upstream_user_id, account.upstream_account_id, account.plan_type,
|
||||
account.authentication_kind, account.credential_revision, account.outbound_proxy_url,
|
||||
@@ -178,6 +178,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
|
||||
settings.max_concurrent_per_account
|
||||
from account_group_accounts membership
|
||||
join provider_accounts account on account.id = membership.provider_account_id
|
||||
left join outbound_proxies proxy on proxy.id = account.outbound_proxy_id
|
||||
cross join runtime_settings settings
|
||||
where settings.id = 1
|
||||
and membership.account_group_id = any($1::text[])
|
||||
@@ -208,7 +209,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message,
|
||||
},
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::*;
|
||||
|
||||
use crate::postgres::{
|
||||
PgProviderAccountRepository, ProviderAccountRepository, ProviderAccountSummary,
|
||||
account_status_projection, load_rate_limited_until,
|
||||
account_status_projection, load_cooldown,
|
||||
};
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
@@ -49,19 +49,15 @@ impl PgObservabilityRepository {
|
||||
)
|
||||
.await?;
|
||||
let now = observed_at.into();
|
||||
let rate_limited_until =
|
||||
load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now).await;
|
||||
let cooldown = load_cooldown(self.cooldowns.as_deref(), &accounts, now).await;
|
||||
let mut metrics = ProviderAccountMetrics {
|
||||
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
|
||||
..ProviderAccountMetrics::default()
|
||||
};
|
||||
let mut normal_accounts = Vec::new();
|
||||
for account in &accounts {
|
||||
let projection = account_status_projection(
|
||||
account,
|
||||
now,
|
||||
rate_limited_until.get(&account.id).copied(),
|
||||
);
|
||||
let projection =
|
||||
account_status_projection(account, now, cooldown.get(&account.id).copied());
|
||||
match projection.status {
|
||||
gateway_core::account::AccountStatus::Normal => {
|
||||
metrics.normal = metrics.normal.saturating_add(1);
|
||||
|
||||
@@ -281,6 +281,7 @@ impl PgAdminAccountStore {
|
||||
async fn commit_prepared_rotation(
|
||||
&self,
|
||||
prepared: PreparedCredentialRotationFacts,
|
||||
settings: Option<UpdateAccount>,
|
||||
context: &MutationContext,
|
||||
action: &str,
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
@@ -288,9 +289,24 @@ impl PgAdminAccountStore {
|
||||
let scope = ProviderAccountAdminScope {
|
||||
provider_kind: prepared.provider_kind.as_str().to_owned(),
|
||||
};
|
||||
let mut changed_fields = vec!["credentials".to_owned()];
|
||||
if let Some(settings) = &settings {
|
||||
changed_fields
|
||||
.extend(["enabled", "concurrency_limit", "weight", "groups"].map(str::to_owned));
|
||||
if settings.model_access.is_some() {
|
||||
changed_fields.push("model_access".to_owned());
|
||||
}
|
||||
if settings.outbound_proxy.is_some() {
|
||||
changed_fields.push("outbound_proxy".to_owned());
|
||||
}
|
||||
if settings.notes.is_some() {
|
||||
changed_fields.push("notes".to_owned());
|
||||
}
|
||||
}
|
||||
let rotation = self
|
||||
.accounts
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
settings,
|
||||
scope,
|
||||
profile: UpdateProviderAccount {
|
||||
id: account_id.as_str().to_owned(),
|
||||
@@ -307,13 +323,14 @@ impl PgAdminAccountStore {
|
||||
has_refresh_token: prepared.has_refresh_token,
|
||||
access_token_expires_at: prepared.access_token_expires_at,
|
||||
next_refresh_at: prepared.next_refresh_at,
|
||||
preserve_profile: prepared.preserve_profile,
|
||||
},
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
action,
|
||||
"provider_account",
|
||||
account_id.as_str(),
|
||||
vec!["credentials".to_owned()],
|
||||
changed_fields,
|
||||
),
|
||||
})
|
||||
.await
|
||||
@@ -390,15 +407,11 @@ impl AccountStore for PgAdminAccountStore {
|
||||
));
|
||||
}
|
||||
let now = Utc::now();
|
||||
let rate_limited_until = runtime
|
||||
.rate_limited_until
|
||||
.into_iter()
|
||||
.map(|(account_id, until)| (account_id, until.into()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let cooldown = runtime.cooldown;
|
||||
let now_system_time = std::time::SystemTime::from(now);
|
||||
let active_rate_limited_ids = rate_limited_until
|
||||
let active_rate_limited_ids = cooldown
|
||||
.iter()
|
||||
.filter(|(_, until)| **until > now_system_time)
|
||||
.filter(|(_, cooldown)| cooldown.is_active(now_system_time))
|
||||
.map(|(account_id, _)| account_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let page =
|
||||
@@ -417,7 +430,7 @@ impl AccountStore for PgAdminAccountStore {
|
||||
let projection = account_status_projection(
|
||||
&summary,
|
||||
now.into(),
|
||||
rate_limited_until.get(&account_id).copied(),
|
||||
cooldown.get(&account_id).copied(),
|
||||
);
|
||||
let mut account = admin_account_record(summary)?;
|
||||
account.groups = groups_by_account.remove(&account_id).unwrap_or_default();
|
||||
@@ -449,12 +462,8 @@ impl AccountStore for PgAdminAccountStore {
|
||||
return Ok(None);
|
||||
};
|
||||
let now = Utc::now();
|
||||
let rate_limited_until = runtime
|
||||
.rate_limited_until
|
||||
.get(account_id)
|
||||
.copied()
|
||||
.map(Into::into);
|
||||
let projection = account_status_projection(&record.summary, now.into(), rate_limited_until);
|
||||
let cooldown = runtime.cooldown.get(account_id).copied();
|
||||
let projection = account_status_projection(&record.summary, now.into(), cooldown);
|
||||
let account_id = record.summary.id.clone();
|
||||
let mut groups = self
|
||||
.account_groups_by_account(std::slice::from_ref(&account_id))
|
||||
@@ -637,7 +646,7 @@ impl AccountStore for PgAdminAccountStore {
|
||||
"reauthorization cannot change account settings",
|
||||
));
|
||||
}
|
||||
self.commit_prepared_rotation(prepared, context, "reauthorize")
|
||||
self.commit_prepared_rotation(prepared, None, context, "reauthorize")
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -648,8 +657,13 @@ impl AccountStore for PgAdminAccountStore {
|
||||
command: CredentialRotationCommit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
self.commit_prepared_rotation(command.prepared, context, "rotate_credential")
|
||||
.await
|
||||
self.commit_prepared_rotation(
|
||||
command.prepared,
|
||||
command.settings,
|
||||
context,
|
||||
"rotate_credential",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn commit_credential_refresh(
|
||||
@@ -657,7 +671,14 @@ impl AccountStore for PgAdminAccountStore {
|
||||
command: CredentialRotationCommit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
self.commit_prepared_rotation(command.prepared, context, "refresh_credential")
|
||||
if command.settings.is_some() {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
ENTITY,
|
||||
"credential refresh cannot change account settings",
|
||||
));
|
||||
}
|
||||
self.commit_prepared_rotation(command.prepared, None, context, "refresh_credential")
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -716,6 +737,69 @@ impl AccountStore for PgAdminAccountStore {
|
||||
})
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &CoreProviderAccountId,
|
||||
limit: AccountConcurrencyLimit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>> {
|
||||
let mut transaction = self.pool.begin().await.map_err(|_| {
|
||||
admin_store_error(ENTITY, postgres_unavailable("begin concurrency reduction"))
|
||||
})?;
|
||||
let result =
|
||||
async {
|
||||
// 与管理写入采用相同锁顺序;锁住默认值后再检查账号最新设置,避免把旧快照写回。
|
||||
let default_limit: i64 = sqlx::query_scalar(
|
||||
"select max_concurrent_per_account from runtime_settings where id = 1 for update"
|
||||
).fetch_one(&mut *transaction).await
|
||||
.map_err(|_| postgres_unavailable("lock default concurrency"))?;
|
||||
let changed = sqlx::query_scalar::<_, String>(
|
||||
"update provider_accounts set concurrency_limit = $2, updated_at = now()
|
||||
where id = $1 and enabled = true and coalesce(concurrency_limit, $3) > $2
|
||||
returning id",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.bind(i64::from(limit.get()))
|
||||
.bind(default_limit)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("lower account concurrency"))?;
|
||||
if changed.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
append_admin_audit_event_in_transaction(
|
||||
&mut transaction,
|
||||
mutation_audit(
|
||||
context,
|
||||
"adapt_concurrency",
|
||||
"provider_account",
|
||||
account_id.as_str(),
|
||||
vec!["concurrency_limit".to_owned()],
|
||||
),
|
||||
revision,
|
||||
)
|
||||
.await?;
|
||||
Ok(Some(revision))
|
||||
}
|
||||
.await;
|
||||
let revision = super::repository::finish_admin_transaction(
|
||||
transaction,
|
||||
result,
|
||||
"lower account concurrency",
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
revision
|
||||
.map(|revision| {
|
||||
Ok(AccountUpdateResult {
|
||||
config_revision: admin_revision(revision)?,
|
||||
account_id: account_id.clone(),
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &CoreProviderAccountId,
|
||||
|
||||
@@ -116,7 +116,7 @@ pub(crate) async fn load_admin_account_page(
|
||||
from runtime_settings
|
||||
where id = 1
|
||||
)
|
||||
select a.outbound_proxy_url, a.id, a.provider_kind, a.name, a.notes, a.email, a.upstream_user_id,
|
||||
select p.location_country, p.location_region, p.location_city, p.location_timezone, a.outbound_proxy_url, a.id, a.provider_kind, a.name, a.notes, a.email, a.upstream_user_id,
|
||||
a.upstream_account_id, a.plan_type, a.authentication_kind,
|
||||
a.credential_revision, a.has_refresh_token, a.access_token_expires_at,
|
||||
a.next_refresh_at, a.enabled, a.concurrency_limit, a.weight, a.model_access_json,
|
||||
@@ -143,6 +143,7 @@ pub(crate) async fn load_admin_account_page(
|
||||
) ordered
|
||||
) page on true
|
||||
left join provider_accounts a on a.id = page.id
|
||||
left join outbound_proxies p on p.id = a.outbound_proxy_id
|
||||
order by page.page_position"
|
||||
);
|
||||
// 动态片段只来自上面的封闭排序枚举与固定 usage predicate;所有请求值仍使用 bind。
|
||||
|
||||
@@ -148,6 +148,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
account_id,
|
||||
expected_revision,
|
||||
profile,
|
||||
preserve_profile,
|
||||
credential,
|
||||
has_refresh_token,
|
||||
access_token_expires_at,
|
||||
@@ -178,22 +179,24 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.and_then(|state| state.message.as_deref());
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update provider_accounts
|
||||
set name = $3, email = $4, plan_type = $5,
|
||||
set name = case when $14 then name else $3 end,
|
||||
email = case when $14 then email else $4 end,
|
||||
plan_type = case when $14 then plan_type else $5 end,
|
||||
provider_credentials_json = $6,
|
||||
credential_revision = credential_revision + 1,
|
||||
has_refresh_token = $7, access_token_expires_at = $8,
|
||||
next_refresh_at = $9,
|
||||
credential_state = case
|
||||
when $10::text is not null and enabled and upstream_user_id is not null
|
||||
when $10::text is not null and enabled
|
||||
then $10 else credential_state end,
|
||||
credential_observed_at = case
|
||||
when $10::text is not null and enabled and upstream_user_id is not null
|
||||
when $10::text is not null and enabled
|
||||
then $11 else credential_observed_at end,
|
||||
last_error_reason = case
|
||||
when $10::text is not null and enabled and upstream_user_id is not null
|
||||
when $10::text is not null and enabled
|
||||
then $12 else last_error_reason end,
|
||||
last_error_message = case
|
||||
when $10::text is not null and enabled and upstream_user_id is not null
|
||||
when $10::text is not null and enabled
|
||||
then $13 else last_error_message end,
|
||||
updated_at = greatest(now(), coalesce($11, now()))
|
||||
where id = $1 and credential_revision = $2
|
||||
@@ -212,6 +215,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.bind(credential_observed_at)
|
||||
.bind(error_reason)
|
||||
.bind(message)
|
||||
.bind(preserve_profile)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?;
|
||||
@@ -236,7 +240,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.map(|account| account.as_str().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let rows = sqlx::query(
|
||||
"select id, credential_revision, provider_quota_json, quota_observed_at, \
|
||||
"select id, credential_revision, plan_type, provider_quota_json, quota_observed_at, \
|
||||
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at \
|
||||
from provider_accounts \
|
||||
where id = any($1) and quota_observed_at is not null",
|
||||
@@ -294,6 +298,9 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
)
|
||||
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
Ok(QuotaObservation {
|
||||
plan_type: row
|
||||
.try_get("plan_type")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
||||
account_id: CoreProviderAccountId::new(account_id)
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
||||
expected_revision: revision,
|
||||
@@ -322,6 +329,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
quota,
|
||||
DateTime::<Utc>::from(observation.observed_at),
|
||||
observation.state,
|
||||
observation.plan_type.as_deref(),
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
|
||||
@@ -28,6 +28,7 @@ pub trait ProviderAccountRepository: Send + Sync {
|
||||
quota: JsonObject,
|
||||
observed_at: DateTime<Utc>,
|
||||
state: QuotaState,
|
||||
plan_type: Option<&str>,
|
||||
) -> StoreResult<bool>;
|
||||
async fn touch_provider_quota_observation(
|
||||
&self,
|
||||
@@ -108,13 +109,15 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
include_disabled: bool,
|
||||
) -> StoreResult<Vec<ProviderAccountSummary>> {
|
||||
let rows = sqlx::query(
|
||||
"select outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
"select location_country, location_region, location_city, location_timezone, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, concurrency_limit, weight, model_access_json, credential_state,
|
||||
credential_observed_at, quota_access_state, quota_evidence,
|
||||
quota_access_observed_at, quota_reset_at,
|
||||
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
|
||||
from provider_accounts
|
||||
left join (select id as location_proxy_id, location_country, location_region, location_city, location_timezone from outbound_proxies) proxy_location
|
||||
on outbound_proxy_id = location_proxy_id
|
||||
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
|
||||
order by provider_kind, name, id",
|
||||
)
|
||||
@@ -128,11 +131,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
|
||||
async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> {
|
||||
account.validate()?;
|
||||
let credential_state = if account.upstream_user_id.is_some() {
|
||||
account.credential_state
|
||||
} else {
|
||||
CredentialState::Unknown
|
||||
};
|
||||
let credential_state = account.credential_state;
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
@@ -255,19 +254,19 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set credential_state = case
|
||||
when enabled and upstream_user_id is not null then $3
|
||||
when enabled then $3
|
||||
else credential_state
|
||||
end,
|
||||
credential_observed_at = case
|
||||
when enabled and upstream_user_id is not null then $4
|
||||
when enabled then $4
|
||||
else credential_observed_at
|
||||
end,
|
||||
last_error_reason = case
|
||||
when enabled and upstream_user_id is not null then $5
|
||||
when enabled then $5
|
||||
else last_error_reason
|
||||
end,
|
||||
last_error_message = case
|
||||
when enabled and upstream_user_id is not null then $6
|
||||
when enabled then $6
|
||||
else last_error_message
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $4) else updated_at end
|
||||
@@ -306,6 +305,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
quota: JsonObject,
|
||||
observed_at: DateTime<Utc>,
|
||||
state: QuotaState,
|
||||
plan_type: Option<&str>,
|
||||
) -> StoreResult<bool> {
|
||||
require_nonempty(ENTITY, "account_id", account_id)?;
|
||||
validate_object_size("provider_quota_json", "a, QUOTA_MAX_BYTES)?;
|
||||
@@ -313,6 +313,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set provider_quota_json = $3, quota_observed_at = $4,
|
||||
plan_type = coalesce($9, plan_type),
|
||||
quota_access_state = case
|
||||
when $7::timestamptz is not null
|
||||
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
|
||||
@@ -341,6 +342,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
.bind(state.evidence().map(QuotaEvidence::as_str))
|
||||
.bind(access_observed_at)
|
||||
.bind(state.reset_at().map(DateTime::<Utc>::from))
|
||||
.bind(plan_type)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("compare and swap provider quota"))?;
|
||||
@@ -543,6 +545,14 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
return Err(invalid("rotated profile and credential account IDs differ"));
|
||||
}
|
||||
validate_credential_update(&command.credential)?;
|
||||
if let Some(settings) = &command.settings {
|
||||
if settings.account_id != command.profile.id {
|
||||
return Err(invalid(
|
||||
"rotated credential and account settings IDs differ",
|
||||
));
|
||||
}
|
||||
validate_batch_update_group_ids(&settings.group_ids)?;
|
||||
}
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
@@ -558,6 +568,30 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
&command.credential,
|
||||
)
|
||||
.await?;
|
||||
// 凭据 CAS、普通设置和审计共享事务,任何设置失败都回滚凭据更新。
|
||||
if let Some(settings) = &command.settings {
|
||||
let ids = std::slice::from_ref(&settings.account_id);
|
||||
update_provider_accounts_scheduling_in_transaction(
|
||||
&mut transaction,
|
||||
ids,
|
||||
Some(settings.enabled),
|
||||
Some(settings.concurrency_limit),
|
||||
Some(settings.weight),
|
||||
settings.model_access.as_ref(),
|
||||
settings.outbound_proxy.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
replace_account_group_assignments_in_transaction(
|
||||
&mut transaction,
|
||||
ids,
|
||||
&settings.group_ids,
|
||||
)
|
||||
.await?;
|
||||
if let Some(notes) = settings.notes.as_deref() {
|
||||
update_provider_account_notes_in_transaction(&mut transaction, ids, notes)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
append_admin_audit_event_in_transaction(
|
||||
&mut transaction,
|
||||
command.audit,
|
||||
@@ -637,7 +671,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
let recovered = sqlx::query_scalar::<_, String>(
|
||||
"update provider_accounts
|
||||
set enabled = true,
|
||||
credential_state = 'ready',
|
||||
credential_state = case when credential_state = 'unknown' then 'unknown' else 'ready' end,
|
||||
credential_observed_at = now(),
|
||||
access_token_expires_at = case
|
||||
when access_token_expires_at <= now() then null
|
||||
@@ -767,11 +801,7 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
account: &NewProviderAccount,
|
||||
) -> StoreResult<String> {
|
||||
account.validate()?;
|
||||
let credential_state = if account.upstream_user_id.is_some() {
|
||||
account.credential_state
|
||||
} else {
|
||||
CredentialState::Unknown
|
||||
};
|
||||
let credential_state = account.credential_state;
|
||||
let proxy_id = match account.outbound_proxy.as_ref() {
|
||||
Some(proxy) => Some(
|
||||
super::super::proxies::ensure_proxy_for_url(transaction, proxy, None)
|
||||
@@ -877,11 +907,12 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
let upstream_user_id = replacement_identity.map(ProviderAccountIdentity::upstream_user_id);
|
||||
let upstream_account_id =
|
||||
replacement_identity.and_then(ProviderAccountIdentity::upstream_account_id);
|
||||
// 事务时间可能早于应用写入的额度观测,保留既有时间下界,避免轮换破坏时间约束。
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update provider_accounts
|
||||
set name = $4,
|
||||
email = $5,
|
||||
plan_type = $6,
|
||||
set name = case when $14 then name else $4 end,
|
||||
email = case when $14 then email else $5 end,
|
||||
plan_type = case when $14 then plan_type else $6 end,
|
||||
provider_credentials_json = $7,
|
||||
credential_revision = credential_revision + 1,
|
||||
has_refresh_token = $8,
|
||||
@@ -891,7 +922,7 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end,
|
||||
credential_state = case
|
||||
when not enabled then credential_state
|
||||
when coalesce($12::text, upstream_user_id) is not null then 'ready'
|
||||
when $11::boolean or credential_state <> 'unknown' then 'ready'
|
||||
else 'unknown'
|
||||
end,
|
||||
credential_observed_at = case
|
||||
@@ -900,7 +931,7 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
end,
|
||||
last_error_reason = case when enabled then null else last_error_reason end,
|
||||
last_error_message = case when enabled then null else last_error_message end,
|
||||
updated_at = now()
|
||||
updated_at = greatest(now(), updated_at)
|
||||
where id = $1 and provider_kind = $2
|
||||
and credential_revision = $3
|
||||
returning credential_revision",
|
||||
@@ -918,6 +949,7 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
.bind(replace_identity)
|
||||
.bind(upstream_user_id)
|
||||
.bind(upstream_account_id)
|
||||
.bind(update.preserve_profile)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user