mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 16:05:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
45954899e5 | ||
|
|
78f533d9d3 | ||
|
|
376305fed4 | ||
|
|
a82bcdb8f8 | ||
|
|
3d072fedcf | ||
|
|
a0e708071a | ||
|
|
e9ad1811dd | ||
|
|
615e8d66f4 | ||
|
|
d79c76a105 | ||
|
|
1bf47f71b6 | ||
|
|
2f3f2975ef | ||
|
|
759935b75d | ||
|
|
fcf4af0ddd | ||
|
|
8fb48dd6c3 | ||
|
|
5a81d61979 | ||
|
|
1fadc75ba2 | ||
|
|
83f5d71853 | ||
|
|
f4eceb076b | ||
|
|
0b02d0d98a | ||
|
|
a014da6916 | ||
|
|
9d4c58f967 | ||
|
|
c67b081ec1 | ||
|
|
1ce9710afa | ||
|
|
447869860a | ||
|
|
815c93a715 | ||
|
|
5843091969 | ||
|
|
1280c437b4 | ||
|
|
d965bc9050 | ||
|
|
bd9b017ed3 | ||
|
|
338cde1505 | ||
|
|
b25f96559a | ||
|
|
a8a96d8919 | ||
|
|
7b8339ede3 | ||
|
|
f439a30612 | ||
|
|
8572873c5f | ||
|
|
57c8ef358f | ||
|
|
70ab8ebc8d | ||
|
|
14ea5234ce | ||
|
|
f8480f3bdc | ||
|
|
52b8103bc9 | ||
|
|
dacdd3f0e2 | ||
|
|
ac36f5bc13 | ||
|
|
e0a022518c | ||
|
|
37f7a55427 | ||
|
|
f6085206eb | ||
|
|
9813c86055 | ||
|
|
260f775039 | ||
|
|
1424cea3a9 | ||
|
|
787ad28424 | ||
|
|
db23560b0e | ||
|
|
d700df8a4f | ||
|
|
c9adfd9a2b | ||
|
|
654b6baf89 | ||
|
|
ae5eaa46b4 | ||
|
|
65d5b32eba | ||
|
|
6c158447be | ||
|
|
81d939bd03 | ||
|
|
4a22479dff | ||
|
|
3ee454b750 | ||
|
|
b5bd63b585 | ||
|
|
286031a441 | ||
|
|
5358de53dd | ||
|
|
d814582c09 | ||
|
|
a9ce01f91f | ||
|
|
0fc25bfce9 | ||
|
|
eed8c93ae2 | ||
|
|
d865a00768 | ||
|
|
a0671fb70b | ||
|
|
fce543d4ce | ||
|
|
219888d5c2 | ||
|
|
5b80340135 | ||
|
|
b2dab66a80 | ||
|
|
f0cf8e7198 | ||
|
|
a9a86071bf | ||
|
|
27cef052c9 | ||
|
|
ef40088303 | ||
|
|
05ba116eef | ||
|
|
3ef4743e1a | ||
|
|
9e02e52fec | ||
|
|
42ff5b74d0 | ||
|
|
eb2eefd51e | ||
|
|
abd60ce60a | ||
|
|
2f96c02ea0 | ||
|
|
0f575e279a | ||
|
|
634b8fc674 | ||
|
|
00bfb1699d | ||
|
|
956ca70a14 | ||
|
|
24dd7b94f4 | ||
|
|
94c769fc9b | ||
|
|
eea16f5275 | ||
|
|
8a406ff594 | ||
|
|
676044c43f | ||
|
|
e7161da277 | ||
|
|
97faca0c45 | ||
|
|
45b473c7e2 | ||
|
|
cc69368f77 | ||
|
|
67691d18d3 | ||
|
|
85c64d35d5 | ||
|
|
9bf86f7a7c | ||
|
|
6c56a23508 | ||
|
|
b5f3cb402c | ||
|
|
57e284d7d1 | ||
|
|
21597f9e98 | ||
|
|
fa0b87dd0a | ||
|
|
389dc63633 | ||
|
|
78b2250faa | ||
|
|
bbb8278281 | ||
|
|
7a5521953f | ||
|
|
f78b1ca487 | ||
|
|
661649cc90 |
@@ -5,6 +5,17 @@ description: Codex Proxy RS 仓库的开发、修复、排障、代码审查和
|
||||
|
||||
# 开发指南
|
||||
|
||||
## 参考仓库的优先级
|
||||
|
||||
**Codex 兼容性优先保持官方行为,以透明转发和最小必要适配为默认方案;这是开发、排障与审查的前置约束,不只是引用资料的要求**
|
||||
|
||||
- 以官方 [openai/codex](https://github.com/openai/codex) 对应版本的源码与调用链为第一手资料,先查用户提供或本机已有的官方仓库,核对来源、版本或提交;资料不足时再查官方对应版本的源码、测试与文档
|
||||
- 先对照官方与 CPR 的实际请求、响应及状态生命周期,确认本项目是否改写、遗漏或破坏官方合同;真实上游报错不自动等于 CPR 缺陷,源码推断、代理链路复现与官方直连结果分别说明
|
||||
- 官方路径已能满足场景时,优先复用已有能力或正确透传合同,不额外解释请求意图、扩展工具权限或改变续接语义;补偿后请求成功、测试通过或第三方采用同一做法,都不能单独证明代理应改变行为
|
||||
- 偏离官方行为必须说明官方路径为何不足、本项目的实际约束、最小必要改动及行为代价,并提供对应验证;依据不足时保持现有合同并报告缺口,不先加兼容分支再以通过测试作为必要性依据;用户明确要求的行为变更限定在授权范围内
|
||||
- 网关独有的调度、隔离和资源管理按本项目职责评估,不能机械照搬单客户端实现,也不能借此任意改变客户端可观察行为;新增默认限制或回收阈值须有实际容量依据,并验证对续接、重试与延迟的影响
|
||||
- 三方仓库只作线索或实现参考,不能替代官方行为依据;结论注明适用版本或提交,区分官方事实、三方参考与本项目推断,不把自定义策略表述为官方要求
|
||||
|
||||
## 加载边界
|
||||
|
||||
- 遵守已提供的 [AGENTS.md](../../../AGENTS.md),上下文已有的规则不重复读取
|
||||
@@ -26,13 +37,6 @@ description: Codex Proxy RS 仓库的开发、修复、排障、代码审查和
|
||||
| 普通代码审查 | [审查标准](../../../CONTRIBUTING.md#审查标准);审查默认只读 |
|
||||
| PR / Issue / 发版 | 按当前意图选 [PR](../cpr-github-pr/SKILL.md)、[Issue](../cpr-github-issue/SKILL.md)或[发版](../cpr-release/SKILL.md),普通提交不创建 PR |
|
||||
|
||||
## 参考仓库的优先级
|
||||
|
||||
- 涉及 Codex 行为、协议或兼容性时,以官方 [openai/codex](https://github.com/openai/codex) 对应版本的源码与调用链为第一手资料,先查官方实现
|
||||
- 优先使用用户提供或本机已有的官方仓库,核对来源、版本或提交;资料不足时再查官方对应版本的源码、测试与文档
|
||||
- 三方仓库只能提供排查线索或实现参考,不能代替官方行为依据;与官方不一致时回到官方源码核实,不直接照搬三方结论
|
||||
- 结论注明适用版本或提交,区分官方事实、三方参考与本项目推断;官方证据不足时保留不确定性
|
||||
|
||||
## 执行与收尾
|
||||
|
||||
1. 确认工作区、任务范围与已有改动,先定位事实归属、同类实现和调用链,再修改;编码前只读[项目约定](../../../CONTRIBUTING.md#项目约定)的对应条目
|
||||
|
||||
@@ -20,7 +20,7 @@ description: 草拟、创建、补充或回复 GitHub Issue,包括缺陷、功
|
||||
- 其他仓库读取它自己的相关规则与实际模板,长文档先定位标题再截取,不预读全部模板或 CONTRIBUTING
|
||||
- 搜索同场景 open/closed Issues、相关 PR 和发布记录,区分主分支修复与已发布修复,不凭标题判断重复;没有新证据时返回已有链接,用户仍要求新建则按其要求处理
|
||||
- 仅当需要额外排查或修改时使用 [cpr-dev-guide](../cpr-dev-guide/SKILL.md),提交症状反馈不以找到根因为前提
|
||||
- 引用上游行为或提出方案时读取[问题与方案依据](../../../CONTRIBUTING.md#问题与方案依据),涉及 Codex 时按[参考仓库优先级](../cpr-dev-guide/SKILL.md#参考仓库的优先级)先查官方 Codex 源码,三方仓库仅作参考;注明适用版本、来源与缺口
|
||||
- 引用上游行为或提出方案时读取[问题与方案依据](../../../CONTRIBUTING.md#问题与方案依据),涉及 Codex 时执行[官方行为优先约束](../cpr-dev-guide/SKILL.md#参考仓库的优先级);注明适用版本、来源与缺口,不把上游报错直接定性为 CPR 缺陷,也不把第三方补偿方案当成预期行为;症状反馈仍不以证明根因为前提
|
||||
|
||||
## 正文与证据
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@ description: 准备、创建、更新或审查 GitHub PR,含回复讨论与处
|
||||
|
||||
本仓库目标分支为 `main`,其他仓库使用其可信基线规则与模板,PR 内修改的规则不能降低审查要求
|
||||
|
||||
涉及 Codex 行为时,优先执行[官方行为优先约束](../cpr-dev-guide/SKILL.md#参考仓库的优先级),先判断是否保持官方合同、偏离是否必要,再评价实现;引用源码和验证通过不能替代方案依据
|
||||
|
||||
## 按模式读取
|
||||
|
||||
只加载当前模式,已有上下文不重读;长文档先定位标题再截取对应章节,不因链接存在而加载全文或所有 references
|
||||
@@ -40,7 +42,7 @@ description: 准备、创建、更新或审查 GitHub PR,含回复讨论与处
|
||||
|
||||
## 创建与审查的公共核对
|
||||
|
||||
- 按[问题与方案依据](../../../CONTRIBUTING.md#问题与方案依据)核实触发条件、发生版本与目标分支现状;涉及 Codex 时按[参考仓库优先级](../cpr-dev-guide/SKILL.md#参考仓库的优先级)先查官方 Codex 源码,三方仓库仅作参考
|
||||
- 按[问题与方案依据](../../../CONTRIBUTING.md#问题与方案依据)核实触发条件、发生版本与目标分支现状,分别判断问题是否成立、是否属于目标项目的职责;涉及 Codex 时核对方案是否满足上述官方行为优先约束
|
||||
- 核对最新目标分支及同场景 open/closed PR、Issue,按实际行为和测试判断重复,不凭标题下结论,也不擅自关闭他人 PR
|
||||
- 自审或审查覆盖完整差异与调用链,文档逐段检查必要性、职责、重复和当前状态表达,构建与链接检查不能代替内容判断
|
||||
- 核对[AI 使用披露](../../../CONTRIBUTING.md#ai-使用披露),填写实际工具与模型,不猜型号;必需截图或型号待补时保留 Draft
|
||||
|
||||
@@ -7,10 +7,12 @@
|
||||
2. 先核实 PR 要解决的问题是否成立:提取触发场景、预期行为和声称的原因,在目标分支基线检查实际行为。
|
||||
按入口的公共核对收集一手证据,区分原有问题、环境或配置差异与 PR 新引入的问题,说明主分支已覆盖的部分。
|
||||
3. 从官方方案评估解决方向:说明官方如何处理同一场景,检查 PR 是否遵循其协议语义和职责归属、
|
||||
是否复用已有能力或正确传递官方合同;核对偏离官方方案的理由与运行证据。
|
||||
是否复用已有能力或正确传递官方合同;按入口的官方行为优先约束核对偏离的必要性、最小范围、行为代价与运行证据。
|
||||
审查 OpenAI 官方上游与 Codex 的对接时,默认官方返回与 Codex 兼容、直接对接正常,重点检查本项目是否误解、
|
||||
改写或破坏官方合同。不得仅凭自行构造的异常上游数据揣测官方不兼容,也不据此报告缺陷、提出改进建议,
|
||||
或要求增加防御分支与测试。讨论特定版本或场景的对接例外须有真实响应或实际复现证据,并区分代理链路与官方直连。
|
||||
真实复现上游错误或补偿后成功,也不能单独证明 CPR 应改写请求;缺少必要性依据的行为偏离须在解决方向中明确指出,
|
||||
不因测试通过而给出方案合理的结论。网关资源策略单独核对容量依据与客户端影响,不以官方客户端没有同类池策略就判错。
|
||||
4. 按贡献文档的审查维度追踪完整调用链,找到共享业务事实、同类实现及已有验证。
|
||||
对既有特殊分支查清服务的场景,再判断本次改动是否引入或加重问题。
|
||||
主动检查本次变更新引入或加重的冗余与代码异味,说明具体位置、维护成本或行为影响及最小修复方向;
|
||||
|
||||
@@ -14,7 +14,7 @@ description: 开发、排查或打包 Codex Proxy RS 独立网关插件;仅改
|
||||
- 确认用途、工程位置、目标宿主版本与平台,沿用已有工程;位置无法推断时只补齐该信息,不把插件源码塞进宿主 workspace
|
||||
- 核对运行宿主、[SDK 版本](../../../backend/crates/gateway-plugin/sdk/Cargo.toml)、[宿主支持清单](../../../backend/crates/gateway-plugin/runtime/plugin-host-compatibility.json)及所用 CLI,清单、进程和能力版本分别判断
|
||||
- `contributes` 声明处理器,`bindings` 选择挂载与匹配范围;安装成功不等于功能生效。当前清单拒绝 `permissions`,安装者完整信任插件,`trustedProcess` 不提供操作系统沙箱或访问域隔离,见[完整信任](../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#完整信任)
|
||||
- 涉及 Codex 时按[参考仓库优先级](../cpr-dev-guide/SKILL.md#参考仓库的优先级)先查官方 Codex 源码,三方仓库仅作参考;宿主能力以目标版本的 SDK 为准
|
||||
- 涉及 Codex 兼容性适配时执行[官方行为优先约束](../cpr-dev-guide/SKILL.md#参考仓库的优先级),不因实现位于插件中扩大请求改写或补偿范围;用户明确要求的业务改写限定在授权范围,宿主能力以目标版本的 SDK 为准
|
||||
|
||||
## 按当前步骤读取
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
## 清单与处理器
|
||||
|
||||
- `publisher` 与机器短名 `name` 派生插件 ID,`displayName` 用于展示;复制示例后同时调整扩展项 ID 与注册结果
|
||||
- `contributes` 每种能力最多一项,普通贡献项可省略派生 ID、能力版本和固定阶段;中间件必须声明版本 `3` 并显式选择阶段。自定义完整扩展项 ID 属于自己的插件命名空间,运行注册与规范化清单一致
|
||||
- `contributes` 每种能力最多一项,普通贡献项可省略派生 ID、能力版本和固定阶段;中间件必须声明版本 `4` 并显式选择阶段,上游适配器必须声明版本 `2`。自定义完整扩展项 ID 属于自己的插件命名空间,运行注册与规范化清单一致
|
||||
- 作者清单用 `Manifest::from_author_slice` 规范化,不手写 `package`、固定阶段或重复注册;CLI 生成平台、协议与资源摘要,版本范围不能使用全版本通配
|
||||
- 安装后宿主会按 `configurationSchema` 准备默认配置,配置完整即可启用;默认绑定的空范围不限制请求。需要业务参数时声明真实必填项,通过 `secretFields` 声明敏感字段,不依赖用户再走一遍自建安装向导
|
||||
- Rust 插件只依赖公开 `gateway-plugin-sdk`;需要异步会话辅助时开启 `io`,使用 `PluginSession` 管理握手、回调关联、流控、取消和关闭
|
||||
|
||||
@@ -31,6 +31,7 @@ description: Codex Proxy RS 的版本规划、发布说明、正式或预发行
|
||||
|
||||
## 必须保持的边界
|
||||
|
||||
- 发行范围涉及 Codex 兼容性变化时,按[官方行为优先约束](../cpr-dev-guide/SKILL.md#参考仓库的优先级)核对已有方案依据和实际验证范围;缺少依据如实列为发行风险,不以 CI 通过代替,不把网关自定义策略写成官方要求
|
||||
- `release/publish` 会提交、打 tag、推送并触发工作流,没有只读预演模式,不能用于探测或演练
|
||||
- 同名版本、超时或中断先查本地与远端状态,不删已发布 tag、不移动 tag、不强推掩盖冲突
|
||||
- 预发行不得覆盖 GitHub Latest 或镜像 `latest`,完成后回读核对
|
||||
|
||||
@@ -42,8 +42,11 @@ else
|
||||
mkdir -p "$(dirname "$directory")"
|
||||
temporary="$(mktemp -d "${directory}.XXXXXX")"
|
||||
trap 'rm -rf "$temporary"' EXIT
|
||||
mkdir "$temporary/context"
|
||||
"${command[@]}" --output "type=local,dest=$temporary/snapshot" "$temporary/context"
|
||||
mkdir "$temporary"/{context,snapshot}
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,直接解包以免额外落盘整份归档。
|
||||
# https://github.com/moby/buildkit/issues/2950
|
||||
"${command[@]}" --output type=tar,dest=- "$temporary/context" \
|
||||
| tar -xf - -C "$temporary/snapshot"
|
||||
if usable "$temporary/snapshot"; then
|
||||
rm -rf "$directory"
|
||||
mv "$temporary/snapshot" "$directory"
|
||||
|
||||
@@ -154,6 +154,21 @@ runs:
|
||||
"${compose[@]}" exec -T codex-proxy-rs sh -c \
|
||||
'touch /app/.runtime/data/.ci-write /app/.runtime/logs/.ci-write && rm /app/.runtime/data/.ci-write /app/.runtime/logs/.ci-write'
|
||||
|
||||
- name: Verify application restart after installer permissions
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" stop --timeout 30 codex-proxy-rs
|
||||
# 使用安装器原函数,验证已有备份暂存目录与日志不会因所有权变更而阻止启动
|
||||
(
|
||||
export INSTALL_DIR="$PWD"
|
||||
source deploy/install.sh
|
||||
fix_runtime_permissions
|
||||
)
|
||||
"${compose[@]}" up -d --no-build --wait codex-proxy-rs
|
||||
|
||||
- name: Verify clean logs and graceful shutdown
|
||||
shell: bash
|
||||
run: |
|
||||
|
||||
@@ -177,8 +177,6 @@ jobs:
|
||||
host_path = "backend/crates/gateway-plugin/runtime/plugin-host-compatibility.json"
|
||||
host = JSON.parse(File.read(host_path))
|
||||
abort "Invalid plugin host compatibility declaration: #{host_path}" unless host["schema_version"] == 2
|
||||
# 保留旧更新器可解析的传输格式,实际支持范围仍取自当前宿主合同。
|
||||
host = host.merge("schema_version" => 1, "permissions" => [])
|
||||
# 清单只依赖版本与兼容声明,平台归档和镜像消费同一份封口结果。
|
||||
manifest = {
|
||||
"schema_version" => 1,
|
||||
|
||||
Generated
+70
-23
@@ -145,6 +145,18 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee19bd99b43e3691acbad4e840420a4881cea6c0b66a208125a824f8fd53f5a1"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-lock"
|
||||
version = "3.4.2"
|
||||
@@ -931,6 +943,23 @@ dependencies = [
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a"
|
||||
dependencies = [
|
||||
"compression-core",
|
||||
"flate2",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compression-core"
|
||||
version = "0.4.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
|
||||
|
||||
[[package]]
|
||||
name = "config"
|
||||
version = "0.15.27"
|
||||
@@ -1681,6 +1710,7 @@ dependencies = [
|
||||
"gateway-core",
|
||||
"heck",
|
||||
"hex",
|
||||
"hkdf",
|
||||
"prettyplease",
|
||||
"quote",
|
||||
"rand_core 0.6.4",
|
||||
@@ -1716,6 +1746,7 @@ dependencies = [
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"regex",
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -1847,6 +1878,7 @@ dependencies = [
|
||||
name = "gateway-plugin-sdk"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -1980,9 +2012,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.19"
|
||||
version = "0.4.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16"
|
||||
checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -2446,9 +2478,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.58.4"
|
||||
version = "0.58.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5761b0ca0a31464030d7c2a98087fde3124ebb5e09b5a112cdb815f03b8ee150"
|
||||
checksum = "f622c574a29fb3034a0a90645afee194dfedc75fc124228801b6e77041075bd1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2474,18 +2506,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.58.5"
|
||||
version = "0.58.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "854e9e22c420535035e95eed240cbfb4a5f4b7da74d122d63144e0c31926dbfd"
|
||||
checksum = "a968fffd387b2ab005bf00c1851f5d26e90df05b6f7ad609ed0d620033cca5e7"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.58.5"
|
||||
version = "0.58.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e3bfac11ac357ec620880025757d38c67da8fd5b9ef5024d19f0f360002b4ca"
|
||||
checksum = "39b7670345e5ce87371339d13705dbb9276b7e3e934bbe1efa2c6734a1f65a96"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2748,6 +2780,18 @@ version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.50.3"
|
||||
@@ -3289,6 +3333,7 @@ dependencies = [
|
||||
"native-tls",
|
||||
"openssl",
|
||||
"openssl-sys",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"roxmltree",
|
||||
"rustls",
|
||||
@@ -3400,9 +3445,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.16"
|
||||
version = "0.11.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
|
||||
checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"getrandom 0.4.3",
|
||||
@@ -3586,9 +3631,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.58.5"
|
||||
version = "0.58.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "590efadb0a669f1712c1e3ab810a55d4c3d20d0127c455f7236dd0710aba0346"
|
||||
checksum = "558f3855529c0a7e78a48e409fa653456e55f1b167eab4109c16a255b7727e3f"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -3603,9 +3648,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.0"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -3748,9 +3793,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
version = "0.23.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -4503,7 +4548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -4741,6 +4786,7 @@ version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -5005,9 +5051,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.26.1"
|
||||
version = "1.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce"
|
||||
checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -5545,9 +5591,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
|
||||
checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
|
||||
dependencies = [
|
||||
"zeroize_derive",
|
||||
]
|
||||
@@ -5598,14 +5644,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.6.0"
|
||||
version = "9.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||
checksum = "a954114b47a54df49f83d2722814e86b23d4bb1b54a1b90fb7d92b41619ddc22"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"nix",
|
||||
"typed-path",
|
||||
]
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ async fn execute(
|
||||
let Environment {
|
||||
host,
|
||||
mut store,
|
||||
timezone,
|
||||
plugin_cache,
|
||||
version,
|
||||
plugin_http,
|
||||
@@ -107,7 +108,7 @@ async fn execute(
|
||||
let settings = gateway_admin::initialize_settings(
|
||||
store.admin_ports().settings(),
|
||||
core.snapshot_control(),
|
||||
providers.admin,
|
||||
providers.admin.clone(),
|
||||
std::sync::Arc::new(gateway_host::pricing::ModelsDevPricing),
|
||||
);
|
||||
let mut services = gateway_admin::public_service::Registry::new(service_middleware(
|
||||
@@ -115,6 +116,22 @@ async fn execute(
|
||||
&plugin_runtime,
|
||||
));
|
||||
or_shutdown!(plugin_runtime, services.register_settings(&settings));
|
||||
let accounts = gateway_admin::initialize_accounts(
|
||||
store.admin_ports().accounts(),
|
||||
store.admin_ports().account_runtime(),
|
||||
store.admin_ports().settings(),
|
||||
timezone,
|
||||
providers.admin.clone(),
|
||||
core.snapshot_control(),
|
||||
core.account_probe(),
|
||||
);
|
||||
or_shutdown!(plugin_runtime, services.register_accounts(&accounts));
|
||||
let usage_data = gateway_admin::initialize_usage_data(
|
||||
store.admin_ports().usage_data(),
|
||||
store.admin_ports().settings(),
|
||||
providers.admin,
|
||||
);
|
||||
or_shutdown!(plugin_runtime, services.register_usage_data(&usage_data));
|
||||
let services = std::sync::Arc::new(services);
|
||||
or_shutdown!(plugin_runtime, plugin_runtime.bind_services(&services));
|
||||
let commands = or_shutdown!(plugin_runtime, plugin_runtime.prepare_command_line().await);
|
||||
|
||||
@@ -56,6 +56,12 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
plugin_runtime,
|
||||
gateway_host::plugin_distribution::HttpPluginDistribution::new(plugin_http)
|
||||
));
|
||||
let analytics_worker = or_shutdown!(
|
||||
plugin_runtime,
|
||||
store.request_analytics_worker(std::sync::Arc::new(
|
||||
gateway_admin::ProviderUsageBillingProjector::new(providers.admin.clone()),
|
||||
))
|
||||
);
|
||||
let mut admin = or_shutdown!(
|
||||
plugin_runtime,
|
||||
gateway_admin::initialize_with_plugin_accounts(
|
||||
@@ -142,6 +148,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
);
|
||||
|
||||
let mut plan = store.take_worker_contributions();
|
||||
plan.push(analytics_worker);
|
||||
plan.push(or_shutdown!(
|
||||
plugin_runtime,
|
||||
gateway_host::retention::worker(store.retention())
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::OnceLock,
|
||||
};
|
||||
|
||||
use syn::{Item, visit::Visit};
|
||||
@@ -26,22 +27,24 @@ pub(super) const WORKSPACE_MEMBERS: &[&str] = &[
|
||||
|
||||
#[test]
|
||||
fn workspace_member_list_matches_the_frozen_dag_scope() {
|
||||
let manifest =
|
||||
fs::read_to_string(backend_root().join("Cargo.toml")).expect("read workspace manifest");
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
assert!(
|
||||
manifest.contains(&format!("\"{member}\"")),
|
||||
"workspace must include {member}"
|
||||
);
|
||||
}
|
||||
let member_count = manifest
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| line.starts_with("\"apps/") || line.starts_with("\"crates/"))
|
||||
.count();
|
||||
let metadata = cargo_metadata_json();
|
||||
let root = backend_root();
|
||||
let actual: BTreeSet<_> = metadata["packages"]
|
||||
.as_array()
|
||||
.expect("Cargo packages")
|
||||
.iter()
|
||||
.map(|package| {
|
||||
Path::new(package["manifest_path"].as_str().expect("package manifest"))
|
||||
.parent()
|
||||
.expect("package directory")
|
||||
.strip_prefix(&root)
|
||||
.expect("workspace member path")
|
||||
.to_path_buf()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
member_count,
|
||||
WORKSPACE_MEMBERS.len(),
|
||||
actual,
|
||||
WORKSPACE_MEMBERS.iter().map(PathBuf::from).collect(),
|
||||
"new workspace members must extend the dependency DAG rules"
|
||||
);
|
||||
}
|
||||
@@ -378,7 +381,7 @@ const ALLOWED_INTERNAL_EDGES: &[(&str, &str)] = &[
|
||||
#[test]
|
||||
fn workspace_internal_dependency_edges_match_frozen_dag() {
|
||||
let metadata = cargo_metadata_json();
|
||||
let actual = workspace_runtime_dependency_edges(&metadata);
|
||||
let actual = workspace_runtime_dependency_edges(metadata);
|
||||
let mut expected: Vec<(String, String)> = ALLOWED_INTERNAL_EDGES
|
||||
.iter()
|
||||
.map(|(from, to)| ((*from).to_owned(), (*to).to_owned()))
|
||||
@@ -474,24 +477,6 @@ fn adapter_public_module_surfaces_match_allowlist() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_sources_do_not_host_tests() {
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
let src = backend_root().join(member).join("src");
|
||||
let sources = super::rust_files(&src);
|
||||
assert!(!sources.is_empty(), "{member} has no production sources");
|
||||
for relative in sources {
|
||||
let path = src.join(relative);
|
||||
let source = fs::read_to_string(&path).expect("read production source");
|
||||
assert!(
|
||||
!source.contains("#[cfg(test)]"),
|
||||
"{} hosts tests in production src",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_modules_follow_conventional_file_layout() {
|
||||
let targets = test_source_roots();
|
||||
@@ -748,40 +733,46 @@ pub(super) fn backend_root() -> PathBuf {
|
||||
.expect("backend workspace root")
|
||||
}
|
||||
|
||||
fn cargo_metadata_json() -> serde_json::Value {
|
||||
let output = std::process::Command::new("cargo")
|
||||
.args(["metadata", "--format-version", "1", "--no-deps"])
|
||||
.current_dir(backend_root())
|
||||
.output()
|
||||
.expect("run cargo metadata");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"cargo metadata failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
serde_json::from_slice(&output.stdout).expect("parse cargo metadata")
|
||||
fn cargo_metadata_json() -> &'static serde_json::Value {
|
||||
static METADATA: OnceLock<serde_json::Value> = OnceLock::new();
|
||||
METADATA.get_or_init(|| {
|
||||
let output = std::process::Command::new("cargo")
|
||||
.args(["metadata", "--format-version", "1", "--no-deps", "--locked"])
|
||||
.current_dir(backend_root())
|
||||
.output()
|
||||
.expect("run cargo metadata");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"cargo metadata failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
serde_json::from_slice(&output.stdout).expect("parse cargo metadata")
|
||||
})
|
||||
}
|
||||
|
||||
/// 提取成员 `[dependencies]` 段内声明的依赖名;段落以下一个 `[` 表头结束
|
||||
/// 复用 Cargo 解析后的实际包名,覆盖重命名依赖与平台条件,排除测试和构建依赖
|
||||
fn dependency_names(member: &str) -> Vec<String> {
|
||||
let manifest = fs::read_to_string(backend_root().join(member).join("Cargo.toml"))
|
||||
.expect("read member manifest");
|
||||
let mut in_dependencies = false;
|
||||
let mut names = Vec::new();
|
||||
for line in manifest.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
in_dependencies = line == "[dependencies]";
|
||||
continue;
|
||||
}
|
||||
if !in_dependencies || line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if let Some((name, _)) = line.split_once('=') {
|
||||
names.push(name.trim().to_owned());
|
||||
}
|
||||
}
|
||||
names
|
||||
let manifest = backend_root().join(member).join("Cargo.toml");
|
||||
let package = cargo_metadata_json()["packages"]
|
||||
.as_array()
|
||||
.expect("Cargo packages")
|
||||
.iter()
|
||||
.find(|package| {
|
||||
Path::new(package["manifest_path"].as_str().expect("package manifest")) == manifest
|
||||
})
|
||||
.expect("workspace member");
|
||||
package["dependencies"]
|
||||
.as_array()
|
||||
.expect("package dependencies")
|
||||
.iter()
|
||||
.filter(|dependency| dependency["kind"].is_null())
|
||||
.map(|dependency| {
|
||||
dependency["name"]
|
||||
.as_str()
|
||||
.expect("dependency name")
|
||||
.to_owned()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn assert_no_dependency(member: &str, forbidden: &[&str]) {
|
||||
|
||||
@@ -78,7 +78,7 @@ async fn proxy_probe_should_use_provider_custom_ca_for_https_proxies() {
|
||||
});
|
||||
let probe = HttpProxyProbe::new("http://unresolvable.invalid/ip")
|
||||
.with_client_builder(provider_openai::build_reqwest_client_with_custom_ca);
|
||||
let result = probe.test(&proxy, false).await;
|
||||
let result = probe.test(Some(&proxy), false).await;
|
||||
assert!(result.success, "{}", result.message);
|
||||
assert_eq!(result.exit_ip.unwrap().to_string(), "203.0.113.8");
|
||||
tokio::time::timeout(Duration::from_secs(5), server)
|
||||
|
||||
@@ -59,9 +59,16 @@ fn cargo_library_root_is_conventional_lib() {
|
||||
fn workspace_production_files_have_no_hidden_modules_or_test_hooks() {
|
||||
for member in architecture::WORKSPACE_MEMBERS {
|
||||
let root = architecture::backend_root().join(member).join("src");
|
||||
for relative in rust_files(&root) {
|
||||
let sources = rust_files(&root);
|
||||
assert!(!sources.is_empty(), "{member} has no production sources");
|
||||
for relative in sources {
|
||||
let path = root.join(relative);
|
||||
let source = fs::read_to_string(&path).expect("read production source");
|
||||
assert!(
|
||||
!source.contains("#[cfg(test)]"),
|
||||
"{} hosts tests in production src",
|
||||
path.display()
|
||||
);
|
||||
assert!(
|
||||
!source.contains("include!("),
|
||||
"{} uses include!",
|
||||
|
||||
@@ -250,8 +250,8 @@ fn source_manifest() -> &'static [u8] {
|
||||
"main":"bin/plugin",
|
||||
"runtime":"trustedProcess",
|
||||
"contributes":{
|
||||
"middleware":{"version":4,"stages":["request"],"inputFormats":["openai"],"outputFormats":["openai"]},
|
||||
"management":{}
|
||||
"middleware":{"version":5,"stages":["request"],"inputFormats":["openai"],"outputFormats":["openai"]},
|
||||
"management":{"version":2}
|
||||
},
|
||||
|
||||
"icon":"assets/icon.png",
|
||||
|
||||
@@ -21,6 +21,7 @@ futures = "0.3.32"
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
heck = "0.5.0"
|
||||
hex = "0.4.3"
|
||||
hkdf = "0.13.0"
|
||||
rand_core = { version = "0.6.4", features = ["getrandom"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
@@ -30,7 +31,7 @@ sha2 = "0.11"
|
||||
subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
uuid = { version = "1.23.3", features = ["v4", "v7"] }
|
||||
cron = "0.17"
|
||||
chrono-tz = "0.10"
|
||||
tokio = { version = "1.52", features = ["rt", "time", "macros", "sync"] }
|
||||
|
||||
@@ -491,11 +491,13 @@ impl BackupTask {
|
||||
)
|
||||
.await
|
||||
{
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
self.fail_uploaded_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("上传到对象存储失败:{}", error.message()),
|
||||
&config,
|
||||
BackupError::new(
|
||||
error.code(),
|
||||
format!("上传到对象存储失败:{}", error.message()),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
@@ -533,16 +535,19 @@ impl BackupTask {
|
||||
sha256_matches,
|
||||
"备份远端对象校验不一致"
|
||||
);
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
self.fail_uploaded_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
match (size_matches, sha256_matches) {
|
||||
(false, true) => "远端对象大小不一致",
|
||||
(true, false) => "远端对象 sha256 校验值不一致",
|
||||
(false, false) => "远端对象大小和 sha256 校验值均不一致",
|
||||
(true, true) => "远端对象校验状态异常",
|
||||
},
|
||||
&config,
|
||||
BackupError::new(
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
match (size_matches, sha256_matches) {
|
||||
(false, true) => "远端对象大小不一致",
|
||||
(true, false) => "远端对象 sha256 校验值不一致",
|
||||
(false, false) => "远端对象大小和 sha256 校验值均不一致",
|
||||
(true, true) => "远端对象校验状态异常",
|
||||
}
|
||||
.to_owned(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -556,11 +561,13 @@ impl BackupTask {
|
||||
.await?;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
self.fail_uploaded_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("远端对象校验失败:{}", error.message()),
|
||||
&config,
|
||||
BackupError::new(
|
||||
error.code(),
|
||||
format!("远端对象校验失败:{}", error.message()),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -568,6 +575,24 @@ impl BackupTask {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 清理失败并入原终态,后续清理沿用备份记录的到期策略
|
||||
async fn fail_uploaded_task(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
config: &BackupStorageConfig,
|
||||
mut error: BackupError,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
if let Err(cleanup) = self
|
||||
.object_store
|
||||
.delete_object(config, &record.object_key)
|
||||
.await
|
||||
{
|
||||
error =
|
||||
error.with_cleanup_failure("DeleteObject", &cleanup, record.expires_at.is_some());
|
||||
}
|
||||
self.fail_task(record, error.code(), error.message()).await
|
||||
}
|
||||
|
||||
/// 把活跃任务写入失败终态并清理暂存
|
||||
async fn fail_task(
|
||||
&self,
|
||||
|
||||
@@ -21,6 +21,7 @@ pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
pub mod public_service;
|
||||
mod reset_credit_automation;
|
||||
mod use_case;
|
||||
pub use use_case::plugins::{PluginDistributionPorts, PluginManagementService, PluginsService};
|
||||
|
||||
@@ -35,10 +36,11 @@ pub use use_case::{
|
||||
client_keys::ClientKeyService,
|
||||
credentials::{CredentialsService, ProviderCredentials},
|
||||
import_tasks::ImportTasksService,
|
||||
observability::ObservabilityService,
|
||||
observability::{ObservabilityService, ProviderUsageBillingProjector},
|
||||
proxies::ProxiesService,
|
||||
settings::SettingsService,
|
||||
system::SystemService,
|
||||
usage_data::UsageDataService,
|
||||
};
|
||||
|
||||
use model::AdminError;
|
||||
@@ -217,6 +219,7 @@ pub struct AdminServices {
|
||||
client_distribution: Arc<dyn ClientDistributionService>,
|
||||
observability: Arc<dyn ObservabilityService>,
|
||||
settings: Arc<dyn SettingsService>,
|
||||
usage_data: Arc<dyn UsageDataService>,
|
||||
system: Arc<dyn SystemService>,
|
||||
credentials: Arc<CredentialsService>,
|
||||
// Runtime 只持有 Weak,这个强引用保障账号端口与 Admin 服务同寿命
|
||||
@@ -285,6 +288,11 @@ impl AdminServices {
|
||||
self.client_distribution.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn usage_data(&self) -> &dyn UsageDataService {
|
||||
self.usage_data.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn observability(&self) -> &dyn ObservabilityService {
|
||||
self.observability.as_ref()
|
||||
@@ -405,9 +413,7 @@ async fn initialize_inner(
|
||||
let registry = providers;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
config.default_username,
|
||||
config.session_ttl_minutes,
|
||||
config.session_absolute_ttl_minutes,
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
@@ -415,13 +421,15 @@ async fn initialize_inner(
|
||||
auth.ensure_default_admin(config.default_password.expose())
|
||||
.await?;
|
||||
|
||||
let accounts = Arc::new(DefaultAccountsService::new(
|
||||
let accounts = initialize_accounts(
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
store.settings(),
|
||||
timezone,
|
||||
registry.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
));
|
||||
);
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
@@ -441,11 +449,14 @@ async fn initialize_inner(
|
||||
plugin_inspector.clone(),
|
||||
));
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
store.observability(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
@@ -466,8 +477,11 @@ async fn initialize_inner(
|
||||
registry.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), registry.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_accounts(&accounts)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
@@ -506,13 +520,15 @@ async fn initialize_inner(
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
store.observability(),
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
@@ -541,6 +557,33 @@ async fn initialize_inner(
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(registration));
|
||||
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
|
||||
let id = WorkerId::try_new(WorkerKind::ResetCreditAutomation, "admin")
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), Duration::from_secs(90))
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 租约配置不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(reset_credit_automation::ResetCreditAutomationTask {
|
||||
accounts,
|
||||
store: store.accounts(),
|
||||
settings: store.settings(),
|
||||
}),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 注册失败"))?,
|
||||
));
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
@@ -629,6 +672,28 @@ fn freeze_recovery_worker_contribution(
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 服务端与 CLI 复用账号用例,不随命令启动后台调度
|
||||
#[must_use]
|
||||
pub fn initialize_accounts(
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
account_runtime: Arc<dyn ports::store::AccountRuntimeStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> Arc<dyn AccountsService> {
|
||||
Arc::new(DefaultAccountsService::new(
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
))
|
||||
}
|
||||
|
||||
/// 设置服务不依赖 Web 管理会话,CLI 与服务器通过同一用例执行事务和发布
|
||||
pub fn initialize_settings(
|
||||
store: Arc<dyn ports::store::SettingsStore>,
|
||||
@@ -643,3 +708,15 @@ pub fn initialize_settings(
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ pub struct AccountGroupCapacity {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupUsage {
|
||||
pub today_usd: DecimalAmount,
|
||||
/// 当前 usage retention 窗口内、按请求发生时 routing group 快照归属的累计成本
|
||||
/// 当前 usage retention 窗口内、当前组成员实际完成请求的累计成本
|
||||
pub retained_total_usd: DecimalAmount,
|
||||
}
|
||||
|
||||
@@ -100,6 +100,7 @@ pub struct AccountGroupListQuery {
|
||||
/// 完整账号分组摘要
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupRecord {
|
||||
pub allow_image_generation: bool,
|
||||
pub fast_mode: FastMode,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
@@ -139,6 +140,7 @@ pub struct AccountGroupOptionsPage {
|
||||
/// 创建账号分组
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreateAccountGroup {
|
||||
pub allow_image_generation: bool,
|
||||
pub fast_mode: FastMode,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
@@ -148,6 +150,7 @@ pub struct CreateAccountGroup {
|
||||
/// 已生成稳定标识、可直接提交存储的创建命令
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct NewAccountGroup {
|
||||
pub allow_image_generation: bool,
|
||||
pub fast_mode: FastMode,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
@@ -158,6 +161,7 @@ pub struct NewAccountGroup {
|
||||
/// 更新账号分组的描述字段
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateAccountGroup {
|
||||
pub allow_image_generation: Option<bool>,
|
||||
pub fast_mode: Option<FastMode>,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
|
||||
@@ -124,6 +124,7 @@ pub struct AccountRecord {
|
||||
pub weight: AccountWeight,
|
||||
pub model_access: gateway_core::account::AccountModelAccess,
|
||||
pub outbound_proxy: Option<gateway_core::account::OutboundProxy>,
|
||||
pub outbound_proxy_enabled: bool,
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
pub quota: QuotaState,
|
||||
@@ -133,6 +134,16 @@ pub struct AccountRecord {
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl AccountRecord {
|
||||
/// 配置保留绑定,执行请求时禁用代理按直连处理
|
||||
#[must_use]
|
||||
pub fn effective_outbound_proxy(&self) -> Option<&gateway_core::account::OutboundProxy> {
|
||||
self.outbound_proxy
|
||||
.as_ref()
|
||||
.filter(|_| self.outbound_proxy_enabled)
|
||||
}
|
||||
}
|
||||
|
||||
pub use super::observability::{
|
||||
AccountModelUsage, AccountPoolMetrics as AccountSummary, AccountRequestBucket,
|
||||
CurrencyCost as AccountCost,
|
||||
@@ -146,9 +157,9 @@ pub struct AccountUsage {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
@@ -283,3 +294,24 @@ pub enum AccountConnectionTestEvent {
|
||||
/// 每次连接测试独占的有限事件流
|
||||
pub type AccountConnectionTestEventStream =
|
||||
Pin<Box<dyn Stream<Item = AccountConnectionTestEvent> + Send + 'static>>;
|
||||
|
||||
/// 单个账号普通并发池的实时租约明细,不关联请求历史
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountConcurrencyEntry {
|
||||
pub lease_id: String,
|
||||
pub request_id: Option<String>,
|
||||
pub client_key_id: Option<String>,
|
||||
pub client_key_name: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub client_ip: Option<String>,
|
||||
pub started_at: Option<DateTime<Utc>>,
|
||||
pub acquired_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 同一次 Redis 读取中的当前占用数和分页明细
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountConcurrencyPage {
|
||||
pub entries: Vec<AccountConcurrencyEntry>,
|
||||
pub total: u64,
|
||||
pub observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
//! 统计投影的新鲜度、覆盖范围与预计算费用合同
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::observability::{DecimalAmount, RequestMetricPoint, UsageOverview};
|
||||
|
||||
/// 数据可见性状态,不以缺失投影冒充零统计
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum AnalyticsState {
|
||||
Ready,
|
||||
#[default]
|
||||
Building,
|
||||
Lagging,
|
||||
Stale,
|
||||
}
|
||||
|
||||
impl AnalyticsState {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Ready => "ready",
|
||||
Self::Building => "building",
|
||||
Self::Lagging => "lagging",
|
||||
Self::Stale => "stale",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 与统计结果在同一读取快照内取得的覆盖和版本事实
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AnalyticsMetadata {
|
||||
pub coverage_start: Option<DateTime<Utc>>,
|
||||
pub coverage_end: Option<DateTime<Utc>>,
|
||||
pub generated_at: DateTime<Utc>,
|
||||
pub lag_seconds: u64,
|
||||
pub state: AnalyticsState,
|
||||
pub percentiles_approximate: bool,
|
||||
pub schema_version: u32,
|
||||
pub histogram_version: u32,
|
||||
/// 派生数据版本供缓存判断失效,不作为源事务提交顺序
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
/// 单次请求经 Provider 费用合同确认后可累加的 USD 情景金额
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct UsageBillingProjection {
|
||||
pub standard_amount: Option<DecimalAmount>,
|
||||
pub no_cache_amount: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
/// 时间桶内的情景费用,未知金额不改写为零
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageProjectedCostPoint {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub standard_amount: Option<DecimalAmount>,
|
||||
pub no_cache_amount: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
/// 同一数据库快照内的完整洞察输入
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageInsightsObservation {
|
||||
pub overview: UsageOverview,
|
||||
pub trend: Vec<RequestMetricPoint>,
|
||||
pub costs: Vec<UsageProjectedCostPoint>,
|
||||
}
|
||||
@@ -76,18 +76,23 @@ impl std::fmt::Debug for LoginCommand {
|
||||
}
|
||||
}
|
||||
|
||||
/// 登录成功后返回的会话事实
|
||||
/// 登录或刷新后仅向当前浏览器交付的会话凭据
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct LoginResult {
|
||||
pub struct SessionGrant {
|
||||
pub session_id: String,
|
||||
pub refresh_token: Option<String>,
|
||||
pub session: AuthSession,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LoginResult {
|
||||
impl std::fmt::Debug for SessionGrant {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("LoginResult")
|
||||
.debug_struct("SessionGrant")
|
||||
.field("session_id", &"[REDACTED]")
|
||||
.field(
|
||||
"refresh_token",
|
||||
&self.refresh_token.as_ref().map(|_| "[REDACTED]"),
|
||||
)
|
||||
.field("session", &self.session)
|
||||
.finish()
|
||||
}
|
||||
@@ -138,12 +143,41 @@ impl std::fmt::Debug for ChangePassword {
|
||||
}
|
||||
}
|
||||
|
||||
/// 会话有效期与管理员续期上限;没有上限字段的旧会话保持固定有效期
|
||||
/// 访问有效期与逻辑会话;没有续期凭据的会话保持固定期限
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AuthSession {
|
||||
pub subject: SessionSubject,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub absolute_expires_at: Option<DateTime<Utc>>,
|
||||
pub renewal: Option<SessionRenewal>,
|
||||
}
|
||||
|
||||
/// 长效登录的凭据摘要与续期窗口;访问到期不删除整段登录
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionRenewal {
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub access_digest: [u8; 32],
|
||||
pub refresh_digest: [u8; 32],
|
||||
pub previous: Option<SessionRefreshReplay>,
|
||||
}
|
||||
|
||||
/// 仅保存上一代摘要及派生盐;响应丢失时须持有上一代凭据才能重建同一响应
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionRefreshReplay {
|
||||
pub digest: [u8; 32],
|
||||
pub salt: [u8; 32],
|
||||
pub expires_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl AuthSession {
|
||||
#[must_use]
|
||||
pub fn retained_until(&self) -> DateTime<Utc> {
|
||||
self.renewal
|
||||
.as_ref()
|
||||
.map_or(self.expires_at, |renewal| renewal.expires_at)
|
||||
}
|
||||
}
|
||||
|
||||
/// 安全审计事件类型
|
||||
|
||||
@@ -474,6 +474,26 @@ impl BackupError {
|
||||
pub fn message(&self) -> &str {
|
||||
&self.message
|
||||
}
|
||||
|
||||
/// 清理失败附加到原始终态,不覆盖上传失败或主动取消的分类
|
||||
#[must_use]
|
||||
pub fn with_cleanup_failure(
|
||||
mut self,
|
||||
stage: &'static str,
|
||||
error: &Self,
|
||||
retry_on_expiration: bool,
|
||||
) -> Self {
|
||||
let recovery = if retry_on_expiration {
|
||||
"等待备份到期清理"
|
||||
} else {
|
||||
"未安排自动重试"
|
||||
};
|
||||
self.message.push_str(&format!(
|
||||
";清理阶段 {stage} 失败({}),{recovery}",
|
||||
error.code()
|
||||
));
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造稳定错误码常量
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
//! 故障诊断导出的目标、已采集证据与缺口
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::{
|
||||
observability::{OpsError, UsageDetail},
|
||||
system::SystemVersion,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DiagnosticTarget {
|
||||
Request(String),
|
||||
Operation(String),
|
||||
}
|
||||
|
||||
impl DiagnosticTarget {
|
||||
#[must_use]
|
||||
pub fn id(&self) -> &str {
|
||||
match self {
|
||||
Self::Request(id) | Self::Operation(id) => id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DiagnosticGap {
|
||||
pub section: String,
|
||||
pub reason: &'static str,
|
||||
}
|
||||
|
||||
pub struct DiagnosticRequest {
|
||||
pub request_id: String,
|
||||
pub detail: Option<UsageDetail>,
|
||||
pub errors: Vec<OpsError>,
|
||||
}
|
||||
|
||||
/// 原始受控详情只能经导出投影脱敏后交付,不参与普通日志或 Debug
|
||||
pub struct DiagnosticBundle {
|
||||
pub target: DiagnosticTarget,
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub environment: Option<SystemVersion>,
|
||||
pub error: Option<OpsError>,
|
||||
pub requests: Vec<DiagnosticRequest>,
|
||||
pub gaps: Vec<DiagnosticGap>,
|
||||
}
|
||||
@@ -4,11 +4,13 @@ use std::num::{NonZeroU16, NonZeroU64};
|
||||
|
||||
pub mod account_groups;
|
||||
pub mod accounts;
|
||||
pub mod analytics;
|
||||
pub mod audit;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_distribution;
|
||||
pub mod client_keys;
|
||||
pub mod diagnostics;
|
||||
pub mod import_tasks;
|
||||
pub mod key_usage;
|
||||
pub mod observability;
|
||||
@@ -20,9 +22,11 @@ pub mod provider_credentials;
|
||||
pub mod proxies;
|
||||
pub mod quota_forecast;
|
||||
pub mod quota_forecast_sampling;
|
||||
pub mod reset_credit_automation;
|
||||
pub mod retention;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod usage_data;
|
||||
|
||||
/// 管理用例对外返回的稳定错误分类
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
||||
@@ -5,6 +5,10 @@ use std::{num::NonZeroU16, str::FromStr};
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
|
||||
use super::AdminModelError;
|
||||
pub use super::analytics::{
|
||||
AnalyticsMetadata, AnalyticsState, UsageBillingProjection, UsageInsightsObservation,
|
||||
UsageProjectedCostPoint,
|
||||
};
|
||||
|
||||
/// 观测记录查询的页大小,由 HTTP、管理用例和存储端口共同使用
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -48,7 +52,7 @@ impl CalendarPeriod {
|
||||
}
|
||||
|
||||
/// 外部观测查询的 UTC 时间范围
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct TimeRange {
|
||||
pub start: DateTime<Utc>,
|
||||
pub end: DateTime<Utc>,
|
||||
@@ -89,7 +93,7 @@ impl TimeRange {
|
||||
}
|
||||
|
||||
/// 请求结果状态
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum RequestOutcome {
|
||||
Running,
|
||||
Succeeded,
|
||||
@@ -100,7 +104,7 @@ pub enum RequestOutcome {
|
||||
}
|
||||
|
||||
/// 未知但有界的请求结果值;私有字段禁止绕过 [`RequestOutcome::new`] 构造
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct OtherRequestOutcome(String);
|
||||
|
||||
impl RequestOutcome {
|
||||
@@ -151,7 +155,7 @@ impl OtherRequestOutcome {
|
||||
}
|
||||
|
||||
/// 用量记录过滤条件
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
|
||||
pub struct UsageFilter {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
@@ -168,9 +172,19 @@ pub struct UsageFilter {
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
/// 请求列表的结果筛选,不改变汇总和计量口径
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum UsageRecordStatus {
|
||||
#[default]
|
||||
Success,
|
||||
Error,
|
||||
All,
|
||||
}
|
||||
|
||||
/// 用量记录分页查询
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageQuery {
|
||||
pub status: UsageRecordStatus,
|
||||
pub range: TimeRange,
|
||||
pub filter: UsageFilter,
|
||||
pub current_page: u32,
|
||||
@@ -178,7 +192,7 @@ pub struct UsageQuery {
|
||||
}
|
||||
|
||||
/// Admin 确定概览的数量、趋势粒度及最近请求筛选,Store 只执行该选择
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct DashboardQuery {
|
||||
pub range: TimeRange,
|
||||
pub granularity: Granularity,
|
||||
@@ -230,7 +244,7 @@ pub struct OpsErrorQuery {
|
||||
}
|
||||
|
||||
/// 用量诊断维度
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum DiagnosticDimension {
|
||||
Provider,
|
||||
Model,
|
||||
@@ -363,8 +377,8 @@ pub struct ProviderBillingInput {
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub total: CurrencyCost,
|
||||
}
|
||||
|
||||
@@ -380,8 +394,8 @@ pub struct UsageCalculatedBillingFact {
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub total: CurrencyCost,
|
||||
}
|
||||
|
||||
@@ -408,7 +422,7 @@ pub struct CalculatedBillingBreakdown {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ImageBillingBreakdown {
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub input_amount: CurrencyCost,
|
||||
pub cache_read_amount: CurrencyCost,
|
||||
pub input_price_per_million: CurrencyCost,
|
||||
@@ -454,9 +468,9 @@ pub struct RequestMetrics {
|
||||
pub caller_error_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub reasoning_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub cache_write_input_tokens: u64,
|
||||
pub reasoning_output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub first_token_latency_sum_ms: u64,
|
||||
pub first_token_latency_count: u64,
|
||||
@@ -496,7 +510,7 @@ pub struct AttemptMetrics {
|
||||
}
|
||||
|
||||
/// 时间序列粒度
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum Granularity {
|
||||
FifteenMinutes,
|
||||
Hour,
|
||||
@@ -556,9 +570,9 @@ pub struct AccountModelUsage {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
@@ -591,9 +605,9 @@ pub struct DashboardAccountUsage {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
@@ -628,7 +642,7 @@ pub struct DashboardRuntimeSlots {
|
||||
pub struct DashboardTotals {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub billing_usd: Option<DecimalAmount>,
|
||||
}
|
||||
@@ -636,6 +650,7 @@ pub struct DashboardTotals {
|
||||
/// 仪表盘所需的当前区间事实及全历史累计
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardObservation {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub range: TimeRange,
|
||||
pub totals: DashboardTotals,
|
||||
pub provider_accounts: AccountPoolMetrics,
|
||||
@@ -648,6 +663,7 @@ pub struct DashboardObservation {
|
||||
/// 使用记录表格的窄读模型
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageListRecord {
|
||||
pub outcome: RequestOutcome,
|
||||
pub client_api_key_name: Option<String>,
|
||||
pub id: String,
|
||||
pub endpoint: String,
|
||||
@@ -657,9 +673,9 @@ pub struct UsageListRecord {
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
/// 账号当前备注,按内部账号 ID 关联,不属于请求历史快照
|
||||
/// 请求选号时冻结的账号备注,未采集时保持未知
|
||||
pub provider_account_notes: Option<String>,
|
||||
/// 账号当前套餐,不属于请求历史快照
|
||||
/// 请求选号时冻结的账号套餐
|
||||
pub provider_account_plan_type: Option<String>,
|
||||
pub provider_account_plan_type_display: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
@@ -669,9 +685,9 @@ pub struct UsageListRecord {
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
@@ -750,9 +766,9 @@ pub struct UsageRecord {
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
@@ -830,9 +846,9 @@ pub struct UsageAttempt {
|
||||
pub message: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_source: Option<String>,
|
||||
pub cost_amount: Option<DecimalAmount>,
|
||||
@@ -840,7 +856,7 @@ pub struct UsageAttempt {
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 一条请求及其全部尝试
|
||||
/// 一条请求及其保留的尝试与诊断轨迹
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageDetail {
|
||||
pub trace: Option<serde_json::Value>,
|
||||
@@ -862,6 +878,7 @@ pub struct ProviderObservation {
|
||||
/// 用量总览
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageOverview {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub range: TimeRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
@@ -878,6 +895,7 @@ pub struct UsageSummary {
|
||||
/// 诊断聚合结果,分母包含截取展示项之前的全部匹配请求
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DiagnosticsObservation {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub total_request_count: u64,
|
||||
pub items: Vec<DiagnosticObservation>,
|
||||
}
|
||||
@@ -1001,7 +1019,7 @@ pub struct TrendSummary {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub service_failure_count: u64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
@@ -1019,6 +1037,7 @@ pub struct TrendSummary {
|
||||
/// 趋势结果;API 再决定图表标签与数字格式
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Trend {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub kind: TrendKind,
|
||||
pub points: Vec<TrendPoint>,
|
||||
pub summary: TrendSummary,
|
||||
@@ -1134,7 +1153,7 @@ pub struct DashboardPeriodMetrics {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub cached_token_rate: f64,
|
||||
pub observed_cached_token_rate: Option<f64>,
|
||||
@@ -1224,7 +1243,7 @@ pub struct UsageInsightsCostPoint {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub standard_cost: Option<DecimalAmount>,
|
||||
@@ -1249,7 +1268,7 @@ pub struct UsageInsightsCost {
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub points: Vec<UsageInsightsCostPoint>,
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
@@ -1259,6 +1278,7 @@ pub struct UsageInsightsCost {
|
||||
/// 用量洞察的完整用例结果;API 只负责标签、时区与字符串格式化
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageInsights {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub granularity: Granularity,
|
||||
pub health: UsageInsightsHealth,
|
||||
pub performance: UsageInsightsPerformance,
|
||||
@@ -1294,6 +1314,7 @@ pub struct DiagnosticsItem {
|
||||
/// 诊断结果
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DiagnosticsResult {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub dimension: DiagnosticDimension,
|
||||
pub items: Vec<DiagnosticsItem>,
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ pub struct PluginManagementResource {
|
||||
|
||||
/// 原始正文可能含业务数据,不能派生内容型 Debug 或写入诊断日志
|
||||
pub struct PluginManagementRequest {
|
||||
pub request_context: Option<std::sync::Arc<gateway_core::engine::extensions::RequestContext>>,
|
||||
pub method: String,
|
||||
pub path: String,
|
||||
pub query: String,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 插件私有状态的归属、代次隔离与迁移数据合同
|
||||
|
||||
use std::{collections::BTreeMap, fmt};
|
||||
use std::{collections::BTreeMap, fmt, sync::Arc};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -173,6 +173,8 @@ pub struct PluginStateTransition {
|
||||
pub instance_id: String,
|
||||
pub artifact_sha256: String,
|
||||
pub namespaces: Vec<PluginStateMigrationNamespace>,
|
||||
/// Store 持有的迁移所有权;最后一个调用持有者释放后允许回收遗留 staging
|
||||
pub guard: Arc<dyn Send + Sync>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
|
||||
@@ -69,6 +69,7 @@ pub struct ProxyTestResult {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProxyRecord {
|
||||
pub enabled: bool,
|
||||
pub auto_location: bool,
|
||||
pub detected_location: Option<DetectedProxyLocation>,
|
||||
pub location: Option<gateway_core::account::RequestLocation>,
|
||||
@@ -103,6 +104,9 @@ pub struct NewProxy {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UpdateProxy {
|
||||
/// 省略保留配置,零值取消限制
|
||||
pub max_websocket_connections: Option<u32>,
|
||||
pub enabled: Option<bool>,
|
||||
pub auto_location: Option<bool>,
|
||||
pub test: Option<ProxyTestResult>,
|
||||
/// 外层为空保留配置,内层为空恢复全局继承
|
||||
@@ -119,6 +123,21 @@ pub struct ProxyMutation {
|
||||
pub record: ProxyRecord,
|
||||
}
|
||||
|
||||
pub const MAX_PROXY_BATCH_SIZE: usize = 100;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TestProxy {
|
||||
pub id: String,
|
||||
pub revision: Revision,
|
||||
pub detect_location: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ProxyBatchResult<T> {
|
||||
Completed(Result<T, super::AdminError>),
|
||||
Skipped,
|
||||
}
|
||||
|
||||
/// 检测失败与连通性分开,双栈冲突不得继续应用旧位置
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)]
|
||||
#[serde(tag = "status", rename_all = "camelCase")]
|
||||
|
||||
@@ -15,7 +15,7 @@ pub struct QuotaForecastUsage {
|
||||
pub tokens: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub missing_token_count: u64,
|
||||
pub known_cost_count: u64,
|
||||
pub unavailable_cost_count: u64,
|
||||
@@ -35,7 +35,9 @@ impl QuotaForecastUsage {
|
||||
tokens: self.tokens.checked_sub(baseline.tokens)?,
|
||||
input_tokens: self.input_tokens.checked_sub(baseline.input_tokens)?,
|
||||
output_tokens: self.output_tokens.checked_sub(baseline.output_tokens)?,
|
||||
cached_tokens: self.cached_tokens.checked_sub(baseline.cached_tokens)?,
|
||||
cached_input_tokens: self
|
||||
.cached_input_tokens
|
||||
.checked_sub(baseline.cached_input_tokens)?,
|
||||
missing_token_count: self
|
||||
.missing_token_count
|
||||
.checked_sub(baseline.missing_token_count)?,
|
||||
@@ -140,7 +142,7 @@ pub fn select_forecast_sample(
|
||||
};
|
||||
history.push(current);
|
||||
// 重置边界不在查询起点时,从首个新观测重新计数,避免把旧周期累计带入总量
|
||||
let mut cycle_baseline = interrupted.then(|| &history[0]);
|
||||
let cycle_baseline = interrupted.then(|| &history[0]);
|
||||
let mut anchors: Vec<&QuotaForecastPoint> = Vec::new();
|
||||
let mut high_water: Option<&QuotaForecastPoint> = None;
|
||||
for point in &history {
|
||||
@@ -153,9 +155,9 @@ pub fn select_forecast_sample(
|
||||
if previous.used_percent - point.used_percent <= 1.0 {
|
||||
continue;
|
||||
}
|
||||
// 同窗口回落不能证明周期重置,只重建采样并保留已记录累计
|
||||
sample.discontinuous = true;
|
||||
anchors.clear();
|
||||
cycle_baseline = Some(point);
|
||||
}
|
||||
}
|
||||
high_water = Some(point);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
//! 自动重置卡的持久计划与查询投影,不保存上游凭据或完整卡片
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PlannedResetCredit {
|
||||
pub id: String,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PendingResetCredit {
|
||||
pub credit: PlannedResetCredit,
|
||||
pub redeem_request_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ResetCreditAutomationState {
|
||||
pub next_discovery_at: DateTime<Utc>,
|
||||
pub credits: Vec<PlannedResetCredit>,
|
||||
pub pending: Option<PendingResetCredit>,
|
||||
pub retry_at: Option<DateTime<Utc>>,
|
||||
pub failures: u32,
|
||||
pub last_result: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ResetCreditAutomation {
|
||||
pub account_id: String,
|
||||
pub enabled: bool,
|
||||
/// 零表示尚未创建;每次持久状态写入都递增版本
|
||||
pub revision: u64,
|
||||
pub next_run_at: DateTime<Utc>,
|
||||
pub state: ResetCreditAutomationState,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResetCreditAutomationStatus {
|
||||
pub enabled: bool,
|
||||
pub lead_minutes: u32,
|
||||
pub next_discovery_at: Option<DateTime<Utc>>,
|
||||
pub next_consume_at: Option<DateTime<Utc>>,
|
||||
pub last_result: Option<String>,
|
||||
}
|
||||
@@ -24,7 +24,7 @@ impl RetentionPolicy {
|
||||
audit_days: u32,
|
||||
) -> Result<(), &'static str> {
|
||||
for (valid, field) in [
|
||||
(usage_days >= 31, "usage_retention_days"),
|
||||
((1..=31).contains(&usage_days), "usage_retention_days"),
|
||||
(ops_days > 0, "ops_event_retention_days"),
|
||||
(audit_days > 0, "audit_retention_days"),
|
||||
] {
|
||||
@@ -38,7 +38,7 @@ impl RetentionPolicy {
|
||||
pub fn try_new(usage_days: u32, ops_days: u32, audit_days: u32) -> Result<Self, AdminError> {
|
||||
if Self::validate_values(usage_days, ops_days, audit_days).is_err() {
|
||||
return Err(AdminError::invalid(
|
||||
"请求保留期至少 31 天,事件与审计保留期必须为正数",
|
||||
"请求保留期必须为 1~31 天,事件与审计保留期必须为正数",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
|
||||
@@ -45,12 +45,12 @@ pub use gateway_core::account::RotationStrategy;
|
||||
/// 运行设置的共同值;版本、Provider 画像更新和数据库秘密由外层类型拥有
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct RuntimeSettingsValues {
|
||||
pub codex_privacy_policy: gateway_core::settings::privacy::CodexPrivacyPolicy,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
pub max_concurrent_per_account: u32,
|
||||
pub request_interval_ms: u64,
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
@@ -75,6 +75,8 @@ pub struct RuntimeSettingsValues {
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
pub reset_credit_lead_minutes: u32,
|
||||
pub reset_credit_max_weekly_remaining_percent: Option<u32>,
|
||||
}
|
||||
|
||||
/// 完整运行设置事实
|
||||
@@ -189,6 +191,15 @@ impl From<RuntimeSettings> for ReplaceRuntimeSettings {
|
||||
impl RuntimeSettingsValues {
|
||||
/// 校验共同事实,返回字段名供边界转换错误;SQL 类型转换仍由 Store 处理
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
if !(1..=1440).contains(&self.reset_credit_lead_minutes) {
|
||||
return Err("reset_credit_lead_minutes");
|
||||
}
|
||||
if self
|
||||
.reset_credit_max_weekly_remaining_percent
|
||||
.is_some_and(|percent| percent > 100)
|
||||
{
|
||||
return Err("reset_credit_max_weekly_remaining_percent");
|
||||
}
|
||||
self.request_location
|
||||
.validate()
|
||||
.map_err(|_| "request_location")?;
|
||||
@@ -211,10 +222,6 @@ impl RuntimeSettingsValues {
|
||||
"refresh_margin_seconds",
|
||||
),
|
||||
(self.refresh_concurrency > 0, "refresh_concurrency"),
|
||||
(
|
||||
i64::try_from(self.request_interval_ms).is_ok(),
|
||||
"request_interval_ms",
|
||||
),
|
||||
] {
|
||||
if !valid {
|
||||
return Err(field);
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
//! 由宿主传入的可信写入主体,不属于插件请求载荷
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageActor {
|
||||
Admin {
|
||||
subject: String,
|
||||
},
|
||||
Plugin {
|
||||
publisher: String,
|
||||
name: String,
|
||||
instance_id: String,
|
||||
},
|
||||
System,
|
||||
}
|
||||
|
||||
impl super::UsageRecordChange {
|
||||
#[must_use]
|
||||
pub fn id(&self) -> &str {
|
||||
match self {
|
||||
Self::Put { id, .. } | Self::Update { id, .. } | Self::Delete { id } => id,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! 按归属时间和维度维护统计贡献,业务来源与调整历史由调用方保存
|
||||
|
||||
use super::{UsageDelta, UsageRecordFilter};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageMetric {
|
||||
RequestCount,
|
||||
SuccessCount,
|
||||
FailureCount,
|
||||
CancelledCount,
|
||||
IncompleteCount,
|
||||
CompletedRequestCount,
|
||||
InputTokens,
|
||||
OutputTokens,
|
||||
CachedInputTokens,
|
||||
CacheWriteInputTokens,
|
||||
ReasoningOutputTokens,
|
||||
ImageInputTokens,
|
||||
ImageOutputTokens,
|
||||
TotalTokens,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageDimensions {
|
||||
pub provider: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub account_ref: Option<String>,
|
||||
pub client_key_ref: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
}
|
||||
|
||||
/// 整个键决定覆盖目标;调用方可将来源标识编码进 key,重试相同键不会累计
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregateKey {
|
||||
pub bucket_start_ms: i64,
|
||||
pub dimensions: UsageDimensions,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
/// 值按 bucket_start_ms 归属,不自动摊分;负值用于相对原生统计的修正
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregate {
|
||||
pub identity: UsageAggregateKey,
|
||||
pub counts: BTreeMap<UsageMetric, UsageDelta>,
|
||||
pub costs: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageAggregateChange {
|
||||
Put { aggregate: UsageAggregate },
|
||||
Delete { identity: UsageAggregateKey },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WriteUsageAggregates {
|
||||
pub changes: Vec<UsageAggregateChange>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregateWriteResult {
|
||||
pub changed: Vec<UsageAggregateKey>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListUsageAggregates {
|
||||
pub filter: UsageRecordFilter,
|
||||
pub cursor: Option<String>,
|
||||
pub limit: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregatePage {
|
||||
pub items: Vec<UsageAggregate>,
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
//! 用量基础操作的错误分类,不保存失败或重试状态
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageErrorKind {
|
||||
InvalidInput,
|
||||
NotFound,
|
||||
Conflict,
|
||||
CapacityExceeded,
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageDataError {
|
||||
pub kind: UsageErrorKind,
|
||||
pub message: String,
|
||||
}
|
||||
impl UsageDataError {
|
||||
#[must_use]
|
||||
pub fn new(kind: UsageErrorKind, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
#[must_use]
|
||||
pub fn invalid(message: impl Into<String>) -> Self {
|
||||
Self::new(UsageErrorKind::InvalidInput, message)
|
||||
}
|
||||
#[must_use]
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::new(UsageErrorKind::Unavailable, message)
|
||||
}
|
||||
}
|
||||
impl std::fmt::Display for UsageDataError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for UsageDataError {}
|
||||
pub type UsageDataResult<T> = Result<T, UsageDataError>;
|
||||
@@ -0,0 +1,22 @@
|
||||
//! 用量明细、聚合维护与纯计价的公开合同
|
||||
|
||||
mod value;
|
||||
pub use value::*;
|
||||
mod record;
|
||||
pub use record::*;
|
||||
mod query;
|
||||
pub use query::*;
|
||||
mod pricing;
|
||||
pub use pricing::*;
|
||||
mod error;
|
||||
pub use error::*;
|
||||
|
||||
pub const USAGE_DATA_MAX_TARGETS: usize = 500;
|
||||
pub const USAGE_DATA_MAX_REQUEST_BYTES: usize = 2 * 1024 * 1024;
|
||||
pub const USAGE_DATA_MAX_PAGE_SIZE: u16 = 200;
|
||||
|
||||
mod actor;
|
||||
pub use actor::*;
|
||||
|
||||
mod aggregate;
|
||||
pub use aggregate::*;
|
||||
@@ -0,0 +1,62 @@
|
||||
//! 纯 Token 计价合同,费用是否采用与额外费用合并由调用方决定
|
||||
|
||||
use super::{UsageCost, UsageTokens};
|
||||
use gateway_core::metering::ModelPriceOverride;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsagePriceSelection {
|
||||
Current,
|
||||
Override { price: Box<ModelPriceOverride> },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsagePricingInput {
|
||||
pub id: String,
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub tokens: UsageTokens,
|
||||
pub service_tier: Option<String>,
|
||||
pub price: UsagePriceSelection,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EvaluateUsagePricing {
|
||||
pub items: Vec<UsagePricingInput>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsagePricingBreakdown {
|
||||
pub input_amount: String,
|
||||
pub output_amount: String,
|
||||
pub cache_read_amount: String,
|
||||
pub cache_write_amount: String,
|
||||
pub standard_amount: String,
|
||||
pub long_context_billing_applied: bool,
|
||||
pub service_tier: Option<String>,
|
||||
pub custom_multiplier_bps: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsagePricingResult {
|
||||
Estimated {
|
||||
id: String,
|
||||
cost: UsageCost,
|
||||
breakdown: UsagePricingBreakdown,
|
||||
},
|
||||
NotEstimable {
|
||||
id: String,
|
||||
reason: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsagePricingEvaluation {
|
||||
pub items: Vec<UsagePricingResult>,
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
//! 请求明细的有界筛选与稳定顺序分页
|
||||
|
||||
use super::UsageRecord;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GetUsageRecord {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecordFilter {
|
||||
pub start_ms: i64,
|
||||
pub end_ms: i64,
|
||||
pub provider: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub account_ref: Option<String>,
|
||||
pub client_key_ref: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListUsageRecords {
|
||||
pub filter: UsageRecordFilter,
|
||||
pub cursor: Option<String>,
|
||||
pub limit: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecordPage {
|
||||
pub items: Vec<UsageRecord>,
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//! 既有请求明细的用量读写合同,不承载插件任务或来源状态
|
||||
|
||||
use super::UsageCount;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Token 使用宿主口径:缓存包含在输入中,推理包含在输出中,未知值为 null
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageTokens {
|
||||
pub input_tokens: Option<UsageCount>,
|
||||
pub output_tokens: Option<UsageCount>,
|
||||
pub cached_input_tokens: Option<UsageCount>,
|
||||
pub cache_write_input_tokens: Option<UsageCount>,
|
||||
pub reasoning_output_tokens: Option<UsageCount>,
|
||||
pub image_input_tokens: Option<UsageCount>,
|
||||
pub image_output_tokens: Option<UsageCount>,
|
||||
pub total_tokens: Option<UsageCount>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageCostSource {
|
||||
ProviderReported,
|
||||
Calculated,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageCost {
|
||||
#[default]
|
||||
Unavailable,
|
||||
Known {
|
||||
source: UsageCostSource,
|
||||
amount: String,
|
||||
currency: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageOutcome {
|
||||
Running,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Cancelled,
|
||||
Incomplete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecordData {
|
||||
pub client_key_ref: String,
|
||||
pub provider: Option<String>,
|
||||
pub account_ref: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub protocol: String,
|
||||
pub operation: String,
|
||||
pub endpoint: String,
|
||||
pub client_transport: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub started_at_ms: i64,
|
||||
pub completed_at_ms: Option<i64>,
|
||||
pub outcome: UsageOutcome,
|
||||
pub tokens: UsageTokens,
|
||||
pub cost: UsageCost,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecord {
|
||||
pub id: String,
|
||||
pub data: UsageRecordData,
|
||||
pub billing_snapshot: Option<serde_json::Value>,
|
||||
pub provider_observation: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageRecordChange {
|
||||
Put {
|
||||
id: String,
|
||||
data: Box<UsageRecordData>,
|
||||
},
|
||||
/// tokens 替换全部 Token 字段;cost 的 unavailable 明确清空费用
|
||||
Update {
|
||||
id: String,
|
||||
tokens: Option<UsageTokens>,
|
||||
cost: Option<UsageCost>,
|
||||
},
|
||||
Delete {
|
||||
id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WriteUsageRecords {
|
||||
pub changes: Vec<UsageRecordChange>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageWriteResult {
|
||||
pub changed_ids: Vec<String>,
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
//! 用量计数使用十进制字符串,避免插件前端丢失整数精度
|
||||
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error};
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct UsageCount(pub u64);
|
||||
impl UsageCount {
|
||||
pub const ZERO: Self = Self(0);
|
||||
#[must_use]
|
||||
pub const fn get(self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
impl From<u64> for UsageCount {
|
||||
fn from(value: u64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
impl fmt::Display for UsageCount {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
self.0.fmt(formatter)
|
||||
}
|
||||
}
|
||||
impl Serialize for UsageCount {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_str(&self.0.to_string())
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for UsageCount {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
let value = String::deserialize(deserializer)?;
|
||||
if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(D::Error::custom("count must be an unsigned decimal string"));
|
||||
}
|
||||
value.parse().map(Self).map_err(D::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
/// 有符号统计贡献受 PostgreSQL bigint 范围限制,JSON 使用十进制字符串
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct UsageDelta(pub i64);
|
||||
impl UsageDelta {
|
||||
#[must_use]
|
||||
pub const fn get(self) -> i64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
impl Serialize for UsageDelta {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_str(&self.0.to_string())
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for UsageDelta {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
let value = String::deserialize(deserializer)?;
|
||||
let digits = value.strip_prefix('-').unwrap_or(&value);
|
||||
if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(D::Error::custom("delta must be a signed decimal string"));
|
||||
}
|
||||
value.parse().map(Self).map_err(D::Error::custom)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,8 @@ pub mod plugins;
|
||||
pub mod pricing;
|
||||
pub mod provider;
|
||||
pub mod proxy;
|
||||
pub mod reset_credit_automation;
|
||||
pub mod retention;
|
||||
pub mod store;
|
||||
pub mod system;
|
||||
pub mod usage_data;
|
||||
|
||||
@@ -26,6 +26,14 @@ use crate::model::{
|
||||
quota_forecast_sampling::QuotaForecastObservation,
|
||||
};
|
||||
|
||||
/// 按原生 Token 口径计算的报价输入,额外费用由调用方处理
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderUsagePricingInput {
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub usage: gateway_core::metering::Usage,
|
||||
}
|
||||
|
||||
/// Provider 管理失败的稳定分类
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProviderAdminErrorKind {
|
||||
@@ -130,11 +138,45 @@ struct ProviderAdminDiagnostic {
|
||||
/// 运行时资源通知只在事务成功后发生
|
||||
#[async_trait]
|
||||
pub trait ProviderAdmin: Send + Sync {
|
||||
fn compile_privacy_policy(
|
||||
&self,
|
||||
_policy: &gateway_core::settings::privacy::CodexPrivacyPolicy,
|
||||
) -> Result<
|
||||
Arc<dyn gateway_core::settings::privacy::CompiledPrivacyPolicy>,
|
||||
gateway_core::settings::privacy::PrivacyError,
|
||||
> {
|
||||
Err(gateway_core::settings::privacy::PrivacyError {
|
||||
rule_index: 0,
|
||||
reason: "Provider 不支持隐私策略",
|
||||
})
|
||||
}
|
||||
|
||||
fn preview_privacy_policy(
|
||||
&self,
|
||||
_request: gateway_core::settings::privacy::PrivacyPreviewRequest,
|
||||
) -> Result<
|
||||
gateway_core::settings::privacy::PrivacyPreviewResult,
|
||||
gateway_core::settings::privacy::PrivacyError,
|
||||
> {
|
||||
Err(gateway_core::settings::privacy::PrivacyError {
|
||||
rule_index: 0,
|
||||
reason: "Provider 不支持隐私策略",
|
||||
})
|
||||
}
|
||||
|
||||
/// 只读内置价目;没有本地计价能力的 Provider 返回空目录
|
||||
fn pricing_catalog(&self) -> crate::model::pricing::ProviderPricingCatalog {
|
||||
Default::default()
|
||||
}
|
||||
|
||||
fn evaluate_usage_pricing(
|
||||
&self,
|
||||
_input: &ProviderUsagePricingInput,
|
||||
_pricing: Option<&gateway_core::metering::ModelPriceOverride>,
|
||||
) -> Result<Option<gateway_core::metering::CalculatedCostBreakdown>, ProviderAdminError> {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
}
|
||||
|
||||
fn provider_kind(&self) -> &ProviderKind;
|
||||
|
||||
/// 只读取原生 Provider 能力及账号类型,不执行网络或数据库查询
|
||||
|
||||
@@ -64,5 +64,6 @@ pub struct ProxyImportReservation {
|
||||
|
||||
#[async_trait]
|
||||
pub trait ProxyProbe: Send + Sync {
|
||||
async fn test(&self, proxy: &OutboundProxy, detect_location: bool) -> ProxyTestResult;
|
||||
/// 未指定代理时探测服务器直连出口,不继承进程代理环境变量
|
||||
async fn test(&self, proxy: Option<&OutboundProxy>, detect_location: bool) -> ProxyTestResult;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
//! 自动重置卡计划的持久化端口,配置变更沿用管理事务与审计
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::store::AdminStoreResult;
|
||||
use crate::model::{MutationContext, reset_credit_automation::ResetCreditAutomation};
|
||||
|
||||
#[async_trait]
|
||||
pub trait ResetCreditAutomationStore: Send + Sync {
|
||||
async fn reset_credit_automation(
|
||||
&self,
|
||||
account_id: &str,
|
||||
) -> AdminStoreResult<Option<ResetCreditAutomation>>;
|
||||
|
||||
/// 只读取到期的本地计划,按到期时间稳定排序
|
||||
async fn due_reset_credit_automations(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<String>>;
|
||||
|
||||
/// 按 record.revision 比较写入,成功返回新版本;配置变更传入审计上下文
|
||||
async fn save_reset_credit_automation(
|
||||
&self,
|
||||
record: &ResetCreditAutomation,
|
||||
context: Option<&MutationContext>,
|
||||
) -> AdminStoreResult<u64>;
|
||||
}
|
||||
@@ -6,7 +6,6 @@ use std::{collections::BTreeMap, net::IpAddr, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::stream::BoxStream;
|
||||
|
||||
use super::backup::BackupStorePorts;
|
||||
use crate::model::{
|
||||
@@ -30,7 +29,8 @@ use crate::model::{
|
||||
observability::{
|
||||
DashboardObservation, DashboardQuery, DiagnosticDimension, DiagnosticsObservation,
|
||||
Granularity, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange,
|
||||
UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageOverview, UsagePage, UsageQuery,
|
||||
UsageBillingProjection, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageInsightsObservation, UsageOverview, UsagePage, UsageQuery,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, CredentialDetails, CredentialImportCommit, CredentialImportResult,
|
||||
@@ -109,7 +109,9 @@ pub type AdminStoreResult<T> = Result<T, AdminStoreError>;
|
||||
|
||||
/// 账号目录与公共账号写操作
|
||||
#[async_trait]
|
||||
pub trait AccountStore: Send + Sync {
|
||||
pub trait AccountStore:
|
||||
super::reset_credit_automation::ResetCreditAutomationStore + Send + Sync
|
||||
{
|
||||
/// 插件回调按授权 Provider/账号在数据库过滤后做稳定 cursor 分页
|
||||
async fn list_plugin_accounts(
|
||||
&self,
|
||||
@@ -128,6 +130,8 @@ pub trait AccountStore: Send + Sync {
|
||||
runtime: AccountRuntimeSnapshot,
|
||||
) -> AdminStoreResult<Option<AccountPageItem>>;
|
||||
|
||||
async fn account_exists(&self, account_id: &str) -> AdminStoreResult<bool>;
|
||||
|
||||
async fn load_account_usage(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
@@ -241,6 +245,13 @@ pub trait AccountStore: Send + Sync {
|
||||
/// 可丢失账号运行态的管理读端口;跨存储编排由 Admin application service 拥有
|
||||
#[async_trait]
|
||||
pub trait AccountRuntimeStore: Send + Sync {
|
||||
async fn account_concurrency(
|
||||
&self,
|
||||
account_id: &str,
|
||||
offset: u64,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<crate::model::accounts::AccountConcurrencyPage>;
|
||||
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot>;
|
||||
|
||||
async fn account_runtime(
|
||||
@@ -294,12 +305,12 @@ pub trait AuthStore: Send + Sync {
|
||||
|
||||
async fn store_session(&self, session_id: &str, session: &AuthSession) -> AdminStoreResult<()>;
|
||||
|
||||
/// 只延长仍存在且匹配的会话;并发续期返回当前值,已退出或过期时不重建
|
||||
async fn renew_session(
|
||||
/// 原子替换或撤销仍匹配的会话并返回当前值;过期或退出后不重新创建
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>>;
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AuthSession>>;
|
||||
@@ -421,13 +432,23 @@ pub trait AccountGroupStore: Send + Sync {
|
||||
) -> AdminStoreResult<AccountGroupMutation>;
|
||||
}
|
||||
|
||||
/// 逐条读取的已计算费用事实;消费结束或丢弃时释放查询资源
|
||||
pub type UsageCalculatedBillingStream<'a> =
|
||||
BoxStream<'a, AdminStoreResult<UsageCalculatedBillingFact>>;
|
||||
/// 后台投影复用 Admin 与 Provider 的费用规则,Store 不重新解释价格
|
||||
pub trait UsageBillingProjector: Send + Sync {
|
||||
fn project(
|
||||
&self,
|
||||
fact: &UsageCalculatedBillingFact,
|
||||
) -> Result<UsageBillingProjection, crate::model::AdminError>;
|
||||
}
|
||||
|
||||
/// 用量、趋势、诊断与运维错误的只读能力
|
||||
#[async_trait]
|
||||
pub trait ObservabilityStore: Send + Sync {
|
||||
/// 读取实现声明的总预算,缓存等待沿用同一配置,避免外层提前截断
|
||||
fn query_timeout(&self) -> std::time::Duration;
|
||||
|
||||
/// 缓存必须与删除和后台投影提交的版本一致
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64>;
|
||||
|
||||
/// 返回历史统计区间和指定观测时刻下的实时账号状态
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
@@ -448,14 +469,13 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>>;
|
||||
|
||||
/// 流式返回可由 Provider 重新校验的已计算费用事实,不保证顺序
|
||||
/// 查询及解码错误由流返回;调用方应逐条聚合,避免收集整个区间
|
||||
fn usage_calculated_billing_facts(
|
||||
/// 同一读取快照中的汇总、趋势和已投影情景费用
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> UsageCalculatedBillingStream<'_>;
|
||||
) -> AdminStoreResult<UsageInsightsObservation>;
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage>;
|
||||
|
||||
@@ -476,6 +496,13 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
) -> AdminStoreResult<DiagnosticsObservation>;
|
||||
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage>;
|
||||
|
||||
/// 按精确目标读取受控错误,不受列表时间窗口与当前页面影响
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
target: &crate::model::diagnostics::DiagnosticTarget,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<Vec<crate::model::observability::OpsError>>;
|
||||
}
|
||||
|
||||
/// Runtime settings 与管理员 API Key 写入
|
||||
@@ -556,9 +583,15 @@ pub struct AdminStorePorts {
|
||||
plugins: Arc<dyn super::plugins::PluginStore>,
|
||||
plugin_state: Arc<dyn super::plugins::PluginStateStore>,
|
||||
plugin_resources: Arc<dyn super::plugin_resources::PluginResourceStore>,
|
||||
usage_data: Arc<dyn super::usage_data::UsageDataStore>,
|
||||
}
|
||||
|
||||
impl AdminStorePorts {
|
||||
#[must_use]
|
||||
pub fn usage_data(&self) -> Arc<dyn super::usage_data::UsageDataStore> {
|
||||
self.usage_data.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn plugin_resources(&self) -> Arc<dyn super::plugin_resources::PluginResourceStore> {
|
||||
self.plugin_resources.clone()
|
||||
@@ -589,6 +622,7 @@ impl AdminStorePorts {
|
||||
plugins: Arc<dyn super::plugins::PluginStore>,
|
||||
plugin_state: Arc<dyn super::plugins::PluginStateStore>,
|
||||
plugin_resources: Arc<dyn super::plugin_resources::PluginResourceStore>,
|
||||
usage_data: Arc<dyn super::usage_data::UsageDataStore>,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
@@ -600,6 +634,7 @@ impl AdminStorePorts {
|
||||
plugins,
|
||||
plugin_state,
|
||||
plugin_resources,
|
||||
usage_data,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
//! 原生请求用量的有界读取和事务写入端口
|
||||
|
||||
use crate::model::{auth::AdminAuditEvent, usage_data::*};
|
||||
use async_trait::async_trait;
|
||||
|
||||
pub struct StoredUsageWrite {
|
||||
pub changes: Vec<(UsageRecordChange, AdminAuditEvent)>,
|
||||
}
|
||||
|
||||
pub struct StoredAggregateWrite {
|
||||
pub changes: Vec<(UsageAggregateChange, AdminAuditEvent)>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait UsageDataStore: Send + Sync {
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregatePage>;
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
command: StoredAggregateWrite,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult>;
|
||||
async fn get_record(&self, request: GetUsageRecord) -> UsageDataResult<UsageRecord>;
|
||||
async fn list_records(&self, request: ListUsageRecords) -> UsageDataResult<UsageRecordPage>;
|
||||
async fn write_records(&self, command: StoredUsageWrite) -> UsageDataResult<UsageWriteResult>;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
//! 账号自动重置的公开操作声明,复用账号用例及宿主签发的审计上下文
|
||||
|
||||
use crate::{AccountsService, model::AdminError};
|
||||
use gateway_core::account::ProviderAccountId;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) fn register(
|
||||
registry: &mut super::Registry,
|
||||
service: &Arc<dyn AccountsService>,
|
||||
) -> Result<(), AdminError> {
|
||||
macro_rules! register {
|
||||
($sdk:ident, $method:ident, $input:ty, $pattern:pat $(, $argument:expr)* $(,)?) => {{
|
||||
let service = service.clone();
|
||||
registry.register(concat!("accounts.", stringify!($method)), move |$pattern: $input| {
|
||||
let service = service.clone();
|
||||
Box::pin(async move { service.$method($($argument),*).await })
|
||||
})?;
|
||||
}};
|
||||
}
|
||||
register!(
|
||||
GetResetCreditAutomation,
|
||||
reset_credit_automation_status,
|
||||
String,
|
||||
account_id,
|
||||
ProviderAccountId::new(account_id).map_err(|_| AdminError::invalid("账号 ID 无效"))?
|
||||
);
|
||||
register!(
|
||||
SetResetCreditAutomation,
|
||||
set_reset_credit_automation,
|
||||
(String, bool),
|
||||
(account_id, enabled),
|
||||
&super::actor::mutation_context()?,
|
||||
ProviderAccountId::new(account_id).map_err(|_| AdminError::invalid("账号 ID 无效"))?,
|
||||
enabled
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
//! 公开服务写入主体由受信宿主边界设置,插件输入不能覆盖
|
||||
|
||||
use crate::model::usage_data::UsageActor;
|
||||
use crate::model::{AdminError, MutationActor, MutationContext};
|
||||
use std::future::Future;
|
||||
|
||||
tokio::task_local! {
|
||||
static ACTOR: Option<UsageActor>;
|
||||
}
|
||||
|
||||
/// 与 middleware Origin 独立传递,嵌套服务不会丢失真实调用主体
|
||||
pub async fn scope_actor<F: Future>(actor: UsageActor, future: F) -> F::Output {
|
||||
scope_captured(Some(actor), future).await
|
||||
}
|
||||
|
||||
pub(super) async fn scope_captured<F: Future>(actor: Option<UsageActor>, future: F) -> F::Output {
|
||||
ACTOR.scope(actor, future).await
|
||||
}
|
||||
|
||||
pub(super) fn current_actor() -> Result<UsageActor, crate::model::usage_data::UsageDataError> {
|
||||
ACTOR.try_with(Clone::clone).ok().flatten().ok_or_else(|| {
|
||||
crate::model::usage_data::UsageDataError::invalid(
|
||||
"usage mutation requires a trusted caller identity",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn mutation_context() -> Result<MutationContext, AdminError> {
|
||||
let actor = ACTOR
|
||||
.try_with(Clone::clone)
|
||||
.ok()
|
||||
.flatten()
|
||||
.ok_or_else(|| AdminError::invalid("服务写入缺少可信调用主体"))?;
|
||||
super::CURRENT
|
||||
.try_with(|origin| {
|
||||
let (actor, request_id) = match actor {
|
||||
UsageActor::Plugin { instance_id, .. } => (
|
||||
MutationActor::System,
|
||||
format!(
|
||||
"plugin:{instance_id}:request:{}:call:{}",
|
||||
origin.request_id, origin.call_id
|
||||
),
|
||||
),
|
||||
UsageActor::Admin { subject } => (
|
||||
MutationActor::AdminSession {
|
||||
admin_user_id: subject,
|
||||
},
|
||||
origin.request_id.clone(),
|
||||
),
|
||||
UsageActor::System => (MutationActor::System, origin.request_id.clone()),
|
||||
};
|
||||
MutationContext { actor, request_id }
|
||||
})
|
||||
.map_err(|_| AdminError::invalid("服务写入缺少调用上下文"))
|
||||
}
|
||||
@@ -1,7 +1,11 @@
|
||||
//! 主动服务调用的统一组合入口;普通业务方法由所属 HTTP 或执行边界包裹
|
||||
|
||||
mod accounts;
|
||||
mod actor;
|
||||
mod registry;
|
||||
mod settings;
|
||||
mod usage_data;
|
||||
pub use actor::scope_actor;
|
||||
pub use registry::Registry;
|
||||
|
||||
use std::{future::Future, sync::Arc};
|
||||
@@ -98,8 +102,10 @@ pub(crate) async fn invoke(
|
||||
plan: plan.clone(),
|
||||
};
|
||||
let inner = context.clone();
|
||||
// next 可在另一回调任务中执行,终端必须保留发起主体及其缺席状态
|
||||
let actor = actor::current_actor().ok();
|
||||
let next = compose(Vec::new(), move |input| {
|
||||
Box::pin(scope(inner, terminal(input)))
|
||||
Box::pin(actor::scope_captured(actor, scope(inner, terminal(input))))
|
||||
});
|
||||
tokio::select! {
|
||||
biased;
|
||||
|
||||
@@ -29,6 +29,13 @@ impl Registry {
|
||||
super::settings::register(self, service)
|
||||
}
|
||||
|
||||
pub fn register_accounts(
|
||||
&mut self,
|
||||
service: &Arc<dyn crate::AccountsService>,
|
||||
) -> Result<(), AdminError> {
|
||||
super::accounts::register(self, service)
|
||||
}
|
||||
|
||||
pub(crate) fn register<I, O>(
|
||||
&mut self,
|
||||
name: &'static str,
|
||||
@@ -54,6 +61,57 @@ impl Registry {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn register_usage_data(
|
||||
&mut self,
|
||||
service: &Arc<dyn crate::UsageDataService>,
|
||||
) -> Result<(), AdminError> {
|
||||
super::usage_data::register(self, service)
|
||||
}
|
||||
|
||||
pub(crate) fn register_usage<I, O>(
|
||||
&mut self,
|
||||
name: &'static str,
|
||||
handler: impl Fn(I) -> BoxFuture<'static, crate::model::usage_data::UsageDataResult<O>>
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static,
|
||||
) -> Result<(), AdminError>
|
||||
where
|
||||
I: DeserializeOwned + Send + 'static,
|
||||
O: Serialize + Send + 'static,
|
||||
{
|
||||
let std::collections::btree_map::Entry::Vacant(entry) = self.operations.entry(name) else {
|
||||
return Err(AdminError::internal(format!("公开服务重复登记:{name}")));
|
||||
};
|
||||
let handler = Arc::new(handler);
|
||||
entry.insert(Arc::new(move |input| {
|
||||
let handler = handler.clone();
|
||||
Box::pin(async move {
|
||||
// 在反序列化之前限制编码后的完整请求,避免单条记录绕过批量行数约束
|
||||
let bytes =
|
||||
serde_json::to_vec(&input).map_err(|_| Error::invalid("服务输入编码无效"))?;
|
||||
if bytes.len() > crate::model::usage_data::USAGE_DATA_MAX_REQUEST_BYTES {
|
||||
return Err(super::usage_data::encode_error(
|
||||
crate::model::usage_data::UsageDataError::new(
|
||||
crate::model::usage_data::UsageErrorKind::CapacityExceeded,
|
||||
"usage input exceeds 2 MiB",
|
||||
),
|
||||
));
|
||||
}
|
||||
let input = serde_json::from_value(input).map_err(|_| {
|
||||
super::usage_data::encode_error(
|
||||
crate::model::usage_data::UsageDataError::invalid("服务输入类型无效"),
|
||||
)
|
||||
})?;
|
||||
let output = handler(input)
|
||||
.await
|
||||
.map_err(super::usage_data::encode_error)?;
|
||||
serde_json::to_value(output).map_err(|_| Error::unavailable("服务结果编码失败"))
|
||||
})
|
||||
}));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn call(&self, operation: &str, input: Value) -> Result<Value, Error> {
|
||||
let (operation, handler) =
|
||||
self.operations
|
||||
|
||||
@@ -25,6 +25,13 @@ pub(super) fn register(
|
||||
}};
|
||||
}
|
||||
register!(Load, load, (), ());
|
||||
register!(
|
||||
PreviewPrivacyPolicy,
|
||||
preview_privacy_policy,
|
||||
gateway_core::settings::privacy::PrivacyPreviewRequest,
|
||||
request,
|
||||
request
|
||||
);
|
||||
register!(
|
||||
Replace,
|
||||
replace,
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
//! 原生用量能力的公开服务登记
|
||||
|
||||
use crate::{
|
||||
UsageDataService,
|
||||
model::{AdminError, usage_data::*},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) fn register(
|
||||
registry: &mut super::Registry,
|
||||
service: &Arc<dyn UsageDataService>,
|
||||
) -> Result<(), AdminError> {
|
||||
macro_rules! read {
|
||||
($name:literal, $method:ident, $input:ty) => {{
|
||||
let service = service.clone();
|
||||
registry.register_usage($name, move |request: $input| {
|
||||
let service = service.clone();
|
||||
Box::pin(async move { service.$method(request).await })
|
||||
})?;
|
||||
}};
|
||||
}
|
||||
read!("usage.records.get", get_record, GetUsageRecord);
|
||||
read!("usage.records.list", list_records, ListUsageRecords);
|
||||
read!(
|
||||
"usage.aggregates.list",
|
||||
list_aggregates,
|
||||
ListUsageAggregates
|
||||
);
|
||||
read!("pricing.evaluate", evaluate_pricing, EvaluateUsagePricing);
|
||||
let aggregates = service.clone();
|
||||
registry.register_usage(
|
||||
"usage.aggregates.write",
|
||||
move |request: WriteUsageAggregates| {
|
||||
let service = aggregates.clone();
|
||||
Box::pin(async move {
|
||||
let actor = super::actor::current_actor()?;
|
||||
service.write_aggregates(&actor, request).await
|
||||
})
|
||||
},
|
||||
)?;
|
||||
let service = service.clone();
|
||||
registry.register_usage("usage.records.write", move |request: WriteUsageRecords| {
|
||||
let service = service.clone();
|
||||
Box::pin(async move {
|
||||
let actor = super::actor::current_actor()?;
|
||||
service.write_records(&actor, request).await
|
||||
})
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn encode_error(
|
||||
error: UsageDataError,
|
||||
) -> gateway_core::engine::middleware::service::Error {
|
||||
gateway_core::engine::middleware::service::Error {
|
||||
kind: serde_json::to_value(error.kind)
|
||||
.ok()
|
||||
.and_then(|v| v.as_str().map(str::to_owned))
|
||||
.unwrap_or_else(|| "unavailable".into()),
|
||||
message: error.message.clone(),
|
||||
details: serde_json::to_value(error).ok(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
//! 每日错峰发现与已知卡片的临期执行,周期只读取本地到期计划
|
||||
|
||||
use crate::{
|
||||
model::reset_credit_automation::{PlannedResetCredit, ResetCreditAutomation},
|
||||
ports::store::{AccountStore, SettingsStore},
|
||||
use_case::accounts::AccountsService,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
task::{ScheduledTask, WorkerCycleContext, WorkerTaskError},
|
||||
time::DeploymentTimeZone,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
/// 当地次日随机发现时间;随机数由调用者提供,结果持久化后不会因重启重抽
|
||||
pub(crate) fn next_discovery(
|
||||
now: DateTime<Utc>,
|
||||
timezone: DeploymentTimeZone,
|
||||
random: u64,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
let start = timezone.days_after(now, 1)?;
|
||||
let end = timezone.days_after(now, 2)?;
|
||||
let seconds = u64::try_from((end - start).num_seconds()).ok()?;
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
start.checked_add_signed(chrono::Duration::seconds(
|
||||
i64::try_from(random % seconds).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// 提前窗口的前 10% 用于账号错峰,最多一分钟,其余时间保留给网络失败恢复
|
||||
pub(crate) fn consume_at(
|
||||
account_id: &str,
|
||||
credit: &PlannedResetCredit,
|
||||
lead_minutes: u32,
|
||||
) -> DateTime<Utc> {
|
||||
let lead_seconds = i64::from(lead_minutes) * 60;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(account_id.as_bytes());
|
||||
hash.update([0]);
|
||||
hash.update(credit.id.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let offset =
|
||||
i64::from(u16::from_be_bytes([digest[0], digest[1]])) % (lead_seconds / 10).clamp(1, 60);
|
||||
credit.expires_at - chrono::Duration::seconds(lead_seconds - offset)
|
||||
}
|
||||
|
||||
pub(crate) fn next_consumption(
|
||||
record: &ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| consume_at(&record.account_id, credit, lead_minutes))
|
||||
.min()
|
||||
}
|
||||
|
||||
pub(crate) fn reschedule(
|
||||
record: &mut ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
now: DateTime<Utc>,
|
||||
) {
|
||||
record.next_run_at = if let Some(retry) = record.state.retry_at {
|
||||
retry
|
||||
} else if record.state.pending.is_some() {
|
||||
now
|
||||
} else {
|
||||
next_consumption(record, lead_minutes).map_or(record.state.next_discovery_at, |due| {
|
||||
due.min(record.state.next_discovery_at)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) struct ResetCreditAutomationTask {
|
||||
pub(crate) accounts: Arc<dyn AccountsService>,
|
||||
pub(crate) store: Arc<dyn AccountStore>,
|
||||
pub(crate) settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
impl ScheduledTask for ResetCreditAutomationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡设置读取失败"))?;
|
||||
// 单轮有界,积压沿到期顺序继续;并发复用已有账号刷新配置
|
||||
let due = self
|
||||
.store
|
||||
.due_reset_credit_automations(Utc::now(), 100)
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡计划读取失败"))?;
|
||||
let work = futures::stream::iter(due)
|
||||
.map(|account_id| async move {
|
||||
let id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡账号 ID 不合法"))?;
|
||||
if let Err(error) = self.accounts.reconcile_reset_credit_automation(id).await {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error_kind = error.kind().as_str(),
|
||||
"自动重置卡任务失败"
|
||||
);
|
||||
}
|
||||
Ok::<_, WorkerTaskError>(())
|
||||
})
|
||||
.buffer_unordered(settings.values.refresh_concurrency.max(1) as usize)
|
||||
.collect::<Vec<_>>();
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
_ = tokio::time::timeout(Duration::from_secs(30), work) => {},
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -160,6 +160,7 @@ impl AccountGroupService for DefaultAccountGroupService {
|
||||
self.store
|
||||
.create_account_group(
|
||||
NewAccountGroup {
|
||||
allow_image_generation: command.allow_image_generation,
|
||||
id,
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
|
||||
+117
-34
@@ -1,5 +1,7 @@
|
||||
//! 统一账号目录与原生 Provider 分派
|
||||
|
||||
mod reset_credit_automation;
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
sync::Arc,
|
||||
@@ -52,6 +54,13 @@ const CONNECTION_TEST_INPUT: &str = "Reply with exactly OK.";
|
||||
/// 统一账号页消费的服务
|
||||
#[async_trait]
|
||||
pub trait AccountsService: Send + Sync {
|
||||
async fn concurrency(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
page: u32,
|
||||
page_size: u32,
|
||||
) -> Result<crate::model::accounts::AccountConcurrencyPage, AdminError>;
|
||||
|
||||
async fn list(&self, query: AccountListQuery) -> Result<AccountDirectoryPage, AdminError>;
|
||||
|
||||
async fn export(
|
||||
@@ -123,6 +132,29 @@ pub trait AccountsService: Send + Sync {
|
||||
Err(AdminError::invalid("当前 Provider 不支持账号头像"))
|
||||
}
|
||||
|
||||
async fn reset_credit_automation_status(
|
||||
&self,
|
||||
_account_id: ProviderAccountId,
|
||||
) -> Result<crate::model::reset_credit_automation::ResetCreditAutomationStatus, AdminError>
|
||||
{
|
||||
Err(AdminError::invalid("账号不支持自动重置卡"))
|
||||
}
|
||||
async fn set_reset_credit_automation(
|
||||
&self,
|
||||
_context: &MutationContext,
|
||||
_account_id: ProviderAccountId,
|
||||
_enabled: bool,
|
||||
) -> Result<crate::model::reset_credit_automation::ResetCreditAutomationStatus, AdminError>
|
||||
{
|
||||
Err(AdminError::invalid("账号不支持自动重置卡"))
|
||||
}
|
||||
async fn reconcile_reset_credit_automation(
|
||||
&self,
|
||||
_account_id: ProviderAccountId,
|
||||
) -> Result<(), AdminError> {
|
||||
Err(AdminError::invalid("账号不支持自动重置卡"))
|
||||
}
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
_context: &MutationContext,
|
||||
@@ -160,6 +192,8 @@ pub trait AccountsService: Send + Sync {
|
||||
pub(crate) struct DefaultAccountsService {
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
account_runtime: Arc<dyn AccountRuntimeStore>,
|
||||
settings: Arc<dyn crate::ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
@@ -172,6 +206,8 @@ impl DefaultAccountsService {
|
||||
pub(crate) fn new(
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
account_runtime: Arc<dyn AccountRuntimeStore>,
|
||||
settings: Arc<dyn crate::ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
@@ -179,6 +215,8 @@ impl DefaultAccountsService {
|
||||
Self {
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
@@ -381,6 +419,31 @@ impl DefaultAccountsService {
|
||||
|
||||
#[async_trait]
|
||||
impl AccountsService for DefaultAccountsService {
|
||||
async fn concurrency(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
page: u32,
|
||||
page_size: u32,
|
||||
) -> Result<crate::model::accounts::AccountConcurrencyPage, AdminError> {
|
||||
let offset = page
|
||||
.checked_sub(1)
|
||||
.and_then(|page| page.checked_mul(page_size))
|
||||
.filter(|_| (1..=100).contains(&page_size))
|
||||
.ok_or_else(|| AdminError::invalid("并发明细分页参数无效"))?;
|
||||
if !self
|
||||
.accounts
|
||||
.account_exists(account_id.as_str())
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "account concurrency"))?
|
||||
{
|
||||
return Err(AdminError::not_found("账号不存在"));
|
||||
}
|
||||
self.account_runtime
|
||||
.account_concurrency(account_id.as_str(), u64::from(offset), page_size)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "account concurrency"))
|
||||
}
|
||||
|
||||
async fn list(&self, query: AccountListQuery) -> Result<AccountDirectoryPage, AdminError> {
|
||||
let providers = &self.providers;
|
||||
let runtime = self
|
||||
@@ -891,27 +954,48 @@ impl AccountsService for DefaultAccountsService {
|
||||
.map_err(|error| map_provider_error(error, "provider profile avatar"))
|
||||
}
|
||||
|
||||
async fn reset_credit_automation_status(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<crate::model::reset_credit_automation::ResetCreditAutomationStatus, AdminError>
|
||||
{
|
||||
self.automation_status(&account_id).await
|
||||
}
|
||||
async fn set_reset_credit_automation(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
enabled: bool,
|
||||
) -> Result<crate::model::reset_credit_automation::ResetCreditAutomationStatus, AdminError>
|
||||
{
|
||||
self.configure_automation(context, &account_id, enabled)
|
||||
.await
|
||||
}
|
||||
async fn reconcile_reset_credit_automation(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<(), AdminError> {
|
||||
self.reconcile_automation(&account_id).await
|
||||
}
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<ProviderResetCredits, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
match provider.reset_credits(&account_id).await {
|
||||
Ok(credits) => Ok(credits),
|
||||
Err(error)
|
||||
if error.kind()
|
||||
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
|
||||
{
|
||||
self.refresh(context, account_id.clone()).await?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
provider
|
||||
.reset_credits(&account_id)
|
||||
.await
|
||||
.map_err(map_reset_credits_error_after_refresh)
|
||||
}
|
||||
Err(error) => Err(map_provider_error(error, "provider reset credits")),
|
||||
let lock = self.reset_credit_lock(&account_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
let credits = self
|
||||
.reset_credits_upstream(context, account_id.clone())
|
||||
.await?;
|
||||
if let Err(error) = self.remember_reset_credits(&account_id, &credits).await {
|
||||
tracing::warn!(
|
||||
account_id = account_id.as_str(),
|
||||
error_kind = error.kind().as_str(),
|
||||
"保存重置卡计划失败"
|
||||
);
|
||||
}
|
||||
Ok(credits)
|
||||
}
|
||||
|
||||
async fn consume_reset_credit(
|
||||
@@ -919,27 +1003,26 @@ impl AccountsService for DefaultAccountsService {
|
||||
context: &MutationContext,
|
||||
command: ConsumeProviderResetCredit,
|
||||
) -> Result<ProviderResetCreditResult, AdminError> {
|
||||
let account_id = command.account_id.clone();
|
||||
// 覆盖 credential refresh + 同键重试的完整账号级临界区,避免 401 两次调用
|
||||
// 之间插入另一笔不可逆消费
|
||||
let lock = self.reset_credit_lock(&account_id).await;
|
||||
let lock = self.reset_credit_lock(&command.account_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
match provider.consume_reset_credit(command.clone()).await {
|
||||
Ok(result) => Ok(result),
|
||||
Err(error)
|
||||
if error.kind()
|
||||
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
|
||||
{
|
||||
self.refresh(context, account_id.clone()).await?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
provider
|
||||
.consume_reset_credit(command)
|
||||
.await
|
||||
.map_err(map_reset_credits_error_after_refresh)
|
||||
}
|
||||
Err(error) => Err(map_provider_error(error, "provider reset-credit consume")),
|
||||
if self
|
||||
.accounts
|
||||
.reset_credit_automation(command.account_id.as_str())
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "reset-credit automation"))?
|
||||
.is_some_and(|record| {
|
||||
record
|
||||
.state
|
||||
.pending
|
||||
.is_some_and(|pending| pending.credit.expires_at > Utc::now())
|
||||
})
|
||||
{
|
||||
return Err(AdminError::new(
|
||||
crate::model::AdminErrorKind::Conflict,
|
||||
"自动重置结果待确认,请稍后刷新",
|
||||
));
|
||||
}
|
||||
self.consume_reset_credit_upstream(context, command).await
|
||||
}
|
||||
|
||||
async fn models(
|
||||
@@ -0,0 +1,513 @@
|
||||
//! 重置卡自动使用的账号锁、持久幂等键、查询缓存与重试编排
|
||||
|
||||
use super::*;
|
||||
|
||||
impl DefaultAccountsService {
|
||||
pub(super) async fn reset_credits_upstream(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<ProviderResetCredits, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
match provider.reset_credits(&account_id).await {
|
||||
Ok(credits) => Ok(credits),
|
||||
Err(error)
|
||||
if error.kind()
|
||||
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
|
||||
{
|
||||
self.refresh(context, account_id.clone()).await?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
provider
|
||||
.reset_credits(&account_id)
|
||||
.await
|
||||
.map_err(map_reset_credits_error_after_refresh)
|
||||
}
|
||||
Err(error) => Err(map_provider_error(error, "provider reset credits")),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn consume_reset_credit_upstream(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: ConsumeProviderResetCredit,
|
||||
) -> Result<ProviderResetCreditResult, AdminError> {
|
||||
let account_id = command.account_id.clone();
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
match provider.consume_reset_credit(command.clone()).await {
|
||||
Ok(result) => Ok(result),
|
||||
Err(error)
|
||||
if error.kind()
|
||||
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
|
||||
{
|
||||
self.refresh(context, account_id.clone()).await?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
provider
|
||||
.consume_reset_credit(command)
|
||||
.await
|
||||
.map_err(map_reset_credits_error_after_refresh)
|
||||
}
|
||||
Err(error) => Err(map_provider_error(error, "provider reset-credit consume")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
use crate::{
|
||||
model::{MutationActor, reset_credit_automation::*},
|
||||
reset_credit_automation::{consume_at, next_consumption, next_discovery, reschedule},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
|
||||
impl DefaultAccountsService {
|
||||
async fn automation_settings(
|
||||
&self,
|
||||
) -> Result<crate::model::settings::RuntimeSettingsValues, AdminError> {
|
||||
Ok(self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "runtime settings"))?
|
||||
.values)
|
||||
}
|
||||
|
||||
pub(super) async fn automation_status(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<ResetCreditAutomationStatus, AdminError> {
|
||||
self.require_reset_credit_automation(account_id).await?;
|
||||
let lead_minutes = self.automation_settings().await?.reset_credit_lead_minutes;
|
||||
let record = self
|
||||
.accounts
|
||||
.reset_credit_automation(account_id.as_str())
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "reset-credit automation"))?;
|
||||
Ok(ResetCreditAutomationStatus {
|
||||
enabled: record.as_ref().is_some_and(|r| r.enabled),
|
||||
lead_minutes,
|
||||
next_discovery_at: record
|
||||
.as_ref()
|
||||
.filter(|r| r.enabled)
|
||||
.map(|r| r.state.next_discovery_at),
|
||||
next_consume_at: record.as_ref().filter(|r| r.enabled).and_then(|r| {
|
||||
next_consumption(r, lead_minutes)
|
||||
.or_else(|| {
|
||||
r.state
|
||||
.pending
|
||||
.as_ref()
|
||||
.map(|pending| consume_at(&r.account_id, &pending.credit, lead_minutes))
|
||||
})
|
||||
.map(|due| r.state.retry_at.map_or(due, |retry| due.max(retry)))
|
||||
}),
|
||||
last_result: record.and_then(|r| {
|
||||
if r.state
|
||||
.pending
|
||||
.as_ref()
|
||||
.is_some_and(|pending| pending.credit.expires_at <= Utc::now())
|
||||
{
|
||||
Some("expired_unknown".to_owned())
|
||||
} else {
|
||||
r.state.last_result
|
||||
}
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
async fn require_reset_credit_automation(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<bool, AdminError> {
|
||||
let (stored, provider) = self.provider_for_account(account_id).await?;
|
||||
let capabilities =
|
||||
provider.account_capabilities(account_id, &stored.account.authentication_kind);
|
||||
if !capabilities.reset_credits || !capabilities.consume_reset_credit {
|
||||
return Err(AdminError::invalid("当前账号不支持自动重置卡"));
|
||||
}
|
||||
Ok(stored.account.enabled)
|
||||
}
|
||||
|
||||
pub(super) async fn configure_automation(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: &ProviderAccountId,
|
||||
enabled: bool,
|
||||
) -> Result<ResetCreditAutomationStatus, AdminError> {
|
||||
let lock = self.reset_credit_lock(account_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
self.require_reset_credit_automation(account_id).await?;
|
||||
let now = Utc::now();
|
||||
let mut record = self
|
||||
.accounts
|
||||
.reset_credit_automation(account_id.as_str())
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "reset-credit automation"))?
|
||||
.unwrap_or_else(|| ResetCreditAutomation {
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
enabled: false,
|
||||
revision: 0,
|
||||
next_run_at: now,
|
||||
state: ResetCreditAutomationState {
|
||||
next_discovery_at: now,
|
||||
credits: vec![],
|
||||
pending: None,
|
||||
retry_at: None,
|
||||
failures: 0,
|
||||
last_result: None,
|
||||
},
|
||||
});
|
||||
if record.enabled != enabled {
|
||||
record.enabled = enabled;
|
||||
// 首次开启错开一分钟;重新开启保留已知到期时间及未决幂等键
|
||||
if enabled && record.revision == 0 {
|
||||
record.state.next_discovery_at =
|
||||
now + chrono::Duration::seconds(i64::from(OsRng.next_u32() % 60));
|
||||
}
|
||||
let lead = self.automation_settings().await?.reset_credit_lead_minutes;
|
||||
reschedule(&mut record, lead, now);
|
||||
self.save_automation(&mut record, Some(context)).await?;
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "runtime settings"))?;
|
||||
publish_committed(self.snapshot.as_ref(), settings.config_revision).await?;
|
||||
}
|
||||
self.automation_status(account_id).await
|
||||
}
|
||||
|
||||
async fn save_automation(
|
||||
&self,
|
||||
record: &mut ResetCreditAutomation,
|
||||
context: Option<&MutationContext>,
|
||||
) -> Result<(), AdminError> {
|
||||
record.revision = self
|
||||
.accounts
|
||||
.save_reset_credit_automation(record, context)
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "reset-credit automation"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_reset_credits(
|
||||
&self,
|
||||
record: &mut ResetCreditAutomation,
|
||||
credits: &ProviderResetCredits,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<(), AdminError> {
|
||||
record.state.credits = credits
|
||||
.credits
|
||||
.iter()
|
||||
.filter(|credit| credit.status.as_deref() == Some("available"))
|
||||
.filter_map(|credit| {
|
||||
credit
|
||||
.expires_at
|
||||
.filter(|expiry| *expiry > now)
|
||||
.map(|expires_at| PlannedResetCredit {
|
||||
id: credit.id.clone(),
|
||||
expires_at,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
record.state.credits.sort_by_key(|credit| credit.expires_at);
|
||||
record.state.credits.dedup_by(|a, b| a.id == b.id);
|
||||
if record.state.next_discovery_at <= now
|
||||
|| self
|
||||
.timezone
|
||||
.local(record.state.next_discovery_at)
|
||||
.date_naive()
|
||||
== self.timezone.local(now).date_naive()
|
||||
{
|
||||
record.state.next_discovery_at =
|
||||
next_discovery(now, self.timezone, OsRng.next_u64())
|
||||
.ok_or_else(|| AdminError::internal("无法计算重置卡发现时间"))?;
|
||||
}
|
||||
if record.state.pending.is_none() {
|
||||
record.state.retry_at = None;
|
||||
record.state.failures = 0;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn remember_reset_credits(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
credits: &ProviderResetCredits,
|
||||
) -> Result<(), AdminError> {
|
||||
let Some(mut record) = self
|
||||
.accounts
|
||||
.reset_credit_automation(account_id.as_str())
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "reset-credit automation"))?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if !record.enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let now = Utc::now();
|
||||
self.observe_reset_credits(&mut record, credits, now)?;
|
||||
reschedule(
|
||||
&mut record,
|
||||
self.automation_settings().await?.reset_credit_lead_minutes,
|
||||
now,
|
||||
);
|
||||
self.save_automation(&mut record, None).await
|
||||
}
|
||||
|
||||
async fn automation_failed(
|
||||
&self,
|
||||
record: &mut ResetCreditAutomation,
|
||||
lead: u32,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<(), AdminError> {
|
||||
if record.state.pending.is_none()
|
||||
&& !next_consumption(record, lead).is_some_and(|due| due <= now)
|
||||
{
|
||||
record.state.next_discovery_at =
|
||||
next_discovery(now, self.timezone, OsRng.next_u64())
|
||||
.ok_or_else(|| AdminError::internal("无法计算重置卡发现时间"))?;
|
||||
record.state.retry_at = None;
|
||||
record.state.last_result = Some("discovery_failed".to_owned());
|
||||
reschedule(record, lead, now);
|
||||
return self.save_automation(record, None).await;
|
||||
}
|
||||
record.state.failures = record.state.failures.saturating_add(1);
|
||||
let delay = (30_i64 * 2_i64.pow(record.state.failures.min(4) - 1)).min(300)
|
||||
+ i64::from(OsRng.next_u32() % 30);
|
||||
record.state.retry_at = Some(now + chrono::Duration::seconds(delay));
|
||||
record.state.last_result = Some(
|
||||
if record.state.pending.is_some() {
|
||||
"pending"
|
||||
} else {
|
||||
"discovery_failed"
|
||||
}
|
||||
.to_owned(),
|
||||
);
|
||||
reschedule(record, lead, now);
|
||||
self.save_automation(record, None).await
|
||||
}
|
||||
|
||||
/// 暂不满足条件不丢弃卡片,也不创建未决消费,允许到期前再次判断
|
||||
async fn defer_automation(
|
||||
&self,
|
||||
record: &mut ResetCreditAutomation,
|
||||
lead: u32,
|
||||
reason: &str,
|
||||
) -> Result<(), AdminError> {
|
||||
let now = Utc::now();
|
||||
let retry = now + chrono::Duration::seconds(30 + i64::from(OsRng.next_u32() % 30));
|
||||
record.state.retry_at = Some(
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| credit.expires_at)
|
||||
.min()
|
||||
.map_or(retry, |expiry| retry.min(expiry)),
|
||||
);
|
||||
record.state.last_result = Some(reason.to_owned());
|
||||
record.state.failures = 0;
|
||||
reschedule(record, lead, now);
|
||||
self.save_automation(record, None).await
|
||||
}
|
||||
|
||||
async fn weekly_reset_credit_gate(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
max_remaining: u32,
|
||||
) -> Result<Option<&'static str>, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(account_id).await?;
|
||||
let quota = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "reset-credit weekly quota"))?;
|
||||
// 复用账号周窗口归属,不把短期、月度或模型专属桶当作剩余周额度
|
||||
let remaining = quota
|
||||
.usage_windows()
|
||||
.find(|(_, period)| *period == AccountUsagePeriod::Weekly)
|
||||
.and_then(|(window, _)| window.used_percent)
|
||||
.filter(|used| used.is_finite())
|
||||
.map(|used| 100.0 - used.clamp(0.0, 100.0));
|
||||
Ok(match remaining {
|
||||
Some(remaining) if remaining > f64::from(max_remaining) => {
|
||||
Some("weekly_quota_above_threshold")
|
||||
}
|
||||
Some(_) => None,
|
||||
None => Some("weekly_quota_unavailable"),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn reconcile_automation(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<(), AdminError> {
|
||||
let lock = self.reset_credit_lock(account_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
let Some(mut record) = self
|
||||
.accounts
|
||||
.reset_credit_automation(account_id.as_str())
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "reset-credit automation"))?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if !record.enabled {
|
||||
return Ok(());
|
||||
}
|
||||
match self.require_reset_credit_automation(account_id).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => return Ok(()),
|
||||
Err(error) => {
|
||||
record.next_run_at = next_discovery(Utc::now(), self.timezone, OsRng.next_u64())
|
||||
.ok_or_else(|| AdminError::internal("无法计算重置卡发现时间"))?;
|
||||
self.save_automation(&mut record, None).await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let now = Utc::now();
|
||||
let settings = self.automation_settings().await?;
|
||||
let lead = settings.reset_credit_lead_minutes;
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.retain(|credit| credit.expires_at > now);
|
||||
if record
|
||||
.state
|
||||
.pending
|
||||
.as_ref()
|
||||
.is_some_and(|pending| pending.credit.expires_at <= now)
|
||||
{
|
||||
record.state.pending = None;
|
||||
record.state.retry_at = None;
|
||||
record.state.last_result = Some("expired_unknown".to_owned());
|
||||
}
|
||||
if record.state.retry_at.is_some_and(|retry| retry > now) {
|
||||
reschedule(&mut record, lead, now);
|
||||
return self.save_automation(&mut record, None).await;
|
||||
}
|
||||
let context = MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "reset-credit-automation".to_owned(),
|
||||
};
|
||||
let due_credit = next_consumption(&record, lead).is_some_and(|due| due <= now);
|
||||
if record.state.pending.is_none()
|
||||
&& (due_credit
|
||||
|| record.state.next_discovery_at <= now
|
||||
|| record.state.retry_at.is_some())
|
||||
{
|
||||
// 先保存下一次发现,查询被取消或进程重启也不会重复刷普通发现接口
|
||||
if record.state.next_discovery_at <= now {
|
||||
record.state.next_discovery_at =
|
||||
next_discovery(now, self.timezone, OsRng.next_u64())
|
||||
.ok_or_else(|| AdminError::internal("无法计算重置卡发现时间"))?;
|
||||
}
|
||||
if due_credit || record.state.retry_at.is_some() {
|
||||
record.state.retry_at = Some(now + chrono::Duration::seconds(30));
|
||||
}
|
||||
reschedule(&mut record, lead, now);
|
||||
self.save_automation(&mut record, None).await?;
|
||||
// 真正消费前重新核对卡片;每日发现结果也可直接承担本次核对
|
||||
match self
|
||||
.reset_credits_upstream(&context, account_id.clone())
|
||||
.await
|
||||
{
|
||||
Ok(credits) => self.observe_reset_credits(&mut record, &credits, Utc::now())?,
|
||||
Err(error) => {
|
||||
self.automation_failed(&mut record, lead, Utc::now())
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
if let Some(credit) = record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.find(|credit| consume_at(account_id.as_str(), credit, lead) <= Utc::now())
|
||||
.cloned()
|
||||
{
|
||||
if let Some(max_remaining) = settings.reset_credit_max_weekly_remaining_percent {
|
||||
// 已发出的未决请求跳过此检查,继续沿原幂等键确认结果
|
||||
match self
|
||||
.weekly_reset_credit_gate(account_id, max_remaining)
|
||||
.await
|
||||
{
|
||||
Ok(Some(reason)) => {
|
||||
return self.defer_automation(&mut record, lead, reason).await;
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
self.defer_automation(&mut record, lead, "weekly_quota_unavailable")
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 刷新额度期间卡片可能到期,不能把已过期卡片持久化为待消费
|
||||
if credit.expires_at <= Utc::now() {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.retain(|credit| credit.expires_at > Utc::now());
|
||||
reschedule(&mut record, lead, Utc::now());
|
||||
return self.save_automation(&mut record, None).await;
|
||||
}
|
||||
record.state.pending = Some(PendingResetCredit {
|
||||
credit,
|
||||
redeem_request_id: uuid::Uuid::new_v4().to_string(),
|
||||
});
|
||||
record.state.last_result = Some("pending".to_owned());
|
||||
record.state.retry_at = Some(Utc::now() + chrono::Duration::seconds(30));
|
||||
reschedule(&mut record, lead, Utc::now());
|
||||
// 必须在发送消费请求前落盘,进程退出或响应丢失继续使用相同幂等键
|
||||
self.save_automation(&mut record, None).await?;
|
||||
}
|
||||
}
|
||||
let Some(pending) = record.state.pending.clone() else {
|
||||
reschedule(&mut record, lead, Utc::now());
|
||||
return self.save_automation(&mut record, None).await;
|
||||
};
|
||||
let command = ConsumeProviderResetCredit {
|
||||
account_id: account_id.clone(),
|
||||
credit_id: Some(pending.credit.id.clone()),
|
||||
redeem_request_id: uuid::Uuid::parse_str(&pending.redeem_request_id)
|
||||
.map_err(|_| AdminError::internal("重置卡幂等键不合法"))?,
|
||||
};
|
||||
match self.consume_reset_credit_upstream(&context, command).await {
|
||||
Ok(result) => {
|
||||
let confirmed = matches!(result.code.as_str(), "reset" | "already_redeemed");
|
||||
record.state.pending = None;
|
||||
if result.code == "nothing_to_reset" {
|
||||
return self
|
||||
.defer_automation(&mut record, lead, "nothing_to_reset")
|
||||
.await;
|
||||
}
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.retain(|credit| credit.id != pending.credit.id);
|
||||
record.state.retry_at = None;
|
||||
record.state.failures = 0;
|
||||
record.state.last_result = Some(if confirmed {
|
||||
result.code
|
||||
} else {
|
||||
"not_redeemed".to_owned()
|
||||
});
|
||||
reschedule(&mut record, lead, Utc::now());
|
||||
// 已确认的结果先落盘,再刷新额度;观测失败不能重放已完成的消费
|
||||
self.save_automation(&mut record, None).await?;
|
||||
if confirmed {
|
||||
let _ = self.quota(account_id, true).await;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.automation_failed(&mut record, lead, Utc::now())
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
//! 控制面认证端口、会话策略与访问和续期共用的身份校验
|
||||
|
||||
use std::{net::IpAddr, sync::Arc};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_core::engine::execution::ClientKeyVerifier;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use super::map_store_error;
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError,
|
||||
auth::{
|
||||
AuthSession, ChangePassword, LoginCommand, LoginError, SessionGrant, SessionSubject,
|
||||
},
|
||||
},
|
||||
ports::store::AuthStore,
|
||||
};
|
||||
|
||||
mod service;
|
||||
mod session;
|
||||
|
||||
/// 所有控制面接口消费同一个会话服务,权限由服务端身份决定
|
||||
#[async_trait]
|
||||
pub trait AuthService: Send + Sync {
|
||||
async fn change_password(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
command: ChangePassword,
|
||||
source_ip: IpAddr,
|
||||
) -> Result<(), AdminError>;
|
||||
async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminError>;
|
||||
async fn session(&self, session_id: Option<&str>) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn renew_session(
|
||||
&self,
|
||||
refresh_token: Option<&str>,
|
||||
access_token: Option<&str>,
|
||||
) -> Result<Option<SessionGrant>, AdminError>;
|
||||
async fn resolve_admin_user_id(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<String>, AdminError>;
|
||||
async fn verify_admin_api_key(&self, key: &str) -> Result<bool, AdminError>;
|
||||
async fn login(
|
||||
&self,
|
||||
command: LoginCommand,
|
||||
source_ip: IpAddr,
|
||||
previous_session_id: Option<&str>,
|
||||
) -> Result<SessionGrant, LoginError>;
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
const MAX_SESSION_TTL_MINUTES: i64 = 366 * 24 * 60;
|
||||
|
||||
pub(crate) struct DefaultAuthService {
|
||||
default_admin_user_id: String,
|
||||
admin_access_ttl: Duration,
|
||||
admin_session_ttl: Duration,
|
||||
admin_session_absolute_ttl: Duration,
|
||||
key_session_ttl: Duration,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
}
|
||||
|
||||
impl DefaultAuthService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
config: &crate::AdminConfig,
|
||||
key_session_ttl_minutes: u64,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
) -> Self {
|
||||
let admin_session_ttl = session_ttl(config.session_ttl_minutes);
|
||||
Self {
|
||||
default_admin_user_id: config.default_username.clone(),
|
||||
// 访问凭据最多有效 24 小时,较短的不活动期限也要留出续期窗口
|
||||
admin_access_ttl: Duration::hours(24).min(admin_session_ttl / 4),
|
||||
admin_session_ttl,
|
||||
admin_session_absolute_ttl: session_ttl(config.session_absolute_ttl_minutes),
|
||||
key_session_ttl: session_ttl(key_session_ttl_minutes),
|
||||
store,
|
||||
verifier,
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_live_session(&self, id: &str) -> Result<Option<AuthSession>, AdminError> {
|
||||
let Some(session) = self
|
||||
.store
|
||||
.load_session(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if session.retained_until() <= Utc::now()
|
||||
|| session
|
||||
.absolute_expires_at
|
||||
.is_some_and(|expiry| expiry <= Utc::now())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let valid = match &session.subject {
|
||||
SessionSubject::Admin {
|
||||
admin_user_id,
|
||||
credential_fingerprint,
|
||||
} => self
|
||||
.store
|
||||
.load_password_hash(admin_user_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator session"))?
|
||||
.is_some_and(|hash| password_fingerprint(&hash) == *credential_fingerprint),
|
||||
SessionSubject::Key { client_key_id } => self
|
||||
.store
|
||||
.client_key_enabled(client_key_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "session key"))?,
|
||||
};
|
||||
if !valid {
|
||||
let _ = self.store.delete_session(id).await;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(session))
|
||||
}
|
||||
}
|
||||
|
||||
fn session_ttl(minutes: u64) -> Duration {
|
||||
Duration::minutes(
|
||||
i64::try_from(minutes)
|
||||
.unwrap_or(MAX_SESSION_TTL_MINUTES)
|
||||
.clamp(1, MAX_SESSION_TTL_MINUTES),
|
||||
)
|
||||
}
|
||||
|
||||
// 指纹只绑定已加盐的密码哈希,不把密码或原始哈希复制到 Redis 会话
|
||||
fn password_fingerprint(password_hash: &str) -> String {
|
||||
URL_SAFE_NO_PAD.encode(Sha256::digest(password_hash.as_bytes()))
|
||||
}
|
||||
+80
-156
@@ -1,93 +1,35 @@
|
||||
//! 控制面登录、会话恢复与身份校验的唯一 owner
|
||||
//! 管理员和 Key 的登录、退出、密码策略与会话访问
|
||||
|
||||
use std::{net::IpAddr, sync::Arc, time::Duration as StdDuration};
|
||||
use std::{net::IpAddr, time::Duration as StdDuration};
|
||||
|
||||
use argon2::{Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_core::engine::execution::ClientKeyVerifier;
|
||||
use chrono::Utc;
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
AuthService, DefaultAuthService, password_fingerprint,
|
||||
session::{self, Credential, storage_id},
|
||||
};
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind,
|
||||
auth::{
|
||||
AdminAuditEvent, AuditActorKind, AuthSession, ChangePassword, LoginCommand, LoginError,
|
||||
LoginResult, SessionSubject,
|
||||
SessionGrant, SessionSubject,
|
||||
},
|
||||
},
|
||||
ports::store::AuthStore,
|
||||
use_case::map_store_error,
|
||||
};
|
||||
|
||||
use super::map_store_error;
|
||||
|
||||
/// 所有控制面接口消费同一个会话服务,权限由服务端身份决定
|
||||
#[async_trait]
|
||||
pub trait AuthService: Send + Sync {
|
||||
async fn change_password(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
command: ChangePassword,
|
||||
source_ip: IpAddr,
|
||||
) -> Result<(), AdminError>;
|
||||
async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminError>;
|
||||
async fn session(&self, session_id: Option<&str>) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn resolve_admin_user_id(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<String>, AdminError>;
|
||||
async fn verify_admin_api_key(&self, key: &str) -> Result<bool, AdminError>;
|
||||
async fn login(
|
||||
&self,
|
||||
command: LoginCommand,
|
||||
source_ip: IpAddr,
|
||||
previous_session_id: Option<&str>,
|
||||
) -> Result<LoginResult, LoginError>;
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
const MAX_SESSION_TTL_MINUTES: i64 = 366 * 24 * 60;
|
||||
const LOGIN_WINDOW: StdDuration = StdDuration::from_secs(60);
|
||||
const LOGIN_ATTEMPTS_PER_SOURCE: u32 = 10;
|
||||
const LOGIN_ATTEMPTS_GLOBAL: u32 = 200;
|
||||
|
||||
pub(crate) struct DefaultAuthService {
|
||||
default_admin_user_id: String,
|
||||
admin_session_ttl: Duration,
|
||||
admin_session_absolute_ttl: Duration,
|
||||
key_session_ttl: Duration,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
}
|
||||
|
||||
impl DefaultAuthService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
default_admin_user_id: impl Into<String>,
|
||||
admin_session_ttl_minutes: u64,
|
||||
admin_session_absolute_ttl_minutes: u64,
|
||||
key_session_ttl_minutes: u64,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
) -> Self {
|
||||
Self {
|
||||
default_admin_user_id: default_admin_user_id.into(),
|
||||
admin_session_ttl: session_ttl(admin_session_ttl_minutes),
|
||||
admin_session_absolute_ttl: session_ttl(admin_session_absolute_ttl_minutes),
|
||||
key_session_ttl: session_ttl(key_session_ttl_minutes),
|
||||
store,
|
||||
verifier,
|
||||
}
|
||||
}
|
||||
|
||||
fn auth_audit(&self, action: &str, admin_user_id: &str) -> AdminAuditEvent {
|
||||
AdminAuditEvent {
|
||||
id: format!("audit_{}", Uuid::now_v7().simple()),
|
||||
@@ -228,50 +170,23 @@ impl AuthService for DefaultAuthService {
|
||||
}
|
||||
|
||||
async fn session(&self, session_id: Option<&str>) -> Result<Option<AuthSession>, AdminError> {
|
||||
let Some(session_id) = session_id.filter(|value| !value.is_empty()) else {
|
||||
let Some(raw) = session_id.filter(|value| !value.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(session) = self
|
||||
.store
|
||||
.load_session(session_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?
|
||||
else {
|
||||
let Some(session) = self.load_live_session(storage_id(raw)).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if session.expires_at <= Utc::now()
|
||||
|| session
|
||||
.absolute_expires_at
|
||||
.is_some_and(|expiry| expiry <= Utc::now())
|
||||
{
|
||||
// Redis 的 TTL 负责过期清理,避免旧读取删除另一个请求刚续期的会话
|
||||
if session.expires_at <= Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if let SessionSubject::Admin {
|
||||
admin_user_id,
|
||||
credential_fingerprint,
|
||||
} = &session.subject
|
||||
{
|
||||
let password_hash = self
|
||||
.store
|
||||
.load_password_hash(admin_user_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator session"))?;
|
||||
if password_hash
|
||||
.is_none_or(|hash| password_fingerprint(&hash) != *credential_fingerprint)
|
||||
{
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
if let Some(renewal) = &session.renewal {
|
||||
let Some(credential) = Credential::parse(raw).filter(|value| !value.refresh) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !credential.owns(renewal) {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
if let SessionSubject::Key { client_key_id } = &session.subject
|
||||
&& !self
|
||||
.store
|
||||
.client_key_enabled(client_key_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "session key"))?
|
||||
{
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
} else if Credential::parse(raw).is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(session))
|
||||
@@ -279,33 +194,25 @@ impl AuthService for DefaultAuthService {
|
||||
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<AuthSession>, AdminError> {
|
||||
let Some(session_id) = session_id else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(session) = self.session(Some(session_id)).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Key 与升级前的会话保留原有期限,不能凭续期获得新的最长登录窗口
|
||||
let Some(absolute) = session
|
||||
.absolute_expires_at
|
||||
.filter(|_| matches!(session.subject, SessionSubject::Admin { .. }))
|
||||
else {
|
||||
return Ok(Some(session));
|
||||
};
|
||||
let now = Utc::now();
|
||||
let expires_at = (now + self.admin_session_ttl).min(absolute);
|
||||
let interval = (self.admin_session_ttl / 4).min(Duration::minutes(1));
|
||||
if expires_at <= session.expires_at
|
||||
|| session.expires_at - now > self.admin_session_ttl - interval
|
||||
{
|
||||
return Ok(Some(session));
|
||||
refresh_token: Option<&str>,
|
||||
access_token: Option<&str>,
|
||||
) -> Result<Option<SessionGrant>, AdminError> {
|
||||
if let Some(refresh) = refresh_token {
|
||||
return self.refresh(refresh).await;
|
||||
}
|
||||
self.store
|
||||
.renew_session(session_id, &session, expires_at)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session renewal"))
|
||||
// Key 与升级前的单凭据仅返回仍有效的原会话,不授予新的续期能力
|
||||
let Some(session) = self
|
||||
.session(access_token)
|
||||
.await?
|
||||
.filter(|session| session.renewal.is_none())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(SessionGrant {
|
||||
session_id: access_token.unwrap_or_default().to_owned(),
|
||||
refresh_token: None,
|
||||
session,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn resolve_admin_user_id(
|
||||
@@ -346,7 +253,7 @@ impl AuthService for DefaultAuthService {
|
||||
command: LoginCommand,
|
||||
source_ip: IpAddr,
|
||||
previous_session_id: Option<&str>,
|
||||
) -> Result<LoginResult, LoginError> {
|
||||
) -> Result<SessionGrant, LoginError> {
|
||||
if let Some(retry_after) = self
|
||||
.store
|
||||
.consume_login_attempt(
|
||||
@@ -370,14 +277,31 @@ impl AuthService for DefaultAuthService {
|
||||
let now = Utc::now();
|
||||
let absolute_expires_at = matches!(subject, SessionSubject::Admin { .. })
|
||||
.then_some(now + self.admin_session_absolute_ttl);
|
||||
let session = AuthSession {
|
||||
subject,
|
||||
expires_at: (now + ttl).min(absolute_expires_at.unwrap_or(now + ttl)),
|
||||
absolute_expires_at,
|
||||
let refresh_expiry = (now + ttl).min(absolute_expires_at.unwrap_or(now + ttl));
|
||||
let grant = if let Some(absolute) = absolute_expires_at {
|
||||
session::issue(
|
||||
subject,
|
||||
(now + self.admin_access_ttl).min(refresh_expiry),
|
||||
refresh_expiry,
|
||||
absolute,
|
||||
)
|
||||
.map_err(|_| LoginError::Unavailable)?
|
||||
} else {
|
||||
SessionGrant {
|
||||
session_id: random_session_token(),
|
||||
refresh_token: None,
|
||||
session: AuthSession {
|
||||
subject,
|
||||
expires_at: refresh_expiry,
|
||||
absolute_expires_at: None,
|
||||
renewal: None,
|
||||
},
|
||||
}
|
||||
};
|
||||
let session_id = random_session_token();
|
||||
let session_id = storage_id(&grant.session_id);
|
||||
let session = &grant.session;
|
||||
self.store
|
||||
.store_session(&session_id, &session)
|
||||
.store_session(session_id, session)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?;
|
||||
if let SessionSubject::Admin { admin_user_id, .. } = &session.subject
|
||||
@@ -387,26 +311,39 @@ impl AuthService for DefaultAuthService {
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
let _ = self.store.delete_session(&session_id).await;
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
return Err(LoginError::Unavailable);
|
||||
}
|
||||
// 新身份验证成功后才撤销旧会话;撤销失败时不向浏览器提交新会话
|
||||
if let Some(previous) = previous_session_id.filter(|value| !value.is_empty())
|
||||
&& self.logout(previous).await.is_err()
|
||||
{
|
||||
let _ = self.store.delete_session(&session_id).await;
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
return Err(LoginError::Unavailable);
|
||||
}
|
||||
Ok(LoginResult {
|
||||
session_id,
|
||||
session,
|
||||
})
|
||||
Ok(grant)
|
||||
}
|
||||
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError> {
|
||||
let id = storage_id(session_id);
|
||||
let Some(stored) = self
|
||||
.store
|
||||
.load_session(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(renewal) = &stored.renewal {
|
||||
if !Credential::parse(session_id).is_some_and(|credential| credential.owns(renewal)) {
|
||||
return Ok(());
|
||||
}
|
||||
} else if Credential::parse(session_id).is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let session = self
|
||||
.store
|
||||
.delete_session(session_id)
|
||||
.delete_session(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?;
|
||||
if let Some(AuthSession {
|
||||
@@ -423,14 +360,6 @@ impl AuthService for DefaultAuthService {
|
||||
}
|
||||
}
|
||||
|
||||
fn session_ttl(minutes: u64) -> Duration {
|
||||
Duration::minutes(
|
||||
i64::try_from(minutes)
|
||||
.unwrap_or(MAX_SESSION_TTL_MINUTES)
|
||||
.clamp(1, MAX_SESSION_TTL_MINUTES),
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_new_password(password: &str) -> Result<(), AdminError> {
|
||||
if password.trim().chars().count() < 12
|
||||
|| password.len() > 1024
|
||||
@@ -470,8 +399,3 @@ fn valid_admin_api_key_shape(value: &str) -> bool {
|
||||
&& value.starts_with("admin-")
|
||||
&& value[6..].bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
// 指纹只绑定已加盐的密码哈希,不把密码或原始哈希复制到 Redis 会话
|
||||
fn password_fingerprint(password_hash: &str) -> String {
|
||||
URL_SAFE_NO_PAD.encode(Sha256::digest(password_hash.as_bytes()))
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
//! 双凭据会话的签发、摘要校验与短窗口重试,不持久化可直接使用的凭据
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
|
||||
use super::DefaultAuthService;
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind,
|
||||
auth::{AuthSession, SessionGrant, SessionRefreshReplay, SessionRenewal, SessionSubject},
|
||||
},
|
||||
use_case::map_store_error,
|
||||
};
|
||||
|
||||
const REPLAY_WINDOW: Duration = Duration::seconds(30);
|
||||
|
||||
pub(super) struct Credential<'a> {
|
||||
pub id: &'a str,
|
||||
pub secret: &'a str,
|
||||
pub refresh: bool,
|
||||
}
|
||||
|
||||
impl<'a> Credential<'a> {
|
||||
pub fn parse(value: &'a str) -> Option<Self> {
|
||||
let mut parts = value.split('.');
|
||||
let refresh = match parts.next()? {
|
||||
"a2" => false,
|
||||
"r2" => true,
|
||||
_ => return None,
|
||||
};
|
||||
let id = parts.next()?;
|
||||
let secret = parts.next()?;
|
||||
if parts.next().is_some() || !valid_secret(id) || !valid_secret(secret) {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
id,
|
||||
secret,
|
||||
refresh,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn digest(&self) -> [u8; 32] {
|
||||
digest(self.secret)
|
||||
}
|
||||
|
||||
pub fn owns(&self, renewal: &SessionRenewal) -> bool {
|
||||
if self.refresh {
|
||||
matches_digest(&renewal.refresh_digest, &self.digest())
|
||||
|| renewal
|
||||
.previous
|
||||
.as_ref()
|
||||
.is_some_and(|previous| matches_digest(&previous.digest, &self.digest()))
|
||||
} else {
|
||||
matches_digest(&renewal.access_digest, &self.digest())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_secret(value: &str) -> bool {
|
||||
value.len() == 43
|
||||
&& URL_SAFE_NO_PAD
|
||||
.decode(value)
|
||||
.is_ok_and(|bytes| bytes.len() == 32)
|
||||
}
|
||||
|
||||
pub(super) fn storage_id(value: &str) -> &str {
|
||||
Credential::parse(value).map_or(value, |credential| credential.id)
|
||||
}
|
||||
|
||||
fn matches_digest(left: &[u8; 32], right: &[u8; 32]) -> bool {
|
||||
bool::from(left.ct_eq(right))
|
||||
}
|
||||
|
||||
fn digest(secret: &str) -> [u8; 32] {
|
||||
Sha256::digest(secret.as_bytes()).into()
|
||||
}
|
||||
|
||||
fn random_bytes() -> [u8; 32] {
|
||||
let mut bytes = [0; 32];
|
||||
OsRng.fill_bytes(&mut bytes);
|
||||
bytes
|
||||
}
|
||||
|
||||
// 使用标准 HKDF 的独立用途标签;只有盐和摘要留在 Store,重试仍须持有原始凭据
|
||||
fn derive(secret: &str, salt: Option<&[u8]>, purpose: &[u8]) -> Result<String, AdminError> {
|
||||
let mut output = [0; 32];
|
||||
hkdf::Hkdf::<Sha256>::new(salt, secret.as_bytes())
|
||||
.expand(purpose, &mut output)
|
||||
.map_err(|_| AdminError::internal("会话凭据派生失败"))?;
|
||||
Ok(URL_SAFE_NO_PAD.encode(output))
|
||||
}
|
||||
|
||||
fn access_secret(refresh: &str) -> Result<String, AdminError> {
|
||||
derive(refresh, None, b"cpr-session-v2/access")
|
||||
}
|
||||
|
||||
pub(super) fn issue(
|
||||
subject: SessionSubject,
|
||||
access_expiry: DateTime<Utc>,
|
||||
refresh_expiry: DateTime<Utc>,
|
||||
absolute: DateTime<Utc>,
|
||||
) -> Result<SessionGrant, AdminError> {
|
||||
let id = URL_SAFE_NO_PAD.encode(random_bytes());
|
||||
let secret = URL_SAFE_NO_PAD.encode(random_bytes());
|
||||
let session = AuthSession {
|
||||
subject,
|
||||
expires_at: access_expiry,
|
||||
absolute_expires_at: Some(absolute),
|
||||
renewal: Some(SessionRenewal {
|
||||
expires_at: refresh_expiry,
|
||||
access_digest: digest(&access_secret(&secret)?),
|
||||
refresh_digest: digest(&secret),
|
||||
previous: None,
|
||||
}),
|
||||
};
|
||||
grant(&id, &secret, session)
|
||||
}
|
||||
|
||||
fn grant(id: &str, refresh: &str, session: AuthSession) -> Result<SessionGrant, AdminError> {
|
||||
Ok(SessionGrant {
|
||||
session_id: format!("a2.{id}.{}", access_secret(refresh)?),
|
||||
refresh_token: Some(format!("r2.{id}.{refresh}")),
|
||||
session,
|
||||
})
|
||||
}
|
||||
|
||||
impl DefaultAuthService {
|
||||
pub(super) async fn refresh(&self, raw: &str) -> Result<Option<SessionGrant>, AdminError> {
|
||||
let Some(credential) = Credential::parse(raw).filter(|value| value.refresh) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut current = self.load_live_session(credential.id).await?;
|
||||
// CAS 冲突只重新解释同一请求的凭据,不对并发退出进行补建
|
||||
for _ in 0..3 {
|
||||
let Some(session) = current else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(renewal) = &session.renewal else {
|
||||
return Ok(None);
|
||||
};
|
||||
let now = Utc::now();
|
||||
if session.retained_until() <= now
|
||||
|| session.absolute_expires_at.is_none_or(|limit| limit <= now)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if matches_digest(&renewal.refresh_digest, &credential.digest()) {
|
||||
// 其他标签页已经刷新或客户端重试时复用当前代,避免 Cookie 响应相互覆盖
|
||||
if session.expires_at > now {
|
||||
return grant(credential.id, credential.secret, session).map(Some);
|
||||
}
|
||||
let absolute = session
|
||||
.absolute_expires_at
|
||||
.ok_or_else(|| AdminError::internal("缺少会话最长有效期"))?;
|
||||
let salt = random_bytes();
|
||||
let next_secret =
|
||||
derive(credential.secret, Some(&salt), b"cpr-session-v2/refresh")?;
|
||||
let refresh_expiry = (now + self.admin_session_ttl).min(absolute);
|
||||
let access_expiry = (now + self.admin_access_ttl).min(refresh_expiry);
|
||||
let replacement = AuthSession {
|
||||
expires_at: access_expiry,
|
||||
renewal: Some(SessionRenewal {
|
||||
expires_at: refresh_expiry,
|
||||
access_digest: digest(&access_secret(&next_secret)?),
|
||||
refresh_digest: digest(&next_secret),
|
||||
previous: Some(SessionRefreshReplay {
|
||||
digest: credential.digest(),
|
||||
salt,
|
||||
expires_at: (now + REPLAY_WINDOW).min(access_expiry),
|
||||
}),
|
||||
}),
|
||||
..session.clone()
|
||||
};
|
||||
current = self
|
||||
.store
|
||||
.compare_exchange_session(credential.id, &session, Some(&replacement))
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session rotation"))?;
|
||||
if current.as_ref() == Some(&replacement) {
|
||||
return grant(credential.id, &next_secret, replacement).map(Some);
|
||||
}
|
||||
} else if let Some(previous) = renewal
|
||||
.previous
|
||||
.as_ref()
|
||||
.filter(|previous| matches_digest(&previous.digest, &credential.digest()))
|
||||
{
|
||||
if previous.expires_at > now {
|
||||
let secret = derive(
|
||||
credential.secret,
|
||||
Some(&previous.salt),
|
||||
b"cpr-session-v2/refresh",
|
||||
)?;
|
||||
return grant(credential.id, &secret, session).map(Some);
|
||||
}
|
||||
// 已知上一代在重试窗口外再次使用,原子撤销整个会话
|
||||
current = self
|
||||
.store
|
||||
.compare_exchange_session(credential.id, &session, None)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session revocation"))?;
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Err(AdminError::new(
|
||||
AdminErrorKind::Unavailable,
|
||||
"会话正在变更,请重试",
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,8 @@ pub(super) async fn pending_authorization(
|
||||
proxy = required_credential(accounts, provider_kind, account_id, resource)
|
||||
.await?
|
||||
.credential
|
||||
.outbound_proxy;
|
||||
.effective_outbound_proxy()
|
||||
.cloned();
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: account_id.clone(),
|
||||
}
|
||||
@@ -121,6 +122,9 @@ pub(super) async fn pending_authorization(
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
proxy_id = Some(record.id);
|
||||
if !record.enabled {
|
||||
return Err(AdminError::invalid("代理已禁用,请启用后再选择"));
|
||||
}
|
||||
Some(record.proxy)
|
||||
}
|
||||
Some(AccountProxySelection::Direct) | None => None,
|
||||
|
||||
@@ -192,10 +192,12 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
.ok_or_else(|| AdminError::internal("时间超出支持范围"))?,
|
||||
end: now,
|
||||
};
|
||||
let (overview, trend, health_points) = futures::try_join!(
|
||||
self.observations.usage_summary(query.range, filter.clone()),
|
||||
self.observations
|
||||
.usage_trend(query.range, filter, Granularity::for_range(query.range)),
|
||||
let (observation, health_points) = futures::try_join!(
|
||||
self.observations.usage_insights_observation(
|
||||
query.range,
|
||||
filter,
|
||||
Granularity::for_range(query.range)
|
||||
),
|
||||
self.observations.usage_trend(
|
||||
today,
|
||||
usage_filter(&id, None),
|
||||
@@ -205,8 +207,8 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
.map_err(|error| map_store_error(error, "key usage overview"))?;
|
||||
Ok(Some(KeyUsageOverview {
|
||||
key,
|
||||
overview,
|
||||
trend,
|
||||
overview: observation.overview,
|
||||
trend: observation.trend,
|
||||
health_timeline: health_timeline_at(&health_points, now, self.timezone)?,
|
||||
}))
|
||||
}
|
||||
@@ -223,6 +225,7 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
KeyUsageRecordKind::Success => KeyUsageRecords::Success(
|
||||
self.observations
|
||||
.list_usage_records(UsageQuery {
|
||||
status: Default::default(),
|
||||
range: query.usage.range,
|
||||
filter: usage_filter(&id, query.usage.model),
|
||||
current_page: query.current_page,
|
||||
|
||||
@@ -18,6 +18,7 @@ pub mod plugins;
|
||||
pub mod proxies;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod usage_data;
|
||||
|
||||
mod credential_mutation;
|
||||
mod error;
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
//! 按故障目标收集诊断证据,补充读取失败不丢弃已取得的记录
|
||||
|
||||
use std::{
|
||||
collections::{HashSet, VecDeque},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use tokio::time::{Instant, timeout_at};
|
||||
|
||||
use super::{DefaultObservabilityService, map_store_error};
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind,
|
||||
diagnostics::{DiagnosticBundle, DiagnosticGap, DiagnosticRequest, DiagnosticTarget},
|
||||
observability::OpsError,
|
||||
};
|
||||
|
||||
const MAX_REQUESTS: usize = 8;
|
||||
const MAX_ERRORS: u16 = 128;
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
pub(super) async fn collect_diagnostics(
|
||||
&self,
|
||||
target: DiagnosticTarget,
|
||||
) -> Result<DiagnosticBundle, AdminError> {
|
||||
if target.id().trim().is_empty() || target.id().len() > 256 {
|
||||
return Err(AdminError::invalid("诊断目标 ID 不合法"));
|
||||
}
|
||||
let deadline = Instant::now() + Duration::from_secs(20);
|
||||
let (collected, environment) = tokio::join!(
|
||||
self.collect_records(target, deadline),
|
||||
timeout_at(deadline, self.system.version()),
|
||||
);
|
||||
let mut bundle = collected?;
|
||||
match environment {
|
||||
Ok(Ok(version)) => bundle.environment = Some(version),
|
||||
_ => gap(&mut bundle, "environment", "collection_failed"),
|
||||
}
|
||||
Ok(bundle)
|
||||
}
|
||||
|
||||
async fn collect_records(
|
||||
&self,
|
||||
target: DiagnosticTarget,
|
||||
deadline: Instant,
|
||||
) -> Result<DiagnosticBundle, AdminError> {
|
||||
let mut bundle = DiagnosticBundle {
|
||||
target,
|
||||
collected_at: chrono::Utc::now(),
|
||||
environment: None,
|
||||
error: None,
|
||||
requests: Vec::new(),
|
||||
gaps: Vec::new(),
|
||||
};
|
||||
let mut remaining_errors = MAX_ERRORS;
|
||||
let first_id = match &bundle.target {
|
||||
DiagnosticTarget::Request(id) => Some(id.clone()),
|
||||
DiagnosticTarget::Operation(_) => {
|
||||
let mut errors = self.export_errors(&bundle.target, 1, deadline).await?;
|
||||
let error = errors
|
||||
.pop()
|
||||
.ok_or_else(|| AdminError::not_found("错误记录不存在或已过期"))?;
|
||||
let id = error.request_id.clone();
|
||||
bundle.error = Some(error);
|
||||
remaining_errors -= 1;
|
||||
id
|
||||
}
|
||||
};
|
||||
let mut pending = VecDeque::from_iter(first_id);
|
||||
let mut seen = HashSet::new();
|
||||
while let Some(id) = pending.pop_front() {
|
||||
if !seen.insert(id.clone()) {
|
||||
continue;
|
||||
}
|
||||
if bundle.requests.len() >= MAX_REQUESTS {
|
||||
gap(&mut bundle, "requests", "related_request_limit");
|
||||
break;
|
||||
}
|
||||
let section = format!("requests.{id}");
|
||||
let target = DiagnosticTarget::Request(id.clone());
|
||||
let selected = bundle
|
||||
.error
|
||||
.as_ref()
|
||||
.filter(|error| error.request_id.as_deref() == Some(id.as_str()));
|
||||
// 事件导出已经持有选中记录,额外读取一个槽位用于去重后判断是否超限
|
||||
let read_limit = remaining_errors + 1 + u16::from(selected.is_some());
|
||||
let (detail, errors) = tokio::join!(
|
||||
timeout_at(deadline, self.store.usage_record_detail(&id)),
|
||||
self.export_errors(&target, read_limit, deadline),
|
||||
);
|
||||
let (detail, detail_error) = match detail {
|
||||
Ok(Ok(detail)) => (Some(detail), None),
|
||||
Ok(Err(error)) => (None, Some(map_store_error(error, "diagnostic request"))),
|
||||
Err(_) => (
|
||||
None,
|
||||
Some(AdminError::new(AdminErrorKind::Unavailable, "诊断采集超时")),
|
||||
),
|
||||
};
|
||||
let mut errors = match errors {
|
||||
Ok(mut errors) => {
|
||||
if let Some(selected) = selected {
|
||||
errors.retain(|error| {
|
||||
error.source != selected.source || error.event_id != selected.event_id
|
||||
});
|
||||
}
|
||||
errors
|
||||
}
|
||||
Err(error) => {
|
||||
if detail.is_none() && bundle.requests.is_empty() && bundle.error.is_none() {
|
||||
return Err(error);
|
||||
}
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.errors"),
|
||||
"collection_failed",
|
||||
);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
if detail.is_none()
|
||||
&& errors.is_empty()
|
||||
&& bundle.requests.is_empty()
|
||||
&& bundle.error.is_none()
|
||||
{
|
||||
return Err(
|
||||
detail_error.unwrap_or_else(|| AdminError::not_found("请求记录不存在或已过期"))
|
||||
);
|
||||
}
|
||||
if errors.len() > usize::from(remaining_errors) {
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.errors"),
|
||||
"error_record_limit",
|
||||
);
|
||||
errors.truncate(usize::from(remaining_errors));
|
||||
}
|
||||
remaining_errors =
|
||||
remaining_errors.saturating_sub(u16::try_from(errors.len()).unwrap_or(MAX_ERRORS));
|
||||
if let Some(record) = &detail {
|
||||
for related in &record.related_requests {
|
||||
if let Some(id) = related.get("requestId").and_then(serde_json::Value::as_str) {
|
||||
pending.push_back(id.to_owned());
|
||||
}
|
||||
}
|
||||
if record.request.outcome == crate::model::observability::RequestOutcome::Running {
|
||||
gap(&mut bundle, §ion, "request_not_finalized");
|
||||
}
|
||||
match &record.trace {
|
||||
None => gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"not_recorded_or_retained",
|
||||
),
|
||||
Some(trace) => {
|
||||
if trace["droppedEvents"].as_u64().unwrap_or(0) > 0 {
|
||||
gap(&mut bundle, &format!("{section}.trace"), "events_evicted");
|
||||
}
|
||||
let events = trace["events"].as_array();
|
||||
if !events.is_some_and(|events| {
|
||||
events.iter().any(|event| {
|
||||
event["stage"] == "request.routed"
|
||||
&& event["data"]["accountPolicy"].is_object()
|
||||
})
|
||||
}) {
|
||||
gap(&mut bundle, §ion, "incident_policy_not_recorded");
|
||||
}
|
||||
for event in events.into_iter().flatten() {
|
||||
if event["stage"] == "capture.gap" || event["data"]["truncated"] == true
|
||||
{
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"capture_gap_or_truncation",
|
||||
);
|
||||
}
|
||||
if event["stage"] == "retry.decided" && event["data"]["phase"].is_null()
|
||||
{
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"legacy_retry_facts_not_recorded",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.attempts"),
|
||||
"best_effort_persistence",
|
||||
);
|
||||
} else {
|
||||
gap(&mut bundle, §ion, "request_detail_unavailable");
|
||||
}
|
||||
bundle.requests.push(DiagnosticRequest {
|
||||
request_id: id,
|
||||
detail,
|
||||
errors,
|
||||
});
|
||||
}
|
||||
Ok(bundle)
|
||||
}
|
||||
|
||||
async fn export_errors(
|
||||
&self,
|
||||
target: &DiagnosticTarget,
|
||||
limit: u16,
|
||||
deadline: Instant,
|
||||
) -> Result<Vec<OpsError>, AdminError> {
|
||||
timeout_at(deadline, self.store.diagnostic_errors(target, limit))
|
||||
.await
|
||||
.map_err(|_| AdminError::new(AdminErrorKind::Unavailable, "诊断采集超时"))?
|
||||
.map_err(|error| map_store_error(error, "diagnostic errors"))
|
||||
}
|
||||
}
|
||||
|
||||
fn gap(bundle: &mut DiagnosticBundle, section: &str, reason: &'static str) {
|
||||
if !bundle
|
||||
.gaps
|
||||
.iter()
|
||||
.any(|gap| gap.section == section && gap.reason == reason)
|
||||
{
|
||||
bundle.gaps.push(DiagnosticGap {
|
||||
section: section.to_owned(),
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
+180
-176
@@ -4,12 +4,10 @@ use std::{
|
||||
collections::{BTreeMap, HashMap, btree_map::Entry},
|
||||
str::FromStr,
|
||||
sync::Arc,
|
||||
time::{Duration as StdDuration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use futures::TryStreamExt;
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind};
|
||||
|
||||
use crate::{
|
||||
@@ -21,16 +19,16 @@ use crate::{
|
||||
DiagnosticDimension, DiagnosticsItem, DiagnosticsResult, Granularity, HealthStatus,
|
||||
HealthTimeline, HealthTimelinePoint, OpsErrorPage, OpsErrorQuery, ProviderBillingInput,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, Trend, TrendKind, TrendPoint,
|
||||
TrendSummary, UsageBilling, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageInsights, UsageInsightsCost, UsageInsightsCostPoint, UsageInsightsHealth,
|
||||
UsageInsightsHealthPoint, UsageInsightsPerformance, UsageInsightsPerformancePoint,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageSummary,
|
||||
TrendSummary, UsageBilling, UsageBillingProjection, UsageCalculatedBillingFact,
|
||||
UsageDetail, UsageFilter, UsageInsights, UsageInsightsCost, UsageInsightsCostPoint,
|
||||
UsageInsightsHealth, UsageInsightsHealthPoint, UsageInsightsPerformance,
|
||||
UsageInsightsPerformancePoint, UsageOverview, UsagePage, UsageQuery, UsageSummary,
|
||||
},
|
||||
provider_credentials::ProviderQuotaRequest,
|
||||
},
|
||||
ports::{
|
||||
provider::{ProviderAdminErrorKind, ProviderAdminRegistry},
|
||||
store::{AccountStore, ObservabilityStore, SettingsStore, UsageCalculatedBillingStream},
|
||||
store::{AccountStore, ObservabilityStore, SettingsStore, UsageBillingProjector},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -39,59 +37,10 @@ use super::{map_provider_error, map_store_error};
|
||||
const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15;
|
||||
const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10;
|
||||
const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3;
|
||||
const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 99.0;
|
||||
const DASHBOARD_QUERY_BUCKET_SECONDS: i64 = 2;
|
||||
const DASHBOARD_QUERY_CACHE_RETENTION: StdDuration = StdDuration::from_secs(2);
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
|
||||
struct DashboardQueryKey {
|
||||
start: DateTime<Utc>,
|
||||
end_bucket: i64,
|
||||
}
|
||||
|
||||
impl DashboardQueryKey {
|
||||
fn new(range: TimeRange) -> Self {
|
||||
Self {
|
||||
start: range.start,
|
||||
end_bucket: range
|
||||
.end
|
||||
.timestamp()
|
||||
.div_euclid(DASHBOARD_QUERY_BUCKET_SECONDS),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DashboardQueryEntry {
|
||||
created_at: Instant,
|
||||
result: Arc<tokio::sync::OnceCell<Result<DashboardResult, AdminError>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DashboardQueryCache {
|
||||
entries: tokio::sync::Mutex<HashMap<DashboardQueryKey, DashboardQueryEntry>>,
|
||||
}
|
||||
|
||||
impl DashboardQueryCache {
|
||||
async fn result_cell(
|
||||
&self,
|
||||
key: DashboardQueryKey,
|
||||
) -> Arc<tokio::sync::OnceCell<Result<DashboardResult, AdminError>>> {
|
||||
let now = Instant::now();
|
||||
let mut entries = self.entries.lock().await;
|
||||
entries.retain(|_, entry| {
|
||||
now.duration_since(entry.created_at) < DASHBOARD_QUERY_CACHE_RETENTION
|
||||
});
|
||||
Arc::clone(
|
||||
&entries
|
||||
.entry(key)
|
||||
.or_insert_with(|| DashboardQueryEntry {
|
||||
created_at: now,
|
||||
result: Arc::new(tokio::sync::OnceCell::new()),
|
||||
})
|
||||
.result,
|
||||
)
|
||||
}
|
||||
}
|
||||
const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 95.0;
|
||||
mod export;
|
||||
mod store_cache;
|
||||
pub(crate) use store_cache::CachedObservabilityStore;
|
||||
|
||||
/// API 消费的观测控制面服务
|
||||
#[async_trait]
|
||||
@@ -122,6 +71,10 @@ pub trait ObservabilityService: Send + Sync {
|
||||
dimension: DiagnosticDimension,
|
||||
) -> Result<DiagnosticsResult, AdminError>;
|
||||
async fn ops_errors(&self, query: OpsErrorQuery) -> Result<OpsErrorPage, AdminError>;
|
||||
async fn export_diagnostics(
|
||||
&self,
|
||||
target: crate::model::diagnostics::DiagnosticTarget,
|
||||
) -> Result<crate::model::diagnostics::DiagnosticBundle, AdminError>;
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultObservabilityService {
|
||||
@@ -130,7 +83,7 @@ pub(crate) struct DefaultObservabilityService {
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
dashboard_queries: DashboardQueryCache,
|
||||
system: Arc<dyn super::system::SystemService>,
|
||||
}
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
@@ -141,6 +94,7 @@ impl DefaultObservabilityService {
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
system: Arc<dyn super::system::SystemService>,
|
||||
) -> Self {
|
||||
Self {
|
||||
timezone,
|
||||
@@ -148,7 +102,7 @@ impl DefaultObservabilityService {
|
||||
accounts,
|
||||
settings,
|
||||
providers,
|
||||
dashboard_queries: DashboardQueryCache::default(),
|
||||
system,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,7 +139,7 @@ impl DefaultObservabilityService {
|
||||
let yesterday = dashboard_period_metrics(&observation.trend, yesterday_start, today_start);
|
||||
let total_billing_usd = observation.totals.billing_usd.clone();
|
||||
let total_cached_token_rate = rate_or_zero(
|
||||
observation.totals.cached_tokens,
|
||||
observation.totals.cached_input_tokens,
|
||||
observation.totals.input_tokens,
|
||||
);
|
||||
let (first_token_latency_sum_ms, first_token_latency_count) = observation
|
||||
@@ -199,7 +153,11 @@ impl DefaultObservabilityService {
|
||||
});
|
||||
let average_first_token_latency_ms =
|
||||
average(first_token_latency_sum_ms, first_token_latency_count);
|
||||
let trend = trend(TrendKind::Usage, observation.trend.clone())?;
|
||||
let trend = trend(
|
||||
TrendKind::Usage,
|
||||
observation.trend.clone(),
|
||||
observation.analytics.clone(),
|
||||
)?;
|
||||
let health_timeline =
|
||||
health_timeline_at(&observation.trend, observation.range.end, self.timezone)?;
|
||||
let wire_profiles = self
|
||||
@@ -242,20 +200,24 @@ impl DefaultObservabilityService {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityService for DefaultObservabilityService {
|
||||
async fn export_diagnostics(
|
||||
&self,
|
||||
target: crate::model::diagnostics::DiagnosticTarget,
|
||||
) -> Result<crate::model::diagnostics::DiagnosticBundle, AdminError> {
|
||||
self.collect_diagnostics(target).await
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
kind: TrendKind,
|
||||
) -> Result<DashboardResult, AdminError> {
|
||||
let cell = self
|
||||
.dashboard_queries
|
||||
.result_cell(DashboardQueryKey::new(range))
|
||||
.await;
|
||||
let mut result = cell
|
||||
.get_or_init(|| self.load_dashboard_summary(range))
|
||||
.await
|
||||
.clone()?;
|
||||
result.trend = trend(kind, result.observation.trend.clone())?;
|
||||
let mut result = self.load_dashboard_summary(range).await?;
|
||||
result.trend = trend(
|
||||
kind,
|
||||
result.observation.trend.clone(),
|
||||
result.observation.analytics.clone(),
|
||||
)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -264,12 +226,16 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
range: TimeRange,
|
||||
kind: TrendKind,
|
||||
) -> Result<Trend, AdminError> {
|
||||
let points = self
|
||||
let observation = self
|
||||
.store
|
||||
.dashboard_trend(range, Granularity::for_range(range))
|
||||
.usage_insights_observation(
|
||||
range,
|
||||
UsageFilter::default(),
|
||||
Granularity::for_range(range),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "dashboard trend"))?;
|
||||
trend(kind, points)
|
||||
trend(kind, observation.trend, observation.overview.analytics)
|
||||
}
|
||||
|
||||
async fn usage_records(&self, query: UsageQuery) -> Result<UsagePage, AdminError> {
|
||||
@@ -321,21 +287,22 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
filter: UsageFilter,
|
||||
) -> Result<UsageInsights, AdminError> {
|
||||
let granularity = Granularity::for_range(range);
|
||||
let aggregates = async {
|
||||
futures::try_join!(
|
||||
self.store.usage_summary(range, filter.clone()),
|
||||
self.store.usage_trend(range, filter.clone(), granularity),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "usage insights"))
|
||||
};
|
||||
let ((overview, trend), standard_costs) = futures::try_join!(
|
||||
aggregates,
|
||||
recover_standard_costs(
|
||||
&self.providers,
|
||||
self.store
|
||||
.usage_calculated_billing_facts(range, filter.clone(), granularity),
|
||||
),
|
||||
)?;
|
||||
let observation = self
|
||||
.store
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage insights"))?;
|
||||
let mut standard_costs = UsageCostScenarios::default();
|
||||
for point in observation.costs {
|
||||
if let Some(amount) = point.standard_amount {
|
||||
standard_costs.standard.add(point.bucket_start, amount)?;
|
||||
}
|
||||
if let Some(amount) = point.no_cache_amount {
|
||||
standard_costs.no_cache.add(point.bucket_start, amount)?;
|
||||
}
|
||||
}
|
||||
let overview = observation.overview;
|
||||
let trend = observation.trend;
|
||||
build_usage_insights(overview, trend, standard_costs)
|
||||
}
|
||||
|
||||
@@ -389,7 +356,11 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Ok(DiagnosticsResult { dimension, items })
|
||||
Ok(DiagnosticsResult {
|
||||
analytics: observation.analytics,
|
||||
dimension,
|
||||
items,
|
||||
})
|
||||
}
|
||||
|
||||
async fn ops_errors(&self, query: OpsErrorQuery) -> Result<OpsErrorPage, AdminError> {
|
||||
@@ -423,8 +394,9 @@ fn build_usage_insights(
|
||||
}
|
||||
|
||||
let requests = &overview.requests;
|
||||
let failed_requests = service_failure_count(requests);
|
||||
let health_requests = requests.success_count.saturating_add(failed_requests);
|
||||
let health_window = HealthWindow::from_metrics(requests);
|
||||
let failed_requests = health_window.failed_requests;
|
||||
let health_requests = health_window.eligible_requests();
|
||||
let health = UsageInsightsHealth {
|
||||
total_requests: requests.request_count,
|
||||
success_requests: requests.success_count,
|
||||
@@ -442,17 +414,19 @@ fn build_usage_insights(
|
||||
points: trend
|
||||
.iter()
|
||||
.map(|point| {
|
||||
let failed_requests = service_failure_count(&point.metrics);
|
||||
let health_requests = point.metrics.success_count.saturating_add(failed_requests);
|
||||
let health_window = HealthWindow::from_metrics(&point.metrics);
|
||||
UsageInsightsHealthPoint {
|
||||
total_requests: point.metrics.request_count,
|
||||
bucket_start: point.bucket_start,
|
||||
success_requests: point.metrics.success_count,
|
||||
failed_requests,
|
||||
failed_requests: health_window.failed_requests,
|
||||
cancelled_requests: point.metrics.cancelled_count,
|
||||
incomplete_requests: point.metrics.incomplete_count,
|
||||
caller_error_requests: point.metrics.caller_error_count,
|
||||
error_rate: rate_or_zero(failed_requests, health_requests),
|
||||
error_rate: rate_or_zero(
|
||||
health_window.non_success_requests(),
|
||||
health_window.eligible_requests(),
|
||||
),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
@@ -533,14 +507,14 @@ fn build_usage_insights(
|
||||
tier_premium,
|
||||
tokens_per_request: rate_or_zero(requests.total_tokens, requests.request_count),
|
||||
cost_per_successful_request,
|
||||
cached_token_rate: rate_or_zero(requests.cached_tokens, requests.input_tokens),
|
||||
cached_token_rate: rate_or_zero(requests.cached_input_tokens, requests.input_tokens),
|
||||
cache_hit_request_rate: ratio(
|
||||
requests.cache_hit_request_count,
|
||||
requests.cache_eligible_request_count,
|
||||
),
|
||||
input_tokens: requests.input_tokens,
|
||||
output_tokens: requests.output_tokens,
|
||||
cached_tokens: requests.cached_tokens,
|
||||
cached_input_tokens: requests.cached_input_tokens,
|
||||
total_tokens: requests.total_tokens,
|
||||
points: trend
|
||||
.iter()
|
||||
@@ -548,7 +522,7 @@ fn build_usage_insights(
|
||||
bucket_start: point.bucket_start,
|
||||
input_tokens: point.metrics.input_tokens,
|
||||
output_tokens: point.metrics.output_tokens,
|
||||
cached_tokens: point.metrics.cached_tokens,
|
||||
cached_input_tokens: point.metrics.cached_input_tokens,
|
||||
total_tokens: point.metrics.total_tokens,
|
||||
estimated_cost: usd_cost(&point.costs),
|
||||
standard_cost: standard_costs_by_bucket.get(&point.bucket_start).cloned(),
|
||||
@@ -558,7 +532,7 @@ fn build_usage_insights(
|
||||
usd_cost(&point.costs).as_ref(),
|
||||
),
|
||||
cached_token_rate: rate_or_zero(
|
||||
point.metrics.cached_tokens,
|
||||
point.metrics.cached_input_tokens,
|
||||
point.metrics.input_tokens,
|
||||
),
|
||||
cache_hit_request_rate: ratio(
|
||||
@@ -571,6 +545,7 @@ fn build_usage_insights(
|
||||
coverage: overview.attempts.cost_coverage.clone(),
|
||||
};
|
||||
Ok(UsageInsights {
|
||||
analytics: overview.analytics,
|
||||
granularity,
|
||||
health,
|
||||
performance,
|
||||
@@ -620,58 +595,65 @@ struct UsageCostScenarios {
|
||||
no_cache: ScenarioCosts,
|
||||
}
|
||||
|
||||
async fn recover_standard_costs(
|
||||
providers: &ProviderAdminRegistry,
|
||||
mut facts: UsageCalculatedBillingStream<'_>,
|
||||
) -> Result<UsageCostScenarios, AdminError> {
|
||||
let mut scenarios = UsageCostScenarios::default();
|
||||
while let Some(mut fact) = facts
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage billing facts"))?
|
||||
{
|
||||
/// 在后台投影时恢复费用场景,查询不再逐请求读取计费快照
|
||||
pub struct ProviderUsageBillingProjector {
|
||||
providers: ProviderAdminRegistry,
|
||||
}
|
||||
|
||||
impl ProviderUsageBillingProjector {
|
||||
#[must_use]
|
||||
pub fn new(providers: ProviderAdminRegistry) -> Self {
|
||||
Self { providers }
|
||||
}
|
||||
}
|
||||
|
||||
impl UsageBillingProjector for ProviderUsageBillingProjector {
|
||||
fn project(
|
||||
&self,
|
||||
fact: &UsageCalculatedBillingFact,
|
||||
) -> Result<UsageBillingProjection, AdminError> {
|
||||
let Ok(provider_kind) = ProviderKind::new(fact.provider_kind.clone()) else {
|
||||
continue;
|
||||
return Ok(UsageBillingProjection::default());
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: fact.upstream_model_id.clone(),
|
||||
service_tier: fact.service_tier.clone(),
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
cache_write_tokens: fact.cache_write_tokens,
|
||||
cached_input_tokens: fact.cached_input_tokens,
|
||||
cache_write_input_tokens: fact.cache_write_input_tokens,
|
||||
total: fact.total.clone(),
|
||||
};
|
||||
let breakdown = match fact
|
||||
.breakdown
|
||||
.take()
|
||||
.filter(|b| b.total_amount == input.total)
|
||||
.as_ref()
|
||||
.filter(|breakdown| breakdown.total_amount == input.total)
|
||||
.cloned()
|
||||
.map_or_else(
|
||||
|| providers.calculated_billing(&provider_kind, &input),
|
||||
|b| Ok(Some(b)),
|
||||
|| self.providers.calculated_billing(&provider_kind, &input),
|
||||
|breakdown| Ok(Some(breakdown)),
|
||||
) {
|
||||
Ok(breakdown) => breakdown,
|
||||
Err(error) if error.kind() == ProviderAdminErrorKind::Unsupported => continue,
|
||||
Err(error) if error.kind() == ProviderAdminErrorKind::Unsupported => {
|
||||
return Ok(UsageBillingProjection::default());
|
||||
}
|
||||
Err(error) => return Err(map_provider_error(error, "usage billing")),
|
||||
};
|
||||
let Some(breakdown) = breakdown else {
|
||||
continue;
|
||||
return Ok(UsageBillingProjection::default());
|
||||
};
|
||||
if !breakdown
|
||||
.standard_amount
|
||||
.currency
|
||||
.eq_ignore_ascii_case("USD")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
scenarios
|
||||
.standard
|
||||
.add(fact.bucket_start, breakdown.standard_amount.amount.clone())?;
|
||||
if let Some(no_cache_cost) = no_cache_cost(&fact, &breakdown) {
|
||||
scenarios.no_cache.add(fact.bucket_start, no_cache_cost)?;
|
||||
return Ok(UsageBillingProjection::default());
|
||||
}
|
||||
Ok(UsageBillingProjection {
|
||||
standard_amount: Some(breakdown.standard_amount.amount.clone()),
|
||||
no_cache_amount: no_cache_cost(fact, &breakdown),
|
||||
})
|
||||
}
|
||||
Ok(scenarios)
|
||||
}
|
||||
|
||||
fn no_cache_cost(
|
||||
@@ -682,19 +664,21 @@ fn no_cache_cost(
|
||||
let input_tokens = fact
|
||||
.input_tokens?
|
||||
.checked_sub(image.map_or(0, |i| i.input_tokens))?;
|
||||
let cached_tokens = fact
|
||||
.cached_tokens
|
||||
let cached_input_tokens = fact
|
||||
.cached_input_tokens
|
||||
.unwrap_or_default()
|
||||
.checked_sub(image.map_or(0, |i| i.cached_tokens))?
|
||||
.checked_sub(image.map_or(0, |i| i.cached_input_tokens))?
|
||||
.min(input_tokens);
|
||||
let cache_write_tokens = fact
|
||||
.cache_write_tokens
|
||||
let cache_write_input_tokens = fact
|
||||
.cache_write_input_tokens
|
||||
.unwrap_or_default()
|
||||
.min(input_tokens.saturating_sub(cached_tokens));
|
||||
.min(input_tokens.saturating_sub(cached_input_tokens));
|
||||
let input_rate = decimal(&breakdown.input_price_per_million.amount)?;
|
||||
let replaced_input_amount = input_rate
|
||||
.scaled()
|
||||
.checked_mul(u128::from(cached_tokens.saturating_add(cache_write_tokens)))?
|
||||
.checked_mul(u128::from(
|
||||
cached_input_tokens.saturating_add(cache_write_input_tokens),
|
||||
))?
|
||||
.checked_div(1_000_000)?;
|
||||
let mut total = decimal(&breakdown.total_amount.amount)?
|
||||
.scaled()
|
||||
@@ -704,7 +688,7 @@ fn no_cache_cost(
|
||||
if let Some(image) = image {
|
||||
let image_no_cache = decimal(&image.input_price_per_million.amount)?
|
||||
.scaled()
|
||||
.checked_mul(u128::from(image.cached_tokens))?
|
||||
.checked_mul(u128::from(image.cached_input_tokens))?
|
||||
.checked_div(1_000_000)?;
|
||||
total = total
|
||||
.checked_sub(decimal(&image.cache_read_amount.amount)?.scaled())?
|
||||
@@ -829,8 +813,8 @@ impl DefaultObservabilityService {
|
||||
service_tier: record.service_tier.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
cache_write_input_tokens: record.cache_write_input_tokens,
|
||||
total: total.clone(),
|
||||
};
|
||||
if let Ok(Some(breakdown)) = self.providers.calculated_billing(&provider_kind, &input) {
|
||||
@@ -861,8 +845,8 @@ impl DefaultObservabilityService {
|
||||
service_tier: record.service_tier.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
cache_write_input_tokens: record.cache_write_input_tokens,
|
||||
total: total.clone(),
|
||||
};
|
||||
if let Ok(Some(breakdown)) = self.providers.calculated_billing(&provider_kind, &input) {
|
||||
@@ -951,7 +935,11 @@ pub(super) fn health_timeline_at(
|
||||
})
|
||||
}
|
||||
|
||||
fn trend(kind: TrendKind, points: Vec<RequestMetricPoint>) -> Result<Trend, AdminError> {
|
||||
fn trend(
|
||||
kind: TrendKind,
|
||||
points: Vec<RequestMetricPoint>,
|
||||
analytics: crate::model::analytics::AnalyticsMetadata,
|
||||
) -> Result<Trend, AdminError> {
|
||||
let mut metrics = RequestMetrics::default();
|
||||
let mut coverage = CostCoverage::default();
|
||||
let mut costs = BTreeMap::<String, DecimalAmount>::new();
|
||||
@@ -986,7 +974,7 @@ fn trend(kind: TrendKind, points: Vec<RequestMetricPoint>) -> Result<Trend, Admi
|
||||
let service_failure_count = service_failure_count(&metrics);
|
||||
let success_rate = ratio(
|
||||
metrics.success_count,
|
||||
metrics.success_count.saturating_add(service_failure_count),
|
||||
HealthWindow::from_metrics(&metrics).eligible_requests(),
|
||||
);
|
||||
let cache_hit_request_rate = ratio(
|
||||
metrics.cache_hit_request_count,
|
||||
@@ -1007,13 +995,14 @@ fn trend(kind: TrendKind, points: Vec<RequestMetricPoint>) -> Result<Trend, Admi
|
||||
.filter_map(|point| point.metrics.output_throughput_p50)
|
||||
.min();
|
||||
Ok(Trend {
|
||||
analytics,
|
||||
kind,
|
||||
points: points.into_iter().map(trend_point).collect(),
|
||||
summary: TrendSummary {
|
||||
request_count: metrics.request_count,
|
||||
input_tokens: metrics.input_tokens,
|
||||
output_tokens: metrics.output_tokens,
|
||||
cached_tokens: metrics.cached_tokens,
|
||||
cached_input_tokens: metrics.cached_input_tokens,
|
||||
total_tokens: metrics.total_tokens,
|
||||
service_failure_count,
|
||||
average_latency_ms,
|
||||
@@ -1044,12 +1033,18 @@ fn trend_point(point: RequestMetricPoint) -> TrendPoint {
|
||||
point.metrics.first_token_latency_sum_ms,
|
||||
point.metrics.first_token_latency_count,
|
||||
),
|
||||
cached_token_rate: rate_or_zero(point.metrics.cached_tokens, point.metrics.input_tokens),
|
||||
cached_token_rate: rate_or_zero(
|
||||
point.metrics.cached_input_tokens,
|
||||
point.metrics.input_tokens,
|
||||
),
|
||||
cache_hit_request_rate: ratio(
|
||||
point.metrics.cache_hit_request_count,
|
||||
point.metrics.cache_eligible_request_count,
|
||||
),
|
||||
success_rate: ratio(point.metrics.success_count, point.metrics.request_count),
|
||||
success_rate: ratio(
|
||||
point.metrics.success_count,
|
||||
HealthWindow::from_metrics(&point.metrics).eligible_requests(),
|
||||
),
|
||||
metrics: point.metrics,
|
||||
cost_coverage: point.cost_coverage,
|
||||
costs: point.costs,
|
||||
@@ -1070,11 +1065,13 @@ fn dashboard_period_metrics(
|
||||
result.request_count = result.request_count.saturating_add(metrics.request_count);
|
||||
result.input_tokens = result.input_tokens.saturating_add(metrics.input_tokens);
|
||||
result.output_tokens = result.output_tokens.saturating_add(metrics.output_tokens);
|
||||
result.cached_tokens = result.cached_tokens.saturating_add(metrics.cached_tokens);
|
||||
result.cached_input_tokens = result
|
||||
.cached_input_tokens
|
||||
.saturating_add(metrics.cached_input_tokens);
|
||||
result.total_tokens = result.total_tokens.saturating_add(metrics.total_tokens);
|
||||
}
|
||||
result.cached_token_rate = rate_or_zero(result.cached_tokens, result.input_tokens);
|
||||
result.observed_cached_token_rate = ratio(result.cached_tokens, result.input_tokens);
|
||||
result.cached_token_rate = rate_or_zero(result.cached_input_tokens, result.input_tokens);
|
||||
result.observed_cached_token_rate = ratio(result.cached_input_tokens, result.input_tokens);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -1091,13 +1088,15 @@ fn add_request_metrics(total: &mut RequestMetrics, value: &RequestMetrics) {
|
||||
.saturating_add(value.caller_error_count);
|
||||
total.input_tokens = total.input_tokens.saturating_add(value.input_tokens);
|
||||
total.output_tokens = total.output_tokens.saturating_add(value.output_tokens);
|
||||
total.cached_tokens = total.cached_tokens.saturating_add(value.cached_tokens);
|
||||
total.cache_write_tokens = total
|
||||
.cache_write_tokens
|
||||
.saturating_add(value.cache_write_tokens);
|
||||
total.reasoning_tokens = total
|
||||
.reasoning_tokens
|
||||
.saturating_add(value.reasoning_tokens);
|
||||
total.cached_input_tokens = total
|
||||
.cached_input_tokens
|
||||
.saturating_add(value.cached_input_tokens);
|
||||
total.cache_write_input_tokens = total
|
||||
.cache_write_input_tokens
|
||||
.saturating_add(value.cache_write_input_tokens);
|
||||
total.reasoning_output_tokens = total
|
||||
.reasoning_output_tokens
|
||||
.saturating_add(value.reasoning_output_tokens);
|
||||
total.total_tokens = total.total_tokens.saturating_add(value.total_tokens);
|
||||
total.first_token_latency_sum_ms = total
|
||||
.first_token_latency_sum_ms
|
||||
@@ -1145,20 +1144,29 @@ struct HealthWindow {
|
||||
}
|
||||
|
||||
impl HealthWindow {
|
||||
fn from_metrics(metrics: &RequestMetrics) -> Self {
|
||||
Self {
|
||||
success_requests: metrics.success_count,
|
||||
failed_requests: service_failure_count(metrics),
|
||||
cancelled_requests: metrics.cancelled_count,
|
||||
incomplete_requests: metrics.incomplete_count,
|
||||
caller_error_requests: metrics.caller_error_count,
|
||||
}
|
||||
}
|
||||
|
||||
// 未完成保留交付终态,但上游失败、超时和进程中断都不能视为服务成功
|
||||
fn non_success_requests(self) -> u64 {
|
||||
self.failed_requests
|
||||
.saturating_add(self.incomplete_requests)
|
||||
}
|
||||
|
||||
fn eligible_requests(self) -> u64 {
|
||||
self.success_requests
|
||||
.saturating_add(self.non_success_requests())
|
||||
}
|
||||
|
||||
fn add_metrics(&mut self, metrics: &RequestMetrics) {
|
||||
self.success_requests = self.success_requests.saturating_add(metrics.success_count);
|
||||
self.failed_requests = self
|
||||
.failed_requests
|
||||
.saturating_add(service_failure_count(metrics));
|
||||
self.cancelled_requests = self
|
||||
.cancelled_requests
|
||||
.saturating_add(metrics.cancelled_count);
|
||||
self.incomplete_requests = self
|
||||
.incomplete_requests
|
||||
.saturating_add(metrics.incomplete_count);
|
||||
self.caller_error_requests = self
|
||||
.caller_error_requests
|
||||
.saturating_add(metrics.caller_error_count);
|
||||
self.add_window(Self::from_metrics(metrics));
|
||||
}
|
||||
|
||||
fn add_window(&mut self, other: Self) {
|
||||
@@ -1177,15 +1185,13 @@ impl HealthWindow {
|
||||
}
|
||||
|
||||
fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus {
|
||||
let eligible_requests = bucket
|
||||
.success_requests
|
||||
.saturating_add(bucket.failed_requests);
|
||||
let eligible_requests = bucket.eligible_requests();
|
||||
if is_future {
|
||||
HealthStatus::Future
|
||||
} else if eligible_requests == 0 {
|
||||
HealthStatus::NoData
|
||||
} else if bucket.success_requests == 0
|
||||
&& bucket.failed_requests >= HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD
|
||||
&& bucket.non_success_requests() >= HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD
|
||||
{
|
||||
HealthStatus::Unavailable
|
||||
} else if eligible_requests < HEALTH_TIMELINE_MIN_SAMPLE_SIZE {
|
||||
@@ -1200,9 +1206,7 @@ fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus {
|
||||
}
|
||||
|
||||
fn health_reliability(bucket: HealthWindow) -> Option<f64> {
|
||||
let eligible_requests = bucket
|
||||
.success_requests
|
||||
.saturating_add(bucket.failed_requests);
|
||||
let eligible_requests = bucket.eligible_requests();
|
||||
(eligible_requests > 0)
|
||||
.then(|| bucket.success_requests as f64 / eligible_requests as f64 * 100.0)
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
//! 有界统计缓存:身份和完整筛选参与键,统计发布代次使旧快照失效
|
||||
|
||||
use crate::model::observability::*;
|
||||
use crate::ports::store::{
|
||||
AdminStoreError, AdminStoreErrorKind, AdminStoreResult, ObservabilityStore,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use std::{collections::HashMap, future::Future, hash::Hash, sync::Arc, time::Duration};
|
||||
use tokio::{sync::Mutex, time::Instant};
|
||||
|
||||
const CAPACITY: usize = 64;
|
||||
const TTL: Duration = Duration::from_secs(15);
|
||||
const MAX_STALE: Duration = Duration::from_secs(60);
|
||||
|
||||
type Slot<T> = Arc<Mutex<Option<Cached<T>>>>;
|
||||
struct Cached<T> {
|
||||
value: T,
|
||||
created: Instant,
|
||||
}
|
||||
struct Entry<T> {
|
||||
slot: Slot<T>,
|
||||
touched: Instant,
|
||||
}
|
||||
struct Cache<K, T> {
|
||||
entries: Mutex<HashMap<K, Entry<T>>>,
|
||||
}
|
||||
impl<K, T> Default for Cache<K, T> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait AnalyticsValue: Clone {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata;
|
||||
}
|
||||
impl AnalyticsValue for DashboardObservation {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.analytics
|
||||
}
|
||||
}
|
||||
impl AnalyticsValue for UsageOverview {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.analytics
|
||||
}
|
||||
}
|
||||
impl AnalyticsValue for UsageInsightsObservation {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.overview.analytics
|
||||
}
|
||||
}
|
||||
impl AnalyticsValue for DiagnosticsObservation {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.analytics
|
||||
}
|
||||
}
|
||||
|
||||
impl<K: Eq + Hash, T: AnalyticsValue> Cache<K, T> {
|
||||
async fn get<F, Fut>(
|
||||
&self,
|
||||
key: K,
|
||||
store: &dyn ObservabilityStore,
|
||||
ttl: Duration,
|
||||
load: F,
|
||||
) -> AdminStoreResult<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = AdminStoreResult<T>>,
|
||||
{
|
||||
// 刷新失败后的同键等待也计入预算,避免串行重试把延迟放大为队列长度倍数
|
||||
tokio::time::timeout(
|
||||
store.query_timeout(),
|
||||
self.get_within_budget(key, store, ttl, load),
|
||||
)
|
||||
.await
|
||||
.map_err(|source| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
"observability cache",
|
||||
"observability cache execution budget is exhausted",
|
||||
)
|
||||
.with_source(source)
|
||||
})?
|
||||
}
|
||||
|
||||
async fn get_within_budget<F, Fut>(
|
||||
&self,
|
||||
key: K,
|
||||
store: &dyn ObservabilityStore,
|
||||
ttl: Duration,
|
||||
load: F,
|
||||
) -> AdminStoreResult<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = AdminStoreResult<T>>,
|
||||
{
|
||||
let slot = {
|
||||
let mut entries = self.entries.lock().await;
|
||||
if !entries.contains_key(&key) && entries.len() >= CAPACITY {
|
||||
let oldest = entries
|
||||
.iter()
|
||||
.filter(|(_, entry)| Arc::strong_count(&entry.slot) == 1)
|
||||
.min_by_key(|(_, entry)| entry.touched)
|
||||
.map(|(_, entry)| entry.touched);
|
||||
if let Some(oldest) = oldest {
|
||||
entries.retain(|_, entry| entry.touched != oldest);
|
||||
}
|
||||
}
|
||||
if !entries.contains_key(&key) && entries.len() >= CAPACITY {
|
||||
drop(entries);
|
||||
return load().await;
|
||||
}
|
||||
let entry = entries.entry(key).or_insert_with(|| Entry {
|
||||
slot: Arc::new(Mutex::new(None)),
|
||||
touched: Instant::now(),
|
||||
});
|
||||
entry.touched = Instant::now();
|
||||
entry.slot.clone()
|
||||
};
|
||||
// 同键刷新只执行一次;失败不会缓存成成功,也不延长旧结果寿命
|
||||
let mut cached = slot.lock().await;
|
||||
// 等锁期间代次可能推进;取得同键锁后再核对,失败时也不能回用已被新统计替代的快照
|
||||
let generation = store.analytics_cache_generation().await?;
|
||||
if let Some(entry) = cached.as_mut()
|
||||
&& entry.value.metadata().generation == generation
|
||||
&& entry.created.elapsed() < ttl
|
||||
{
|
||||
return Ok(with_age(entry.value.clone(), false));
|
||||
}
|
||||
match load().await {
|
||||
Ok(value) => {
|
||||
*cached = Some(Cached {
|
||||
value: value.clone(),
|
||||
created: Instant::now(),
|
||||
});
|
||||
Ok(value)
|
||||
}
|
||||
Err(error) => {
|
||||
if ttl == TTL
|
||||
&& error.kind() == AdminStoreErrorKind::Unavailable
|
||||
&& let Some(entry) = cached.as_mut()
|
||||
&& entry.created.elapsed() <= MAX_STALE
|
||||
&& Utc::now()
|
||||
.signed_duration_since(entry.value.metadata().generated_at)
|
||||
.to_std()
|
||||
.unwrap_or_default()
|
||||
<= MAX_STALE
|
||||
&& entry.value.metadata().state != AnalyticsState::Building
|
||||
&& entry.value.metadata().generation
|
||||
== store.analytics_cache_generation().await?
|
||||
{
|
||||
return Ok(with_age(entry.value.clone(), true));
|
||||
}
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn with_age<T: AnalyticsValue>(mut value: T, stale: bool) -> T {
|
||||
let metadata = value.metadata();
|
||||
metadata.lag_seconds = metadata.lag_seconds.saturating_add(
|
||||
Utc::now()
|
||||
.signed_duration_since(metadata.generated_at)
|
||||
.num_seconds()
|
||||
.max(0) as u64,
|
||||
);
|
||||
if stale {
|
||||
metadata.state = AnalyticsState::Stale;
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
type UsageKey = (TimeRange, UsageFilter);
|
||||
pub(crate) struct CachedObservabilityStore {
|
||||
inner: Arc<dyn ObservabilityStore>,
|
||||
dashboard: Cache<DashboardQuery, DashboardObservation>,
|
||||
summaries: Cache<UsageKey, UsageOverview>,
|
||||
insights: Cache<(UsageKey, Granularity), UsageInsightsObservation>,
|
||||
diagnostics: Cache<(UsageKey, DiagnosticDimension, u16), DiagnosticsObservation>,
|
||||
}
|
||||
impl CachedObservabilityStore {
|
||||
pub(crate) fn new(inner: Arc<dyn ObservabilityStore>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
dashboard: Cache::default(),
|
||||
summaries: Cache::default(),
|
||||
insights: Cache::default(),
|
||||
diagnostics: Cache::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for CachedObservabilityStore {
|
||||
fn query_timeout(&self) -> Duration {
|
||||
self.inner.query_timeout()
|
||||
}
|
||||
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
self.inner.analytics_cache_generation().await
|
||||
}
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
query: DashboardQuery,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<DashboardObservation> {
|
||||
// 复合概览还包含账号容量和最近请求,保持原有两秒实时刷新上限
|
||||
self.dashboard
|
||||
.get(
|
||||
query.clone(),
|
||||
self.inner.as_ref(),
|
||||
Duration::from_secs(2),
|
||||
|| self.inner.dashboard_summary(query, observed_at),
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn dashboard_trend(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
||||
self.inner.dashboard_trend(range, granularity).await
|
||||
}
|
||||
async fn usage_trend(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
||||
self.inner.usage_trend(range, filter, granularity).await
|
||||
}
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.insights
|
||||
.get(
|
||||
((range, filter.clone()), granularity),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
self.summaries
|
||||
.get((range, filter.clone()), self.inner.as_ref(), TTL, || {
|
||||
self.inner.usage_summary(range, filter)
|
||||
})
|
||||
.await
|
||||
}
|
||||
async fn usage_diagnostics(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
dimension: DiagnosticDimension,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<DiagnosticsObservation> {
|
||||
self.diagnostics
|
||||
.get(
|
||||
((range, filter.clone()), dimension, limit),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_diagnostics(range, filter, dimension, limit)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
self.inner.list_usage_records(query).await
|
||||
}
|
||||
async fn usage_record_detail(&self, request_id: &str) -> AdminStoreResult<UsageDetail> {
|
||||
self.inner.usage_record_detail(request_id).await
|
||||
}
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
||||
self.inner.list_ops_errors(query).await
|
||||
}
|
||||
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
target: &crate::model::diagnostics::DiagnosticTarget,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<Vec<crate::model::observability::OpsError>> {
|
||||
self.inner.diagnostic_errors(target, limit).await
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,7 @@ impl PluginResourceAccess for DefaultPluginResourceAccess {
|
||||
owner,
|
||||
resource_key,
|
||||
NewAccountGroup {
|
||||
allow_image_generation: command.allow_image_generation,
|
||||
id,
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
|
||||
@@ -305,6 +305,9 @@ impl PluginsService {
|
||||
.get(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
if !record.enabled {
|
||||
return Err(AdminError::invalid("下载代理已禁用,请启用或改为直连"));
|
||||
}
|
||||
Ok(Some(PluginDistributionEgress::new(
|
||||
record.id,
|
||||
record.revision,
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
//! 账号出站代理的管理、账号解绑与连通性测试用例
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures::{StreamExt, stream};
|
||||
@@ -27,6 +30,16 @@ use crate::{
|
||||
|
||||
#[async_trait]
|
||||
pub trait ProxiesService: Send + Sync {
|
||||
async fn create_batch(
|
||||
&self,
|
||||
commands: Vec<Result<NewProxy, AdminError>>,
|
||||
context: &MutationContext,
|
||||
) -> Result<Vec<ProxyBatchResult<ProxyMutation>>, AdminError>;
|
||||
async fn test_batch(
|
||||
&self,
|
||||
commands: Vec<Result<TestProxy, AdminError>>,
|
||||
context: &MutationContext,
|
||||
) -> Result<Vec<ProxyBatchResult<ProxyRecord>>, AdminError>;
|
||||
async fn list(&self, query: ProxyListQuery) -> Result<ProxyPage, AdminError>;
|
||||
async fn list_accounts(
|
||||
&self,
|
||||
@@ -61,10 +74,10 @@ pub trait ProxiesService: Send + Sync {
|
||||
detect_location: bool,
|
||||
context: &MutationContext,
|
||||
) -> Result<ProxyRecord, AdminError>;
|
||||
/// 探测未保存的连接地址,不写入代理记录或修改账号绑定
|
||||
/// 探测未保存的连接地址或服务器直连出口,不写入配置或修改账号绑定
|
||||
async fn probe(
|
||||
&self,
|
||||
proxy: &OutboundProxy,
|
||||
proxy: Option<&OutboundProxy>,
|
||||
detect_location: bool,
|
||||
) -> Result<ProxyTestResult, AdminError>;
|
||||
}
|
||||
@@ -96,6 +109,75 @@ impl DefaultProxiesService {
|
||||
|
||||
#[async_trait]
|
||||
impl ProxiesService for DefaultProxiesService {
|
||||
async fn create_batch(
|
||||
&self,
|
||||
commands: Vec<Result<NewProxy, AdminError>>,
|
||||
context: &MutationContext,
|
||||
) -> Result<Vec<ProxyBatchResult<ProxyMutation>>, AdminError> {
|
||||
validate_batch_size(commands.len())?;
|
||||
let mut stopped = false;
|
||||
let mut results = Vec::with_capacity(commands.len());
|
||||
for command in commands {
|
||||
if stopped {
|
||||
results.push(ProxyBatchResult::Skipped);
|
||||
continue;
|
||||
}
|
||||
let result = match command {
|
||||
Ok(command) => self.create(command, context).await,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
stopped = result.as_ref().is_err_and(batch_should_stop);
|
||||
results.push(ProxyBatchResult::Completed(result));
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn test_batch(
|
||||
&self,
|
||||
commands: Vec<Result<TestProxy, AdminError>>,
|
||||
context: &MutationContext,
|
||||
) -> Result<Vec<ProxyBatchResult<ProxyRecord>>, AdminError> {
|
||||
validate_batch_size(commands.len())?;
|
||||
let mut ids = std::collections::HashSet::with_capacity(commands.len());
|
||||
if commands
|
||||
.iter()
|
||||
.filter_map(|command| command.as_ref().ok())
|
||||
.any(|command| !ids.insert(&command.id))
|
||||
{
|
||||
return Err(AdminError::invalid("批量测试不能包含重复代理"));
|
||||
}
|
||||
let stopped = AtomicBool::new(false);
|
||||
// 与单条测试共用全局容量,批次最多占用两槽,保留交互操作的容量
|
||||
Ok(stream::iter(commands)
|
||||
.map(|command| {
|
||||
let stopped = &stopped;
|
||||
async move {
|
||||
if stopped.load(Ordering::Relaxed) {
|
||||
return ProxyBatchResult::Skipped;
|
||||
}
|
||||
let result = match command {
|
||||
Ok(command) => {
|
||||
self.test(
|
||||
&command.id,
|
||||
command.revision,
|
||||
command.detect_location,
|
||||
context,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
if result.as_ref().is_err_and(batch_should_stop) {
|
||||
stopped.store(true, Ordering::Relaxed);
|
||||
}
|
||||
ProxyBatchResult::Completed(result)
|
||||
}
|
||||
})
|
||||
.buffered(2)
|
||||
.collect()
|
||||
.await)
|
||||
}
|
||||
|
||||
async fn remove_account(
|
||||
&self,
|
||||
proxy_id: &str,
|
||||
@@ -194,7 +276,7 @@ impl ProxiesService for DefaultProxiesService {
|
||||
.transpose()
|
||||
.map_err(|_| AdminError::invalid("代理位置不合法"))?;
|
||||
command.test = if command.auto_location {
|
||||
Some(self.probe(&command.proxy, true).await?)
|
||||
Some(self.probe(Some(&command.proxy), true).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -233,7 +315,7 @@ impl ProxiesService for DefaultProxiesService {
|
||||
if command.auto_location.unwrap_or(current.auto_location)
|
||||
&& (!current.auto_location || proxy != ¤t.proxy)
|
||||
{
|
||||
command.test = Some(self.probe(proxy, true).await?);
|
||||
command.test = Some(self.probe(Some(proxy), true).await?);
|
||||
}
|
||||
}
|
||||
let result = self
|
||||
@@ -262,7 +344,7 @@ impl ProxiesService for DefaultProxiesService {
|
||||
|
||||
async fn probe(
|
||||
&self,
|
||||
proxy: &OutboundProxy,
|
||||
proxy: Option<&OutboundProxy>,
|
||||
detect_location: bool,
|
||||
) -> Result<ProxyTestResult, AdminError> {
|
||||
let _permit = self.test_slots.try_acquire().map_err(|_| {
|
||||
@@ -292,7 +374,7 @@ impl ProxiesService for DefaultProxiesService {
|
||||
// 手动解析只为本次测试请求位置,不改变自动跟随的持久配置
|
||||
let result = self
|
||||
.probe
|
||||
.test(&record.proxy, record.auto_location || detect_location)
|
||||
.test(Some(&record.proxy), record.auto_location || detect_location)
|
||||
.await;
|
||||
let mutation = self
|
||||
.store
|
||||
@@ -313,3 +395,22 @@ fn validate_name(value: &str) -> Result<String, AdminError> {
|
||||
}
|
||||
Ok(value.to_owned())
|
||||
}
|
||||
|
||||
fn validate_batch_size(size: usize) -> Result<(), AdminError> {
|
||||
if size == 0 || size > MAX_PROXY_BATCH_SIZE {
|
||||
return Err(AdminError::invalid("一次需要提交 1 至 100 条代理"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn batch_should_stop(error: &AdminError) -> bool {
|
||||
matches!(
|
||||
error.kind(),
|
||||
AdminErrorKind::Unauthorized
|
||||
| AdminErrorKind::Forbidden
|
||||
| AdminErrorKind::BadGateway
|
||||
| AdminErrorKind::UpstreamResultUnknown
|
||||
| AdminErrorKind::Unavailable
|
||||
| AdminErrorKind::Internal
|
||||
)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,13 @@ use super::{map_store_error, publish_committed};
|
||||
/// API 消费的 Runtime settings 管理服务
|
||||
#[async_trait]
|
||||
pub trait SettingsService: Send + Sync {
|
||||
async fn preview_privacy_policy(
|
||||
&self,
|
||||
_request: gateway_core::settings::privacy::PrivacyPreviewRequest,
|
||||
) -> Result<gateway_core::settings::privacy::PrivacyPreviewResult, AdminError> {
|
||||
Err(AdminError::invalid("隐私预览不可用"))
|
||||
}
|
||||
|
||||
async fn preview_pricing_sync(
|
||||
&self,
|
||||
) -> Result<crate::model::pricing::PricingSyncPreview, AdminError>;
|
||||
@@ -117,6 +124,21 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
|
||||
|
||||
#[async_trait]
|
||||
impl SettingsService for DefaultSettingsService {
|
||||
async fn preview_privacy_policy(
|
||||
&self,
|
||||
request: gateway_core::settings::privacy::PrivacyPreviewRequest,
|
||||
) -> Result<gateway_core::settings::privacy::PrivacyPreviewResult, AdminError> {
|
||||
self.profile_provider("openai")?
|
||||
.preview_privacy_policy(request)
|
||||
.map_err(|error| {
|
||||
AdminError::invalid(format!(
|
||||
"第 {} 条规则:{}",
|
||||
error.rule_index + 1,
|
||||
error.reason
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
async fn preview_pricing_sync(
|
||||
&self,
|
||||
) -> Result<crate::model::pricing::PricingSyncPreview, AdminError> {
|
||||
@@ -238,10 +260,12 @@ impl SettingsService for DefaultSettingsService {
|
||||
PricingChange::Replace(pricing) => {
|
||||
pricing.validate().map_err(AdminError::invalid)?;
|
||||
if command.provider == "xai"
|
||||
&& pricing
|
||||
.bands
|
||||
.keys()
|
||||
.any(|band| matches!(band.as_str(), "image" | "flex" | "long_flex"))
|
||||
&& pricing.bands.keys().any(|band| {
|
||||
matches!(
|
||||
band.as_str(),
|
||||
"image" | "flex" | "long_flex" | "ultrafast" | "long_ultrafast"
|
||||
)
|
||||
})
|
||||
{
|
||||
return Err(AdminError::invalid("xAI 不支持此价格档位"));
|
||||
}
|
||||
@@ -342,6 +366,17 @@ impl SettingsService for DefaultSettingsService {
|
||||
mut command: ReplaceRuntimeSettings,
|
||||
) -> Result<RuntimeSettings, AdminError> {
|
||||
validate_settings(&command)?;
|
||||
if command.values.codex_privacy_policy != Default::default() {
|
||||
self.profile_provider("openai")?
|
||||
.compile_privacy_policy(&command.values.codex_privacy_policy)
|
||||
.map_err(|error| {
|
||||
AdminError::invalid(format!(
|
||||
"第 {} 条规则:{}",
|
||||
error.rule_index + 1,
|
||||
error.reason
|
||||
))
|
||||
})?;
|
||||
}
|
||||
if command
|
||||
.request_profile_updates
|
||||
.values()
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
//! 公开原生用量读写与纯计价,插件拥有跨调用流程和恢复策略
|
||||
|
||||
mod pricing;
|
||||
mod validation;
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
auth::{AdminAuditEvent, AuditActorKind},
|
||||
usage_data::*,
|
||||
},
|
||||
ports::{provider::ProviderAdminRegistry, store::SettingsStore, usage_data::*},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[async_trait]
|
||||
pub trait UsageDataService: Send + Sync {
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregatePage>;
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult>;
|
||||
async fn get_record(&self, request: GetUsageRecord) -> UsageDataResult<UsageRecord>;
|
||||
async fn list_records(&self, request: ListUsageRecords) -> UsageDataResult<UsageRecordPage>;
|
||||
async fn write_records(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageRecords,
|
||||
) -> UsageDataResult<UsageWriteResult>;
|
||||
async fn evaluate_pricing(
|
||||
&self,
|
||||
request: EvaluateUsagePricing,
|
||||
) -> UsageDataResult<UsagePricingEvaluation>;
|
||||
}
|
||||
|
||||
pub struct DefaultUsageDataService {
|
||||
store: Arc<dyn UsageDataStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
}
|
||||
impl DefaultUsageDataService {
|
||||
pub fn new(
|
||||
store: Arc<dyn UsageDataStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
settings,
|
||||
providers,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl UsageDataService for DefaultUsageDataService {
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregatePage> {
|
||||
validation::query(&request.filter, request.cursor.as_deref(), request.limit)?;
|
||||
self.store.list_aggregates(request).await
|
||||
}
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult> {
|
||||
validation::bounded(&request, request.changes.len())?;
|
||||
let mut changes = Vec::with_capacity(request.changes.len());
|
||||
for change in request.changes {
|
||||
validation::aggregate(&change)?;
|
||||
let identity = match &change {
|
||||
UsageAggregateChange::Put { aggregate } => &aggregate.identity,
|
||||
UsageAggregateChange::Delete { identity } => identity,
|
||||
};
|
||||
let action = match &change {
|
||||
UsageAggregateChange::Put { .. } => "usage.aggregate.put",
|
||||
UsageAggregateChange::Delete { .. } => "usage.aggregate.delete",
|
||||
};
|
||||
let audit = audit_event(
|
||||
actor,
|
||||
action,
|
||||
"usage_aggregate",
|
||||
&identity.key,
|
||||
vec!["aggregate".into()],
|
||||
);
|
||||
changes.push((change, audit));
|
||||
}
|
||||
self.store
|
||||
.write_aggregates(StoredAggregateWrite { changes })
|
||||
.await
|
||||
}
|
||||
async fn get_record(&self, request: GetUsageRecord) -> UsageDataResult<UsageRecord> {
|
||||
validation::text(&request.id, 128)?;
|
||||
self.store.get_record(request).await
|
||||
}
|
||||
async fn list_records(&self, request: ListUsageRecords) -> UsageDataResult<UsageRecordPage> {
|
||||
validation::query(&request.filter, request.cursor.as_deref(), request.limit)?;
|
||||
self.store.list_records(request).await
|
||||
}
|
||||
async fn write_records(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageRecords,
|
||||
) -> UsageDataResult<UsageWriteResult> {
|
||||
validation::bounded(&request, request.changes.len())?;
|
||||
let mut changes = Vec::with_capacity(request.changes.len());
|
||||
for change in request.changes {
|
||||
validation::change(&change)?;
|
||||
let action = match &change {
|
||||
UsageRecordChange::Put { .. } => "usage.put",
|
||||
UsageRecordChange::Update { .. } => "usage.update",
|
||||
UsageRecordChange::Delete { .. } => "usage.delete",
|
||||
};
|
||||
let fields = match &change {
|
||||
UsageRecordChange::Update { tokens, cost, .. } => tokens
|
||||
.as_ref()
|
||||
.map(|_| "tokens".to_owned())
|
||||
.into_iter()
|
||||
.chain(cost.as_ref().map(|_| "cost".to_owned()))
|
||||
.collect(),
|
||||
_ => vec!["record".into()],
|
||||
};
|
||||
let audit = audit_event(actor, action, "model_request", change.id(), fields);
|
||||
changes.push((change, audit));
|
||||
}
|
||||
self.store.write_records(StoredUsageWrite { changes }).await
|
||||
}
|
||||
async fn evaluate_pricing(
|
||||
&self,
|
||||
request: EvaluateUsagePricing,
|
||||
) -> UsageDataResult<UsagePricingEvaluation> {
|
||||
validation::bounded(&request, request.items.len())?;
|
||||
let stored = self
|
||||
.settings
|
||||
.load_pricing()
|
||||
.await
|
||||
.map_err(|_| UsageDataError::unavailable("读取价目失败"))?;
|
||||
let prices = gateway_core::metering::merge_pricing(stored.synced, &stored.overrides);
|
||||
let mut items = Vec::with_capacity(request.items.len());
|
||||
for input in request.items {
|
||||
validation::text(&input.id, 128)?;
|
||||
items.push(self.evaluate_one(input, &prices)?);
|
||||
}
|
||||
Ok(UsagePricingEvaluation { items })
|
||||
}
|
||||
}
|
||||
|
||||
fn audit_event(
|
||||
actor: &UsageActor,
|
||||
action: &str,
|
||||
entity_kind: &str,
|
||||
id: &str,
|
||||
changed_fields: Vec<String>,
|
||||
) -> AdminAuditEvent {
|
||||
let actor_ref = match actor {
|
||||
UsageActor::Plugin {
|
||||
instance_id,
|
||||
publisher,
|
||||
name,
|
||||
..
|
||||
} => format!("plugin:{publisher}/{name}:{instance_id}"),
|
||||
UsageActor::Admin { subject } => format!("admin:{subject}"),
|
||||
UsageActor::System => "system".into(),
|
||||
};
|
||||
AdminAuditEvent {
|
||||
id: format!("audit_{}", uuid::Uuid::now_v7().simple()),
|
||||
actor_kind: AuditActorKind::System,
|
||||
actor_admin_user_id: None,
|
||||
actor_ref,
|
||||
request_id: None,
|
||||
action: action.into(),
|
||||
entity_kind: entity_kind.into(),
|
||||
entity_ref: id.to_owned(),
|
||||
config_revision: None,
|
||||
changed_fields,
|
||||
occurred_at: chrono::Utc::now(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! 复用 Provider 的单请求 Token 计价规则,不写入历史或账本
|
||||
|
||||
use super::{DefaultUsageDataService, validation};
|
||||
use crate::{
|
||||
model::usage_data::*,
|
||||
ports::provider::{ProviderAdminErrorKind, ProviderUsagePricingInput},
|
||||
};
|
||||
use gateway_core::{
|
||||
metering::{PricingOverrides, Usage},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
|
||||
impl DefaultUsageDataService {
|
||||
pub(super) fn evaluate_one(
|
||||
&self,
|
||||
input: UsagePricingInput,
|
||||
prices: &PricingOverrides,
|
||||
) -> UsageDataResult<UsagePricingResult> {
|
||||
validation::text(&input.provider, 128)?;
|
||||
validation::text(&input.model, 128)?;
|
||||
validation::tokens(&input.tokens)?;
|
||||
let unavailable = |reason: &str| UsagePricingResult::NotEstimable {
|
||||
id: input.id.clone(),
|
||||
reason: reason.into(),
|
||||
};
|
||||
let kind = ProviderKind::new(input.provider.clone())
|
||||
.map_err(|_| UsageDataError::invalid("Provider 无效"))?;
|
||||
let Ok(provider) = self.providers.require(&kind) else {
|
||||
return Ok(unavailable("Provider 没有可用的计价规则"));
|
||||
};
|
||||
let tokens = &input.tokens;
|
||||
let price = match &input.price {
|
||||
UsagePriceSelection::Current => prices
|
||||
.get(&input.provider)
|
||||
.and_then(|models| models.get(&input.model)),
|
||||
UsagePriceSelection::Override { price } => {
|
||||
price.validate().map_err(UsageDataError::invalid)?;
|
||||
Some(price.as_ref())
|
||||
}
|
||||
};
|
||||
let request = ProviderUsagePricingInput {
|
||||
upstream_model_id: input.model.clone(),
|
||||
service_tier: input.service_tier.clone(),
|
||||
usage: Usage {
|
||||
input_tokens: tokens.input_tokens.map(UsageCount::get),
|
||||
output_tokens: tokens.output_tokens.map(UsageCount::get),
|
||||
cached_input_tokens: tokens.cached_input_tokens.map(UsageCount::get),
|
||||
cache_write_input_tokens: tokens.cache_write_input_tokens.map(UsageCount::get),
|
||||
reasoning_output_tokens: tokens.reasoning_output_tokens.map(UsageCount::get),
|
||||
image_input_tokens: tokens.image_input_tokens.map(UsageCount::get),
|
||||
image_output_tokens: tokens.image_output_tokens.map(UsageCount::get),
|
||||
total_tokens: tokens.total_tokens.map(UsageCount::get),
|
||||
},
|
||||
};
|
||||
let calculated = match provider.evaluate_usage_pricing(&request, price) {
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => return Ok(unavailable("用量不完整、包含图像或价格不可用")),
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderAdminErrorKind::Unsupported | ProviderAdminErrorKind::Invalid
|
||||
) =>
|
||||
{
|
||||
return Ok(unavailable("Provider 不支持该计价输入"));
|
||||
}
|
||||
Err(_) => return Err(UsageDataError::unavailable("Provider 计价失败")),
|
||||
};
|
||||
Ok(UsagePricingResult::Estimated {
|
||||
id: input.id,
|
||||
cost: UsageCost::Known {
|
||||
source: UsageCostSource::Calculated,
|
||||
amount: calculated.total_amount().amount().canonical(),
|
||||
currency: calculated.total_amount().currency().as_str().into(),
|
||||
},
|
||||
breakdown: UsagePricingBreakdown {
|
||||
input_amount: calculated.input_amount().amount().canonical(),
|
||||
output_amount: calculated.output_amount().amount().canonical(),
|
||||
cache_read_amount: calculated.cache_read_amount().amount().canonical(),
|
||||
cache_write_amount: calculated.cache_write_amount().amount().canonical(),
|
||||
standard_amount: calculated.standard_amount().amount().canonical(),
|
||||
long_context_billing_applied: calculated.long_context_billing_applied(),
|
||||
service_tier: calculated.service_tier().map(str::to_owned),
|
||||
custom_multiplier_bps: calculated.custom_multiplier_bps(),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
//! 用量输入沿用原生记录的数值、时间和资源边界
|
||||
|
||||
use crate::model::usage_data::*;
|
||||
use gateway_core::metering::Decimal;
|
||||
use serde::Serialize;
|
||||
|
||||
pub(super) fn text(value: &str, max: usize) -> UsageDataResult<()> {
|
||||
if value.trim().is_empty() || value.len() > max || value.chars().any(char::is_control) {
|
||||
return Err(UsageDataError::invalid("文本为空、过长或包含控制字符"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn bounded(value: &impl Serialize, count: usize) -> UsageDataResult<()> {
|
||||
if count == 0
|
||||
|| count > USAGE_DATA_MAX_TARGETS
|
||||
|| serde_json::to_vec(value)
|
||||
.map_err(|_| UsageDataError::invalid("请求编码失败"))?
|
||||
.len()
|
||||
> USAGE_DATA_MAX_REQUEST_BYTES
|
||||
{
|
||||
return Err(UsageDataError::new(
|
||||
UsageErrorKind::CapacityExceeded,
|
||||
"每次操作接受 1~500 项且不超过 2 MiB",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn tokens(value: &UsageTokens) -> UsageDataResult<()> {
|
||||
if [
|
||||
value.input_tokens,
|
||||
value.output_tokens,
|
||||
value.cached_input_tokens,
|
||||
value.cache_write_input_tokens,
|
||||
value.reasoning_output_tokens,
|
||||
value.image_input_tokens,
|
||||
value.image_output_tokens,
|
||||
value.total_tokens,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|v| v.get() > i64::MAX as u64)
|
||||
{
|
||||
return Err(UsageDataError::invalid("Token 超出原生存储范围"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn cost(value: &UsageCost) -> UsageDataResult<()> {
|
||||
if let UsageCost::Known {
|
||||
amount, currency, ..
|
||||
} = value
|
||||
{
|
||||
let _ = amount
|
||||
.parse::<Decimal>()
|
||||
.map_err(|_| UsageDataError::invalid("费用不是有效十进制数"))?;
|
||||
if currency.len() != 3 || !currency.bytes().all(|b| b.is_ascii_uppercase()) {
|
||||
return Err(UsageDataError::invalid("费用或币种不符合原生存储范围"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn change(value: &UsageRecordChange) -> UsageDataResult<()> {
|
||||
text(value.id(), 128)?;
|
||||
match value {
|
||||
UsageRecordChange::Put { data, .. } => {
|
||||
text(&data.client_key_ref, 128)?;
|
||||
for v in [
|
||||
&data.provider,
|
||||
&data.account_ref,
|
||||
&data.requested_model,
|
||||
&data.model,
|
||||
&data.service_tier,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
text(v, 128)?;
|
||||
}
|
||||
for v in [
|
||||
&data.protocol,
|
||||
&data.operation,
|
||||
&data.endpoint,
|
||||
&data.client_transport,
|
||||
] {
|
||||
text(v, 256)?;
|
||||
}
|
||||
if data.outcome == UsageOutcome::Running
|
||||
|| data
|
||||
.completed_at_ms
|
||||
.is_none_or(|end| end < data.started_at_ms)
|
||||
{
|
||||
return Err(UsageDataError::invalid("只允许写入已结束的请求明细"));
|
||||
}
|
||||
for at in [Some(data.started_at_ms), data.completed_at_ms]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if chrono::DateTime::from_timestamp_millis(at).is_none() {
|
||||
return Err(UsageDataError::invalid("请求时间无效"));
|
||||
}
|
||||
}
|
||||
tokens(&data.tokens)?;
|
||||
cost(&data.cost)?;
|
||||
}
|
||||
UsageRecordChange::Update {
|
||||
tokens: new_tokens,
|
||||
cost: new_cost,
|
||||
..
|
||||
} => {
|
||||
if new_tokens.is_none() && new_cost.is_none() {
|
||||
return Err(UsageDataError::invalid("修改必须指定 Token 或费用"));
|
||||
}
|
||||
if let Some(value) = new_tokens {
|
||||
tokens(value)?;
|
||||
}
|
||||
if let Some(value) = new_cost {
|
||||
cost(value)?;
|
||||
}
|
||||
}
|
||||
UsageRecordChange::Delete { .. } => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn query(
|
||||
filter: &UsageRecordFilter,
|
||||
cursor: Option<&str>,
|
||||
limit: Option<u16>,
|
||||
) -> UsageDataResult<()> {
|
||||
if limit.is_some_and(|v| v == 0 || v > USAGE_DATA_MAX_PAGE_SIZE)
|
||||
|| filter.start_ms >= filter.end_ms
|
||||
|| chrono::DateTime::from_timestamp_millis(filter.start_ms).is_none()
|
||||
|| chrono::DateTime::from_timestamp_millis(filter.end_ms).is_none()
|
||||
{
|
||||
return Err(UsageDataError::invalid("查询时间范围或分页大小无效"));
|
||||
}
|
||||
for v in [
|
||||
&filter.provider,
|
||||
&filter.model,
|
||||
&filter.account_ref,
|
||||
&filter.client_key_ref,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
text(v, 128)?;
|
||||
}
|
||||
if cursor.is_some_and(|v| v.len() > 4096) {
|
||||
return Err(UsageDataError::invalid("游标过长"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn aggregate(value: &UsageAggregateChange) -> UsageDataResult<()> {
|
||||
let identity = match value {
|
||||
UsageAggregateChange::Put { aggregate } => &aggregate.identity,
|
||||
UsageAggregateChange::Delete { identity } => identity,
|
||||
};
|
||||
text(&identity.key, 128)?;
|
||||
if chrono::DateTime::from_timestamp_millis(identity.bucket_start_ms).is_none() {
|
||||
return Err(UsageDataError::invalid("聚合归属时间无效"));
|
||||
}
|
||||
let d = &identity.dimensions;
|
||||
for value in [
|
||||
&d.provider,
|
||||
&d.model,
|
||||
&d.account_ref,
|
||||
&d.client_key_ref,
|
||||
&d.operation,
|
||||
&d.transport,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
text(value, 128)?;
|
||||
}
|
||||
if let UsageAggregateChange::Put { aggregate } = value {
|
||||
if aggregate.costs.len() > 32 {
|
||||
return Err(UsageDataError::invalid("费用币种数量超过限制"));
|
||||
}
|
||||
for (currency, amount) in &aggregate.costs {
|
||||
if currency.len() != 3
|
||||
|| !currency.bytes().all(|b| b.is_ascii_uppercase())
|
||||
|| amount.len() > 128
|
||||
{
|
||||
return Err(UsageDataError::invalid("聚合费用格式无效"));
|
||||
}
|
||||
amount
|
||||
.strip_prefix('-')
|
||||
.unwrap_or(amount)
|
||||
.parse::<Decimal>()
|
||||
.map_err(|_| UsageDataError::invalid("聚合费用不是有效十进制数"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -536,6 +536,9 @@ impl DatabaseDumpPort for FakeDumpPort {
|
||||
pub(crate) struct FakeObjectStore {
|
||||
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
|
||||
upload_error: Option<BackupError>,
|
||||
head_error: Option<BackupError>,
|
||||
delete_error: Option<BackupError>,
|
||||
connection_error: Option<BackupError>,
|
||||
}
|
||||
|
||||
impl FakeObjectStore {
|
||||
@@ -543,6 +546,9 @@ impl FakeObjectStore {
|
||||
Self {
|
||||
objects: Mutex::new(std::collections::HashMap::new()),
|
||||
upload_error: None,
|
||||
head_error: None,
|
||||
delete_error: None,
|
||||
connection_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -552,6 +558,21 @@ impl FakeObjectStore {
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn fail_head(mut self, code: &'static str) -> Self {
|
||||
self.head_error = Some(BackupError::new(code, "HEAD failed".into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn fail_delete(mut self, code: &'static str) -> Self {
|
||||
self.delete_error = Some(BackupError::new(code, "DELETE failed".into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn fail_connection(mut self, code: &'static str) -> Self {
|
||||
self.connection_error = Some(BackupError::new(code, "DELETE failed".into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
|
||||
self.objects.lock().expect("objects").get(key).cloned()
|
||||
}
|
||||
@@ -563,6 +584,14 @@ impl BackupObjectStorePort for FakeObjectStore {
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.connection_error {
|
||||
return Ok(ConnectionTestResult {
|
||||
ok: false,
|
||||
stage: "deleteObject",
|
||||
code: Some(error.code()),
|
||||
message: error.message().into(),
|
||||
});
|
||||
}
|
||||
Ok(ConnectionTestResult {
|
||||
ok: true,
|
||||
stage: "deleteObject",
|
||||
@@ -597,6 +626,9 @@ impl BackupObjectStorePort for FakeObjectStore {
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.head_error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let Some(content) = self.object(object_key) else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -613,6 +645,9 @@ impl BackupObjectStorePort for FakeObjectStore {
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.delete_error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
self.objects.lock().expect("objects").remove(object_key);
|
||||
Ok(())
|
||||
}
|
||||
@@ -690,11 +725,11 @@ impl AuthStore for FakeAuthStore {
|
||||
) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &gateway_admin::model::auth::AuthSession,
|
||||
_: chrono::DateTime<chrono::Utc>,
|
||||
_: Option<&gateway_admin::model::auth::AuthSession>,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AuthSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
@@ -324,3 +324,58 @@ async fn completed_recovery_removes_staging_for_dumping_and_uploading_tasks() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_backup_preserves_primary_and_final_delete_failures() {
|
||||
for (upload_fails, expiring) in [(false, false), (true, false), (false, true)] {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let store = FakeObjectStore::new().fail_delete(code::S3_PERMISSION_DENIED);
|
||||
let store = Arc::new(if upload_fails {
|
||||
store.fail_upload(code::S3_UPLOAD_FAILED, "上传失败")
|
||||
} else {
|
||||
store.fail_head(code::REMOTE_VERIFICATION_FAILED)
|
||||
});
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("cleanup-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/cleanup-failure.dump".into(),
|
||||
expires_at: expiring.then(|| Utc::now() + Duration::hours(1)),
|
||||
};
|
||||
repository.insert_backup_record(seed.clone()).await.unwrap();
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), store.clone());
|
||||
task.run_cycle(&CancellationToken::new()).await.unwrap();
|
||||
let records = repository.all_records();
|
||||
let record = &records[0];
|
||||
assert_eq!(record.status, BackupStatus::Failed);
|
||||
assert_eq!(
|
||||
record.error_code.as_deref(),
|
||||
Some(if upload_fails {
|
||||
code::S3_UPLOAD_FAILED
|
||||
} else {
|
||||
code::REMOTE_VERIFICATION_FAILED
|
||||
})
|
||||
);
|
||||
let message = record.error_message.as_deref().unwrap();
|
||||
assert!(message.contains(if upload_fails {
|
||||
"上传失败"
|
||||
} else {
|
||||
"HEAD failed"
|
||||
}));
|
||||
assert!(message.contains("DeleteObject"));
|
||||
assert!(message.contains(code::S3_PERMISSION_DENIED));
|
||||
assert!(message.contains(if expiring {
|
||||
"等待备份到期清理"
|
||||
} else {
|
||||
"未安排自动重试"
|
||||
}));
|
||||
assert_eq!(store.object(&seed.object_key).is_some(), !upload_fails);
|
||||
assert!(dump.inspect_staging(&seed.id).await.unwrap().is_none());
|
||||
task.run_cycle(&CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(
|
||||
repository.all_records()[0].error_message,
|
||||
record.error_message
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,9 +170,9 @@ fn usage_window_should_not_use_daily_rolling_usage_for_weekly_statistics() {
|
||||
success_count: 5,
|
||||
input_tokens: Some(100),
|
||||
output_tokens: Some(20),
|
||||
cached_tokens: Some(80),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(80),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
image_request_count: 0,
|
||||
|
||||
@@ -115,9 +115,9 @@ fn usage_window(key: &str, group: &str, seconds: u64) -> ProviderQuotaWindow {
|
||||
success_count: 0,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
|
||||
@@ -32,9 +32,9 @@ fn window(key: &str, days: u64) -> ProviderQuotaWindow {
|
||||
success_count: 10,
|
||||
input_tokens: Some(900),
|
||||
output_tokens: Some(100),
|
||||
cached_tokens: Some(800),
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: Some(800),
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
@@ -84,7 +84,7 @@ fn samples(quota: &ProviderQuota) -> Vec<QuotaForecastSample> {
|
||||
tokens: usage.total_tokens.unwrap_or(0),
|
||||
input_tokens: usage.input_tokens.unwrap_or(0),
|
||||
output_tokens: usage.output_tokens.unwrap_or(0),
|
||||
cached_tokens: usage.cached_tokens.unwrap_or(0),
|
||||
cached_input_tokens: usage.cached_input_tokens.unwrap_or(0),
|
||||
missing_token_count: u64::from(usage.total_tokens.is_none()),
|
||||
known_cost_count: if usd.is_some() {
|
||||
usage.cost_coverage.known_count()
|
||||
|
||||
@@ -73,14 +73,14 @@ fn snapshot_at_a_block_boundary_keeps_only_three_recent_blocks() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_percentage_reversal_restarts_sampling_without_crossing_old_consumption() {
|
||||
fn large_percentage_reversal_restarts_only_sampling_without_crossing_old_consumption() {
|
||||
let sample = select(
|
||||
vec![point(1, 40.0, 100), point(2, 50.0, 200), point(3, 1.0, 210)],
|
||||
point(4, 8.0, 280),
|
||||
);
|
||||
assert_eq!(sample.baseline_percent, 1.0);
|
||||
assert_eq!(sample.usage.tokens, 70);
|
||||
assert_eq!(sample.cycle_usage.tokens, 70);
|
||||
assert_eq!(sample.cycle_usage.tokens, 280);
|
||||
assert_eq!(sample.sampled_percent, 7.0);
|
||||
assert!(!sample.discontinuous);
|
||||
let waiting = select(
|
||||
@@ -90,6 +90,30 @@ fn large_percentage_reversal_restarts_sampling_without_crossing_old_consumption(
|
||||
assert!(waiting.discontinuous);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_window_reversal_preserves_all_recorded_cycle_usage() {
|
||||
let current = point(3, 35.0, 140);
|
||||
let recorded = current.usage.clone();
|
||||
let sample = select(vec![point(1, 32.0, 100), point(2, 30.0, 120)], current);
|
||||
assert_eq!(sample.cycle_usage, recorded);
|
||||
assert_eq!(sample.usage.tokens, 20);
|
||||
assert!((sample.usage.usd - 0.02).abs() < 1e-10);
|
||||
assert!(!sample.discontinuous);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_reversals_after_full_usage_preserve_cycle_totals_and_cumulative_forecasts() {
|
||||
let current = point(3, 99.5, 10_020);
|
||||
let recorded = current.usage.clone();
|
||||
let sample = select(
|
||||
vec![point(1, 100.0, 10_000), point(2, 99.0, 10_010)],
|
||||
current,
|
||||
);
|
||||
assert_eq!(sample.cycle_usage, recorded);
|
||||
assert_eq!(sample.method, QuotaForecastMethod::Cumulative);
|
||||
assert!(!sample.discontinuous);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_point_reversal_is_skipped_without_counting_an_extra_block() {
|
||||
let sample = select(
|
||||
@@ -128,17 +152,41 @@ fn counter_regressions_fail_closed_and_future_points_are_not_used() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_history_is_retained_to_detect_a_reset_with_an_unchanged_deadline() {
|
||||
fn same_window_recovery_restarts_sampling_but_preserves_recorded_cycle_usage() {
|
||||
let sample = select(
|
||||
vec![point(1, 100.0, 10_000), point(2, 0.0, 10_000)],
|
||||
point(3, 10.0, 10_100),
|
||||
);
|
||||
assert_eq!(sample.cycle_usage.tokens, 100);
|
||||
assert_eq!(sample.cycle_usage.tokens, 10_100);
|
||||
assert!((sample.cycle_usage.usd - 10.1).abs() < 1e-10);
|
||||
assert_eq!(sample.usage.tokens, 100);
|
||||
assert_eq!(sample.sampled_percent, 10.0);
|
||||
assert!(!sample.discontinuous);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_single_non_exhausted_observation_does_not_restart_cycle_usage() {
|
||||
let sample = select(vec![point(1, 100.0, 10_000)], point(2, 0.0, 10_000));
|
||||
assert_eq!(sample.cycle_usage.tokens, 10_000);
|
||||
assert!(sample.discontinuous);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_same_window_recoveries_preserve_recorded_cycle_usage() {
|
||||
let sample = select(
|
||||
vec![
|
||||
point(1, 100.0, 10_000),
|
||||
point(2, 0.0, 10_000),
|
||||
point(3, 100.0, 10_100),
|
||||
point(4, 0.0, 10_100),
|
||||
],
|
||||
point(5, 10.0, 10_200),
|
||||
);
|
||||
assert_eq!(sample.cycle_usage.tokens, 10_200);
|
||||
assert_eq!(sample.usage.tokens, 100);
|
||||
assert!(!sample.discontinuous);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_window_restarts_cycle_totals_at_first_matching_observation() {
|
||||
let sample = select_forecast_sample(
|
||||
|
||||
@@ -4,7 +4,13 @@ use gateway_admin::model::retention::{RetentionPolicy, RetentionTarget};
|
||||
|
||||
#[test]
|
||||
fn retention_policy_rejects_invalid_windows_and_preserves_independent_windows() {
|
||||
for (usage, ops, audit) in [(30, 1, 1), (31, 0, 1), (31, 1, 0)] {
|
||||
for (usage, ops, audit) in [
|
||||
(0, 1, 1),
|
||||
(32, 1, 1),
|
||||
(u32::MAX, 1, 1),
|
||||
(1, 0, 1),
|
||||
(1, 1, 0),
|
||||
] {
|
||||
assert!(RetentionPolicy::try_new(usage, ops, audit).is_err());
|
||||
}
|
||||
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
|
||||
@@ -12,3 +18,13 @@ fn retention_policy_rejects_invalid_windows_and_preserves_independent_windows()
|
||||
assert_eq!(policy.days(RetentionTarget::OpsEvents), 7);
|
||||
assert_eq!(policy.days(RetentionTarget::AdminAuditEvents), 90);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_retention_accepts_short_windows_without_changing_other_targets() {
|
||||
for days in [1, 3, 7, 15, 30, 31] {
|
||||
let policy = RetentionPolicy::try_new(days, 30, 90).unwrap();
|
||||
assert_eq!(policy.days(RetentionTarget::ModelRequests), days);
|
||||
assert_eq!(policy.days(RetentionTarget::OpsEvents), 30);
|
||||
assert_eq!(policy.days(RetentionTarget::AdminAuditEvents), 90);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -273,6 +273,19 @@ struct FakeRuntimeStore {
|
||||
|
||||
#[async_trait]
|
||||
impl AccountRuntimeStore for FakeRuntimeStore {
|
||||
async fn account_concurrency(
|
||||
&self,
|
||||
_: &str,
|
||||
_: u64,
|
||||
_: u32,
|
||||
) -> AdminStoreResult<gateway_admin::model::accounts::AccountConcurrencyPage> {
|
||||
Err(gateway_admin::ports::store::AdminStoreError::new(
|
||||
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"account concurrency",
|
||||
"not configured in this fixture",
|
||||
))
|
||||
}
|
||||
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
@@ -317,6 +330,7 @@ impl AccountRuntimeStore for FakeRuntimeStore {
|
||||
fn group_record() -> AccountGroupRecord {
|
||||
let now = Utc::now();
|
||||
AccountGroupRecord {
|
||||
allow_image_generation: true,
|
||||
fast_mode: FastMode::Default,
|
||||
id: group_id(),
|
||||
name: "Primary".to_owned(),
|
||||
@@ -390,6 +404,7 @@ async fn invalid_group_fields_are_rejected_before_store_mutations() {
|
||||
.create(
|
||||
&context,
|
||||
CreateAccountGroup {
|
||||
allow_image_generation: true,
|
||||
fast_mode: FastMode::Default,
|
||||
name: " invalid".to_owned(),
|
||||
description: None,
|
||||
@@ -403,6 +418,7 @@ async fn invalid_group_fields_are_rejected_before_store_mutations() {
|
||||
.update(
|
||||
&context,
|
||||
UpdateAccountGroup {
|
||||
allow_image_generation: None,
|
||||
fast_mode: None,
|
||||
id: record.id,
|
||||
name: "valid".to_owned(),
|
||||
|
||||
+221
-10
@@ -1,5 +1,7 @@
|
||||
//! 账号管理用例测试及共享 Provider、存储替身
|
||||
|
||||
mod reset_credit_automation;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex},
|
||||
@@ -87,6 +89,11 @@ pub(super) struct FakeProviderAdmin {
|
||||
quota_refresh_account: Mutex<Option<(Arc<FakeAccountStore>, AccountRecord)>>,
|
||||
current_credential_revision: Mutex<Revision>,
|
||||
reset_credit_commands: Mutex<Vec<ConsumeProviderResetCredit>>,
|
||||
reset_credits: Mutex<Vec<gateway_admin::model::provider_credentials::ProviderResetCredit>>,
|
||||
reset_credit_failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
reset_credit_result: Mutex<String>,
|
||||
reset_credit_query_started: tokio::sync::Notify,
|
||||
reset_credit_query_gate: Mutex<Option<tokio::sync::oneshot::Receiver<()>>>,
|
||||
profile_result: Mutex<Result<ProviderProfileStatistics, ProviderAdminErrorKind>>,
|
||||
subscription_result: Mutex<Result<Option<ProviderSubscription>, ProviderAdminErrorKind>>,
|
||||
personal_info_barrier: Mutex<Option<Arc<tokio::sync::Barrier>>>,
|
||||
@@ -112,6 +119,11 @@ impl FakeProviderAdmin {
|
||||
quota_refresh_account: Mutex::new(None),
|
||||
current_credential_revision: Mutex::new(revision(1)),
|
||||
reset_credit_commands: Mutex::new(Vec::new()),
|
||||
reset_credits: Mutex::default(),
|
||||
reset_credit_failure: Mutex::default(),
|
||||
reset_credit_result: Mutex::new("reset".into()),
|
||||
reset_credit_query_started: tokio::sync::Notify::new(),
|
||||
reset_credit_query_gate: Mutex::new(None),
|
||||
profile_result: Mutex::new(Ok(empty_profile_statistics())),
|
||||
subscription_result: Mutex::new(Ok(None)),
|
||||
personal_info_barrier: Mutex::new(None),
|
||||
@@ -498,6 +510,28 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
Ok(self.quota.lock().expect("provider quota").clone())
|
||||
}
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
) -> Result<gateway_admin::model::provider_credentials::ProviderResetCredits, ProviderAdminError>
|
||||
{
|
||||
self.record("provider.reset_credits");
|
||||
self.require_available()?;
|
||||
self.reset_credit_query_started.notify_one();
|
||||
let gate = self.reset_credit_query_gate.lock().unwrap().take();
|
||||
if let Some(gate) = gate {
|
||||
let _ = gate.await;
|
||||
}
|
||||
|
||||
let credits = self.reset_credits.lock().unwrap().clone();
|
||||
Ok(
|
||||
gateway_admin::model::provider_credentials::ProviderResetCredits {
|
||||
available_count: credits.len() as u64,
|
||||
credits,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async fn consume_reset_credit(
|
||||
&self,
|
||||
command: ConsumeProviderResetCredit,
|
||||
@@ -508,8 +542,11 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
.expect("provider reset-credit commands")
|
||||
.push(command);
|
||||
self.require_available()?;
|
||||
if let Some(kind) = self.reset_credit_failure.lock().unwrap().take() {
|
||||
return Err(ProviderAdminError::new(kind));
|
||||
}
|
||||
Ok(ProviderResetCreditResult {
|
||||
code: "reset".to_owned(),
|
||||
code: self.reset_credit_result.lock().unwrap().clone(),
|
||||
credit: None,
|
||||
})
|
||||
}
|
||||
@@ -577,6 +614,9 @@ pub(super) struct FakeAccountStore {
|
||||
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
|
||||
quota_forecast_history: Mutex<QuotaForecastHistory>,
|
||||
update_commands: Mutex<Vec<UpdateAccount>>,
|
||||
automations: Mutex<
|
||||
BTreeMap<String, gateway_admin::model::reset_credit_automation::ResetCreditAutomation>,
|
||||
>,
|
||||
pub(super) lowered_limits: Mutex<Vec<(String, u32)>>,
|
||||
}
|
||||
|
||||
@@ -599,6 +639,7 @@ impl FakeAccountStore {
|
||||
quota_forecast_history: Mutex::new(QuotaForecastHistory::default()),
|
||||
update_commands: Mutex::new(Vec::new()),
|
||||
lowered_limits: Mutex::new(Vec::new()),
|
||||
automations: Mutex::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -693,6 +734,16 @@ impl FakeAccountStore {
|
||||
|
||||
#[async_trait]
|
||||
impl AccountStore for FakeAccountStore {
|
||||
async fn account_exists(&self, account_id: &str) -> AdminStoreResult<bool> {
|
||||
self.record("store.account_exists");
|
||||
Ok(self
|
||||
.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.any(|account| account.id == account_id))
|
||||
}
|
||||
|
||||
async fn list_plugin_accounts(
|
||||
&self,
|
||||
_: gateway_admin::model::provider_credentials::PluginAccountListQuery,
|
||||
@@ -1086,7 +1137,10 @@ impl AccountStore for FakeAccountStore {
|
||||
}
|
||||
}
|
||||
|
||||
struct StaticSettingsStore;
|
||||
#[derive(Default)]
|
||||
struct StaticSettingsStore {
|
||||
reset_credit_max_weekly_remaining_percent: Option<u32>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SettingsStore for StaticSettingsStore {
|
||||
@@ -1115,12 +1169,12 @@ impl SettingsStore for StaticSettingsStore {
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
updated_at: Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 1,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
@@ -1145,6 +1199,9 @@ impl SettingsStore for StaticSettingsStore {
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: self
|
||||
.reset_credit_max_weekly_remaining_percent,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1840,7 +1897,7 @@ async fn accounts_batch_update_should_commit_once_and_notify_each_provider() {
|
||||
store.set_accounts(vec![openai_account, xai_account]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.settings(Arc::new(StaticSettingsStore::default()))
|
||||
.provider(openai)
|
||||
.provider(xai)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
@@ -2048,7 +2105,7 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
store.set_accounts(vec![account_record("openai"), xai_account]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.settings(Arc::new(StaticSettingsStore::default()))
|
||||
.provider(openai)
|
||||
.provider(failing)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
@@ -2398,6 +2455,93 @@ async fn quota_forecast_only_queries_the_two_selected_source_windows() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_forecast_same_window_reversal_keeps_recorded_usage_in_the_report() {
|
||||
use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistoryPoint;
|
||||
|
||||
for (percents, tokens, expected_tokens, expected_usd) in [
|
||||
([32.0, 30.0, 35.0], [100, 120, 140], 400, 0.4),
|
||||
(
|
||||
[100.0, 99.0, 99.5],
|
||||
[10_000, 10_010, 10_020],
|
||||
10_070,
|
||||
10.02 / 0.995,
|
||||
),
|
||||
] {
|
||||
let now = Utc::now();
|
||||
let observed = now - TimeDelta::minutes(1);
|
||||
let reset = now + TimeDelta::days(1);
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
plan_type: Some("pro".to_owned()),
|
||||
observed_at: Some(observed),
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "week".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "周额度".to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(604_800),
|
||||
used_percent: Some(percents[2]),
|
||||
reset_at: Some(reset),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
..empty_quota()
|
||||
});
|
||||
let mut account = account_record("openai");
|
||||
account.created_at = now - TimeDelta::days(7);
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let usage = |requests, tokens| QuotaForecastUsage {
|
||||
request_count: requests,
|
||||
tokens,
|
||||
input_tokens: tokens,
|
||||
known_cost_count: requests,
|
||||
usd: tokens as f64 / 1_000.0,
|
||||
..Default::default()
|
||||
};
|
||||
*store.quota_forecast_history.lock().unwrap() = QuotaForecastHistory {
|
||||
points: [(3, percents[0], tokens[0]), (2, percents[1], tokens[1])]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(
|
||||
|(index, (minutes, percent, tokens))| QuotaForecastHistoryPoint {
|
||||
started_at: now - TimeDelta::minutes(5),
|
||||
completed_at: now - TimeDelta::minutes(minutes),
|
||||
usage: usage(index as u64 + 1, tokens),
|
||||
provider_observation: ProviderDocument::new(OpaqueProviderData::new(
|
||||
serde_json::json!({
|
||||
"percent": percent, "reset": reset.to_rfc3339(), "plan": "pro",
|
||||
})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
)),
|
||||
},
|
||||
)
|
||||
.collect(),
|
||||
usage: usage(3, tokens[2]),
|
||||
pending_request_count: 0,
|
||||
};
|
||||
let services = accounts_service(provider, store).await;
|
||||
let result = services
|
||||
.accounts()
|
||||
.quota_forecast(&ProviderAccountId::new("acct_test").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let forecast = &result.forecasts[0];
|
||||
let source = forecast.source.as_ref().unwrap();
|
||||
assert_eq!(source.tokens, Some(tokens[2]));
|
||||
assert!((source.usd.unwrap() - tokens[2] as f64 / 1_000.0).abs() < 1e-10);
|
||||
assert_eq!(forecast.estimated_tokens, Some(expected_tokens));
|
||||
assert!((forecast.estimated_usd.unwrap() - expected_usd).abs() < 1e-10);
|
||||
assert!(forecast.unavailable_reason.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_changed_bucket() {
|
||||
use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistoryPoint;
|
||||
@@ -3031,9 +3175,9 @@ fn quota_local_usage(account_id: &str, total_tokens: u64) -> AccountUsage {
|
||||
success_count: 1,
|
||||
input_tokens: Some(total_tokens),
|
||||
output_tokens: Some(0),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(0),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
image_request_count: 0,
|
||||
@@ -3050,6 +3194,7 @@ fn quota_local_usage(account_id: &str, total_tokens: u64) -> AccountUsage {
|
||||
pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
let now = Utc::now();
|
||||
AccountRecord {
|
||||
outbound_proxy_enabled: true,
|
||||
notes: None,
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
@@ -3317,7 +3462,7 @@ async fn accounts_service_with_probe(
|
||||
) -> AdminServices {
|
||||
super::AdminHarness::new()
|
||||
.accounts(store)
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.settings(Arc::new(StaticSettingsStore::default()))
|
||||
.provider(provider)
|
||||
.probe(probe)
|
||||
.build()
|
||||
@@ -3486,6 +3631,19 @@ struct CapacityRuntime {
|
||||
|
||||
#[async_trait]
|
||||
impl AccountRuntimeStore for CapacityRuntime {
|
||||
async fn account_concurrency(
|
||||
&self,
|
||||
_: &str,
|
||||
_: u64,
|
||||
_: u32,
|
||||
) -> AdminStoreResult<gateway_admin::model::accounts::AccountConcurrencyPage> {
|
||||
Err(gateway_admin::ports::store::AdminStoreError::new(
|
||||
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"account concurrency",
|
||||
"not configured in this fixture",
|
||||
))
|
||||
}
|
||||
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
@@ -3548,7 +3706,7 @@ async fn account_capacity_should_batch_page_ids_and_distinguish_idle_from_unavai
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store)
|
||||
.account_runtime(runtime.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.settings(Arc::new(StaticSettingsStore::default()))
|
||||
.provider(FakeProviderAdmin::new("openai", events()))
|
||||
.build()
|
||||
.await;
|
||||
@@ -3579,3 +3737,56 @@ async fn account_capacity_should_batch_page_ids_and_distinguish_idle_from_unavai
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl gateway_admin::ports::reset_credit_automation::ResetCreditAutomationStore
|
||||
for FakeAccountStore
|
||||
{
|
||||
async fn reset_credit_automation(
|
||||
&self,
|
||||
account_id: &str,
|
||||
) -> AdminStoreResult<
|
||||
Option<gateway_admin::model::reset_credit_automation::ResetCreditAutomation>,
|
||||
> {
|
||||
Ok(self.automations.lock().unwrap().get(account_id).cloned())
|
||||
}
|
||||
async fn due_reset_credit_automations(
|
||||
&self,
|
||||
now: chrono::DateTime<Utc>,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<String>> {
|
||||
let records = self.automations.lock().unwrap();
|
||||
let mut due = records
|
||||
.values()
|
||||
.filter(|r| r.enabled && r.next_run_at <= now)
|
||||
.collect::<Vec<_>>();
|
||||
due.sort_by_key(|r| r.next_run_at);
|
||||
Ok(due
|
||||
.into_iter()
|
||||
.take(limit as usize)
|
||||
.map(|r| r.account_id.clone())
|
||||
.collect())
|
||||
}
|
||||
async fn save_reset_credit_automation(
|
||||
&self,
|
||||
record: &gateway_admin::model::reset_credit_automation::ResetCreditAutomation,
|
||||
context: Option<&MutationContext>,
|
||||
) -> AdminStoreResult<u64> {
|
||||
let mut records = self.automations.lock().unwrap();
|
||||
if records.get(&record.account_id).map_or(0, |r| r.revision) != record.revision {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::Conflict,
|
||||
"automation",
|
||||
"stale revision",
|
||||
));
|
||||
}
|
||||
let mut next = record.clone();
|
||||
next.revision += 1;
|
||||
let revision = next.revision;
|
||||
records.insert(next.account_id.clone(), next);
|
||||
if let Some(context) = context {
|
||||
self.record_context(context);
|
||||
}
|
||||
Ok(revision)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,969 @@
|
||||
//! 自动重置的持久幂等恢复、低频发现与账号独立错峰
|
||||
|
||||
use super::*;
|
||||
use gateway_admin::{
|
||||
model::reset_credit_automation::*,
|
||||
ports::reset_credit_automation::ResetCreditAutomationStore as _,
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::middleware::{self, FrozenMiddlewarePlan, MiddlewarePlan},
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ServiceForwarder;
|
||||
|
||||
impl ExtensionSetLease for ServiceForwarder {
|
||||
fn is_ready(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl MiddlewarePlan for ServiceForwarder {
|
||||
fn has_service(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn handle_service(
|
||||
&self,
|
||||
_: middleware::service::Context,
|
||||
input: middleware::service::Value,
|
||||
next: middleware::service::Next,
|
||||
) -> BoxFuture<'static, Result<middleware::service::Value, middleware::service::Error>> {
|
||||
Box::pin(async move {
|
||||
// 模拟 RPC 回调跨任务执行 next,且该任务带有另一插件的身份
|
||||
tokio::spawn(gateway_admin::public_service::scope_actor(
|
||||
gateway_admin::model::usage_data::UsageActor::Plugin {
|
||||
publisher: "test".into(),
|
||||
name: "forwarder".into(),
|
||||
instance_id: "forwarding-plugin".into(),
|
||||
},
|
||||
next.run(input),
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
})
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
_: middleware::MiddlewareContext,
|
||||
request: middleware::MiddlewareRequest,
|
||||
next: middleware::MiddlewareNext,
|
||||
) -> BoxFuture<'static, Result<middleware::MiddlewareResponse, middleware::MiddlewareError>>
|
||||
{
|
||||
next.run(request)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn public_service_reads_and_updates_reset_credit_automation_with_host_identity() {
|
||||
use gateway_admin::{
|
||||
model::usage_data::UsageActor,
|
||||
public_service::{Origin, Plan, scope, scope_actor},
|
||||
};
|
||||
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = accounts_service(provider, store.clone()).await;
|
||||
let registry = services.public_services();
|
||||
let read = "accounts.reset_credit_automation_status";
|
||||
let write = "accounts.set_reset_credit_automation";
|
||||
let initial = registry.call(read, json!("acct_test")).await.unwrap();
|
||||
assert_eq!(initial["enabled"], false);
|
||||
assert_eq!(initial["lead_minutes"], 10);
|
||||
|
||||
let actor = UsageActor::Plugin {
|
||||
publisher: "test".into(),
|
||||
name: "automation".into(),
|
||||
instance_id: "installed-plugin".into(),
|
||||
};
|
||||
let middleware = Arc::new(ServiceForwarder);
|
||||
let plan = FrozenMiddlewarePlan::new(
|
||||
middleware.clone(),
|
||||
ExtensionSetReference::new(
|
||||
ExtensionSetId::new("service-forwarder".into()).unwrap(),
|
||||
middleware,
|
||||
),
|
||||
);
|
||||
let origin = Origin {
|
||||
request_id: "plugin-request".into(),
|
||||
call_id: "plugin-parent".into(),
|
||||
cancellation: Default::default(),
|
||||
extensions: Default::default(),
|
||||
plan: Plan::Frozen(Some(plan)),
|
||||
};
|
||||
let error = scope(
|
||||
origin.clone(),
|
||||
registry.call(write, json!(["acct_test", true])),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind, "invalid");
|
||||
let enabled = scope_actor(
|
||||
actor.clone(),
|
||||
scope(
|
||||
origin.clone(),
|
||||
registry.call(write, json!(["acct_test", true])),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status: ResetCreditAutomationStatus = serde_json::from_value(enabled.clone()).unwrap();
|
||||
assert!(status.enabled);
|
||||
assert!(status.next_discovery_at.is_some());
|
||||
assert_eq!(
|
||||
registry.call(read, json!("acct_test")).await.unwrap(),
|
||||
enabled
|
||||
);
|
||||
assert!(store.automations.lock().unwrap()["acct_test"].enabled);
|
||||
let audit = store.audit_requests();
|
||||
assert_eq!(audit.len(), 1);
|
||||
assert!(audit[0].starts_with("plugin:installed-plugin:request:plugin-request:call:"));
|
||||
|
||||
let repeated = scope_actor(
|
||||
actor.clone(),
|
||||
scope(
|
||||
origin.clone(),
|
||||
registry.call(write, json!(["acct_test", true])),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(repeated, enabled);
|
||||
assert_eq!(store.audit_requests(), audit);
|
||||
|
||||
let disabled = scope_actor(
|
||||
actor,
|
||||
scope(origin, registry.call(write, json!(["acct_test", false]))),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(disabled["enabled"], false);
|
||||
assert!(disabled["next_discovery_at"].is_null());
|
||||
assert!(disabled["next_consume_at"].is_null());
|
||||
assert!(!store.automations.lock().unwrap()["acct_test"].enabled);
|
||||
assert_eq!(store.audit_requests().len(), 2);
|
||||
assert!(!recorded(&events).contains(&"provider.reset_credits"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn public_service_automation_rejects_invalid_and_cancelled_writes_without_side_effects() {
|
||||
use gateway_admin::{
|
||||
model::usage_data::UsageActor,
|
||||
public_service::{Origin, Plan, scope, scope_actor},
|
||||
};
|
||||
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events);
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let registry = services.public_services();
|
||||
let write = "accounts.set_reset_credit_automation";
|
||||
// 直接服务调用也必须有可信主体,输入不能自报管理员身份
|
||||
let error = registry
|
||||
.call(write, json!(["acct_test", true]))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind, "invalid");
|
||||
let error = scope_actor(
|
||||
UsageActor::System,
|
||||
registry.call(
|
||||
write,
|
||||
json!([{"actor": "admin_api_key"}, "acct_test", true]),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind, "invalid");
|
||||
let error = scope_actor(
|
||||
UsageActor::System,
|
||||
registry.call(write, json!(["acct_test", true])),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind, "invalid");
|
||||
assert_eq!(error.message, "当前账号不支持自动重置卡");
|
||||
|
||||
enable_capability(&provider);
|
||||
store.set_accounts(Vec::new());
|
||||
let error = scope_actor(
|
||||
UsageActor::System,
|
||||
registry.call(write, json!(["acct_test", true])),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind, "not_found");
|
||||
let cancellation = gateway_core::lifecycle::CancellationToken::new();
|
||||
cancellation.cancel();
|
||||
let error = scope_actor(
|
||||
UsageActor::System,
|
||||
scope(
|
||||
Origin {
|
||||
request_id: "cancelled".into(),
|
||||
call_id: "parent".into(),
|
||||
cancellation,
|
||||
extensions: Default::default(),
|
||||
plan: Plan::Frozen(None),
|
||||
},
|
||||
registry.call(write, json!(["acct_test", true])),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind, "unavailable");
|
||||
assert!(store.automations.lock().unwrap().is_empty());
|
||||
assert!(store.audit_requests().is_empty());
|
||||
}
|
||||
|
||||
fn enable_capability(provider: &FakeProviderAdmin) {
|
||||
let mut capabilities = provider.capabilities.lock().unwrap();
|
||||
capabilities.reset_credits = true;
|
||||
capabilities.consume_reset_credit = true;
|
||||
}
|
||||
|
||||
fn plan(expires_at: chrono::DateTime<Utc>) -> ResetCreditAutomation {
|
||||
ResetCreditAutomation {
|
||||
account_id: "acct_test".into(),
|
||||
enabled: true,
|
||||
revision: 1,
|
||||
next_run_at: Utc::now(),
|
||||
state: ResetCreditAutomationState {
|
||||
next_discovery_at: Utc::now() + TimeDelta::days(1),
|
||||
credits: vec![PlannedResetCredit {
|
||||
id: "card_1".into(),
|
||||
expires_at,
|
||||
}],
|
||||
pending: None,
|
||||
retry_at: None,
|
||||
failures: 0,
|
||||
last_result: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn card(
|
||||
expires_at: chrono::DateTime<Utc>,
|
||||
) -> gateway_admin::model::provider_credentials::ProviderResetCredit {
|
||||
gateway_admin::model::provider_credentials::ProviderResetCredit {
|
||||
id: "card_1".into(),
|
||||
status: Some("available".into()),
|
||||
expires_at: Some(expires_at),
|
||||
title: None,
|
||||
reset_type: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_restores_pending_idempotency_after_service_restart() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let expiry = Utc::now() + TimeDelta::minutes(5);
|
||||
provider.reset_credits.lock().unwrap().push(card(expiry));
|
||||
*provider.reset_credit_failure.lock().unwrap() = Some(ProviderAdminErrorKind::Ambiguous);
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), plan(expiry));
|
||||
let account_id = ProviderAccountId::new("acct_test").unwrap();
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id.clone())
|
||||
.await
|
||||
.expect_err("unknown consume result");
|
||||
let persisted = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let pending = persisted.state.pending.as_ref().unwrap();
|
||||
let command = provider.reset_credit_commands()[0].clone();
|
||||
assert_eq!(
|
||||
pending.redeem_request_id,
|
||||
command.redeem_request_id.to_string()
|
||||
);
|
||||
assert_eq!(command.redeem_request_id.get_version_num(), 4);
|
||||
assert_eq!(provider.quota_requests().len(), 0);
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
provider.reset_credit_commands().len(),
|
||||
1,
|
||||
"wait until persisted retry time"
|
||||
);
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("acct_test")
|
||||
.unwrap()
|
||||
.state
|
||||
.retry_at = Some(Utc::now() - TimeDelta::seconds(1));
|
||||
drop(services);
|
||||
let restarted = threshold_services(provider.clone(), store.clone(), Some(0)).await;
|
||||
restarted
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(provider.reset_credit_commands(), [command.clone(), command]);
|
||||
assert_eq!(
|
||||
recorded(&events)
|
||||
.iter()
|
||||
.filter(|e| **e == "provider.reset_credits")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let record = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(record.state.pending.is_none());
|
||||
assert!(record.state.credits.is_empty());
|
||||
assert_eq!(record.state.last_result.as_deref(), Some("reset"));
|
||||
assert!(provider.quota_requests().iter().any(|q| q.refresh));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_only_queries_due_accounts_and_rechecks_credit_before_consume() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut record = plan(Utc::now() + TimeDelta::hours(6));
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), record.clone());
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let account_id = ProviderAccountId::new("acct_test").unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!recorded(&events).contains(&"provider.reset_credits"));
|
||||
record.state.credits[0].expires_at = Utc::now() + TimeDelta::minutes(5);
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), record);
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(recorded(&events).contains(&"provider.reset_credits"));
|
||||
assert!(
|
||||
provider.reset_credit_commands().is_empty(),
|
||||
"disappeared credit must never be consumed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_defaults_off_and_disabled_pending_survives_reenable() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let account_id = ProviderAccountId::new("acct_test").unwrap();
|
||||
assert!(
|
||||
!services
|
||||
.accounts()
|
||||
.reset_credit_automation_status(account_id.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
services
|
||||
.accounts()
|
||||
.set_reset_credit_automation(&context("enable"), account_id.clone(), true)
|
||||
.await
|
||||
.unwrap();
|
||||
let first = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(first.state.next_discovery_at >= Utc::now() - TimeDelta::seconds(1));
|
||||
let pending = PendingResetCredit {
|
||||
credit: PlannedResetCredit {
|
||||
id: "pending".into(),
|
||||
expires_at: Utc::now() + TimeDelta::hours(1),
|
||||
},
|
||||
redeem_request_id: uuid::Uuid::new_v4().to_string(),
|
||||
};
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("acct_test")
|
||||
.unwrap()
|
||||
.state
|
||||
.pending = Some(pending.clone());
|
||||
services
|
||||
.accounts()
|
||||
.set_reset_credit_automation(&context("disable"), account_id.clone(), false)
|
||||
.await
|
||||
.unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(provider.reset_credit_commands().is_empty());
|
||||
services
|
||||
.accounts()
|
||||
.set_reset_credit_automation(&context("reenable"), account_id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
let next = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(next.state.pending, Some(pending));
|
||||
assert_eq!(next.state.next_discovery_at, first.state.next_discovery_at);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_schedules_each_account_inside_the_lead_window_and_next_local_day() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events);
|
||||
let now = Utc::now();
|
||||
let expiry = now + TimeDelta::days(2);
|
||||
provider.reset_credits.lock().unwrap().push(card(expiry));
|
||||
let accounts = (0..100)
|
||||
.map(|n| {
|
||||
let mut account = account_record("openai");
|
||||
account.id = format!("acct_{n}");
|
||||
let mut record = plan(expiry);
|
||||
record.account_id = account.id.clone();
|
||||
record.state.next_discovery_at = now - TimeDelta::seconds(1);
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(account.id.clone(), record);
|
||||
account
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
store.set_accounts(accounts.clone());
|
||||
let services = accounts_service(provider, store.clone()).await;
|
||||
let timezone = gateway_core::time::DeploymentTimeZone::default();
|
||||
let start = timezone.days_after(now, 1).unwrap();
|
||||
let end = timezone.days_after(now, 2).unwrap();
|
||||
let mut times = std::collections::BTreeSet::new();
|
||||
for account in accounts {
|
||||
let id = ProviderAccountId::new(account.id.clone()).unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let status = services
|
||||
.accounts()
|
||||
.reset_credit_automation_status(id)
|
||||
.await
|
||||
.unwrap();
|
||||
let discovery = status.next_discovery_at.unwrap();
|
||||
assert!(discovery >= start && discovery < end);
|
||||
times.insert(status.next_consume_at.unwrap());
|
||||
}
|
||||
assert!(
|
||||
times.len() > 30,
|
||||
"accounts must not all fire at the same instant"
|
||||
);
|
||||
assert!(times.iter().all(
|
||||
|due| *due >= expiry - TimeDelta::minutes(10) && *due < expiry - TimeDelta::minutes(9)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_discards_externally_used_credit_and_stops_on_already_redeemed() {
|
||||
for already_redeemed_after_query in [false, true] {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let expiry = Utc::now() + TimeDelta::minutes(5);
|
||||
let mut remote_credit = card(expiry);
|
||||
if !already_redeemed_after_query {
|
||||
remote_credit.status = Some("redeemed".into());
|
||||
}
|
||||
provider.reset_credits.lock().unwrap().push(remote_credit);
|
||||
*provider.reset_credit_result.lock().unwrap() = "already_redeemed".into();
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), plan(expiry));
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let account_id = ProviderAccountId::new("acct_test").unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(account_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
provider.reset_credit_commands().len(),
|
||||
usize::from(already_redeemed_after_query)
|
||||
);
|
||||
let stored = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(stored.state.credits.is_empty());
|
||||
assert!(stored.state.pending.is_none());
|
||||
if already_redeemed_after_query {
|
||||
assert_eq!(
|
||||
stored.state.last_result.as_deref(),
|
||||
Some("already_redeemed")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_daily_discovery_is_not_polled_again_after_success_or_failure() {
|
||||
for failed in [false, true] {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
if failed {
|
||||
provider.fail_next(ProviderAdminErrorKind::Unavailable);
|
||||
}
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut record = plan(Utc::now() + TimeDelta::days(3));
|
||||
record.state.credits.clear();
|
||||
record.state.next_discovery_at = Utc::now() - TimeDelta::seconds(1);
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), record);
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let id = ProviderAccountId::new("acct_test").unwrap();
|
||||
let first = services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id.clone())
|
||||
.await;
|
||||
assert_eq!(first.is_err(), failed);
|
||||
let scheduled = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.state
|
||||
.next_discovery_at;
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
recorded(&events)
|
||||
.iter()
|
||||
.filter(|e| **e == "provider.reset_credits")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.state
|
||||
.next_discovery_at,
|
||||
scheduled
|
||||
);
|
||||
assert!(scheduled > Utc::now());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_cancelled_discovery_preserves_next_random_time_across_restart() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut record = plan(Utc::now() + TimeDelta::days(3));
|
||||
record.state.credits.clear();
|
||||
record.state.next_discovery_at = Utc::now() - TimeDelta::seconds(1);
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), record);
|
||||
let (release, gate) = tokio::sync::oneshot::channel();
|
||||
*provider.reset_credit_query_gate.lock().unwrap() = Some(gate);
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let task = tokio::spawn(async move {
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(ProviderAccountId::new("acct_test").unwrap())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
provider.reset_credit_query_started.notified(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
drop(release);
|
||||
let before = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(before.state.next_discovery_at > Utc::now());
|
||||
let restarted = accounts_service(provider.clone(), store.clone()).await;
|
||||
restarted
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(ProviderAccountId::new("acct_test").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let after = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
before.state.next_discovery_at,
|
||||
after.state.next_discovery_at
|
||||
);
|
||||
assert_eq!(
|
||||
recorded(&events)
|
||||
.iter()
|
||||
.filter(|e| **e == "provider.reset_credits")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_automation_only_blocks_manual_consume_until_pending_credit_expires() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
enable_capability(&provider);
|
||||
let store = FakeAccountStore::new("openai", events);
|
||||
let mut record = plan(Utc::now() + TimeDelta::minutes(5));
|
||||
record.enabled = false;
|
||||
record.state.pending = Some(PendingResetCredit {
|
||||
credit: record.state.credits[0].clone(),
|
||||
redeem_request_id: uuid::Uuid::new_v4().to_string(),
|
||||
});
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), record.clone());
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let command = ConsumeProviderResetCredit {
|
||||
account_id: ProviderAccountId::new("acct_test").unwrap(),
|
||||
credit_id: Some("another_card".into()),
|
||||
redeem_request_id: uuid::Uuid::new_v4(),
|
||||
};
|
||||
services
|
||||
.accounts()
|
||||
.consume_reset_credit(&context("manual-with-pending"), command.clone())
|
||||
.await
|
||||
.expect_err("unresolved automation is still active");
|
||||
assert!(provider.reset_credit_commands().is_empty());
|
||||
record.state.pending.as_mut().unwrap().credit.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), record);
|
||||
services
|
||||
.accounts()
|
||||
.consume_reset_credit(&context("manual-with-pending"), command)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(provider.reset_credit_commands().len(), 1);
|
||||
}
|
||||
|
||||
async fn threshold_services(
|
||||
provider: Arc<FakeProviderAdmin>,
|
||||
store: Arc<FakeAccountStore>,
|
||||
max_remaining: Option<u32>,
|
||||
) -> AdminServices {
|
||||
super::super::AdminHarness::new()
|
||||
.accounts(store)
|
||||
.settings(Arc::new(StaticSettingsStore {
|
||||
reset_credit_max_weekly_remaining_percent: max_remaining,
|
||||
}))
|
||||
.provider(provider)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.build()
|
||||
.await
|
||||
}
|
||||
|
||||
fn weekly_quota(used_percent: f64) -> ProviderQuota {
|
||||
ProviderQuota {
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "week".into(),
|
||||
group: "shortTerm".into(),
|
||||
label: "周额度".into(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(7 * 86_400),
|
||||
used_percent: Some(used_percent),
|
||||
reset_at: Some(Utc::now() + TimeDelta::days(1)),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
..empty_quota()
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule_due_credit(provider: &FakeProviderAdmin, store: &FakeAccountStore) {
|
||||
enable_capability(provider);
|
||||
let expiry = Utc::now() + TimeDelta::minutes(5);
|
||||
provider.reset_credits.lock().unwrap().push(card(expiry));
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("acct_test".into(), plan(expiry));
|
||||
}
|
||||
|
||||
fn advance_retry(store: &FakeAccountStore) {
|
||||
store
|
||||
.automations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("acct_test")
|
||||
.unwrap()
|
||||
.state
|
||||
.retry_at = Some(Utc::now() - TimeDelta::seconds(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_weekly_threshold_is_optional_inclusive_and_uses_remaining_quota() {
|
||||
for (threshold, used, consume) in [
|
||||
(None, 0.0, true),
|
||||
(Some(70), 29.9, false),
|
||||
(Some(70), 30.0, true),
|
||||
(Some(0), 99.9, false),
|
||||
(Some(0), 100.0, true),
|
||||
(Some(100), 0.0, true),
|
||||
] {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
schedule_due_credit(&provider, &store);
|
||||
provider.set_quota(weekly_quota(used));
|
||||
let services = threshold_services(provider.clone(), store.clone(), threshold).await;
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(ProviderAccountId::new("acct_test").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
!provider.reset_credit_commands().is_empty(),
|
||||
consume,
|
||||
"threshold={threshold:?}, used={used}"
|
||||
);
|
||||
let quota_requests = provider.quota_requests();
|
||||
// 消费成功已有一次回读;未设置上限时不增加消费前查询
|
||||
assert_eq!(
|
||||
quota_requests.len(),
|
||||
usize::from(threshold.is_some()) + usize::from(consume)
|
||||
);
|
||||
assert!(quota_requests.iter().all(|request| request.refresh));
|
||||
let record = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(record.state.pending.is_none());
|
||||
assert_eq!(record.state.credits.is_empty(), consume);
|
||||
if !consume {
|
||||
assert_eq!(
|
||||
record.state.last_result.as_deref(),
|
||||
Some("weekly_quota_above_threshold")
|
||||
);
|
||||
assert!(record.state.retry_at.unwrap() > Utc::now());
|
||||
assert!(record.state.retry_at.unwrap() <= record.state.credits[0].expires_at);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_retains_skipped_credit_and_refreshes_quota_on_next_attempt() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
schedule_due_credit(&provider, &store);
|
||||
provider.set_quota(weekly_quota(10.0));
|
||||
let services = threshold_services(provider.clone(), store.clone(), Some(70)).await;
|
||||
let id = ProviderAccountId::new("acct_test").unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(provider.reset_credit_commands().is_empty());
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
provider.set_quota(weekly_quota(40.0));
|
||||
advance_retry(&store);
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(provider.reset_credit_commands().len(), 1);
|
||||
let record = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(record.state.credits.is_empty());
|
||||
assert!(record.state.retry_at.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_defers_missing_invalid_or_failed_weekly_quota() {
|
||||
let weekly = weekly_quota(90.0).windows.remove(0);
|
||||
let mut short = weekly.clone();
|
||||
short.window_seconds = Some(18_000);
|
||||
let mut model = weekly.clone();
|
||||
model.local_usage_attribution = QuotaLocalUsageAttribution::Unavailable;
|
||||
let mut month = weekly.clone();
|
||||
month.group = "monthly".into();
|
||||
month.window_seconds = Some(30 * 86_400);
|
||||
let mut missing_percent = weekly.clone();
|
||||
missing_percent.used_percent = None;
|
||||
for (quota, failure) in [
|
||||
(empty_quota(), None),
|
||||
(
|
||||
ProviderQuota {
|
||||
windows: vec![short, model, month],
|
||||
..empty_quota()
|
||||
},
|
||||
None,
|
||||
),
|
||||
(
|
||||
ProviderQuota {
|
||||
windows: vec![missing_percent],
|
||||
..empty_quota()
|
||||
},
|
||||
None,
|
||||
),
|
||||
(weekly_quota(f64::NAN), None),
|
||||
(
|
||||
weekly_quota(90.0),
|
||||
Some(ProviderAdminErrorKind::Unavailable),
|
||||
),
|
||||
] {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
schedule_due_credit(&provider, &store);
|
||||
provider.set_quota(quota);
|
||||
*provider.quota_failure.lock().unwrap() = failure;
|
||||
let services = threshold_services(provider.clone(), store.clone(), Some(70)).await;
|
||||
let result = services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(ProviderAccountId::new("acct_test").unwrap())
|
||||
.await;
|
||||
assert_eq!(result.is_err(), failure.is_some());
|
||||
assert!(provider.reset_credit_commands().is_empty());
|
||||
let record = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
record.state.last_result.as_deref(),
|
||||
Some("weekly_quota_unavailable")
|
||||
);
|
||||
assert!(record.state.pending.is_none());
|
||||
assert_eq!(record.state.credits.len(), 1);
|
||||
assert!(record.state.retry_at.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automation_nothing_to_reset_retains_credit_and_retries_with_a_new_key() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
schedule_due_credit(&provider, &store);
|
||||
*provider.reset_credit_result.lock().unwrap() = "nothing_to_reset".into();
|
||||
let services = accounts_service(provider.clone(), store.clone()).await;
|
||||
let id = ProviderAccountId::new("acct_test").unwrap();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let record = store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
record.state.last_result.as_deref(),
|
||||
Some("nothing_to_reset")
|
||||
);
|
||||
assert!(record.state.pending.is_none());
|
||||
assert_eq!(record.state.credits.len(), 1);
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(provider.reset_credit_commands().len(), 1);
|
||||
advance_retry(&store);
|
||||
*provider.reset_credit_result.lock().unwrap() = "reset".into();
|
||||
services
|
||||
.accounts()
|
||||
.reconcile_reset_credit_automation(id)
|
||||
.await
|
||||
.unwrap();
|
||||
let commands = provider.reset_credit_commands();
|
||||
assert_eq!(commands.len(), 2);
|
||||
assert_eq!(commands[0].credit_id, commands[1].credit_id);
|
||||
assert_ne!(commands[0].redeem_request_id, commands[1].redeem_request_id);
|
||||
assert!(
|
||||
store
|
||||
.reset_credit_automation("acct_test")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.state
|
||||
.credits
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
+47
-110
@@ -1,4 +1,4 @@
|
||||
//! 验证管理员认证、密码变更、会话绑定与限流行为
|
||||
//! 验证管理员认证、密码变更与限流,提供会话轮换的内存存储辅助
|
||||
|
||||
use std::{collections::BTreeMap, sync::Mutex};
|
||||
|
||||
@@ -69,6 +69,7 @@ async fn password_change_obeys_rate_limit_and_rejects_unbound_legacy_sessions()
|
||||
admin_user_id: "admin".into(),
|
||||
credential_fingerprint: String::new(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + TimeDelta::hours(1),
|
||||
},
|
||||
@@ -138,19 +139,24 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
if session.retained_until() <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
if let Some(replacement) = replacement {
|
||||
*session = replacement.clone();
|
||||
} else {
|
||||
sessions.remove(session_id);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
@@ -190,8 +196,13 @@ impl AuthStore for MemoryAuthStore {
|
||||
#[tokio::test]
|
||||
async fn successful_login_should_create_expiring_session_and_audit() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new().auth(store.clone()).build().await;
|
||||
let services = super::AdminHarness::new()
|
||||
.session_ttl_minutes(7 * 24 * 60)
|
||||
.auth(store.clone())
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let before = Utc::now();
|
||||
let result = services
|
||||
.auth()
|
||||
.login(
|
||||
@@ -205,6 +216,17 @@ async fn successful_login_should_create_expiring_session_and_audit() {
|
||||
.await
|
||||
.expect("login");
|
||||
|
||||
let after = Utc::now();
|
||||
for (expiry, ttl) in [
|
||||
(result.session.expires_at, TimeDelta::hours(24)),
|
||||
(result.session.retained_until(), TimeDelta::days(7)),
|
||||
(
|
||||
result.session.absolute_expires_at.unwrap(),
|
||||
TimeDelta::days(30),
|
||||
),
|
||||
] {
|
||||
assert!((before + ttl..=after + ttl).contains(&expiry));
|
||||
}
|
||||
assert!(
|
||||
services
|
||||
.auth()
|
||||
@@ -293,6 +315,7 @@ async fn expired_sessions_are_rejected_and_store_outages_are_not_treated_as_logo
|
||||
credential_fingerprint: String::new(),
|
||||
admin_user_id: "admin".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() - TimeDelta::seconds(1),
|
||||
},
|
||||
@@ -362,7 +385,7 @@ async fn failed_rotation_discards_the_new_session_and_leaves_the_old_session_ret
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!format!("{original:?}").contains(&original.session_id));
|
||||
*store.reject_delete.lock().unwrap() = Some(original.session_id.clone());
|
||||
*store.reject_delete.lock().unwrap() = Some(stored_id(&original.session_id).to_owned());
|
||||
assert_eq!(
|
||||
services
|
||||
.auth()
|
||||
@@ -391,109 +414,23 @@ async fn failed_rotation_discards_the_new_session_and_leaves_the_old_session_ret
|
||||
assert!(store.sessions.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_is_throttled_and_capped_by_the_original_absolute_expiry() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new()
|
||||
.auth(store.clone())
|
||||
.session_ttl_minutes(60)
|
||||
.build()
|
||||
.await;
|
||||
let auth = services.auth();
|
||||
let login = auth
|
||||
.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(login.session.clone())
|
||||
);
|
||||
let limit = Utc::now() + TimeDelta::minutes(30);
|
||||
{
|
||||
let mut sessions = store.sessions.lock().unwrap();
|
||||
let session = sessions.get_mut(&login.session_id).unwrap();
|
||||
session.expires_at = Utc::now() + TimeDelta::minutes(1);
|
||||
session.absolute_expires_at = Some(limit);
|
||||
}
|
||||
let renewed = auth
|
||||
.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(renewed.expires_at, limit);
|
||||
assert_eq!(renewed.absolute_expires_at, Some(limit));
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(renewed)
|
||||
);
|
||||
fn stored_id(token: &str) -> &str {
|
||||
token.split('.').nth(1).unwrap_or(token)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_never_revives_expired_revoked_or_legacy_sessions() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = auth
|
||||
.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut session = login.session.clone();
|
||||
session.absolute_expires_at = None;
|
||||
session.expires_at = Utc::now() + TimeDelta::seconds(30);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session.clone());
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(session.clone())
|
||||
);
|
||||
session.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session);
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let mut session = login.session;
|
||||
session.absolute_expires_at = Some(Utc::now() - TimeDelta::seconds(1));
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session);
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
auth.logout(&login.session_id).await.unwrap();
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(store.sessions.lock().unwrap().is_empty());
|
||||
async fn admin_login(
|
||||
auth: &dyn gateway_admin::AuthService,
|
||||
) -> gateway_admin::model::auth::SessionGrant {
|
||||
auth.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
mod session;
|
||||
@@ -0,0 +1,268 @@
|
||||
//! 验证双凭据续期、响应重试、并发撤销与固定期限边界
|
||||
|
||||
use super::super::AdminHarness;
|
||||
use super::{MemoryAuthStore, admin_login, stored_id};
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_admin::model::auth::AuthSession;
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_access_is_restored_by_refresh_and_duplicate_refresh_replays_the_same_grant() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
let refresh = login.refresh_token.as_deref().unwrap();
|
||||
assert!(auth.session(Some(refresh)).await.unwrap().is_none());
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.renew_session(None, Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(refresh), None).await.unwrap(),
|
||||
Some(login.clone())
|
||||
);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
assert!(
|
||||
auth.session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let (first, concurrent) = tokio::join!(
|
||||
auth.renew_session(Some(refresh), None),
|
||||
auth.renew_session(Some(refresh), None)
|
||||
);
|
||||
let first = first.unwrap().unwrap();
|
||||
assert_eq!(concurrent.unwrap().as_ref(), Some(&first));
|
||||
assert_ne!(first.session_id, login.session_id);
|
||||
assert_ne!(first.refresh_token, login.refresh_token);
|
||||
assert_eq!(
|
||||
first.session.absolute_expires_at,
|
||||
login.session.absolute_expires_at
|
||||
);
|
||||
assert!(
|
||||
auth.session(Some(&first.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(
|
||||
auth.renew_session(first.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(first.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(refresh), None).await.unwrap(),
|
||||
Some(first.clone())
|
||||
);
|
||||
let persisted = format!("{:?}", store.sessions.lock().unwrap());
|
||||
for token in [
|
||||
&first.session_id,
|
||||
first.refresh_token.as_ref().unwrap(),
|
||||
&login.session_id,
|
||||
login.refresh_token.as_ref().unwrap(),
|
||||
] {
|
||||
assert!(!persisted.contains(token));
|
||||
assert!(!persisted.contains(token.rsplit('.').next().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_refresh_reuse_revokes_the_family_but_unknown_secrets_do_not() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
let next = auth
|
||||
.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let unknown = format!("r2.{}.{}", stored_id(&login.session_id), "A".repeat(43));
|
||||
assert!(
|
||||
auth.renew_session(Some(&unknown), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
auth.logout(&unknown).await.unwrap();
|
||||
auth.logout(stored_id(&login.session_id)).await.unwrap();
|
||||
assert!(
|
||||
auth.session(Some(&next.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.renewal
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.previous
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.session(Some(&next.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.renew_session(next.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_respects_idle_and_absolute_expiry_and_logout_after_access_expiry() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
for absolute in [false, true] {
|
||||
let login = admin_login(auth).await;
|
||||
{
|
||||
let mut sessions = store.sessions.lock().unwrap();
|
||||
let session = sessions.get_mut(stored_id(&login.session_id)).unwrap();
|
||||
if absolute {
|
||||
session.absolute_expires_at = Some(Utc::now() - TimeDelta::seconds(1));
|
||||
} else {
|
||||
session.renewal.as_mut().unwrap().expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
let login = admin_login(auth).await;
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
auth.logout(login.refresh_token.as_deref().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refreshed_expiry_is_capped_and_store_outages_preserve_the_session() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
let limit = Utc::now() + TimeDelta::minutes(1);
|
||||
{
|
||||
let mut sessions = store.sessions.lock().unwrap();
|
||||
let session = sessions.get_mut(stored_id(&login.session_id)).unwrap();
|
||||
session.absolute_expires_at = Some(limit);
|
||||
session.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
}
|
||||
store
|
||||
.unavailable
|
||||
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.unavailable
|
||||
.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
let refreshed = auth
|
||||
.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(refreshed.session.expires_at, limit);
|
||||
assert_eq!(refreshed.session.retained_until(), limit);
|
||||
assert_eq!(refreshed.session.absolute_expires_at, Some(limit));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fixed_sessions_cannot_acquire_refresh_credentials() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
let fixed = AuthSession {
|
||||
renewal: None,
|
||||
..login.session
|
||||
};
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("fixed".into(), fixed.clone());
|
||||
let grant = auth
|
||||
.renew_session(None, Some("fixed"))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(grant.session, fixed);
|
||||
assert!(grant.refresh_token.is_none());
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("fixed")
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
assert!(
|
||||
auth.renew_session(None, Some("fixed"))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
@@ -285,19 +285,24 @@ impl AuthStore for MemoryKeyAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
if session.retained_until() <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
if let Some(replacement) = replacement {
|
||||
*session = replacement.clone();
|
||||
} else {
|
||||
sessions.remove(session_id);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
|
||||
@@ -102,3 +102,51 @@ async fn backup_service_rejects_download_of_non_completed_record() {
|
||||
.expect_err("non-completed record must be rejected");
|
||||
assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_delete_probe_does_not_refresh_verification() {
|
||||
use gateway_admin::{model::backup::code, ports::backup::BackupRepository as _};
|
||||
for succeeds in [false, true] {
|
||||
let mut settings = configured_settings();
|
||||
settings.last_verified_at = None;
|
||||
let repository = Arc::new(FakeBackupRepository::new(settings));
|
||||
let auth = Arc::new(FakeAuthStore::new());
|
||||
let store = FakeObjectStore::new();
|
||||
let store = if succeeds {
|
||||
store
|
||||
} else {
|
||||
store.fail_connection(code::S3_PERMISSION_DENIED)
|
||||
};
|
||||
let services = AdminHarness::new()
|
||||
.auth(auth.clone())
|
||||
.backup(BackupStorePorts::new(
|
||||
repository.clone(),
|
||||
Arc::new(FakeDumpPort::new()),
|
||||
Arc::new(store),
|
||||
))
|
||||
.build()
|
||||
.await;
|
||||
let result = services
|
||||
.backups()
|
||||
.test_connection(&system_context())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.ok, succeeds);
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.last_verified_at
|
||||
.is_some(),
|
||||
succeeds
|
||||
);
|
||||
assert_eq!(
|
||||
auth.audit_actions()
|
||||
.iter()
|
||||
.filter(|action| action.as_str() == "backup.s3_connection_tested")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,12 +32,12 @@ fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> Runti
|
||||
rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart,
|
||||
updated_at: Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
@@ -62,6 +62,8 @@ fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> Runti
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -158,6 +160,19 @@ impl FreezeRuntimeStore {
|
||||
|
||||
#[async_trait]
|
||||
impl AccountRuntimeStore for FreezeRuntimeStore {
|
||||
async fn account_concurrency(
|
||||
&self,
|
||||
_: &str,
|
||||
_: u64,
|
||||
_: u32,
|
||||
) -> AdminStoreResult<gateway_admin::model::accounts::AccountConcurrencyPage> {
|
||||
Err(gateway_admin::ports::store::AdminStoreError::new(
|
||||
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"account concurrency",
|
||||
"not configured in this fixture",
|
||||
))
|
||||
}
|
||||
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ mod plugins;
|
||||
mod proxies;
|
||||
mod settings;
|
||||
mod system;
|
||||
mod usage_data;
|
||||
|
||||
use std::{
|
||||
str::FromStr,
|
||||
@@ -95,6 +96,7 @@ pub(super) struct AdminHarness {
|
||||
client_session_ttl_minutes: u64,
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
proxies: Arc<dyn gateway_admin::ports::proxy::ProxyStore>,
|
||||
proxy_probe: Arc<dyn gateway_admin::ports::proxy::ProxyProbe>,
|
||||
account_runtime: Arc<dyn AccountRuntimeStore>,
|
||||
account_groups: Arc<dyn AccountGroupStore>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
@@ -120,6 +122,7 @@ impl AdminHarness {
|
||||
client_session_ttl_minutes: 1_440,
|
||||
accounts: unavailable.clone(),
|
||||
proxies: Arc::new(proxies::TestProxies::default()),
|
||||
proxy_probe: Arc::new(proxies::TestProxies::default()),
|
||||
account_runtime: unavailable.clone(),
|
||||
account_groups: Arc::new(UnavailableAccountGroupStore),
|
||||
auth: Arc::new(BootstrapAuthStore::default()),
|
||||
@@ -273,6 +276,7 @@ impl AdminHarness {
|
||||
self.plugin_store,
|
||||
Arc::new(plugins::TestPluginPorts),
|
||||
Arc::new(plugins::TestPluginPorts),
|
||||
Arc::new(plugins::TestPluginPorts),
|
||||
),
|
||||
gateway_admin::AdminRuntimePorts {
|
||||
timezone: Default::default(),
|
||||
@@ -289,7 +293,7 @@ impl AdminHarness {
|
||||
.unwrap(),
|
||||
snapshot: Arc::new(NoopSnapshot),
|
||||
account_probe: self.probe,
|
||||
proxy_probe: Arc::new(proxies::TestProxies::default()),
|
||||
proxy_probe: self.proxy_probe,
|
||||
client_distribution: Arc::new(NoopClientDistribution),
|
||||
system: self.system,
|
||||
client_key_verifier: self.client_key_verifier,
|
||||
@@ -369,11 +373,11 @@ impl AuthStore for BootstrapAuthStore {
|
||||
Err(unavailable("admin session"))
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &gateway_admin::model::auth::AuthSession,
|
||||
_: chrono::DateTime<chrono::Utc>,
|
||||
_: Option<&gateway_admin::model::auth::AuthSession>,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AuthSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
@@ -474,6 +478,10 @@ impl AccountGroupStore for UnavailableAccountGroupStore {
|
||||
|
||||
#[async_trait]
|
||||
impl AccountStore for UnavailableStore {
|
||||
async fn account_exists(&self, _: &str) -> AdminStoreResult<bool> {
|
||||
Err(unavailable("account"))
|
||||
}
|
||||
|
||||
async fn list_plugin_accounts(
|
||||
&self,
|
||||
_: gateway_admin::model::provider_credentials::PluginAccountListQuery,
|
||||
@@ -643,6 +651,19 @@ impl AccountStore for UnavailableStore {
|
||||
|
||||
#[async_trait]
|
||||
impl AccountRuntimeStore for UnavailableStore {
|
||||
async fn account_concurrency(
|
||||
&self,
|
||||
_: &str,
|
||||
_: u64,
|
||||
_: u32,
|
||||
) -> AdminStoreResult<gateway_admin::model::accounts::AccountConcurrencyPage> {
|
||||
Err(gateway_admin::ports::store::AdminStoreError::new(
|
||||
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"account concurrency",
|
||||
"not configured in this fixture",
|
||||
))
|
||||
}
|
||||
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
@@ -747,6 +768,18 @@ impl ClientKeyStore for UnavailableStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnavailableStore {
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
_target: &gateway_admin::model::diagnostics::DiagnosticTarget,
|
||||
_limit: u16,
|
||||
) -> AdminStoreResult<Vec<gateway_admin::model::observability::OpsError>> {
|
||||
Err(unavailable("diagnostic errors"))
|
||||
}
|
||||
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
_: DashboardQuery,
|
||||
@@ -772,15 +805,17 @@ impl ObservabilityStore for UnavailableStore {
|
||||
Err(unavailable("usage trend"))
|
||||
}
|
||||
|
||||
fn usage_calculated_billing_facts(
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
Err(unavailable("analytics generation"))
|
||||
}
|
||||
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
_: Granularity,
|
||||
) -> gateway_admin::ports::store::UsageCalculatedBillingStream<'_> {
|
||||
Box::pin(futures::stream::once(async {
|
||||
Err(unavailable("usage billing facts"))
|
||||
}))
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
Err(unavailable("usage insights"))
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, _: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
@@ -1144,3 +1179,31 @@ fn unavailable_system() -> SystemOperationError {
|
||||
)
|
||||
}
|
||||
mod service_contract;
|
||||
|
||||
#[async_trait]
|
||||
impl gateway_admin::ports::reset_credit_automation::ResetCreditAutomationStore
|
||||
for UnavailableStore
|
||||
{
|
||||
async fn reset_credit_automation(
|
||||
&self,
|
||||
_: &str,
|
||||
) -> AdminStoreResult<
|
||||
Option<gateway_admin::model::reset_credit_automation::ResetCreditAutomation>,
|
||||
> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn due_reset_credit_automations(
|
||||
&self,
|
||||
_: chrono::DateTime<chrono::Utc>,
|
||||
_: u32,
|
||||
) -> AdminStoreResult<Vec<String>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn save_reset_credit_automation(
|
||||
&self,
|
||||
_: &gateway_admin::model::reset_credit_automation::ResetCreditAutomation,
|
||||
_: Option<&gateway_admin::model::MutationContext>,
|
||||
) -> AdminStoreResult<u64> {
|
||||
Err(unavailable("reset-credit automation"))
|
||||
}
|
||||
}
|
||||
|
||||
+401
-80
@@ -17,13 +17,13 @@ use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, CostCoverage, CurrencyCost, DashboardAccountUsage,
|
||||
DashboardObservation, DashboardQuery, DashboardRuntimeSlots, DiagnosticDimension,
|
||||
DiagnosticObservation, DiagnosticsObservation, Granularity, HealthStatus,
|
||||
LatencyPercentiles, ObservabilityPageSize, OpsErrorPage, OpsErrorQuery,
|
||||
AccountPoolMetrics, AnalyticsState, AttemptMetrics, CostCoverage, CurrencyCost,
|
||||
DashboardAccountUsage, DashboardObservation, DashboardQuery, DashboardRuntimeSlots,
|
||||
DiagnosticDimension, DiagnosticObservation, DiagnosticsObservation, Granularity,
|
||||
HealthStatus, LatencyPercentiles, ObservabilityPageSize, OpsErrorPage, OpsErrorQuery,
|
||||
PercentileMilliseconds, RequestMetricPoint, RequestMetrics, TimeRange, TrendKind,
|
||||
UsageBilling, UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageListRecord,
|
||||
UsageOverview, UsagePage, UsageQuery,
|
||||
UsageBilling, UsageDetail, UsageFilter, UsageInsightsObservation, UsageListRecord,
|
||||
UsageOverview, UsagePage, UsageProjectedCostPoint, UsageQuery,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -148,7 +148,7 @@ async fn dashboard_summary_should_derive_first_token_average_from_trend() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_timeline_should_match_legacy_status_precedence_and_thresholds() {
|
||||
async fn health_timeline_should_apply_status_precedence_and_thresholds() {
|
||||
let now = Utc::now();
|
||||
let current_slot = quarter_hour_start(now);
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
@@ -158,13 +158,28 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds()
|
||||
RequestMetrics {
|
||||
failure_count: 1,
|
||||
cancelled_count: 1,
|
||||
incomplete_count: 1,
|
||||
caller_error_count: 1,
|
||||
..RequestMetrics::default()
|
||||
},
|
||||
HealthStatus::NoData,
|
||||
None,
|
||||
),
|
||||
(
|
||||
RequestMetrics {
|
||||
incomplete_count: 1,
|
||||
..RequestMetrics::default()
|
||||
},
|
||||
HealthStatus::LowSample,
|
||||
Some(0.0),
|
||||
),
|
||||
(
|
||||
RequestMetrics {
|
||||
incomplete_count: 3,
|
||||
..RequestMetrics::default()
|
||||
},
|
||||
HealthStatus::Unavailable,
|
||||
Some(0.0),
|
||||
),
|
||||
(
|
||||
RequestMetrics {
|
||||
failure_count: 3,
|
||||
@@ -184,21 +199,21 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds()
|
||||
),
|
||||
(
|
||||
RequestMetrics {
|
||||
success_count: 98,
|
||||
failure_count: 2,
|
||||
success_count: 94,
|
||||
failure_count: 6,
|
||||
..RequestMetrics::default()
|
||||
},
|
||||
HealthStatus::Unstable,
|
||||
Some(98.0),
|
||||
Some(94.0),
|
||||
),
|
||||
(
|
||||
RequestMetrics {
|
||||
success_count: 99,
|
||||
failure_count: 1,
|
||||
success_count: 95,
|
||||
incomplete_count: 5,
|
||||
..RequestMetrics::default()
|
||||
},
|
||||
HealthStatus::Stable,
|
||||
Some(99.0),
|
||||
Some(95.0),
|
||||
),
|
||||
];
|
||||
|
||||
@@ -224,6 +239,92 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate() {
|
||||
let now = Utc::now();
|
||||
let current_slot = quarter_hour_start(now);
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let fixtures = [
|
||||
(162, 0, 2, 0, 0, 98.780_487_804_878_05),
|
||||
(297, 0, 2, 5, 0, 99.331_103_678_929_76),
|
||||
(467, 1, 5, 0, 0, 98.731_501_057_082_45),
|
||||
(95, 2, 3, 4, 1, 95.0),
|
||||
(100, 0, 0, 0, 0, 100.0),
|
||||
];
|
||||
|
||||
for (index, (success, failed, incomplete, cancelled, caller_error, expected_percent)) in
|
||||
fixtures.into_iter().enumerate()
|
||||
{
|
||||
let range = observation_range(
|
||||
now + Duration::seconds(i64::try_from(index).expect("fixture index fits i64") * 2),
|
||||
);
|
||||
let metrics = RequestMetrics {
|
||||
request_count: success + failed + incomplete + cancelled + caller_error,
|
||||
success_count: success,
|
||||
failure_count: failed + caller_error,
|
||||
incomplete_count: incomplete,
|
||||
cancelled_count: cancelled,
|
||||
caller_error_count: caller_error,
|
||||
..RequestMetrics::default()
|
||||
};
|
||||
store.replace_trend(vec![health_metric_point(current_slot, metrics.clone())]);
|
||||
store.replace_overview(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range,
|
||||
requests: metrics,
|
||||
// 中间尝试失败不会增加最终失败请求数
|
||||
attempts: AttemptMetrics {
|
||||
failure_count: 2,
|
||||
..AttemptMetrics::default()
|
||||
},
|
||||
providers: Vec::new(),
|
||||
});
|
||||
|
||||
let dashboard = services
|
||||
.observability()
|
||||
.dashboard_summary(range, TrendKind::Usage)
|
||||
.await
|
||||
.expect("dashboard summary");
|
||||
let timeline = dashboard.health_timeline;
|
||||
let point = timeline
|
||||
.points
|
||||
.iter()
|
||||
.find(|point| point.bucket_start == current_slot)
|
||||
.expect("current health slot");
|
||||
assert_eq!(point.status, HealthStatus::Stable);
|
||||
assert_eq!(point.failed_requests, failed);
|
||||
assert_eq!(point.incomplete_requests, incomplete);
|
||||
assert_eq!(point.cancelled_requests, cancelled);
|
||||
assert_eq!(point.caller_error_requests, caller_error);
|
||||
for reliability in [timeline.reliability_percent, point.reliability_percent] {
|
||||
assert!((reliability.expect("health reliability") - expected_percent).abs() < 1e-10);
|
||||
}
|
||||
for success_rate in [
|
||||
dashboard.trend.summary.success_rate,
|
||||
dashboard.trend.points[0].success_rate,
|
||||
] {
|
||||
assert!(
|
||||
(success_rate.expect("trend success rate") * 100.0 - expected_percent).abs()
|
||||
< 1e-10
|
||||
);
|
||||
}
|
||||
|
||||
let insights = services
|
||||
.observability()
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.expect("usage insights");
|
||||
assert_eq!(insights.health.failed_requests, failed);
|
||||
assert_eq!(insights.health.incomplete_requests, incomplete);
|
||||
assert!((insights.health.success_rate * 100.0 - expected_percent).abs() < 1e-10);
|
||||
assert!(
|
||||
(insights.health.points[0].error_rate * 100.0 - (100.0 - expected_percent)).abs()
|
||||
< 1e-10
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
|
||||
let now = Utc::now();
|
||||
@@ -300,7 +401,7 @@ async fn dashboard_summary_should_observe_current_accounts_for_a_historical_rang
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_coalesce_concurrent_requests_in_one_short_window() {
|
||||
async fn dashboard_summary_should_keep_distinct_exact_ranges_in_one_short_window() {
|
||||
let end_bucket = Utc::now().timestamp().div_euclid(2) * 2;
|
||||
let first_end = DateTime::from_timestamp(end_bucket, 100_000_000).expect("first end");
|
||||
let second_end = DateTime::from_timestamp(end_bucket, 900_000_000).expect("second end");
|
||||
@@ -330,7 +431,7 @@ async fn dashboard_summary_should_coalesce_concurrent_requests_in_one_short_wind
|
||||
|
||||
first.expect("first dashboard summary");
|
||||
second.expect("second dashboard summary");
|
||||
assert_eq!(store.dashboard_summary_calls(), 1);
|
||||
assert_eq!(store.dashboard_summary_calls(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -403,6 +504,7 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
..RequestMetrics::default()
|
||||
};
|
||||
store.replace_overview(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range,
|
||||
requests: metrics.clone(),
|
||||
attempts: AttemptMetrics {
|
||||
@@ -431,23 +533,13 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
.expect("bucket USD cost"),
|
||||
}],
|
||||
}]);
|
||||
store.replace_calculated_billing_facts(vec![UsageCalculatedBillingFact {
|
||||
breakdown: None,
|
||||
store.replace_projected_costs(vec![UsageProjectedCostPoint {
|
||||
bucket_start: quarter_hour_start(now),
|
||||
provider_kind: "openai".to_owned(),
|
||||
upstream_model_id: "gpt-5.5".to_owned(),
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
||||
.expect("calculated total"),
|
||||
},
|
||||
standard_amount: Some("1".parse().expect("standard cost")),
|
||||
no_cache_amount: Some("1.25".parse().expect("no-cache cost")),
|
||||
}]);
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 4,
|
||||
items: vec![diagnostic("openai", 3), diagnostic("xai", 1)],
|
||||
});
|
||||
@@ -543,6 +635,7 @@ async fn diagnostics_should_preserve_request_order_when_a_small_sample_fails() {
|
||||
failure.success_count = 0;
|
||||
failure.failure_count = 1;
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 101,
|
||||
items: vec![success, failure],
|
||||
});
|
||||
@@ -566,6 +659,7 @@ async fn diagnostics_should_use_the_total_before_truncating_groups() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 200,
|
||||
items: (0..100)
|
||||
.map(|index| diagnostic(&format!("model-{index}"), 1))
|
||||
@@ -591,6 +685,7 @@ async fn diagnostics_should_count_each_retried_request_once() {
|
||||
item.retry_count = 6;
|
||||
item.retried_request_count = 2;
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 8,
|
||||
items: vec![item],
|
||||
});
|
||||
@@ -691,26 +786,16 @@ async fn usage_records_should_enrich_provider_reported_totals_when_pricing_match
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_insights_should_reject_partial_costs_when_billing_stream_fails() {
|
||||
async fn usage_insights_should_reject_partial_costs_when_insights_fail() {
|
||||
let now = Utc::now();
|
||||
let range = observation_range(now);
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.replace_calculated_billing_facts(vec![UsageCalculatedBillingFact {
|
||||
breakdown: None,
|
||||
store.replace_projected_costs(vec![UsageProjectedCostPoint {
|
||||
bucket_start: quarter_hour_start(now),
|
||||
provider_kind: "openai".to_owned(),
|
||||
upstream_model_id: "gpt-5.5".to_owned(),
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: "1.25".parse().expect("cost"),
|
||||
},
|
||||
standard_amount: Some("1".parse().expect("standard cost")),
|
||||
no_cache_amount: Some("1.25".parse().expect("no-cache cost")),
|
||||
}]);
|
||||
store.billing_stream_fails.store(true, Ordering::SeqCst);
|
||||
store.insights_fail.store(true, Ordering::SeqCst);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
assert!(
|
||||
services
|
||||
@@ -741,9 +826,9 @@ async fn dashboard_should_preserve_provider_plan_names() {
|
||||
success_count: 0,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
@@ -783,8 +868,9 @@ async fn dashboard_should_preserve_provider_plan_names() {
|
||||
struct FixtureObservabilityStore {
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
overview: Mutex<UsageOverview>,
|
||||
calculated_billing_facts: Mutex<Vec<UsageCalculatedBillingFact>>,
|
||||
billing_stream_fails: AtomicBool,
|
||||
projected_costs: Mutex<Vec<UsageProjectedCostPoint>>,
|
||||
insights_fail: AtomicBool,
|
||||
summary_fail: AtomicBool,
|
||||
diagnostics: Mutex<DiagnosticsObservation>,
|
||||
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
||||
summary_observed_at: Mutex<Option<DateTime<Utc>>>,
|
||||
@@ -794,6 +880,7 @@ struct FixtureObservabilityStore {
|
||||
account_usage: Mutex<Vec<DashboardAccountUsage>>,
|
||||
dashboard_delay: Mutex<StdDuration>,
|
||||
dashboard_summary_calls: AtomicUsize,
|
||||
usage_summary_filters: Mutex<Vec<UsageFilter>>,
|
||||
}
|
||||
|
||||
impl FixtureObservabilityStore {
|
||||
@@ -801,13 +888,15 @@ impl FixtureObservabilityStore {
|
||||
Self {
|
||||
trend: Mutex::new(Vec::new()),
|
||||
overview: Mutex::new(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range,
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
providers: Vec::new(),
|
||||
}),
|
||||
calculated_billing_facts: Mutex::new(Vec::new()),
|
||||
billing_stream_fails: AtomicBool::new(false),
|
||||
projected_costs: Mutex::new(Vec::new()),
|
||||
insights_fail: AtomicBool::new(false),
|
||||
summary_fail: AtomicBool::new(false),
|
||||
diagnostics: Mutex::new(DiagnosticsObservation::default()),
|
||||
runtime_slots: Mutex::new(None),
|
||||
summary_observed_at: Mutex::new(None),
|
||||
@@ -817,6 +906,7 @@ impl FixtureObservabilityStore {
|
||||
account_usage: Mutex::new(Vec::new()),
|
||||
dashboard_delay: Mutex::new(StdDuration::ZERO),
|
||||
dashboard_summary_calls: AtomicUsize::new(0),
|
||||
usage_summary_filters: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -836,11 +926,8 @@ impl FixtureObservabilityStore {
|
||||
*self.overview.lock().expect("overview") = overview;
|
||||
}
|
||||
|
||||
fn replace_calculated_billing_facts(&self, facts: Vec<UsageCalculatedBillingFact>) {
|
||||
*self
|
||||
.calculated_billing_facts
|
||||
.lock()
|
||||
.expect("calculated billing facts") = facts;
|
||||
fn replace_projected_costs(&self, facts: Vec<UsageProjectedCostPoint>) {
|
||||
*self.projected_costs.lock().expect("projected costs") = facts;
|
||||
}
|
||||
|
||||
fn replace_diagnostics(&self, diagnostics: DiagnosticsObservation) {
|
||||
@@ -858,6 +945,18 @@ impl FixtureObservabilityStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for FixtureObservabilityStore {
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
_target: &gateway_admin::model::diagnostics::DiagnosticTarget,
|
||||
_limit: u16,
|
||||
) -> AdminStoreResult<Vec<gateway_admin::model::observability::OpsError>> {
|
||||
Err(super::unavailable("diagnostic errors"))
|
||||
}
|
||||
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
query: DashboardQuery,
|
||||
@@ -874,6 +973,7 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
tokio::time::sleep(dashboard_delay).await;
|
||||
}
|
||||
Ok(DashboardObservation {
|
||||
analytics: self.overview.lock().expect("overview").analytics.clone(),
|
||||
range: query.range,
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
@@ -902,25 +1002,29 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
Ok(self.trend.lock().expect("trend").clone())
|
||||
}
|
||||
|
||||
fn usage_calculated_billing_facts(
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
Ok(self.overview.lock().expect("overview").analytics.generation)
|
||||
}
|
||||
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> gateway_admin::ports::store::UsageCalculatedBillingStream<'_> {
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.usage_granularities.lock().unwrap().push(granularity);
|
||||
let facts = self
|
||||
.calculated_billing_facts
|
||||
.lock()
|
||||
.expect("calculated billing facts")
|
||||
.clone();
|
||||
let error = self
|
||||
.billing_stream_fails
|
||||
.load(Ordering::SeqCst)
|
||||
.then(|| Err(super::unavailable("billing stream failed")));
|
||||
Box::pin(futures::stream::iter(
|
||||
facts.into_iter().map(Ok).chain(error),
|
||||
))
|
||||
if self.insights_fail.load(Ordering::SeqCst) {
|
||||
return Err(super::unavailable("projection read failed"));
|
||||
}
|
||||
Ok(UsageInsightsObservation {
|
||||
overview: self.overview.lock().expect("overview").clone(),
|
||||
trend: self.trend.lock().expect("trend").clone(),
|
||||
costs: self
|
||||
.projected_costs
|
||||
.lock()
|
||||
.expect("projected costs")
|
||||
.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
@@ -938,8 +1042,26 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
Err(super::unavailable("usage detail"))
|
||||
}
|
||||
|
||||
async fn usage_summary(&self, _: TimeRange, _: UsageFilter) -> AdminStoreResult<UsageOverview> {
|
||||
Ok(self.overview.lock().expect("overview").clone())
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
self.usage_summary_filters
|
||||
.lock()
|
||||
.expect("summary filters")
|
||||
.push(filter);
|
||||
let delay = *self.dashboard_delay.lock().expect("query delay");
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
if self.summary_fail.load(Ordering::SeqCst) {
|
||||
return Err(super::unavailable("projection summary failed"));
|
||||
}
|
||||
Ok(UsageOverview {
|
||||
range,
|
||||
..self.overview.lock().expect("overview").clone()
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_diagnostics(
|
||||
@@ -988,12 +1110,12 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
updated_at: Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: self.max_concurrent_per_account,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
@@ -1018,6 +1140,8 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1080,6 +1204,7 @@ fn observation_range(end: DateTime<Utc>) -> TimeRange {
|
||||
|
||||
fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
UsageQuery {
|
||||
status: Default::default(),
|
||||
range: observation_range(now),
|
||||
filter: UsageFilter::default(),
|
||||
current_page: 1,
|
||||
@@ -1094,6 +1219,7 @@ fn total_record(
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Succeeded,
|
||||
client_api_key_name: Some("Production".to_owned()),
|
||||
id: id.to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
@@ -1113,9 +1239,9 @@ fn total_record(
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(0),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1_000),
|
||||
@@ -1228,9 +1354,204 @@ async fn admin_selects_dashboard_defaults_and_matching_usage_buckets_at_range_bo
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
*store.usage_granularities.lock().unwrap(),
|
||||
vec![expected, expected]
|
||||
assert_eq!(*store.usage_granularities.lock().unwrap(), vec![expected]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_summary_cache_should_coalesce_identical_queries_and_invalidate_deleted_generation() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.set_dashboard_delay(StdDuration::from_millis(20));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
let filter = UsageFilter {
|
||||
client_api_key_ref: Some("deleted-key".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let (first, second) = tokio::join!(
|
||||
service.usage_summary(range, filter.clone()),
|
||||
service.usage_summary(range, filter.clone()),
|
||||
);
|
||||
first.expect("first summary");
|
||||
second.expect("coalesced summary");
|
||||
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 1);
|
||||
{
|
||||
let mut overview = store.overview.lock().unwrap();
|
||||
overview.analytics.generation = 1;
|
||||
overview.requests.request_count = 7;
|
||||
}
|
||||
let refreshed = service
|
||||
.usage_summary(range, filter)
|
||||
.await
|
||||
.expect("generation refresh");
|
||||
assert_eq!(refreshed.overview.analytics.generation, 1);
|
||||
assert_eq!(refreshed.overview.requests.request_count, 7);
|
||||
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn usage_summary_cache_should_coalesce_when_generation_advances_during_the_first_load() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.set_dashboard_delay(StdDuration::from_millis(20));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
let (first, second, ()) = tokio::join!(
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
async {
|
||||
tokio::time::sleep(StdDuration::from_millis(5)).await;
|
||||
store.overview.lock().unwrap().analytics.generation = 1;
|
||||
},
|
||||
);
|
||||
assert_eq!(
|
||||
first.expect("first summary").overview.analytics.generation,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
second
|
||||
.expect("coalesced summary")
|
||||
.overview
|
||||
.analytics
|
||||
.generation,
|
||||
1
|
||||
);
|
||||
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_summary_cache_should_isolate_key_identity_and_filters() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
for (key, model) in [
|
||||
("key-a", None),
|
||||
("key-a", None),
|
||||
("key-b", None),
|
||||
("key-a", Some("model-a")),
|
||||
] {
|
||||
service
|
||||
.usage_summary(
|
||||
range,
|
||||
UsageFilter {
|
||||
client_api_key_ref: Some(key.to_owned()),
|
||||
model: model.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("isolated summary");
|
||||
}
|
||||
let calls = store.usage_summary_filters.lock().unwrap();
|
||||
assert_eq!(calls.len(), 3);
|
||||
assert_eq!(calls[0].client_api_key_ref.as_deref(), Some("key-a"));
|
||||
assert_eq!(calls[1].client_api_key_ref.as_deref(), Some("key-b"));
|
||||
assert_eq!(calls[2].model.as_deref(), Some("model-a"));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow_recovery() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.set_dashboard_delay(store.query_timeout().mul_f64(0.7));
|
||||
store.summary_fail.store(true, Ordering::SeqCst);
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
let started = tokio::time::Instant::now();
|
||||
let (first, second, third, fourth) = tokio::join!(
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
);
|
||||
assert!(first.is_err() && second.is_err() && third.is_err() && fourth.is_err());
|
||||
assert!(
|
||||
started.elapsed() <= store.query_timeout(),
|
||||
"同键失败不能使后续请求逐个等待完整数据库超时: {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
|
||||
store.summary_fail.store(false, Ordering::SeqCst);
|
||||
store.set_dashboard_delay(StdDuration::ZERO);
|
||||
service
|
||||
.usage_summary(range, UsageFilter::default())
|
||||
.await
|
||||
.expect("cancelled cache refresh releases its slot for recovery");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn insights_cache_should_bound_stale_lifetime_without_renewing_failed_refreshes() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
{
|
||||
let mut overview = store.overview.lock().unwrap();
|
||||
overview.analytics.state = AnalyticsState::Ready;
|
||||
overview.analytics.generated_at = Utc::now();
|
||||
overview.analytics.lag_seconds = 7;
|
||||
overview.requests.request_count = 19;
|
||||
}
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
store.insights_fail.store(true, Ordering::SeqCst);
|
||||
tokio::time::advance(StdDuration::from_secs(16)).await;
|
||||
let stale = service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stale.analytics.state, AnalyticsState::Stale);
|
||||
assert!(stale.analytics.lag_seconds >= 7);
|
||||
assert_eq!(stale.health.total_requests, 19);
|
||||
tokio::time::advance(StdDuration::from_secs(45)).await;
|
||||
assert!(
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn insights_cache_should_reject_deleted_building_and_old_generation_times() {
|
||||
for reason in ["deleted", "building", "generated-too-long-ago"] {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
{
|
||||
let mut overview = store.overview.lock().unwrap();
|
||||
overview.analytics.state = if reason == "building" {
|
||||
AnalyticsState::Building
|
||||
} else {
|
||||
AnalyticsState::Ready
|
||||
};
|
||||
overview.analytics.generated_at = Utc::now()
|
||||
- Duration::seconds(if reason == "generated-too-long-ago" {
|
||||
61
|
||||
} else {
|
||||
0
|
||||
});
|
||||
}
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
store.insights_fail.store(true, Ordering::SeqCst);
|
||||
if reason == "deleted" {
|
||||
store.overview.lock().unwrap().analytics.generation += 1;
|
||||
}
|
||||
tokio::time::advance(StdDuration::from_secs(16)).await;
|
||||
assert!(
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"{reason}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -534,6 +534,7 @@ fn preview_fixture() -> Fixture {
|
||||
fn source_proxy() -> ProxyRecord {
|
||||
let now = chrono::Utc::now();
|
||||
ProxyRecord {
|
||||
enabled: true,
|
||||
auto_location: false,
|
||||
detected_location: None,
|
||||
location: None,
|
||||
@@ -608,6 +609,38 @@ async fn saved_source_proxy_is_resolved_for_queries_and_frozen_into_verified_sou
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_source_proxy_rejects_queries_and_downloads_before_network_access() {
|
||||
let mut fixture = preview_fixture();
|
||||
fixture.binding.get_mut().unwrap().outbound_proxy_id = Some("source-proxy".into());
|
||||
let mut proxy = source_proxy();
|
||||
proxy.enabled = false;
|
||||
*fixture.proxy.get_mut().unwrap() = Some(proxy);
|
||||
let fixture = Arc::new(fixture);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.service()
|
||||
.check_update("test.example", &[])
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Invalid
|
||||
);
|
||||
let mut request = preview_request();
|
||||
request.outbound_proxy_id = Some("source-proxy".into());
|
||||
assert_eq!(
|
||||
fixture
|
||||
.service()
|
||||
.verify_remote(request)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Invalid
|
||||
);
|
||||
assert!(fixture.egresses.lock().unwrap().is_empty());
|
||||
assert_eq!(*fixture.downloads.lock().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_revision_change_during_download_rejects_the_verified_candidate() {
|
||||
let mut fixture = preview_fixture();
|
||||
|
||||
@@ -434,6 +434,7 @@ impl PluginStateStore for LifecycleFixture {
|
||||
from_schema_version: 1,
|
||||
to_schema_version: 2,
|
||||
}],
|
||||
guard: Arc::new(()),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 插件管理用例的测试入口与共用端口替身
|
||||
|
||||
mod usage_data;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_admin::model::plugins::distribution::{
|
||||
DownloadedPlugin, GithubReleaseQuery, PluginRelease, RemotePluginLocation, SourceCredential,
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
//! 通用用量端口的插件管理测试夹具,非本测试调用路径禁止访问
|
||||
use super::TestPluginPorts;
|
||||
use gateway_admin::{model::usage_data::*, ports::usage_data::*};
|
||||
#[async_trait::async_trait]
|
||||
impl UsageDataStore for TestPluginPorts {
|
||||
async fn get_record(&self, _: GetUsageRecord) -> UsageDataResult<UsageRecord> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn list_records(&self, _: ListUsageRecords) -> UsageDataResult<UsageRecordPage> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn write_records(&self, _: StoredUsageWrite) -> UsageDataResult<UsageWriteResult> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn list_aggregates(&self, _: ListUsageAggregates) -> UsageDataResult<UsageAggregatePage> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
_: StoredAggregateWrite,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ pub(super) struct TestProxies {
|
||||
pub events: Option<super::accounts::EventLog>,
|
||||
pub accounts: Option<Vec<ProxyAccountRef>>,
|
||||
pub record: Option<ProxyRecord>,
|
||||
pub records: Vec<ProxyRecord>,
|
||||
}
|
||||
|
||||
struct ImportGuard(super::accounts::EventLog);
|
||||
@@ -77,9 +78,13 @@ impl ProxyStore for TestProxies {
|
||||
self.record
|
||||
.clone()
|
||||
.filter(|record| record.id == id)
|
||||
.or_else(|| self.records.iter().find(|record| record.id == id).cloned())
|
||||
.ok_or_else(|| super::unavailable("proxy"))
|
||||
}
|
||||
async fn create(&self, _: NewProxy, _: &MutationContext) -> AdminStoreResult<ProxyMutation> {
|
||||
if let Some(events) = &self.events {
|
||||
events.lock().unwrap().push("proxy.create");
|
||||
}
|
||||
Err(super::unavailable("proxy"))
|
||||
}
|
||||
async fn update(&self, _: UpdateProxy, _: &MutationContext) -> AdminStoreResult<ProxyMutation> {
|
||||
@@ -95,18 +100,23 @@ impl ProxyStore for TestProxies {
|
||||
}
|
||||
async fn record_test(
|
||||
&self,
|
||||
_: &str,
|
||||
id: &str,
|
||||
_: Revision,
|
||||
_: ProxyTestResult,
|
||||
result: ProxyTestResult,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<ProxyMutation> {
|
||||
Err(super::unavailable("proxy"))
|
||||
let mut record = self.get(id).await?;
|
||||
record.last_test = Some(result);
|
||||
Ok(ProxyMutation {
|
||||
config_revision: record.revision,
|
||||
record,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProxyProbe for TestProxies {
|
||||
async fn test(&self, _: &OutboundProxy, _: bool) -> ProxyTestResult {
|
||||
async fn test(&self, _: Option<&OutboundProxy>, _: bool) -> ProxyTestResult {
|
||||
panic!("unexpected proxy probe")
|
||||
}
|
||||
}
|
||||
@@ -126,6 +136,7 @@ async fn authorization_uses_selected_proxy_regardless_of_probe_status() {
|
||||
.accounts(FakeAccountStore::new(kind, events.clone()))
|
||||
.proxies(Arc::new(TestProxies {
|
||||
record: Some(ProxyRecord {
|
||||
enabled: true,
|
||||
auto_location: false,
|
||||
detected_location: None,
|
||||
location: None,
|
||||
@@ -309,3 +320,163 @@ async fn credential_import_keeps_proxy_reserved_until_commit_and_releases_on_err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn batch_create_stops_after_uncertain_store_failure_without_replaying() {
|
||||
use super::accounts::{context, events, recorded};
|
||||
use gateway_admin::model::AdminErrorKind;
|
||||
let events = events();
|
||||
let services = super::AdminHarness::new()
|
||||
.proxies(std::sync::Arc::new(TestProxies {
|
||||
events: Some(events.clone()),
|
||||
..Default::default()
|
||||
}))
|
||||
.build()
|
||||
.await;
|
||||
let commands = [" ", "First", "Skipped"]
|
||||
.into_iter()
|
||||
.map(|name| {
|
||||
Ok(NewProxy {
|
||||
name: name.to_owned(),
|
||||
proxy: OutboundProxy::parse("http://proxy.example:8080").unwrap(),
|
||||
auto_location: false,
|
||||
location: None,
|
||||
test: None,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let results = services
|
||||
.proxies()
|
||||
.create_batch(commands, &context("batch-create"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(&results[0], ProxyBatchResult::Completed(Err(error)) if error.kind() == AdminErrorKind::Invalid)
|
||||
);
|
||||
assert!(
|
||||
matches!(&results[1], ProxyBatchResult::Completed(Err(error)) if error.kind() == AdminErrorKind::Unavailable)
|
||||
);
|
||||
assert!(matches!(&results[2], ProxyBatchResult::Skipped));
|
||||
assert_eq!(recorded(&events), vec!["proxy.create"]);
|
||||
}
|
||||
|
||||
struct GatedProxyProbe {
|
||||
started: tokio::sync::mpsc::UnboundedSender<()>,
|
||||
release: tokio::sync::Semaphore,
|
||||
active: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
struct ActiveProbe<'a>(&'a std::sync::atomic::AtomicUsize);
|
||||
impl Drop for ActiveProbe<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProxyProbe for GatedProxyProbe {
|
||||
async fn test(&self, _: Option<&OutboundProxy>, _: bool) -> ProxyTestResult {
|
||||
self.active
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let _active = ActiveProbe(&self.active);
|
||||
self.started.send(()).unwrap();
|
||||
self.release.acquire().await.unwrap().forget();
|
||||
ProxyTestResult {
|
||||
location: Default::default(),
|
||||
success: true,
|
||||
latency_ms: 1,
|
||||
exit_ip: None,
|
||||
exit_ipv4: None,
|
||||
exit_ipv6: None,
|
||||
message: "ok".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn batch_tests_share_capacity_preserve_order_and_release_slots_on_cancellation() {
|
||||
use super::accounts::context;
|
||||
use std::sync::{Arc, atomic::Ordering};
|
||||
let (started, mut receiver) = tokio::sync::mpsc::unbounded_channel();
|
||||
let probe = Arc::new(GatedProxyProbe {
|
||||
started,
|
||||
release: tokio::sync::Semaphore::new(0),
|
||||
active: Default::default(),
|
||||
});
|
||||
let now = chrono::Utc::now();
|
||||
let records = (0..4)
|
||||
.map(|i| ProxyRecord {
|
||||
id: format!("proxy_{i}"),
|
||||
name: format!("Proxy {i}"),
|
||||
proxy: OutboundProxy::parse("http://proxy.example:8080").unwrap(),
|
||||
revision: Revision::new(1).unwrap(),
|
||||
enabled: true,
|
||||
auto_location: false,
|
||||
location: None,
|
||||
detected_location: None,
|
||||
account_count: 0,
|
||||
last_test_at: None,
|
||||
last_test: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})
|
||||
.collect();
|
||||
let mut harness = super::AdminHarness::new().proxies(Arc::new(TestProxies {
|
||||
records,
|
||||
..Default::default()
|
||||
}));
|
||||
harness.proxy_probe = probe.clone();
|
||||
let services = harness.build().await;
|
||||
let commands = || {
|
||||
(0..4)
|
||||
.map(|i| {
|
||||
Ok(TestProxy {
|
||||
id: format!("proxy_{i}"),
|
||||
revision: Revision::new(1).unwrap(),
|
||||
detect_location: true,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let context = context("batch-test");
|
||||
let mut batch = Box::pin(services.proxies().test_batch(commands(), &context));
|
||||
tokio::select! {
|
||||
_ = &mut batch => panic!("batch finished before probes were released"),
|
||||
_ = async { receiver.recv().await.unwrap(); receiver.recv().await.unwrap(); } => {}
|
||||
}
|
||||
assert_eq!(probe.active.load(Ordering::Relaxed), 2);
|
||||
let mut single_a = Box::pin(services.proxies().probe(None, false));
|
||||
let mut single_b = Box::pin(services.proxies().probe(None, false));
|
||||
tokio::select! {
|
||||
_ = &mut batch => panic!("batch unexpectedly finished"),
|
||||
_ = &mut single_a => panic!("single probe unexpectedly finished"),
|
||||
_ = &mut single_b => panic!("single probe unexpectedly finished"),
|
||||
_ = async { receiver.recv().await.unwrap(); receiver.recv().await.unwrap(); } => {}
|
||||
}
|
||||
assert_eq!(probe.active.load(Ordering::Relaxed), 4);
|
||||
assert_eq!(
|
||||
services
|
||||
.proxies()
|
||||
.probe(None, false)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
gateway_admin::model::AdminErrorKind::RateLimited
|
||||
);
|
||||
drop((batch, single_a, single_b));
|
||||
assert_eq!(probe.active.load(Ordering::Relaxed), 0);
|
||||
probe.release.add_permits(4);
|
||||
let results = services
|
||||
.proxies()
|
||||
.test_batch(commands(), &context)
|
||||
.await
|
||||
.unwrap();
|
||||
for (index, result) in results.into_iter().enumerate() {
|
||||
let ProxyBatchResult::Completed(Ok(record)) = result else {
|
||||
panic!("probe failed after cancellation")
|
||||
};
|
||||
assert_eq!(record.id, format!("proxy_{index}"));
|
||||
assert!(record.last_test.unwrap().success);
|
||||
}
|
||||
assert_eq!(probe.active.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ impl VisitMut for WireTypes {
|
||||
let last = path.path.segments.last().unwrap().clone();
|
||||
*ty = match last.ident.to_string().as_str() {
|
||||
"DateTime" | "Tz" | "TokenPrice" | "AdminApiKey" => parse_quote!(String),
|
||||
"Value" => parse_quote!(serde_json::Value),
|
||||
"OpaqueProviderData" => parse_quote!(serde_json::Map<String, serde_json::Value>),
|
||||
_ => Type::Path(syn::TypePath {
|
||||
attrs: std::mem::take(&mut path.attrs),
|
||||
@@ -79,6 +80,11 @@ fn contract_type(mut item: Item, name: &str) -> Item {
|
||||
| "ReplaceRuntimeSettings"
|
||||
| "AdminApiKeyMutation"
|
||||
| "RegeneratedAdminApiKey"
|
||||
| "CodexPrivacyPolicy"
|
||||
| "PrivacyRule"
|
||||
| "PrivacyStep"
|
||||
| "PrivacyPreviewRequest"
|
||||
| "PrivacyPreviewResult"
|
||||
);
|
||||
(
|
||||
&mut item.attrs,
|
||||
@@ -107,16 +113,120 @@ fn contract_type(mut item: Item, name: &str) -> Item {
|
||||
field.vis = parse_quote!(pub);
|
||||
}
|
||||
}
|
||||
if let Item::Enum(item) = &mut item {
|
||||
for variant in &mut item.variants {
|
||||
variant
|
||||
.attrs
|
||||
.retain(|attr| !attr.path().is_ident("default"));
|
||||
}
|
||||
}
|
||||
WireTypes.visit_item_mut(&mut item);
|
||||
item
|
||||
}
|
||||
|
||||
fn service_operations(
|
||||
root: &Path,
|
||||
namespace: &str,
|
||||
service_path: &str,
|
||||
service_name: &str,
|
||||
) -> (Vec<syn::File>, usize) {
|
||||
let service = source(root, service_path)
|
||||
.items
|
||||
.into_iter()
|
||||
.find_map(|item| match item {
|
||||
Item::Trait(item) if item.ident == service_name => Some(item),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
let outputs: BTreeMap<_, _> = service
|
||||
.items
|
||||
.into_iter()
|
||||
.filter_map(|item| match item {
|
||||
syn::TraitItem::Fn(method) => {
|
||||
let syn::ReturnType::Type(_, output) = method.sig.output else {
|
||||
panic!("服务必须返回 Result")
|
||||
};
|
||||
let Type::Path(result) = *output else {
|
||||
panic!("服务必须返回 Result")
|
||||
};
|
||||
let segment = result.path.segments.last().unwrap();
|
||||
assert_eq!(segment.ident, "Result");
|
||||
let syn::PathArguments::AngleBracketed(args) = &segment.arguments else {
|
||||
panic!("Result 缺少类型参数")
|
||||
};
|
||||
let syn::GenericArgument::Type(mut output) = args.args.first().unwrap().clone()
|
||||
else {
|
||||
panic!("Result 缺少输出类型")
|
||||
};
|
||||
WireTypes.visit_type_mut(&mut output);
|
||||
Some((method.sig.ident.to_string(), output))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let register = source(root, &format!("src/public_service/{namespace}.rs"))
|
||||
.items
|
||||
.into_iter()
|
||||
.find_map(|item| match item {
|
||||
Item::Fn(item) if item.sig.ident == "register" => Some(item),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
let operations: Vec<syn::File> = register
|
||||
.block
|
||||
.stmts
|
||||
.into_iter()
|
||||
.filter_map(|stmt| {
|
||||
let syn::Stmt::Macro(stmt) = stmt else {
|
||||
return None;
|
||||
};
|
||||
if !stmt.mac.path.is_ident("register") {
|
||||
return None;
|
||||
}
|
||||
let Operation {
|
||||
name,
|
||||
method,
|
||||
mut input,
|
||||
} = syn::parse2(stmt.mac.tokens).unwrap();
|
||||
WireTypes.visit_type_mut(&mut input);
|
||||
let output = outputs.get(&method.to_string()).unwrap();
|
||||
let operation = format!("{namespace}.{method}");
|
||||
Some(
|
||||
syn::parse2(quote! {
|
||||
pub struct #name;
|
||||
impl Operation for #name {
|
||||
const NAME: &'static str = #operation;
|
||||
type Input = #input;
|
||||
type Output = #output;
|
||||
}
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
(operations, outputs.len())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdk_settings_contract_matches_host_declarations() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let mut types = Vec::new();
|
||||
let mut replacement_fields = Vec::new();
|
||||
for (path, names) in [
|
||||
(
|
||||
"../gateway-core/src/settings/privacy.rs",
|
||||
&[
|
||||
"CodexPrivacyPolicy",
|
||||
"PrivacyFailureMode",
|
||||
"PrivacyScope",
|
||||
"PrivacyAction",
|
||||
"PrivacyRule",
|
||||
"PrivacyStep",
|
||||
"PrivacyRuleOutcome",
|
||||
"PrivacyPreviewRequest",
|
||||
"PrivacyPreviewResult",
|
||||
][..],
|
||||
),
|
||||
(
|
||||
"src/model/settings.rs",
|
||||
&[
|
||||
@@ -213,78 +323,13 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
}
|
||||
}
|
||||
|
||||
let service = source(root, "src/use_case/settings.rs")
|
||||
.items
|
||||
.into_iter()
|
||||
.find_map(|item| match item {
|
||||
Item::Trait(item) if item.ident == "SettingsService" => Some(item),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
let outputs: BTreeMap<_, _> = service
|
||||
.items
|
||||
.into_iter()
|
||||
.filter_map(|item| match item {
|
||||
syn::TraitItem::Fn(method) => {
|
||||
let syn::ReturnType::Type(_, output) = method.sig.output else {
|
||||
panic!("服务必须返回 Result")
|
||||
};
|
||||
let Type::Path(result) = *output else {
|
||||
panic!("服务必须返回 Result")
|
||||
};
|
||||
let segment = result.path.segments.last().unwrap();
|
||||
assert_eq!(segment.ident, "Result");
|
||||
let syn::PathArguments::AngleBracketed(args) = &segment.arguments else {
|
||||
panic!("Result 缺少类型参数")
|
||||
};
|
||||
let syn::GenericArgument::Type(mut output) = args.args.first().unwrap().clone()
|
||||
else {
|
||||
panic!("Result 缺少输出类型")
|
||||
};
|
||||
WireTypes.visit_type_mut(&mut output);
|
||||
Some((method.sig.ident.to_string(), output))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let register = source(root, "src/public_service/settings.rs")
|
||||
.items
|
||||
.into_iter()
|
||||
.find_map(|item| match item {
|
||||
Item::Fn(item) if item.sig.ident == "register" => Some(item),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
let operations: Vec<_> = register
|
||||
.block
|
||||
.stmts
|
||||
.into_iter()
|
||||
.filter_map(|stmt| {
|
||||
let syn::Stmt::Macro(stmt) = stmt else {
|
||||
return None;
|
||||
};
|
||||
if !stmt.mac.path.is_ident("register") {
|
||||
return None;
|
||||
}
|
||||
let Operation {
|
||||
name,
|
||||
method,
|
||||
mut input,
|
||||
} = syn::parse2(stmt.mac.tokens).unwrap();
|
||||
WireTypes.visit_type_mut(&mut input);
|
||||
let output = outputs.get(&method.to_string()).unwrap();
|
||||
let operation = format!("settings.{method}");
|
||||
Some(quote! {
|
||||
pub struct #name;
|
||||
impl Operation for #name {
|
||||
const NAME: &'static str = #operation;
|
||||
type Input = #input;
|
||||
type Output = #output;
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(operations.len(), outputs.len(), "公开服务方法必须完整声明");
|
||||
let (operations, method_count) = service_operations(
|
||||
root,
|
||||
"settings",
|
||||
"src/use_case/settings.rs",
|
||||
"SettingsService",
|
||||
);
|
||||
assert_eq!(operations.len(), method_count, "公开服务方法必须完整声明");
|
||||
let assignments = replacement_fields.iter().map(|field| match field.to_string().as_str() {
|
||||
"expected_revision" => quote!(expected_revision: settings.config_revision),
|
||||
"request_profile_updates" => quote!(request_profile_updates: settings.request_profiles.into_iter().map(|(provider, value)| (provider, Some(value))).collect()),
|
||||
@@ -313,14 +358,48 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
}
|
||||
}
|
||||
};
|
||||
let output = root.join("../gateway-plugin/sdk/src/call/services/settings.rs");
|
||||
let expected = prettyplease::unparse(&syn::parse2::<syn::File>(generated).unwrap());
|
||||
check_contract(root, "settings", syn::parse2(generated).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdk_accounts_contract_matches_host_declarations() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let status = source(root, "src/model/reset_credit_automation.rs")
|
||||
.items
|
||||
.into_iter()
|
||||
.find(|item| matches!(item, Item::Struct(item) if item.ident == "ResetCreditAutomationStatus"))
|
||||
.unwrap();
|
||||
let status = contract_type(status, "ResetCreditAutomationStatus");
|
||||
let (operations, _) = service_operations(
|
||||
root,
|
||||
"accounts",
|
||||
"src/use_case/accounts/mod.rs",
|
||||
"AccountsService",
|
||||
);
|
||||
let generated = quote! {
|
||||
//! 账号自动重置的类型化操作与状态合同
|
||||
//!
|
||||
//! 从宿主状态类型与 public_service/accounts.rs 生成;更新命令见 SDK 维护说明
|
||||
|
||||
use super::Operation;
|
||||
use serde::{Deserialize, Serialize};
|
||||
#status
|
||||
#(#operations)*
|
||||
};
|
||||
check_contract(root, "accounts", syn::parse2(generated).unwrap());
|
||||
}
|
||||
|
||||
fn check_contract(root: &Path, namespace: &str, generated: syn::File) {
|
||||
let output = root.join(format!(
|
||||
"../gateway-plugin/sdk/src/call/services/{namespace}.rs"
|
||||
));
|
||||
let expected = prettyplease::unparse(&generated);
|
||||
if std::env::var_os("CPR_UPDATE_SERVICE_CONTRACT").is_some() {
|
||||
std::fs::write(&output, &expected).unwrap();
|
||||
}
|
||||
let actual = syn::parse_file(&std::fs::read_to_string(output).unwrap()).unwrap();
|
||||
assert!(
|
||||
prettyplease::unparse(&actual) == expected,
|
||||
"SDK 服务合同已过期:CPR_UPDATE_SERVICE_CONTRACT=1 cargo test -p gateway-admin --test main sdk_settings_contract_matches_host_declarations"
|
||||
"SDK 服务合同已过期:CPR_UPDATE_SERVICE_CONTRACT=1 cargo test -p gateway-admin --test main sdk_{namespace}_contract_matches_host_declarations"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -100,6 +100,51 @@ async fn client_profile_preview_rejects_unknown_provider_before_loading_settings
|
||||
assert_eq!(error.kind(), AdminErrorKind::Invalid);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pricing_accepts_ultrafast_for_openai_but_rejects_it_for_xai() {
|
||||
for provider in ["openai", "xai"] {
|
||||
let store = Arc::new(PricingSettingsStore::default());
|
||||
let services = super::AdminHarness::new()
|
||||
.settings(store.clone())
|
||||
.provider(super::test_provider(provider))
|
||||
.build()
|
||||
.await;
|
||||
let context = MutationContext {
|
||||
actor: gateway_admin::model::MutationActor::System,
|
||||
request_id: "request-ultrafast-pricing".to_owned(),
|
||||
};
|
||||
for band in ["ultrafast", "long_ultrafast"] {
|
||||
let pricing = serde_json::from_value(serde_json::json!({
|
||||
"multiplierBps": 10000,
|
||||
"bands": {
|
||||
"standard": {"input":"2", "output":"10", "cacheRead":"0.1", "cacheWrite":"2.5"},
|
||||
(band): {"input":"12", "output":"60", "cacheRead":"0.6", "cacheWrite":"15"}
|
||||
}
|
||||
}))
|
||||
.expect("Ultrafast override");
|
||||
let update = UpdatePricing {
|
||||
provider: provider.to_owned(),
|
||||
models: vec!["example-model".to_owned()],
|
||||
change: PricingChange::Replace(pricing),
|
||||
};
|
||||
let result = services
|
||||
.settings()
|
||||
.update_pricing(&context, update.clone())
|
||||
.await;
|
||||
if provider == "openai" {
|
||||
result.expect("OpenAI accepts Ultrafast overrides");
|
||||
assert_eq!(store.updated.lock().unwrap().as_ref(), Some(&update));
|
||||
} else {
|
||||
assert_eq!(
|
||||
result.expect_err("xAI rejects Ultrafast").kind(),
|
||||
AdminErrorKind::Invalid
|
||||
);
|
||||
assert!(store.updated.lock().unwrap().is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pricing_accepts_native_provider_and_rejects_unknown_sync_provider() {
|
||||
let store = Arc::new(PricingSettingsStore::default());
|
||||
@@ -280,12 +325,12 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
model_mappings: Default::default(),
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 0,
|
||||
refresh_concurrency: 1,
|
||||
max_concurrent_per_account: 1,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
@@ -310,6 +355,8 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
//! 用量服务的 wire 精度、可信主体和独立 SDK 合同校验
|
||||
use gateway_admin::model::usage_data::*;
|
||||
use serde_json::json;
|
||||
#[test]
|
||||
fn counters_preserve_integer_precision_and_reject_json_numbers() {
|
||||
assert_eq!(
|
||||
serde_json::from_value::<UsageCount>(json!(u64::MAX.to_string()))
|
||||
.unwrap()
|
||||
.get(),
|
||||
u64::MAX
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<UsageDelta>(json!(i64::MIN.to_string()))
|
||||
.unwrap()
|
||||
.get(),
|
||||
i64::MIN
|
||||
);
|
||||
for value in [json!(1), json!("1.5"), json!("+1")] {
|
||||
assert!(serde_json::from_value::<UsageCount>(value.clone()).is_err());
|
||||
assert!(serde_json::from_value::<UsageDelta>(value).is_err());
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn usage_mutations_require_trusted_actor_and_old_workflow_operations_are_absent() {
|
||||
let admin = super::AdminHarness::new().build().await;
|
||||
let registry = admin.public_services();
|
||||
let denied = registry
|
||||
.call("usage.records.write", json!({"changes":[]}))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(denied.message.contains("trusted caller"));
|
||||
assert!(
|
||||
registry
|
||||
.call(
|
||||
"usage.records.write",
|
||||
json!({"changes":[],"actor":{"kind":"system"}})
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let ops = registry
|
||||
.operations()
|
||||
.filter(|op| op.starts_with("usage.") || *op == "pricing.evaluate")
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
ops,
|
||||
std::collections::BTreeSet::from([
|
||||
"usage.records.get",
|
||||
"usage.records.list",
|
||||
"usage.records.write",
|
||||
"usage.aggregates.list",
|
||||
"usage.aggregates.write",
|
||||
"pricing.evaluate"
|
||||
])
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn sdk_wire_types_match_host_contracts() {
|
||||
use quote::ToTokens as _;
|
||||
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let items = |path: std::path::PathBuf| {
|
||||
syn::parse_file(&std::fs::read_to_string(path).unwrap())
|
||||
.unwrap()
|
||||
.items
|
||||
.into_iter()
|
||||
.filter(|item| {
|
||||
matches!(
|
||||
item,
|
||||
syn::Item::Struct(_)
|
||||
| syn::Item::Enum(_)
|
||||
| syn::Item::Impl(_)
|
||||
| syn::Item::Type(_)
|
||||
)
|
||||
})
|
||||
.map(|item| item.into_token_stream().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for name in [
|
||||
"value.rs",
|
||||
"record.rs",
|
||||
"query.rs",
|
||||
"aggregate.rs",
|
||||
"pricing.rs",
|
||||
"error.rs",
|
||||
] {
|
||||
assert_eq!(
|
||||
items(root.join("src/model/usage_data").join(name)),
|
||||
items(
|
||||
root.join("../gateway-plugin/sdk/src/call/services/usage")
|
||||
.join(name)
|
||||
),
|
||||
"{name}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn batch_and_wire_limits_fail_before_reaching_storage() {
|
||||
let admin = super::AdminHarness::new().build().await;
|
||||
let registry = admin.public_services();
|
||||
let changes = (0..501)
|
||||
.map(|i| json!({"kind":"delete","id":format!("item_{i}")}))
|
||||
.collect::<Vec<_>>();
|
||||
let too_many = gateway_admin::public_service::scope_actor(
|
||||
UsageActor::System,
|
||||
registry.call("usage.records.write", json!({"changes":changes})),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(too_many.kind, "capacity_exceeded");
|
||||
let oversized = registry
|
||||
.call(
|
||||
"usage.records.get",
|
||||
json!({"id":"x".repeat(USAGE_DATA_MAX_REQUEST_BYTES)}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(oversized.kind, "capacity_exceeded");
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user