mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9039cef3d8 | ||
|
|
a9a77f5f9d | ||
|
|
6f7d92594e | ||
|
|
6859520969 | ||
|
|
21d84af2d8 | ||
|
|
311e4d228f | ||
|
|
d604adfbeb | ||
|
|
46b4665c5b | ||
|
|
1daa0623a0 | ||
|
|
860fb39622 | ||
|
|
f5c5a6e5b7 | ||
|
|
bdfc201795 | ||
|
|
789609dd53 | ||
|
|
10e25ce25f | ||
|
|
6b362a5029 | ||
|
|
13374e2a70 | ||
|
|
19ef484969 | ||
|
|
242ffb2324 | ||
|
|
53a09bbfe9 | ||
|
|
239def9a5b | ||
|
|
5eaca14a64 | ||
|
|
78e943f406 | ||
|
|
0663df14df | ||
|
|
39aa20a13a | ||
|
|
621b1faceb | ||
|
|
28eb4708b5 | ||
|
|
aa003f7d90 | ||
|
|
4c57a0405d | ||
|
|
49cee2397f | ||
|
|
1915806bd1 | ||
|
|
5100f6333e | ||
|
|
187af36306 | ||
|
|
be35f8a6ee | ||
|
|
317c05c74a | ||
|
|
12c32fcdec | ||
|
|
26de671226 | ||
|
|
a92b95c48c | ||
|
|
e7c24781fb | ||
|
|
3b4b547e9c | ||
|
|
2016bda031 | ||
|
|
790f3147e0 | ||
|
|
96e8443ede | ||
|
|
42badf64ca | ||
|
|
35afe1123f | ||
|
|
2ac4265832 | ||
|
|
d22ec29824 | ||
|
|
7a32c6cec2 | ||
|
|
0ff1a72919 | ||
|
|
79083a5632 | ||
|
|
90f7456db8 | ||
|
|
6f71b27c3c | ||
|
|
7b05b14a2b | ||
|
|
f9b5b5ff77 | ||
|
|
4d5778a84f | ||
|
|
6d22d73506 | ||
|
|
d8ab207282 | ||
|
|
3816b46ba1 | ||
|
|
4a7c06f895 | ||
|
|
e0b7a841ca | ||
|
|
96a292dd0f | ||
|
|
6f138c8256 | ||
|
|
c8ffb1a8c8 | ||
|
|
79df82a4f0 | ||
|
|
62516d9769 | ||
|
|
892cba68ae | ||
|
|
fada9bc0bd | ||
|
|
27b9bbe994 | ||
|
|
db849c1251 | ||
|
|
c57af9b0dd |
@@ -5,6 +5,13 @@ description: Codex Proxy RS 开发、修复、排障、代码审查和文档修
|
||||
|
||||
# 开发指南
|
||||
|
||||
## 首要写作要求:言简意赅
|
||||
|
||||
所有面向人的文字都按此要求编写和审查,包括界面、通知、后端错误、任务状态、命令行、日志、注释、文档和交付说明
|
||||
|
||||
- 先写读者需要知道的事实或动作。删掉一句不影响理解、判断或操作,就删;标题、正文和通知不重复同一信息
|
||||
- 保留会改变决定的条件、权限、后果和恢复步骤。错误通知只说已知问题,有明确办法才补一句;后端返回的提示在后端改,前端不重复解释。上游原文、用户内容和诊断事实保持原样
|
||||
|
||||
## 先确定领域与必读规范
|
||||
|
||||
按用户意图、受影响行为与对象初判,定位后按实际文件复核,不要求用户说出“前端”“后端”或技能名。**所有命中行共同构成本次必读规范,先读完再确定方案、修改或给出审查结论**;跨领域叠加,不能只选最像当前步骤的一行
|
||||
@@ -39,7 +46,7 @@ description: Codex Proxy RS 开发、修复、排障、代码审查和文档修
|
||||
2. 以最终差异重新匹配上表,补齐新增领域的规范并检查实际实现,再处理本次引入的职责越界、重复规则、冗余状态与无依据分支;删除或替换功能时一并清理失效的配置、类型、调用、测试与文档,已部署数据和公开合同仍遵守所属迁移及兼容规则;仅审查时报告,不自行修复
|
||||
3. 按最终行为执行[文档检查](references/documentation.md),核对所属正文及相关概述、示例和引用,在本任务内完成必要同步;不以尚无文档差异跳过检查。内部重构等未改变文档所述事实时无需改文档;常规文档只写当前状态,普通开发不自动修改 `release/notes.md`
|
||||
4. 按变更范围执行[验证](../../../CONTRIBUTING.md#验证)及其停止条件,后端命令见[验证命令](../../../docs/architecture.md#验证命令);沿用本轮有效证据与用户已确定的验收分工,区分通过、失败、跳过与未执行;提交时补读[提交约定](../../../CONTRIBUTING.md#项目约定)
|
||||
5. 交付实际结果与验证缺口;前端按[前端流程](references/frontend.md)核对最终差异和运行证据,后端按上表检查生命周期,不把构建通过当作行为验收
|
||||
5. 交付前按[首要写作要求](#首要写作要求言简意赅)通读本次新增或修改的文字。交代实际结果与验证缺口;前端按[前端流程](references/frontend.md)核对运行证据,后端按上表检查生命周期,不把构建通过当作行为验收
|
||||
|
||||
## 测试维护
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@ description: 准备、创建、更新或审查 GitHub PR,含回复讨论与处
|
||||
|
||||
本仓库目标分支为 `main`,其他仓库使用其可信基线规则与模板,PR 内修改的规则不能降低审查要求
|
||||
|
||||
为本仓库拟写或更新 PR 标题时,完整标题使用英文,沿用提交信息的 Conventional Commits 格式,例如 `fix(store): preserve client key ledgers after deletion`
|
||||
|
||||
涉及上游行为的判断按[问题与方案依据](../../../CONTRIBUTING.md#问题与方案依据)执行
|
||||
|
||||
## 按模式读取
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# SQLx migration checksums are byte-sensitive; keep migration inputs portable across hosts.
|
||||
backend/migrations/*.sql text eol=lf
|
||||
backend/migrations/.frozen-sha256 text eol=lf
|
||||
@@ -139,7 +139,6 @@ body:
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
|
||||
@@ -108,7 +108,6 @@ body:
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: 预期行为
|
||||
description: 说明原本期望得到的结果。
|
||||
validations:
|
||||
required: true
|
||||
|
||||
|
||||
@@ -7,21 +7,18 @@ RUN --mount=type=bind,target=/cache \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& cp -a /cache/registry/. /usr/local/cargo/registry/ \
|
||||
&& cp -a /cache/git/. /usr/local/cargo/git/ \
|
||||
&& cp -a /cache/target/. /app/backend/target/
|
||||
&& tar -xzf /cache/registry.tar.gz -C /usr/local/cargo/registry \
|
||||
&& tar -xzf /cache/git.tar.gz -C /usr/local/cargo/git \
|
||||
&& tar -xzf /cache/target.tar.gz -C /app/backend/target
|
||||
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS collect
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS snapshot
|
||||
ARG CACHE_STAMP
|
||||
# 一次导出三个 mount,避免逐目录创建镜像、载入 Docker 再复制文件。
|
||||
# 直接压缩 mount 内容,构建层和 runner 只保存归档,避免多份展开的编译产物占满磁盘
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& mkdir -p /cache/registry /cache/git /cache/target \
|
||||
&& cp -a /usr/local/cargo/registry/. /cache/registry/ \
|
||||
&& cp -a /usr/local/cargo/git/. /cache/git/ \
|
||||
&& cp -a /app/backend/target/. /cache/target/
|
||||
|
||||
FROM scratch AS snapshot
|
||||
COPY --from=collect /cache/ /
|
||||
&& mkdir -p /cache \
|
||||
&& tar -czf /cache/registry.tar.gz -C /usr/local/cargo/registry . \
|
||||
&& tar -czf /cache/git.tar.gz -C /usr/local/cargo/git . \
|
||||
&& tar -czf /cache/target.tar.gz -C /app/backend/target .
|
||||
|
||||
@@ -14,24 +14,42 @@ case "$architecture" in amd64|arm64) ;; *) exit 2 ;; esac
|
||||
dockerfile="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/Dockerfile"
|
||||
|
||||
usable() {
|
||||
local root="$1" path crates=false libraries=false
|
||||
# 目录或 buildstamp 存在不代表编译发生过,必须同时有下载包与目标依赖。
|
||||
for path in "$root"/registry/cache/*/*.crate; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then crates=true; break; fi
|
||||
done
|
||||
for path in "$root"/target/"$architecture"/release/deps/*.rlib; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then libraries=true; break; fi
|
||||
done
|
||||
[ "$crates" = true ] && [ "$libraries" = true ]
|
||||
# 流式检查归档中的非空依赖文件,校验时不在 runner 展开编译产物
|
||||
python3 - "$1" "$architecture" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
architecture = sys.argv[2]
|
||||
|
||||
def contains(filename, pattern):
|
||||
with tarfile.open(root / filename, mode="r|gz") as archive:
|
||||
return any(
|
||||
member.isfile() and member.size > 0
|
||||
and re.fullmatch(pattern, member.name.removeprefix("./"))
|
||||
for member in archive
|
||||
)
|
||||
|
||||
try:
|
||||
valid = (
|
||||
(root / "git.tar.gz").stat().st_size > 0
|
||||
and contains("registry.tar.gz", r"cache/[^/]+/[^/]+\.crate")
|
||||
and contains("target.tar.gz", rf"{architecture}/release/deps/[^/]+\.rlib")
|
||||
)
|
||||
except (OSError, tarfile.TarError):
|
||||
valid = False
|
||||
sys.exit(0 if valid else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
command=(docker buildx build --builder "$builder" --file "$dockerfile" --progress plain)
|
||||
# mount 的内容不参与层指纹,用时间戳重跑复制;--no-cache 会重置 cache mount。
|
||||
# mount 的内容不参与层指纹,用时间戳重跑归档操作;--no-cache 会重置 cache mount
|
||||
command+=(--build-arg "CACHE_STAMP=$(date +%s%N)" --target "$operation")
|
||||
valid=false
|
||||
if [ "$operation" = restore ]; then
|
||||
if usable "$directory"; then
|
||||
mkdir -p "$directory"/{registry,git,target}
|
||||
"${command[@]}" --output type=cacheonly "$directory"
|
||||
valid=true
|
||||
else
|
||||
@@ -43,10 +61,11 @@ else
|
||||
temporary="$(mktemp -d "${directory}.XXXXXX")"
|
||||
trap 'rm -rf "$temporary"' EXIT
|
||||
mkdir "$temporary"/{context,snapshot}
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,直接解包以免额外落盘整份归档。
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,只接收 /cache 下的压缩归档
|
||||
# 直接导出打包阶段,避免再创建一层归档副本
|
||||
# https://github.com/moby/buildkit/issues/2950
|
||||
"${command[@]}" --output type=tar,dest=- "$temporary/context" \
|
||||
| tar -xf - -C "$temporary/snapshot"
|
||||
| tar -xf - -C "$temporary/snapshot" --strip-components=1 cache/
|
||||
if usable "$temporary/snapshot"; then
|
||||
rm -rf "$directory"
|
||||
mv "$temporary/snapshot" "$directory"
|
||||
|
||||
@@ -36,10 +36,9 @@ jobs:
|
||||
with:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
# 同一平台和依赖复用快照,源码变化由 Cargo 指纹判定是否重编译
|
||||
key: container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
key: container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
restore-keys: |
|
||||
container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v2-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-
|
||||
|
||||
- name: Restore Cargo mounts to BuildKit
|
||||
id: cargo-restore
|
||||
@@ -50,6 +49,9 @@ jobs:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
architecture: amd64
|
||||
|
||||
- name: Release restored Cargo snapshot
|
||||
run: rm -rf "$RUNNER_TEMP/container-cargo"
|
||||
|
||||
- name: Build runtime image
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
||||
with:
|
||||
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
cargo nextest run \
|
||||
--cargo-metadata "$RUNNER_TEMP/nextest-cargo-metadata.json" \
|
||||
--binaries-metadata "$RUNNER_TEMP/nextest-binaries.json" \
|
||||
--no-tests fail
|
||||
--test-threads 8 --no-tests fail
|
||||
|
||||
- name: Store tests with Redis ACL authentication
|
||||
working-directory: backend
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
cargo nextest run \
|
||||
--cargo-metadata "$RUNNER_TEMP/nextest-cargo-metadata.json" \
|
||||
--binaries-metadata "$RUNNER_TEMP/nextest-binaries.json" \
|
||||
--no-tests fail -E 'package(=gateway-store) & test(/^redis::/)'
|
||||
--test-threads 8 --no-tests fail -E 'package(=gateway-store) & test(/^redis::/)'
|
||||
|
||||
frontend:
|
||||
if: inputs.run_frontend
|
||||
|
||||
@@ -171,6 +171,12 @@ jobs:
|
||||
toolchain: ${{ needs.release-info.outputs.rust }}
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Configure Windows runtime linkage
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
# 发行包静态链接 MSVC 运行库,解压运行时无需另外安装 VC++ Redistributable
|
||||
run: echo 'RUSTFLAGS=-C target-feature=+crt-static' >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache Cargo
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
@@ -186,6 +192,8 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
cargo build --release --locked --target "${{ matrix.target }}" --bin codex-proxy-rs
|
||||
"target/${{ matrix.target }}/release/${{ matrix.binary }}" --help
|
||||
"target/${{ matrix.target }}/release/${{ matrix.binary }}" --version
|
||||
mkdir -p "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}"
|
||||
cp "target/${{ matrix.target }}/release/${{ matrix.binary }}" "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/${{ matrix.binary }}"
|
||||
working-directory: backend
|
||||
@@ -195,6 +203,16 @@ jobs:
|
||||
CPR_BUILD_TIME: ${{ needs.release-info.outputs.build_time }}
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Verify Windows platform behavior
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
working-directory: backend
|
||||
env:
|
||||
RUST_MIN_STACK: '16777216'
|
||||
run: |
|
||||
cargo test --release --locked --target "${{ matrix.target }}" -p gateway-admin --test main use_case::plugins::official::
|
||||
cargo test --release --locked --target "${{ matrix.target }}" -p gateway-host --test main system_update::
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
|
||||
@@ -40,6 +40,8 @@
|
||||
|
||||
以下命令适用于 Linux amd64/arm64,需要 Docker Engine、Docker Compose Plugin、curl 和 OpenSSL
|
||||
|
||||
Windows x64 / ARM64 使用发行包部署,步骤见 [Windows 二进制部署](deploy/README.md#windows-二进制部署)
|
||||
|
||||
已有部署请先看 [升级说明](deploy/README.md#镜像升级与源码构建),不要覆盖原配置
|
||||
|
||||
### 一键安装
|
||||
@@ -89,7 +91,7 @@ API Key 持有者可在登录页切换身份,查看自己的用量与额度,
|
||||
|
||||
1. 在「账号」中添加账号,完成授权或导入
|
||||
2. 按需建立账号分组,再创建客户端密钥并选择可用分组,**不选分组表示可使用全部账号**
|
||||
3. 打开密钥的「使用密钥」,复制客户端配置
|
||||
3. [连接客户端](#客户端接入)
|
||||
|
||||
## 客户端接入
|
||||
|
||||
|
||||
Generated
+40
-38
@@ -286,9 +286,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.155.0"
|
||||
version = "1.156.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a542f0e9261b9ce56708d591a62062079fa3c58a362ffbf49a2e48a4f6ced16"
|
||||
checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -322,9 +322,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.6.3"
|
||||
version = "1.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16"
|
||||
checksum = "f385da9b7757ed4c8214797ad663ff534fb622006143530aa80e21f8e06ff85c"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -387,9 +387,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.64.2"
|
||||
version = "0.64.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398"
|
||||
checksum = "19af609dd5edf8b056cd0bb001c706c8b3b1458055328b334ec13d7f46f6aaa8"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -408,9 +408,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.5.1"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e"
|
||||
checksum = "859a81364f8185349c584fb41cd8765a400cf982f447adada590cf008a9f3b85"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -432,9 +432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.65.0"
|
||||
version = "0.65.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666"
|
||||
checksum = "b45f9fc77ffe92c92c341165a01a565bd30c39ecf5aa8c0719b41719009604f9"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -452,9 +452,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.16.1"
|
||||
version = "1.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a"
|
||||
checksum = "4cb80fb19c084da5fca9d797845735e184445e5b958599440d67529bf6ef1b64"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -475,9 +475,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.19.1"
|
||||
version = "1.19.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4"
|
||||
checksum = "8c4e85cffe9944b10d576b075432bd8f98dbc081e8681cb37d5b2d87dc27bc84"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -492,9 +492,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api-macros"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
|
||||
checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -514,9 +514,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822"
|
||||
checksum = "cf183d70774ea2cfcc101ca74a8884e4c76be8b9df65cd303f6a1175c88e8f80"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -537,9 +537,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.64.0"
|
||||
version = "0.64.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413"
|
||||
checksum = "590fdc1a70546b2f7f2061c42eaf173fb6f26d935330d069edc04dbd8f01d924"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -2485,9 +2485,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.60.0"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e7305938cac5ee144352ceb71583ffc6cfeac7d20fcb20ef865a4d5c314e499"
|
||||
checksum = "a42a11b47492e6a2cc76e9c7bcc8077776cb31dac444e0795d7f61bcc6441c08"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2513,18 +2513,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f8a848331614144c03e815fb931f238c9e342bd7ab501b29e35d647e7cd5688"
|
||||
checksum = "107aeb6ff14b8deb5610096a8ce5c554ec89806b521147eaeb3cae05b07edb7a"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5a947ef798eeacacb43edf29b1d0ab64bdf5e2c9fbcaad10ee8663d922b544"
|
||||
checksum = "e281d6b177bb4834f2ffd6f92aa0443cffe29c996b251c8d94eb6f456186feba"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -3578,9 +3578,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.7.1"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9"
|
||||
checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -3634,9 +3634,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "922268812b6f9e20b4ab0f1efe8e07036958eed7346ceddb31c777b5531fd11f"
|
||||
checksum = "f42818036d3b37cda6bd4c494543158a36e431cb717015ef7c93acc6ba214d3b"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -3759,9 +3759,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "roxmltree"
|
||||
version = "0.21.1"
|
||||
version = "0.21.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
|
||||
checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@@ -4953,9 +4953,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
@@ -5052,9 +5052,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.28.0"
|
||||
version = "1.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606"
|
||||
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -5646,16 +5646,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.0.5"
|
||||
version = "9.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
|
||||
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"nix",
|
||||
"typed-path",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -49,6 +49,10 @@ opt-level = 2
|
||||
[profile.dev.package."miniz_oxide@0.9.1"]
|
||||
opt-level = 2
|
||||
|
||||
# workspace 的 WebSocket deflate 特性会让 flate2 选用 zlib-rs
|
||||
[profile.dev.package.zlib-rs]
|
||||
opt-level = 2
|
||||
|
||||
[profile.release]
|
||||
debug = false
|
||||
strip = "symbols"
|
||||
|
||||
@@ -146,6 +146,10 @@ async fn execute(
|
||||
providers.admin,
|
||||
);
|
||||
or_shutdown!(plugin_runtime, services.register_usage_data(&usage_data));
|
||||
or_shutdown!(
|
||||
plugin_runtime,
|
||||
plugin_runtime.bind_usage_queries(&usage_data)
|
||||
);
|
||||
let services = std::sync::Arc::new(services);
|
||||
or_shutdown!(plugin_runtime, plugin_runtime.bind_services(&services));
|
||||
let commands = or_shutdown!(plugin_runtime, plugin_runtime.prepare_command_line().await);
|
||||
|
||||
@@ -160,6 +160,10 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
plugin_runtime,
|
||||
plugin_runtime.bind_services(&admin.services().public_services())
|
||||
);
|
||||
or_shutdown!(
|
||||
plugin_runtime,
|
||||
plugin_runtime.bind_usage_queries(&admin.services().usage_data())
|
||||
);
|
||||
|
||||
let mut plan = store.take_worker_contributions();
|
||||
plan.push(analytics_worker);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//! 网关命令行入口,选择启动模式并创建异步运行时
|
||||
|
||||
use std::{
|
||||
env,
|
||||
error::Error,
|
||||
io::{self, Write},
|
||||
process::ExitCode,
|
||||
@@ -21,7 +20,7 @@ fn main() -> ExitCode {
|
||||
|
||||
fn run() -> Result<ExitCode, Box<dyn Error + Send + Sync>> {
|
||||
ensure_rustls_provider();
|
||||
let mut arguments = env::args_os().skip(1);
|
||||
let mut arguments = gateway_host::system_update::startup_arguments()?;
|
||||
let command = arguments
|
||||
.next()
|
||||
.map(|value| {
|
||||
@@ -92,7 +91,7 @@ fn runtime() -> Result<tokio::runtime::Runtime, io::Error> {
|
||||
|
||||
fn reject_extra_arguments(mut arguments: impl Iterator) -> Result<(), io::Error> {
|
||||
if arguments.next().is_some() {
|
||||
return Err(invalid_cli("unexpected extra command arguments"));
|
||||
return Err(invalid_cli("too many arguments"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -70,6 +70,7 @@ fn gateway_core_depends_on_no_http_db_redis_or_provider_crate() {
|
||||
#[test]
|
||||
fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
let root = backend_root().join("crates/gateway-core/src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
for relative in super::rust_files(&root) {
|
||||
let allowed: Option<&[&str]> = match relative.to_str().expect("source path") {
|
||||
"validation.rs" => Some(&[]),
|
||||
@@ -96,7 +97,7 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
let Some(allowed) = allowed else { continue };
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
let syntax = syn::parse_file(&source).expect("parse core source");
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
let mut references = CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
references.visit_file(&syntax);
|
||||
// 相对路径中的 self 指向当前 owner,不是跨 owner 依赖
|
||||
references.owners.remove(&references.module_path[0]);
|
||||
@@ -111,17 +112,24 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_owner_paths_form_a_dag() {
|
||||
let root = backend_root().join("crates/gateway-core/src");
|
||||
fn domain_owner_paths_form_a_dag() {
|
||||
for member in ["crates/gateway-core", "crates/gateway-admin"] {
|
||||
assert_owner_paths_form_a_dag(member);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_owner_paths_form_a_dag(member: &str) {
|
||||
let root = backend_root().join(member).join("src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
let mut graph = BTreeMap::<String, BTreeSet<String>>::new();
|
||||
for relative in super::rust_files(&root) {
|
||||
if relative == Path::new("lib.rs") {
|
||||
continue;
|
||||
}
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
let mut references = CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
let owner = references.module_path[0].clone();
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse core source"));
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read owner source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse owner source"));
|
||||
references.owners.remove(&owner);
|
||||
graph.entry(owner).or_default().extend(references.owners);
|
||||
}
|
||||
@@ -131,7 +139,7 @@ fn core_owner_paths_form_a_dag() {
|
||||
while let Some(dependency) = pending.pop() {
|
||||
assert_ne!(
|
||||
&dependency, owner,
|
||||
"Core owner dependency cycle reaches {owner}"
|
||||
"{member} owner dependency cycle reaches {owner}"
|
||||
);
|
||||
if visited.insert(dependency.clone())
|
||||
&& let Some(next) = graph.get(&dependency)
|
||||
@@ -143,7 +151,44 @@ fn core_owner_paths_form_a_dag() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
fn shared_owners_do_not_depend_on_their_callers() {
|
||||
let boundaries: &[(&str, &str, &[&str])] = &[
|
||||
("crates/providers/xai", "transport", &["credential"]),
|
||||
("crates/providers/openai", "transport", &["provider"]),
|
||||
("crates/gateway-api", "time", &["admin"]),
|
||||
(
|
||||
"crates/gateway-api",
|
||||
"client_error",
|
||||
&["admin", "openai", "middleware"],
|
||||
),
|
||||
("crates/gateway-api", "middleware", &["openai"]),
|
||||
("crates/gateway-host", "system_update", &["config"]),
|
||||
("crates/gateway-plugin/runtime", "package", &["callback"]),
|
||||
];
|
||||
for &(member, owner, forbidden) in boundaries {
|
||||
let root = backend_root().join(member).join("src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
for relative in super::rust_files(&root) {
|
||||
let mut references =
|
||||
CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
if references.module_path.first().map(String::as_str) != Some(owner) {
|
||||
continue;
|
||||
}
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read owner source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse owner source"));
|
||||
for dependency in forbidden {
|
||||
assert!(
|
||||
!references.owners.contains(*dependency),
|
||||
"{member}/src/{} must not depend on crate::{dependency}",
|
||||
relative.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owner_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
let syntax = syn::parse_file(
|
||||
r#"
|
||||
pub use crate::{engine::AttemptContext, settings::SettingsValues as Values};
|
||||
@@ -167,13 +212,86 @@ fn core_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CrateReferences {
|
||||
owners: BTreeSet<String>,
|
||||
module_path: Vec<String>,
|
||||
#[test]
|
||||
fn owner_reference_scan_resolves_crate_root_reexports() {
|
||||
let root = syn::parse_file(
|
||||
r#"
|
||||
mod provider;
|
||||
mod transport;
|
||||
mod secret;
|
||||
pub use provider::{OFFICIAL_URL, Options as ProviderOptions};
|
||||
pub use self::transport::{self as network, Request};
|
||||
pub use crate::secret::SecretValue;
|
||||
pub use external::Unrelated;
|
||||
"#,
|
||||
)
|
||||
.expect("parse root reexport fixture");
|
||||
let syntax = syn::parse_file(
|
||||
r#"
|
||||
use crate::{OFFICIAL_URL, ProviderOptions, network::Client};
|
||||
use super::super::SecretValue;
|
||||
fn request() -> crate::Request { unimplemented!() }
|
||||
"#,
|
||||
)
|
||||
.expect("parse reexport consumer fixture");
|
||||
let reexports = root_reexport_owners(&root);
|
||||
assert!(!reexports.contains_key("Unrelated"));
|
||||
let mut references = CrateReferences::in_module(Path::new("consumer/request.rs"))
|
||||
.with_root_reexports(&reexports);
|
||||
references.visit_file(&syntax);
|
||||
assert_eq!(
|
||||
references.owners,
|
||||
BTreeSet::from([
|
||||
"provider".to_owned(),
|
||||
"transport".to_owned(),
|
||||
"secret".to_owned(),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for CrateReferences {
|
||||
fn read_root_reexport_owners(root: &Path) -> BTreeMap<String, String> {
|
||||
let source = fs::read_to_string(root.join("lib.rs")).expect("read crate root");
|
||||
root_reexport_owners(&syn::parse_file(&source).expect("parse crate root"))
|
||||
}
|
||||
|
||||
// 只解析根模块的显式重导出,不推断 glob、宏展开或完整 Rust 名称解析
|
||||
fn root_reexport_owners(syntax: &syn::File) -> BTreeMap<String, String> {
|
||||
let modules = syntax
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
Item::Mod(module) => Some(module.ident.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mut reexports = BTreeMap::new();
|
||||
for item in &syntax.items {
|
||||
if let Item::Use(item) = item
|
||||
&& !matches!(item.vis, syn::Visibility::Inherited)
|
||||
{
|
||||
visit_use_leaves(&item.tree, &mut Vec::new(), &mut |path, name| {
|
||||
let owner = path
|
||||
.iter()
|
||||
.find(|segment| !matches!(segment.as_str(), "crate" | "self"));
|
||||
if let (Some(owner), Some(name)) = (owner, name)
|
||||
&& modules.contains(owner)
|
||||
{
|
||||
reexports.insert(name.to_owned(), owner.clone());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
reexports
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CrateReferences<'a> {
|
||||
owners: BTreeSet<String>,
|
||||
module_path: Vec<String>,
|
||||
root_reexports: Option<&'a BTreeMap<String, String>>,
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for CrateReferences<'_> {
|
||||
fn visit_path(&mut self, path: &'ast syn::Path) {
|
||||
self.record(
|
||||
&path
|
||||
@@ -186,11 +304,13 @@ impl<'ast> Visit<'ast> for CrateReferences {
|
||||
}
|
||||
|
||||
fn visit_item_use(&mut self, item: &'ast syn::ItemUse) {
|
||||
self.use_tree(&item.tree, &mut Vec::new());
|
||||
visit_use_leaves(&item.tree, &mut Vec::new(), &mut |path, _| {
|
||||
self.record(path);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl CrateReferences {
|
||||
impl<'a> CrateReferences<'a> {
|
||||
fn in_module(relative: &Path) -> Self {
|
||||
let mut module = relative
|
||||
.with_extension("")
|
||||
@@ -208,13 +328,26 @@ impl CrateReferences {
|
||||
Self {
|
||||
owners: BTreeSet::new(),
|
||||
module_path: module,
|
||||
root_reexports: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_root_reexports(mut self, reexports: &'a BTreeMap<String, String>) -> Self {
|
||||
self.root_reexports = Some(reexports);
|
||||
self
|
||||
}
|
||||
|
||||
fn root_owner(&self, name: &str) -> String {
|
||||
self.root_reexports
|
||||
.and_then(|reexports| reexports.get(name))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| name.to_owned())
|
||||
}
|
||||
|
||||
fn record(&mut self, path: &[String]) {
|
||||
let Some(first) = path.first() else { return };
|
||||
let owner = if first == "crate" {
|
||||
path.get(1).cloned()
|
||||
path.get(1).map(|name| self.root_owner(name))
|
||||
} else if first == "self" || first == "super" {
|
||||
let mut module = self.module_path.clone();
|
||||
let mut rest = path;
|
||||
@@ -227,7 +360,10 @@ impl CrateReferences {
|
||||
}
|
||||
rest = &rest[1..];
|
||||
}
|
||||
module.first().or_else(|| rest.first()).cloned()
|
||||
module
|
||||
.first()
|
||||
.cloned()
|
||||
.or_else(|| rest.first().map(|name| self.root_owner(name)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -235,31 +371,41 @@ impl CrateReferences {
|
||||
self.owners.insert(owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn use_tree(&mut self, tree: &syn::UseTree, prefix: &mut Vec<String>) {
|
||||
match tree {
|
||||
syn::UseTree::Path(path) => {
|
||||
prefix.push(path.ident.to_string());
|
||||
self.use_tree(&path.tree, prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Name(name) => {
|
||||
prefix.push(name.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Rename(rename) => {
|
||||
prefix.push(rename.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Group(group) => {
|
||||
for item in &group.items {
|
||||
self.use_tree(item, prefix);
|
||||
}
|
||||
}
|
||||
syn::UseTree::Glob(_) => self.record(prefix),
|
||||
fn visit_use_leaves(
|
||||
tree: &syn::UseTree,
|
||||
prefix: &mut Vec<String>,
|
||||
visit: &mut impl FnMut(&[String], Option<&str>),
|
||||
) {
|
||||
match tree {
|
||||
syn::UseTree::Path(path) => {
|
||||
prefix.push(path.ident.to_string());
|
||||
visit_use_leaves(&path.tree, prefix, visit);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Name(name) => {
|
||||
let name = name.ident.to_string();
|
||||
let export = if name == "self" {
|
||||
prefix.last().cloned()
|
||||
} else {
|
||||
Some(name.clone())
|
||||
};
|
||||
prefix.push(name);
|
||||
visit(prefix, export.as_deref());
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Rename(rename) => {
|
||||
prefix.push(rename.ident.to_string());
|
||||
visit(prefix, Some(&rename.rename.to_string()));
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Group(group) => {
|
||||
for item in &group.items {
|
||||
visit_use_leaves(item, prefix, visit);
|
||||
}
|
||||
}
|
||||
syn::UseTree::Glob(_) => visit(prefix, None),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ cargo +1.99.1 install --locked --path backend/apps/plugin-cli --root /path/to/lo
|
||||
|
||||
## 打包
|
||||
|
||||
先构建目标平台的插件二进制;插件提供管理页面时,还需构建页面静态资源。打包命令传入作者清单、二进制和所需的资源目录映射,通过 `cpr-plugin package --help` 查看完整参数:
|
||||
先构建目标平台的二进制;有管理页面时,也构建静态资源。全部参数见 `cpr-plugin package --help`
|
||||
|
||||
```bash
|
||||
cpr-plugin package \
|
||||
|
||||
@@ -11,11 +11,15 @@ use chrono::{DateTime, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
task::{DaemonTask, WorkerDaemonContext, WorkerTaskError},
|
||||
task::{
|
||||
DaemonRestartPolicy, DaemonTask, WorkerContribution, WorkerDaemonContext, WorkerId,
|
||||
WorkerKind, WorkerRegistration, WorkerRunnable, WorkerTaskError,
|
||||
},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::{
|
||||
model::AdminError,
|
||||
model::backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupStatus, BackupStatusTransition,
|
||||
BackupStorageConfig, BackupTriggerKind, build_backup_seed, code,
|
||||
@@ -34,6 +38,8 @@ impl From<BackupError> for WorkerTaskError {
|
||||
|
||||
use super::policy::{BackupSchedule, decide_retention};
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
|
||||
/// 循环间隔:管理员修改 Cron 后该时长内生效;无工作时可取消等待
|
||||
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// 每个循环最多执行的保留删除数量
|
||||
@@ -72,6 +78,22 @@ impl BackupTask {
|
||||
self.timezone = timezone;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("备份 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("备份 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("备份 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for BackupTask {
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
//! 管理服务与后台任务的启动组装,窄端口供服务器和 CLI 复用
|
||||
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use gateway_core::{engine::probe::AccountProbe, runtime::SnapshotControl};
|
||||
|
||||
use crate::{
|
||||
AccountsService, AdminBundle, AdminConfig, AdminRuntimePorts, AdminServices, ClientConfig,
|
||||
ClientKeyService, CredentialsService, PluginDistributionPorts, PluginManagementService,
|
||||
PluginsService, SettingsService, UsageDataService, backup, freeze_recovery,
|
||||
model::AdminError,
|
||||
ports::{
|
||||
self, plugin_accounts::PluginAccountAccess, plugin_client_keys::PluginClientKeyAccess,
|
||||
provider::ProviderAdminRegistry, store::AdminStorePorts,
|
||||
},
|
||||
public_service, reset_credit_worker, use_case,
|
||||
use_case::{
|
||||
account_groups::DefaultAccountGroupService,
|
||||
accounts::DefaultAccountsService,
|
||||
auth::{AuthService, DefaultAuthService},
|
||||
backup::DefaultBackupService,
|
||||
client_distribution::DefaultClientDistributionService,
|
||||
client_keys::DefaultClientKeyService,
|
||||
observability::DefaultObservabilityService,
|
||||
settings::DefaultSettingsService,
|
||||
system::DefaultSystemService,
|
||||
},
|
||||
};
|
||||
|
||||
/// 校验配置、接入已组装的 Provider 注册表并完成默认管理员幂等初始化
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 配置非法或默认管理员初始化失败时返回错误
|
||||
pub async fn initialize(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, None).await
|
||||
}
|
||||
|
||||
/// 使用组合根已绑定给 Runtime 的同一账号窄端口,避免为完整 Admin 重建第二实例
|
||||
pub async fn initialize_with_plugin_accounts(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Arc<dyn PluginAccountAccess>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, Some(plugin_accounts)).await
|
||||
}
|
||||
|
||||
async fn initialize_inner(
|
||||
mut config: AdminConfig,
|
||||
mut client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Option<Arc<dyn PluginAccountAccess>>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
let AdminRuntimePorts {
|
||||
upstream_info,
|
||||
timezone,
|
||||
service_middleware,
|
||||
plugin_preparation,
|
||||
plugin_management,
|
||||
published_snapshot,
|
||||
plugin_distribution,
|
||||
plugin_inspector,
|
||||
pricing_source,
|
||||
providers,
|
||||
snapshot,
|
||||
account_probe: probe,
|
||||
proxy_probe,
|
||||
client_distribution,
|
||||
system,
|
||||
client_key_verifier,
|
||||
} = runtime;
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
client_config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
));
|
||||
auth.ensure_default_admin(config.default_password.expose())
|
||||
.await?;
|
||||
|
||||
let accounts = initialize_accounts(
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
store.settings(),
|
||||
timezone,
|
||||
providers.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
upstream_info.clone(),
|
||||
);
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
timezone,
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
)
|
||||
.with_timezone(timezone);
|
||||
let system_preflight = Arc::new(
|
||||
use_case::system_preflight::PluginSystemUpdatePreflight::new(
|
||||
store.plugins(),
|
||||
plugin_inspector.clone(),
|
||||
),
|
||||
);
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
let credentials = Arc::new(CredentialsService::new(
|
||||
providers.clone(),
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
));
|
||||
let plugin_accounts = plugin_accounts.unwrap_or_else(|| {
|
||||
initialize_plugin_accounts(
|
||||
providers.clone(),
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
)
|
||||
});
|
||||
let import_tasks = use_case::import_tasks::DefaultImportTasksService::new(credentials.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let settings = initialize_settings(
|
||||
store.settings(),
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), providers.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_accounts(&accounts)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
peer_link: Arc::new(use_case::peer_link::DefaultPeerLinkService::new(
|
||||
store.client_keys(),
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
providers.clone(),
|
||||
)),
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
plugin_management: Arc::new(PluginManagementService::new(
|
||||
plugin_management,
|
||||
store.plugins(),
|
||||
published_snapshot.clone(),
|
||||
)),
|
||||
plugins: Arc::new(PluginsService::new(
|
||||
store.plugins(),
|
||||
plugin_inspector,
|
||||
PluginDistributionPorts::new(plugin_distribution, store.proxies()),
|
||||
snapshot.clone(),
|
||||
plugin_preparation,
|
||||
published_snapshot,
|
||||
store.plugin_state(),
|
||||
)),
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
store.proxies(),
|
||||
proxy_probe,
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
upstream_info.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
upstream_info.clone(),
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
client_keys: Arc::new(DefaultClientKeyService::new(
|
||||
store.client_keys(),
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
providers.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
upstream_info,
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let worker_contributions = vec![
|
||||
backup_task.into_worker()?,
|
||||
import_task.into_worker()?,
|
||||
freeze_recovery.into_worker()?,
|
||||
reset_credit_worker::ResetCreditAutomationTask {
|
||||
accounts,
|
||||
store: store.accounts(),
|
||||
settings: store.settings(),
|
||||
}
|
||||
.into_worker()?,
|
||||
];
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// CLI 只组合账号用例,不初始化管理员、管理服务或后台任务;写入仍复用同一事务与审计
|
||||
pub fn initialize_plugin_accounts(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
snapshot: Arc<dyn gateway_core::runtime::SnapshotControl>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn PluginAccountAccess> {
|
||||
Arc::new(use_case::plugin_accounts::DefaultPluginAccountAccess::new(
|
||||
providers,
|
||||
accounts,
|
||||
snapshot,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 为 Runtime 创建非秘密 Client Key 目录与预算重置的窄端口
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_client_keys(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
store: Arc<dyn ports::store::ClientKeyStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn PluginClientKeyAccess> {
|
||||
let service: Arc<dyn ClientKeyService> =
|
||||
Arc::new(DefaultClientKeyService::new(store, snapshot, providers));
|
||||
Arc::new(use_case::plugin_client_keys::DefaultPluginClientKeyAccess::new(service))
|
||||
}
|
||||
|
||||
/// 为 Runtime 组合实例自有资源写入;权限和归属在同一存储事务复核
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_resources(
|
||||
store: Arc<dyn ports::plugin_resources::PluginResourceStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn ports::plugin_resources::PluginResourceAccess> {
|
||||
Arc::new(use_case::plugin_resources::DefaultPluginResourceAccess { store, snapshot })
|
||||
}
|
||||
|
||||
/// 服务端与 CLI 复用账号用例,不随命令启动后台调度
|
||||
#[must_use]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "账号存储、发布、探测与互联分别通过独立端口注入"
|
||||
)]
|
||||
pub fn initialize_accounts(
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
account_runtime: Arc<dyn ports::store::AccountRuntimeStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn AccountsService> {
|
||||
Arc::new(DefaultAccountsService::new(
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 设置服务不依赖 Web 管理会话,CLI 与服务器通过同一用例执行事务和发布
|
||||
pub fn initialize_settings(
|
||||
store: Arc<dyn ports::store::SettingsStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
providers: ProviderAdminRegistry,
|
||||
pricing_source: Arc<dyn ports::pricing::PricingSource>,
|
||||
) -> Arc<dyn SettingsService> {
|
||||
Arc::new(DefaultSettingsService::new(
|
||||
store,
|
||||
snapshot,
|
||||
providers,
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
@@ -7,10 +7,15 @@ use std::time::{Duration, SystemTime};
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::account::{AccountErrorProtection, ProviderAccountId};
|
||||
use gateway_core::engine::probe::AccountProbeTarget;
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError};
|
||||
use gateway_core::task::{
|
||||
ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerId, WorkerKind,
|
||||
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::model::AdminError;
|
||||
use crate::model::accounts::{
|
||||
AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot,
|
||||
};
|
||||
@@ -42,6 +47,34 @@ impl FreezeRecoveryTask {
|
||||
Self { deps }
|
||||
}
|
||||
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
FREEZE_RECOVERY_INTERVAL,
|
||||
WORKER_INITIAL_BACKOFF,
|
||||
WORKER_MAXIMUM_BACKOFF,
|
||||
WORKER_LEASE_TTL,
|
||||
WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
|
||||
/// 无法读取或校验配置时跳过本轮,不能把未知状态当作允许解冻
|
||||
async fn freeze_policy(&self) -> Option<AccountErrorProtection> {
|
||||
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
|
||||
@@ -129,7 +162,7 @@ impl FreezeRecoveryTask {
|
||||
let probe_succeeded = match self
|
||||
.deps
|
||||
.accounts
|
||||
.test_connection(account.clone(), model)
|
||||
.test_connection(AccountProbeTarget::FreezeRecovery(account.clone()), model)
|
||||
.await
|
||||
{
|
||||
Ok(events) => drain_probe(events).await,
|
||||
|
||||
@@ -2,26 +2,22 @@
|
||||
//!
|
||||
//! 本 crate 不包含 HTTP wire、数据库实现或具体 Provider 实现
|
||||
|
||||
use std::{fmt, path::Path, sync::Arc, time::Duration};
|
||||
use std::{fmt, path::Path, sync::Arc};
|
||||
|
||||
use gateway_core::{
|
||||
engine::execution::ClientKeyVerifier,
|
||||
engine::probe::AccountProbe,
|
||||
runtime::SnapshotControl,
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest,
|
||||
WorkerRegistration, WorkerRunnable, WorkerSchedule,
|
||||
},
|
||||
engine::execution::ClientKeyVerifier, engine::probe::AccountProbe, runtime::SnapshotControl,
|
||||
task::WorkerContribution,
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
mod bootstrap;
|
||||
pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
pub mod public_service;
|
||||
mod reset_credit_automation;
|
||||
mod reset_credit_worker;
|
||||
mod use_case;
|
||||
pub use use_case::plugins::{PluginDistributionPorts, PluginManagementService, PluginsService};
|
||||
|
||||
@@ -44,18 +40,15 @@ pub use use_case::{
|
||||
usage_data::UsageDataService,
|
||||
};
|
||||
|
||||
use model::AdminError;
|
||||
pub use bootstrap::{
|
||||
initialize, initialize_accounts, initialize_plugin_accounts, initialize_plugin_client_keys,
|
||||
initialize_plugin_resources, initialize_settings, initialize_usage_data,
|
||||
initialize_with_plugin_accounts,
|
||||
};
|
||||
|
||||
use ports::{
|
||||
client_distribution::ClientDistributionResolver, plugin_accounts::PluginAccountAccess,
|
||||
plugin_client_keys::PluginClientKeyAccess, provider::ProviderAdminRegistry,
|
||||
store::AdminStorePorts, system::SystemOperations,
|
||||
};
|
||||
use use_case::{
|
||||
account_groups::DefaultAccountGroupService, accounts::DefaultAccountsService,
|
||||
auth::DefaultAuthService, backup::DefaultBackupService,
|
||||
client_distribution::DefaultClientDistributionService, client_keys::DefaultClientKeyService,
|
||||
observability::DefaultObservabilityService, settings::DefaultSettingsService,
|
||||
system::DefaultSystemService,
|
||||
provider::ProviderAdminRegistry, system::SystemOperations,
|
||||
};
|
||||
|
||||
const MINIMUM_INITIAL_PASSWORD_BYTES: usize = 12;
|
||||
@@ -70,7 +63,6 @@ const WEAK_ADMIN_PASSWORDS: &[&str] = &[
|
||||
"codex-proxy-rs",
|
||||
];
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
const DEFAULT_CLIENT_SESSION_TTL_MINUTES: u64 = 24 * 60;
|
||||
|
||||
/// 只用于首次幂等创建默认管理员的启动密码
|
||||
@@ -296,8 +288,8 @@ impl AdminServices {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn usage_data(&self) -> &dyn UsageDataService {
|
||||
self.usage_data.as_ref()
|
||||
pub fn usage_data(&self) -> Arc<dyn UsageDataService> {
|
||||
self.usage_data.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -366,395 +358,3 @@ pub struct AdminRuntimePorts {
|
||||
pub system: Arc<dyn SystemOperations>,
|
||||
pub client_key_verifier: Arc<dyn ClientKeyVerifier>,
|
||||
}
|
||||
|
||||
/// 校验配置、接入已组装的 Provider 注册表并完成默认管理员幂等初始化
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 配置非法或默认管理员初始化失败时返回错误
|
||||
pub async fn initialize(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, None).await
|
||||
}
|
||||
|
||||
/// 使用组合根已绑定给 Runtime 的同一账号窄端口,避免为完整 Admin 重建第二实例
|
||||
pub async fn initialize_with_plugin_accounts(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Arc<dyn PluginAccountAccess>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, Some(plugin_accounts)).await
|
||||
}
|
||||
|
||||
async fn initialize_inner(
|
||||
mut config: AdminConfig,
|
||||
mut client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Option<Arc<dyn PluginAccountAccess>>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
let AdminRuntimePorts {
|
||||
upstream_info,
|
||||
timezone,
|
||||
service_middleware,
|
||||
plugin_preparation,
|
||||
plugin_management,
|
||||
published_snapshot,
|
||||
plugin_distribution,
|
||||
plugin_inspector,
|
||||
pricing_source,
|
||||
providers,
|
||||
snapshot,
|
||||
account_probe: probe,
|
||||
proxy_probe,
|
||||
client_distribution,
|
||||
system,
|
||||
client_key_verifier,
|
||||
} = runtime;
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
client_config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
let registry = providers;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
));
|
||||
auth.ensure_default_admin(config.default_password.expose())
|
||||
.await?;
|
||||
|
||||
let accounts = initialize_accounts(
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
store.settings(),
|
||||
timezone,
|
||||
registry.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
upstream_info.clone(),
|
||||
);
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
timezone,
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
)
|
||||
.with_timezone(timezone);
|
||||
let system_preflight = Arc::new(use_case::plugin_update::PluginSystemUpdatePreflight::new(
|
||||
store.plugins(),
|
||||
plugin_inspector.clone(),
|
||||
));
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
let credentials = Arc::new(CredentialsService::new(
|
||||
registry.clone(),
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
));
|
||||
let plugin_accounts = plugin_accounts.unwrap_or_else(|| {
|
||||
initialize_plugin_accounts(
|
||||
registry.clone(),
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
)
|
||||
});
|
||||
let import_tasks = use_case::import_tasks::DefaultImportTasksService::new(credentials.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let settings = initialize_settings(
|
||||
store.settings(),
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), registry.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_accounts(&accounts)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
peer_link: Arc::new(use_case::peer_link::DefaultPeerLinkService::new(
|
||||
store.client_keys(),
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
registry.clone(),
|
||||
)),
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
plugin_management: Arc::new(PluginManagementService::new(
|
||||
plugin_management,
|
||||
store.plugins(),
|
||||
published_snapshot.clone(),
|
||||
)),
|
||||
plugins: Arc::new(PluginsService::new(
|
||||
store.plugins(),
|
||||
plugin_inspector,
|
||||
PluginDistributionPorts::new(plugin_distribution, store.proxies()),
|
||||
snapshot.clone(),
|
||||
plugin_preparation,
|
||||
published_snapshot,
|
||||
store.plugin_state(),
|
||||
)),
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
store.proxies(),
|
||||
proxy_probe,
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
upstream_info.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
client_keys: Arc::new(DefaultClientKeyService::new(
|
||||
store.client_keys(),
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
upstream_info,
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let mut worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(import_task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(registration));
|
||||
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
|
||||
let id = WorkerId::try_new(WorkerKind::ResetCreditAutomation, "admin")
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), Duration::from_secs(90))
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 租约配置不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(reset_credit_automation::ResetCreditAutomationTask {
|
||||
accounts,
|
||||
store: store.accounts(),
|
||||
settings: store.settings(),
|
||||
}),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 注册失败"))?,
|
||||
));
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// CLI 只组合账号用例,不初始化管理员、管理服务或后台任务;写入仍复用同一事务与审计
|
||||
pub fn initialize_plugin_accounts(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
snapshot: Arc<dyn gateway_core::runtime::SnapshotControl>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn PluginAccountAccess> {
|
||||
Arc::new(use_case::plugin_accounts::DefaultPluginAccountAccess::new(
|
||||
providers,
|
||||
accounts,
|
||||
snapshot,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 为 Runtime 创建非秘密 Client Key 目录与预算重置的窄端口
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_client_keys(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
store: Arc<dyn ports::store::ClientKeyStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn PluginClientKeyAccess> {
|
||||
let service: Arc<dyn ClientKeyService> =
|
||||
Arc::new(DefaultClientKeyService::new(store, snapshot, providers));
|
||||
Arc::new(use_case::plugin_client_keys::DefaultPluginClientKeyAccess::new(service))
|
||||
}
|
||||
|
||||
/// 为 Runtime 组合实例自有资源写入;权限和归属在同一存储事务复核
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_resources(
|
||||
store: Arc<dyn ports::plugin_resources::PluginResourceStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn ports::plugin_resources::PluginResourceAccess> {
|
||||
Arc::new(use_case::plugin_resources::DefaultPluginResourceAccess { store, snapshot })
|
||||
}
|
||||
|
||||
/// Backup Worker 注册:单个可取消 Daemon,owner 固定为 `backup`
|
||||
fn backup_worker_contribution(
|
||||
task: backup::task::BackupTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("备份 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("备份 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("备份 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定
|
||||
fn freeze_recovery_worker_contribution(
|
||||
task: freeze_recovery::FreezeRecoveryTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
freeze_recovery::FREEZE_RECOVERY_INTERVAL,
|
||||
freeze_recovery::WORKER_INITIAL_BACKOFF,
|
||||
freeze_recovery::WORKER_MAXIMUM_BACKOFF,
|
||||
freeze_recovery::WORKER_LEASE_TTL,
|
||||
freeze_recovery::WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 服务端与 CLI 复用账号用例,不随命令启动后台调度
|
||||
#[must_use]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "账号存储、发布、探测与互联分别通过独立端口注入"
|
||||
)]
|
||||
pub fn initialize_accounts(
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
account_runtime: Arc<dyn ports::store::AccountRuntimeStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn AccountsService> {
|
||||
Arc::new(DefaultAccountsService::new(
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 设置服务不依赖 Web 管理会话,CLI 与服务器通过同一用例执行事务和发布
|
||||
pub fn initialize_settings(
|
||||
store: Arc<dyn ports::store::SettingsStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
providers: ProviderAdminRegistry,
|
||||
pricing_source: Arc<dyn ports::pricing::PricingSource>,
|
||||
) -> Arc<dyn SettingsService> {
|
||||
Arc::new(DefaultSettingsService::new(
|
||||
store,
|
||||
snapshot,
|
||||
providers,
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -73,6 +73,7 @@ pub struct AccountGroupUsage {
|
||||
/// 当前页账号组 membership 对应的持久账号事实;运行态在 Admin query service 合并
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupMemberFact {
|
||||
pub inherited_version: Option<gateway_core::account::QuotaObservationVersion>,
|
||||
pub group_id: AccountGroupId,
|
||||
pub account_id: String,
|
||||
pub status: AccountStatusFacts,
|
||||
|
||||
@@ -219,7 +219,7 @@ pub struct AccountPageItem {
|
||||
pub struct AccountCapacity {
|
||||
/// 实时租约读取失败时为 `None`,不能当作空闲
|
||||
pub used_slots: Option<u64>,
|
||||
/// 应用账号覆盖或全局默认值后的上限;`None` 表示不限
|
||||
/// 应用账号覆盖或继承默认值后的普通并发上限;`None` 表示不限
|
||||
pub total_slots: Option<u64>,
|
||||
}
|
||||
|
||||
|
||||
@@ -472,6 +472,80 @@ impl CostCoverage {
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求健康计数与服务成功率,调用方错误和取消不计入有效样本
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct RequestHealth {
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
}
|
||||
|
||||
impl RequestHealth {
|
||||
#[must_use]
|
||||
pub fn from_counts(
|
||||
success: u64,
|
||||
failed: u64,
|
||||
cancelled: u64,
|
||||
incomplete: u64,
|
||||
caller_error: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
success_requests: success,
|
||||
failed_requests: failed.saturating_sub(caller_error),
|
||||
cancelled_requests: cancelled,
|
||||
incomplete_requests: incomplete,
|
||||
caller_error_requests: caller_error,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn from_metrics(metrics: &RequestMetrics) -> Self {
|
||||
Self::from_counts(
|
||||
metrics.success_count,
|
||||
metrics.failure_count,
|
||||
metrics.cancelled_count,
|
||||
metrics.incomplete_count,
|
||||
metrics.caller_error_count,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn success_rate(self) -> Option<f64> {
|
||||
let total = self.eligible_requests();
|
||||
(total > 0).then(|| self.success_requests as f64 / total as f64 * 100.0)
|
||||
}
|
||||
|
||||
// 未完成保留交付终态,但上游失败、超时和进程中断都不能视为服务成功
|
||||
pub(crate) fn non_success_requests(self) -> u64 {
|
||||
self.failed_requests
|
||||
.saturating_add(self.incomplete_requests)
|
||||
}
|
||||
|
||||
pub(crate) fn eligible_requests(self) -> u64 {
|
||||
self.success_requests
|
||||
.saturating_add(self.non_success_requests())
|
||||
}
|
||||
|
||||
pub(crate) fn add_metrics(&mut self, metrics: &RequestMetrics) {
|
||||
self.add_window(Self::from_metrics(metrics));
|
||||
}
|
||||
|
||||
pub(crate) fn add_window(&mut self, other: Self) {
|
||||
self.success_requests = self.success_requests.saturating_add(other.success_requests);
|
||||
self.failed_requests = self.failed_requests.saturating_add(other.failed_requests);
|
||||
self.cancelled_requests = self
|
||||
.cancelled_requests
|
||||
.saturating_add(other.cancelled_requests);
|
||||
self.incomplete_requests = self
|
||||
.incomplete_requests
|
||||
.saturating_add(other.incomplete_requests);
|
||||
self.caller_error_requests = self
|
||||
.caller_error_requests
|
||||
.saturating_add(other.caller_error_requests);
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求级聚合指标
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct RequestMetrics {
|
||||
@@ -622,7 +696,7 @@ pub struct AccountPoolMetrics {
|
||||
pub struct AccountModelUsage {
|
||||
pub model: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub health: RequestHealth,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
@@ -635,7 +709,7 @@ pub struct AccountModelUsage {
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
pub last_used_at: DateTime<Utc>,
|
||||
pub last_used_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 账号在单小时窗口内的请求数
|
||||
@@ -684,8 +758,10 @@ pub struct DashboardAccountUsage {
|
||||
}
|
||||
|
||||
/// Dashboard 当前账号池的可重建运行时槽位事实
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DashboardRuntimeSlots {
|
||||
/// 仅未设置本地覆盖的账号可能继承有效互联观测
|
||||
pub inherited_versions: Vec<gateway_core::account::QuotaObservationVersion>,
|
||||
/// 继承运行参数默认并发上限的可调度账号数
|
||||
pub inherited_accounts: u64,
|
||||
/// 已配置账号级并发上限的可调度账号槽位总数
|
||||
@@ -724,6 +800,11 @@ pub struct UsageListRecord {
|
||||
pub outcome: RequestOutcome,
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub error_kind: Option<String>,
|
||||
pub websocket_close_code: Option<u16>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub recovered: bool,
|
||||
pub client_status_code: Option<u16>,
|
||||
pub client_api_key_name: Option<String>,
|
||||
pub id: String,
|
||||
@@ -739,6 +820,8 @@ pub struct UsageListRecord {
|
||||
/// 请求选号时冻结的账号套餐
|
||||
pub provider_account_plan_type: Option<String>,
|
||||
pub provider_account_plan_type_display: Option<String>,
|
||||
/// 查询时的互联识别结果,缺少当前摘要时为未知
|
||||
pub provider_account_upstream_peer: Option<bool>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
@@ -780,6 +863,23 @@ pub struct UsageListRecord {
|
||||
pub started_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 受控请求详情中的上游错误字段,不参与错误分类或普通日志输出
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub struct UsageUpstreamError {
|
||||
pub code: Option<String>,
|
||||
pub error_type: Option<String>,
|
||||
pub message: Option<String>,
|
||||
pub param: Option<String>,
|
||||
pub close_reason: Option<String>,
|
||||
pub last_event_type: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for UsageUpstreamError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("UsageUpstreamError(<restricted>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// 一次完整模型请求的公共观测记录
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageRecord {
|
||||
@@ -818,6 +918,11 @@ pub struct UsageRecord {
|
||||
pub error_kind: Option<String>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub upstream_error: Option<UsageUpstreamError>,
|
||||
pub websocket_close_code: Option<u16>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub recovered: bool,
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
@@ -1439,6 +1544,8 @@ pub struct DiagnosticsItem {
|
||||
pub name: String,
|
||||
pub account_authentication_kind: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
/// 仅账号维度补充当前互联识别结果
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
|
||||
@@ -17,6 +17,7 @@ pub struct PeerLinkLimits {
|
||||
pub struct PeerLinkExport {
|
||||
pub observed_at: DateTime<Utc>,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub key_limits: gateway_core::policy::RateLimits,
|
||||
pub budget: ClientBudgetStatus,
|
||||
pub budget_access: ClientBudgetAccess,
|
||||
pub budget_observed_at: DateTime<Utc>,
|
||||
|
||||
@@ -165,8 +165,9 @@ pub struct ConfigurePluginInstance {
|
||||
pub artifact_sha256: String,
|
||||
pub enabled: bool,
|
||||
pub configuration: serde_json::Value,
|
||||
/// 缺省保留旧值;显式空对象清除全部 secret
|
||||
pub secrets: Option<BTreeMap<String, SecretString>>,
|
||||
/// 按字段更新,省略的字段保留,null 清除对应字段
|
||||
#[serde(default)]
|
||||
pub secrets: BTreeMap<String, Option<SecretString>>,
|
||||
pub bindings: Vec<PluginCapabilityBinding>,
|
||||
}
|
||||
|
||||
|
||||
@@ -806,11 +806,12 @@ pub struct ProviderQuotaWindow {
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
/// 账号用量统计周期,按周优先、月次之选择;短期限流窗口不参与统计面板
|
||||
/// 账号用量统计周期,完整周/月额度优先,随后使用 Provider 提供的本地滚动日统计
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum AccountUsagePeriod {
|
||||
Weekly,
|
||||
Monthly,
|
||||
Daily,
|
||||
}
|
||||
|
||||
/// Provider 解释 quota 所需的公共请求事实
|
||||
@@ -998,16 +999,27 @@ impl ProviderQuota {
|
||||
}
|
||||
}
|
||||
|
||||
/// 选择账号用量统计的周/月窗口,同时返回展示所需的周期事实
|
||||
/// 选择账号用量统计窗口,同时返回展示所需的周期事实
|
||||
///
|
||||
/// 只使用边界完整、可归属到整个账号的窗口;周优先于月,同周期保持
|
||||
/// Provider 投影顺序
|
||||
/// 没有符合条件的窗口时不回退到短期或历史累计
|
||||
/// 周/月额度使用边界完整、可归属到整个账号的窗口
|
||||
/// 周优先于月,同周期保持 Provider 投影顺序
|
||||
/// Provider 提供的无固定重置点的滚动日统计可独立展示
|
||||
#[must_use]
|
||||
pub fn usage_window(&self) -> Option<(&ProviderQuotaWindow, AccountUsagePeriod)> {
|
||||
self.usage_windows()
|
||||
.filter(|(window, _)| window.local_usage.is_some())
|
||||
.min_by_key(|(_, period)| *period)
|
||||
.or_else(|| {
|
||||
self.windows
|
||||
.iter()
|
||||
.find(|window| {
|
||||
window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
|
||||
&& window.window_seconds == Some(86_400)
|
||||
&& window.reset_at.is_none()
|
||||
&& window.local_usage.is_some()
|
||||
})
|
||||
.map(|window| (window, AccountUsagePeriod::Daily))
|
||||
})
|
||||
}
|
||||
|
||||
/// 互联账号按最长的 Key 消费周期统计本地用量,池内账号窗口不代表此 Key 的周期
|
||||
@@ -1025,7 +1037,7 @@ impl ProviderQuota {
|
||||
.max_by_key(|window| window.window_seconds)
|
||||
}
|
||||
|
||||
/// 统计与预测共用窗口归属规则,避免把模型专属桶或短期限流当作账号容量
|
||||
/// 周/月额度的归属规则,供额度周期统计与容量预测共用
|
||||
pub(crate) fn usage_windows(
|
||||
&self,
|
||||
) -> impl Iterator<Item = (&ProviderQuotaWindow, AccountUsagePeriod)> {
|
||||
|
||||
@@ -62,14 +62,15 @@ pub fn account_quota_forecasts(
|
||||
now: DateTime<Utc>,
|
||||
samples: &[QuotaForecastSample],
|
||||
) -> [AccountQuotaForecast; 2] {
|
||||
[AccountUsagePeriod::Weekly, AccountUsagePeriod::Monthly].map(|period| {
|
||||
[
|
||||
(AccountUsagePeriod::Weekly, 7 * DAY_SECONDS),
|
||||
(AccountUsagePeriod::Monthly, 30 * DAY_SECONDS),
|
||||
]
|
||||
.map(|(period, target_seconds)| {
|
||||
let selected = quota_forecast_source_window(quota, period);
|
||||
let mut forecast = AccountQuotaForecast {
|
||||
period,
|
||||
target_seconds: match period {
|
||||
AccountUsagePeriod::Weekly => 7 * DAY_SECONDS,
|
||||
AccountUsagePeriod::Monthly => 30 * DAY_SECONDS,
|
||||
},
|
||||
target_seconds,
|
||||
extrapolated: false,
|
||||
source: None,
|
||||
unavailable_reason: Some("缺少周/月额度窗口,请刷新账号额度"),
|
||||
|
||||
@@ -54,6 +54,7 @@ pub struct RuntimeSettingsValues {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
|
||||
@@ -4,11 +4,14 @@ use chrono::{DateTime, Utc};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderUpstreamInfo {
|
||||
/// 当前连接的互联识别结果,明确关闭共享或不支持协议时清除
|
||||
pub discovered: bool,
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub stale: bool,
|
||||
pub connection_revision: String,
|
||||
pub last_error: Option<String>,
|
||||
pub max_concurrency: Option<u64>,
|
||||
pub requests_per_minute: Option<u64>,
|
||||
pub pool: Option<UpstreamPool>,
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ pub enum UsageErrorKind {
|
||||
Conflict,
|
||||
CapacityExceeded,
|
||||
Unavailable,
|
||||
Cancelled,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
||||
@@ -20,3 +20,6 @@ pub use actor::*;
|
||||
|
||||
mod aggregate;
|
||||
pub use aggregate::*;
|
||||
|
||||
mod read;
|
||||
pub use read::*;
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
//! 历史用量的窄事实、分组统计与调用内分批读取合同
|
||||
|
||||
use super::{UsageCount, UsageMetric, UsageRecordData, UsageRecordFilter};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub const USAGE_READ_MAX_BATCH_ROWS: u16 = 1024;
|
||||
pub const USAGE_READ_MAX_BATCH_BYTES: usize = 2 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReadFilter {
|
||||
pub records: UsageRecordFilter,
|
||||
pub routing_group_ref: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ScanUsageRecords {
|
||||
pub filter: UsageReadFilter,
|
||||
pub batch_size: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageGrouping {
|
||||
Model,
|
||||
Key,
|
||||
KeyModel,
|
||||
RoutingGroup,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct QueryUsageStatistics {
|
||||
pub filter: UsageReadFilter,
|
||||
pub group_by: UsageGrouping,
|
||||
pub batch_size: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum UsageReadQuery {
|
||||
Records(ScanUsageRecords),
|
||||
Statistics(QueryUsageStatistics),
|
||||
}
|
||||
|
||||
impl UsageReadQuery {
|
||||
pub fn filter(&self) -> &UsageReadFilter {
|
||||
match self {
|
||||
Self::Records(query) => &query.filter,
|
||||
Self::Statistics(query) => &query.filter,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn batch_size(&self) -> u16 {
|
||||
match self {
|
||||
Self::Records(query) => query.batch_size,
|
||||
Self::Statistics(query) => query.batch_size,
|
||||
}
|
||||
.unwrap_or(USAGE_READ_MAX_BATCH_ROWS)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageRecordFact {
|
||||
pub id: String,
|
||||
pub data: UsageRecordData,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub request_kind: Option<String>,
|
||||
pub routing_scope: Option<String>,
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub recovered: bool,
|
||||
/// 由宿主现行规则派生,原始审计中的恢复请求仍可读取
|
||||
pub request_counted: bool,
|
||||
/// 包含完整交付、预热排除及恢复关联规则
|
||||
pub usage_counted: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageStatisticsGroup {
|
||||
pub client_key_ref: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub routing_group_ref: Option<String>,
|
||||
pub unassigned_contribution: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageStatisticsCost {
|
||||
pub currency: String,
|
||||
pub amount: String,
|
||||
pub native_amount: Option<String>,
|
||||
/// 外部修正允许负值,原生费用来源覆盖单独返回
|
||||
pub contribution_amount: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageStatisticsCostCoverage {
|
||||
pub provider_reported: UsageCount,
|
||||
pub calculated: UsageCount,
|
||||
pub unavailable: UsageCount,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageStatisticsMetrics {
|
||||
/// 缺失的 Token 指标表示未知,计数指标始终返回
|
||||
pub values: BTreeMap<UsageMetric, UsageCount>,
|
||||
pub native_request_count: UsageCount,
|
||||
pub contribution_count: UsageCount,
|
||||
pub native_token_samples: BTreeMap<UsageMetric, UsageCount>,
|
||||
/// 数量表示提供该指标的贡献记录数,不能用作请求覆盖率的分母
|
||||
pub contribution_token_samples: BTreeMap<UsageMetric, UsageCount>,
|
||||
pub cost_coverage: UsageStatisticsCostCoverage,
|
||||
pub costs: Vec<UsageStatisticsCost>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageStatisticsRow {
|
||||
/// null 表示全局总计,分组重叠不会重复计入该行
|
||||
pub group: Option<UsageStatisticsGroup>,
|
||||
pub metrics: UsageStatisticsMetrics,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
|
||||
pub enum UsageReadItem {
|
||||
Record(Box<UsageRecordFact>),
|
||||
Statistics(Box<UsageStatisticsRow>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReadAnalytics {
|
||||
pub coverage_start_ms: Option<i64>,
|
||||
pub coverage_end_ms: Option<i64>,
|
||||
pub lag_seconds: u64,
|
||||
pub state: String,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReadObservation {
|
||||
pub generated_at_ms: i64,
|
||||
pub analytics: Option<UsageReadAnalytics>,
|
||||
pub total_groups: Option<UsageCount>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReadBatch {
|
||||
pub observation: UsageReadObservation,
|
||||
pub items: Vec<UsageReadItem>,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReadOpened {
|
||||
pub reader: Option<String>,
|
||||
pub batch: UsageReadBatch,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReadHandle {
|
||||
pub reader: String,
|
||||
}
|
||||
@@ -38,6 +38,13 @@ pub trait PeerTransport: Send + Sync {
|
||||
/// 账号用例读取公共互联观测,显式刷新等待额度查询完成
|
||||
#[async_trait]
|
||||
pub trait PeerUpstreamService: gateway_core::account::UpstreamInfoSource {
|
||||
/// 仅在指定持久观测与当前调度缓存一致时估算容量
|
||||
fn observed_concurrency_limit(
|
||||
&self,
|
||||
version: &gateway_core::account::QuotaObservationVersion,
|
||||
now: std::time::SystemTime,
|
||||
) -> Option<gateway_core::account::AccountConcurrencyLimit>;
|
||||
|
||||
async fn account_changed(
|
||||
&self,
|
||||
id: &gateway_core::account::ProviderAccountId,
|
||||
|
||||
@@ -222,6 +222,15 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
}
|
||||
|
||||
/// Key 可以继承当前全局画像;额外字段的合法性由对应 Provider 判定
|
||||
fn preview_key_client_profile(
|
||||
&self,
|
||||
key: &gateway_core::account::OpaqueProviderData,
|
||||
_global: Option<&gateway_core::account::OpaqueProviderData>,
|
||||
) -> Result<gateway_core::account::OpaqueProviderData, ProviderAdminError> {
|
||||
self.preview_client_profile(key)
|
||||
}
|
||||
|
||||
/// 提供最终套餐展示名称;未覆盖时采用通用的大驼峰格式
|
||||
fn plan_type_display(&self, plan_type: &str) -> String {
|
||||
default_plan_type_display(plan_type)
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
|
||||
use crate::model::{auth::AdminAuditEvent, usage_data::*};
|
||||
use async_trait::async_trait;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
|
||||
#[async_trait]
|
||||
pub trait UsageReadSession: Send {
|
||||
async fn next_batch(&mut self) -> UsageDataResult<UsageReadBatch>;
|
||||
fn is_finished(&self) -> bool;
|
||||
async fn close(&mut self);
|
||||
}
|
||||
|
||||
pub struct StoredUsageWrite {
|
||||
pub changes: Vec<(UsageRecordChange, AdminAuditEvent)>,
|
||||
@@ -13,6 +21,11 @@ pub struct StoredAggregateWrite {
|
||||
|
||||
#[async_trait]
|
||||
pub trait UsageDataStore: Send + Sync {
|
||||
async fn open_read(
|
||||
&self,
|
||||
query: UsageReadQuery,
|
||||
cancellation: CancellationToken,
|
||||
) -> UsageDataResult<Box<dyn UsageReadSession>>;
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
//! 每日错峰发现与已知卡片的临期执行,周期只读取本地到期计划
|
||||
|
||||
use crate::{
|
||||
model::reset_credit_automation::{PlannedResetCredit, ResetCreditAutomation},
|
||||
ports::store::{AccountStore, SettingsStore},
|
||||
use_case::accounts::AccountsService,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
task::{ScheduledTask, WorkerCycleContext, WorkerTaskError},
|
||||
time::DeploymentTimeZone,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
/// 当地次日随机发现时间;随机数由调用者提供,结果持久化后不会因重启重抽
|
||||
pub(crate) fn next_discovery(
|
||||
now: DateTime<Utc>,
|
||||
timezone: DeploymentTimeZone,
|
||||
random: u64,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
let start = timezone.days_after(now, 1)?;
|
||||
let end = timezone.days_after(now, 2)?;
|
||||
let seconds = u64::try_from((end - start).num_seconds()).ok()?;
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
start.checked_add_signed(chrono::Duration::seconds(
|
||||
i64::try_from(random % seconds).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// 提前窗口的前 10% 用于账号错峰,最多一分钟,其余时间保留给网络失败恢复
|
||||
pub(crate) fn consume_at(
|
||||
account_id: &str,
|
||||
credit: &PlannedResetCredit,
|
||||
lead_minutes: u32,
|
||||
) -> DateTime<Utc> {
|
||||
let lead_seconds = i64::from(lead_minutes) * 60;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(account_id.as_bytes());
|
||||
hash.update([0]);
|
||||
hash.update(credit.id.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let offset =
|
||||
i64::from(u16::from_be_bytes([digest[0], digest[1]])) % (lead_seconds / 10).clamp(1, 60);
|
||||
credit.expires_at - chrono::Duration::seconds(lead_seconds - offset)
|
||||
}
|
||||
|
||||
pub(crate) fn next_consumption(
|
||||
record: &ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| consume_at(&record.account_id, credit, lead_minutes))
|
||||
.min()
|
||||
}
|
||||
|
||||
pub(crate) fn reschedule(
|
||||
record: &mut ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
now: DateTime<Utc>,
|
||||
) {
|
||||
record.next_run_at = if let Some(retry) = record.state.retry_at {
|
||||
retry
|
||||
} else if record.state.pending.is_some() {
|
||||
now
|
||||
} else {
|
||||
next_consumption(record, lead_minutes).map_or(record.state.next_discovery_at, |due| {
|
||||
due.min(record.state.next_discovery_at)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) struct ResetCreditAutomationTask {
|
||||
pub(crate) accounts: Arc<dyn AccountsService>,
|
||||
pub(crate) store: Arc<dyn AccountStore>,
|
||||
pub(crate) settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
impl ScheduledTask for ResetCreditAutomationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡设置读取失败"))?;
|
||||
// 单轮有界,积压沿到期顺序继续;并发复用已有账号刷新配置
|
||||
let due = self
|
||||
.store
|
||||
.due_reset_credit_automations(Utc::now(), 100)
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡计划读取失败"))?;
|
||||
let work = futures::stream::iter(due)
|
||||
.map(|account_id| async move {
|
||||
let id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡账号 ID 不合法"))?;
|
||||
if let Err(error) = self.accounts.reconcile_reset_credit_automation(id).await {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error_kind = error.kind().as_str(),
|
||||
"自动重置卡任务失败"
|
||||
);
|
||||
}
|
||||
Ok::<_, WorkerTaskError>(())
|
||||
})
|
||||
.buffer_unordered(settings.values.refresh_concurrency.max(1) as usize)
|
||||
.collect::<Vec<_>>();
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
_ = tokio::time::timeout(Duration::from_secs(30), work) => {},
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//! 自动重置卡 Worker 的到期扫描、并发预算与取消,消费编排由账号用例负责
|
||||
|
||||
use crate::{
|
||||
model::AdminError,
|
||||
ports::store::{AccountStore, SettingsStore},
|
||||
use_case::accounts::AccountsService,
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
task::{
|
||||
ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerId, WorkerKind,
|
||||
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
|
||||
},
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
pub(crate) struct ResetCreditAutomationTask {
|
||||
pub(crate) accounts: Arc<dyn AccountsService>,
|
||||
pub(crate) store: Arc<dyn AccountStore>,
|
||||
pub(crate) settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
impl ResetCreditAutomationTask {
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::ResetCreditAutomation, "admin")
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), Duration::from_secs(90))
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 注册失败"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl ScheduledTask for ResetCreditAutomationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡设置读取失败"))?;
|
||||
// 单轮有界,积压沿到期顺序继续;并发复用已有账号刷新配置
|
||||
let due = self
|
||||
.store
|
||||
.due_reset_credit_automations(Utc::now(), 100)
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡计划读取失败"))?;
|
||||
let work = futures::stream::iter(due)
|
||||
.map(|account_id| async move {
|
||||
let id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡账号 ID 不合法"))?;
|
||||
if let Err(error) = self.accounts.reconcile_reset_credit_automation(id).await {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error_kind = error.kind().as_str(),
|
||||
"自动重置卡任务失败"
|
||||
);
|
||||
}
|
||||
Ok::<_, WorkerTaskError>(())
|
||||
})
|
||||
.buffer_unordered(settings.values.refresh_concurrency.max(1) as usize)
|
||||
.collect::<Vec<_>>();
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
_ = tokio::time::timeout(Duration::from_secs(30), work) => {},
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,7 @@ pub trait AccountGroupService: Send + Sync {
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultAccountGroupService {
|
||||
upstream_info: Option<Arc<dyn crate::ports::peer::PeerUpstreamService>>,
|
||||
store: Arc<dyn AccountGroupStore>,
|
||||
runtime: Arc<dyn AccountRuntimeStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
@@ -68,11 +69,13 @@ pub(crate) struct DefaultAccountGroupService {
|
||||
impl DefaultAccountGroupService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
upstream_info: Option<Arc<dyn crate::ports::peer::PeerUpstreamService>>,
|
||||
store: Arc<dyn AccountGroupStore>,
|
||||
runtime: Arc<dyn AccountRuntimeStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Self {
|
||||
Self {
|
||||
upstream_info,
|
||||
store,
|
||||
runtime,
|
||||
snapshot,
|
||||
@@ -105,7 +108,7 @@ impl DefaultAccountGroupService {
|
||||
Default::default()
|
||||
}
|
||||
};
|
||||
project_group_runtime(records, &members, &runtime);
|
||||
project_group_runtime(records, &members, &runtime, self.upstream_info.as_deref());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -215,6 +218,7 @@ fn project_group_runtime(
|
||||
records: &mut [AccountGroupRecord],
|
||||
members: &[AccountGroupMemberFact],
|
||||
runtime: &crate::model::accounts::AccountRuntimeSnapshot,
|
||||
upstream: Option<&dyn crate::ports::peer::PeerUpstreamService>,
|
||||
) {
|
||||
let now = SystemTime::now();
|
||||
let mut status_by_account = BTreeMap::new();
|
||||
@@ -228,7 +232,14 @@ fn project_group_runtime(
|
||||
.or_insert_with(|| resolve_account_status(&status, now).status);
|
||||
slots_by_account
|
||||
.entry(member.account_id.as_str())
|
||||
.or_insert(member.total_slots);
|
||||
.or_insert_with(|| {
|
||||
member
|
||||
.inherited_version
|
||||
.as_ref()
|
||||
.and_then(|version| upstream?.observed_concurrency_limit(version, now))
|
||||
.map(|limit| u64::from(limit.get()))
|
||||
.or(member.total_slots)
|
||||
});
|
||||
accounts_by_group
|
||||
.entry(member.group_id.as_str())
|
||||
.or_default()
|
||||
|
||||
@@ -12,7 +12,7 @@ use chrono::{Duration, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
engine::probe::{AccountProbe, AccountProbeRequest},
|
||||
engine::probe::{AccountProbe, AccountProbeRequest, AccountProbeTarget},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
runtime::SnapshotControl,
|
||||
};
|
||||
@@ -178,7 +178,7 @@ pub trait AccountsService: Send + Sync {
|
||||
|
||||
async fn test_connection(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
target: AccountProbeTarget,
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError>;
|
||||
}
|
||||
@@ -504,7 +504,11 @@ impl DefaultAccountsService {
|
||||
});
|
||||
Ok(AccountDirectoryItem {
|
||||
protection_rules: stored.protection_rules,
|
||||
capacity: stored.capacity,
|
||||
capacity: super::peer_link::upstream::account_capacity(
|
||||
&stored.account,
|
||||
stored.capacity,
|
||||
"a,
|
||||
),
|
||||
capabilities: provider
|
||||
.account_capabilities(account_id, &stored.account.authentication_kind),
|
||||
plan_type_display: self.providers.resolve_account_plan(
|
||||
@@ -618,7 +622,11 @@ impl AccountsService for DefaultAccountsService {
|
||||
used_slots: in_flight
|
||||
.as_ref()
|
||||
.map(|counts| counts.get(&item.account.id).copied().unwrap_or(0)),
|
||||
..item.capacity
|
||||
..super::peer_link::upstream::account_capacity(
|
||||
&item.account,
|
||||
item.capacity,
|
||||
"a,
|
||||
)
|
||||
},
|
||||
capabilities,
|
||||
plan_type_display: providers.resolve_account_plan(
|
||||
@@ -1123,10 +1131,10 @@ impl AccountsService for DefaultAccountsService {
|
||||
|
||||
async fn test_connection(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
target: AccountProbeTarget,
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError> {
|
||||
let (stored, provider) = self.provider_for_account(&account_id).await?;
|
||||
let (stored, provider) = self.provider_for_account(target.account_id()).await?;
|
||||
let account = stored.account;
|
||||
let model = upstream_model.as_str().to_owned();
|
||||
let operation = provider
|
||||
@@ -1149,7 +1157,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
let result = probe
|
||||
.probe(
|
||||
AccountProbeRequest {
|
||||
account_id,
|
||||
target,
|
||||
provider_kind: account.provider_kind,
|
||||
upstream_model,
|
||||
operation,
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
//! 重置卡自动使用的账号锁、持久幂等键、查询缓存与重试编排
|
||||
//! 重置卡查询、消费与自动使用排程,统一账号锁、持久幂等键和重试编排
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::model::{MutationActor, reset_credit_automation::*};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::time::DeploymentTimeZone;
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
impl DefaultAccountsService {
|
||||
pub(super) async fn reset_credits_upstream(
|
||||
&self,
|
||||
@@ -51,13 +57,6 @@ impl DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
use crate::{
|
||||
model::{MutationActor, reset_credit_automation::*},
|
||||
reset_credit_automation::{consume_at, next_consumption, next_discovery, reschedule},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
|
||||
impl DefaultAccountsService {
|
||||
async fn automation_settings(
|
||||
&self,
|
||||
@@ -511,3 +510,54 @@ impl DefaultAccountsService {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 当地次日随机发现时间;随机数由调用者提供,结果持久化后不会因重启重抽
|
||||
fn next_discovery(
|
||||
now: DateTime<Utc>,
|
||||
timezone: DeploymentTimeZone,
|
||||
random: u64,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
let start = timezone.days_after(now, 1)?;
|
||||
let end = timezone.days_after(now, 2)?;
|
||||
let seconds = u64::try_from((end - start).num_seconds()).ok()?;
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
start.checked_add_signed(chrono::Duration::seconds(
|
||||
i64::try_from(random % seconds).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// 提前窗口的前 10% 用于账号错峰,最多一分钟,其余时间保留给网络失败恢复
|
||||
fn consume_at(account_id: &str, credit: &PlannedResetCredit, lead_minutes: u32) -> DateTime<Utc> {
|
||||
let lead_seconds = i64::from(lead_minutes) * 60;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(account_id.as_bytes());
|
||||
hash.update([0]);
|
||||
hash.update(credit.id.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let offset =
|
||||
i64::from(u16::from_be_bytes([digest[0], digest[1]])) % (lead_seconds / 10).clamp(1, 60);
|
||||
credit.expires_at - chrono::Duration::seconds(lead_seconds - offset)
|
||||
}
|
||||
|
||||
fn next_consumption(record: &ResetCreditAutomation, lead_minutes: u32) -> Option<DateTime<Utc>> {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| consume_at(&record.account_id, credit, lead_minutes))
|
||||
.min()
|
||||
}
|
||||
|
||||
fn reschedule(record: &mut ResetCreditAutomation, lead_minutes: u32, now: DateTime<Utc>) {
|
||||
record.next_run_at = if let Some(retry) = record.state.retry_at {
|
||||
retry
|
||||
} else if record.state.pending.is_some() {
|
||||
now
|
||||
} else {
|
||||
next_consumption(record, lead_minutes).map_or(record.state.next_discovery_at, |due| {
|
||||
due.min(record.state.next_discovery_at)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ impl BackupService for DefaultBackupService {
|
||||
ok: false,
|
||||
stage: "putObject",
|
||||
code: Some(code::INVALID_CONFIG),
|
||||
message: "备份存储配置不完整,无法执行连接测试".to_owned(),
|
||||
message: "备份存储配置不完整".to_owned(),
|
||||
});
|
||||
};
|
||||
let result = self
|
||||
@@ -198,14 +198,10 @@ impl BackupService for DefaultBackupService {
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
if !settings.storage_configured() {
|
||||
return Err(AdminError::bad_request(
|
||||
"备份存储配置不完整,请先保存存储配置",
|
||||
));
|
||||
return Err(AdminError::bad_request("请先补全并保存备份存储配置"));
|
||||
}
|
||||
if !settings.storage_verified() {
|
||||
return Err(AdminError::bad_request(
|
||||
"备份存储尚未通过连接测试,请先测试存储配置",
|
||||
));
|
||||
return Err(AdminError::bad_request("请先通过备份存储连接测试"));
|
||||
}
|
||||
let prefix = settings.prefix.as_deref().unwrap_or_default();
|
||||
let seed = build_backup_seed(
|
||||
@@ -241,9 +237,7 @@ impl BackupService for DefaultBackupService {
|
||||
) -> Result<DownloadUrlResult, AdminError> {
|
||||
let record = self.require_record(backup_id).await?;
|
||||
if record.status != BackupStatus::Completed {
|
||||
return Err(AdminError::conflict(
|
||||
"只有 completed 的备份才能创建下载地址",
|
||||
));
|
||||
return Err(AdminError::conflict("只能下载已完成的备份"));
|
||||
}
|
||||
let settings = self
|
||||
.repository
|
||||
@@ -280,9 +274,7 @@ impl BackupService for DefaultBackupService {
|
||||
) -> Result<BackupRecord, AdminError> {
|
||||
let record = self.require_record(backup_id).await?;
|
||||
if !record.status.can_be_deleted() {
|
||||
return Err(AdminError::conflict(
|
||||
"只有 completed 或 failed 的备份才能删除",
|
||||
));
|
||||
return Err(AdminError::conflict("只能删除已完成或失败的备份"));
|
||||
}
|
||||
let now = Utc::now();
|
||||
let deleting = self
|
||||
@@ -290,7 +282,7 @@ impl BackupService for DefaultBackupService {
|
||||
.transition_to_deleting(backup_id, now, Some(context))
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?
|
||||
.ok_or_else(|| AdminError::conflict("备份记录已进入删除流程"))?;
|
||||
.ok_or_else(|| AdminError::conflict("备份状态已变更,请刷新列表"))?;
|
||||
Ok(deleting)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
for (provider, profile) in &command.request_profile_overrides {
|
||||
self.providers
|
||||
.require(provider)
|
||||
.and_then(|provider| provider.preview_client_profile(profile))
|
||||
.and_then(|provider| provider.preview_key_client_profile(profile, None))
|
||||
.map_err(|error| super::map_provider_error(error, "client profile"))?;
|
||||
}
|
||||
validate_peer_sharing(command.peer_sharing, &command.group_ids)?;
|
||||
@@ -209,7 +209,7 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
if let Some(profile) = profile {
|
||||
self.providers
|
||||
.require(provider)
|
||||
.and_then(|provider| provider.preview_client_profile(profile))
|
||||
.and_then(|provider| provider.preview_key_client_profile(profile, None))
|
||||
.map_err(|error| super::map_provider_error(error, "client profile"))?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,10 @@ use std::{
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::{FutureExt as _, StreamExt as _, future::BoxFuture, stream::FuturesUnordered};
|
||||
use gateway_core::task::{DaemonTask, WorkerDaemonContext, WorkerTaskError};
|
||||
use gateway_core::task::{
|
||||
DaemonRestartPolicy, DaemonTask, WorkerContribution, WorkerDaemonContext, WorkerId, WorkerKind,
|
||||
WorkerRegistration, WorkerRunnable, WorkerTaskError,
|
||||
};
|
||||
use tokio::{sync::Notify, time::Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -204,7 +207,7 @@ impl DefaultImportTasksService {
|
||||
}
|
||||
Err(_) => {
|
||||
item.status = ImportItemStatus::Unknown;
|
||||
item.message = Some("执行中断,请先核对账号列表再决定是否重新导入".to_owned());
|
||||
item.message = Some("导入结果未知,重新导入前请核对账号列表".to_owned());
|
||||
}
|
||||
}
|
||||
entry.finish_if_done();
|
||||
@@ -328,6 +331,24 @@ impl ImportTasksService for DefaultImportTasksService {
|
||||
|
||||
pub(crate) struct ImportTaskWorker(pub Arc<DefaultImportTasksService>);
|
||||
|
||||
impl ImportTaskWorker {
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for ImportTaskWorker {
|
||||
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
|
||||
@@ -14,11 +14,11 @@ pub mod peer_link;
|
||||
pub mod plugin_accounts;
|
||||
pub mod plugin_client_keys;
|
||||
pub mod plugin_resources;
|
||||
pub(crate) mod plugin_update;
|
||||
pub mod plugins;
|
||||
pub mod proxies;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub(crate) mod system_preflight;
|
||||
pub mod usage_data;
|
||||
|
||||
mod credential_mutation;
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::{
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind};
|
||||
|
||||
use crate::{
|
||||
@@ -18,8 +19,8 @@ use crate::{
|
||||
DashboardPeriodMetrics, DashboardQuery, DashboardResult, DecimalAmount,
|
||||
DiagnosticDimension, DiagnosticsItem, DiagnosticsResult, Granularity, HealthStatus,
|
||||
HealthTimeline, HealthTimelinePoint, OpsErrorPage, OpsErrorQuery, ProviderBillingInput,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, Trend, TrendKind, TrendPoint,
|
||||
TrendSummary, UsageBilling, UsageBillingProjection, UsageCacheEfficiency,
|
||||
RequestHealth, RequestMetricPoint, RequestMetrics, TimeRange, Trend, TrendKind,
|
||||
TrendPoint, TrendSummary, UsageBilling, UsageBillingProjection, UsageCacheEfficiency,
|
||||
UsageCalculatedBillingFact, UsageConcurrencyPoint, UsageCostComparison,
|
||||
UsageCostSummary, UsageDetail, UsageFilter, UsageInsight, UsageInsightData,
|
||||
UsageInsightMetric, UsageInsights, UsageInsightsCost, UsageInsightsCostPoint,
|
||||
@@ -132,8 +133,9 @@ impl DefaultObservabilityService {
|
||||
self.settings.load_runtime_settings(),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "dashboard"))?;
|
||||
let runtime_slots = observation.runtime_slots;
|
||||
self.enrich_list_records(&mut observation.recent_requests);
|
||||
let runtime_slots = observation.runtime_slots.as_ref();
|
||||
self.enrich_list_records(&mut observation.recent_requests)
|
||||
.await;
|
||||
self.enrich_dashboard_quotas(&mut observation.account_usage)
|
||||
.await;
|
||||
for account in &mut observation.account_usage {
|
||||
@@ -178,11 +180,19 @@ impl DefaultObservabilityService {
|
||||
.providers
|
||||
.dashboard_wire_profiles(&settings.request_profiles);
|
||||
let max_concurrent_per_account = u64::from(settings.values.max_concurrent_per_account);
|
||||
let (inherited_accounts, overridden_slots) = runtime_slots
|
||||
.as_ref()
|
||||
let (mut inherited_accounts, mut overridden_slots) = runtime_slots
|
||||
.map_or((observation.provider_accounts.normal, 0), |slots| {
|
||||
(slots.inherited_accounts, slots.overridden_slots)
|
||||
});
|
||||
if let (Some(peer), Some(slots)) = (&self.upstream_info, runtime_slots) {
|
||||
let now = std::time::SystemTime::now();
|
||||
for version in &slots.inherited_versions {
|
||||
if let Some(limit) = peer.observed_concurrency_limit(version, now) {
|
||||
inherited_accounts -= 1;
|
||||
overridden_slots = overridden_slots.saturating_add(u64::from(limit.get()));
|
||||
}
|
||||
}
|
||||
}
|
||||
let total_slots = (max_concurrent_per_account > 0 || inherited_accounts == 0).then(|| {
|
||||
inherited_accounts
|
||||
.saturating_mul(max_concurrent_per_account)
|
||||
@@ -259,7 +269,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.list_usage_records(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage records"))?;
|
||||
self.enrich_list_records(&mut page.items);
|
||||
self.enrich_list_records(&mut page.items).await;
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
@@ -336,6 +346,12 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.usage_diagnostics(range, filter, dimension, 100)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage diagnostics"))?;
|
||||
let peers = if dimension == DiagnosticDimension::Account {
|
||||
self.current_upstream_peers(observation.items.iter().map(|item| item.key.as_str()))
|
||||
.await
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
let items = observation
|
||||
.items
|
||||
.into_iter()
|
||||
@@ -345,6 +361,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
rate_or_zero(item.non_completion_count, item.request_count);
|
||||
let retry_rate = rate_or_zero(item.retried_request_count, item.request_count);
|
||||
DiagnosticsItem {
|
||||
account_upstream_peer: peers.get(&item.key).copied(),
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
account_authentication_kind: item.account_authentication_kind,
|
||||
@@ -557,7 +574,7 @@ fn build_usage_insights(
|
||||
}
|
||||
|
||||
let requests = &overview.requests;
|
||||
let health_window = HealthWindow::from_metrics(requests);
|
||||
let health_window = RequestHealth::from_metrics(requests);
|
||||
let failed_requests = health_window.failed_requests;
|
||||
let health_requests = health_window.eligible_requests();
|
||||
let health = UsageInsightsHealth {
|
||||
@@ -577,7 +594,7 @@ fn build_usage_insights(
|
||||
points: trend
|
||||
.iter()
|
||||
.map(|point| {
|
||||
let health_window = HealthWindow::from_metrics(&point.metrics);
|
||||
let health_window = RequestHealth::from_metrics(&point.metrics);
|
||||
UsageInsightsHealthPoint {
|
||||
total_requests: point.metrics.request_count,
|
||||
bucket_start: point.bucket_start,
|
||||
@@ -912,6 +929,29 @@ fn usd_cost(costs: &[CurrencyCost]) -> Option<DecimalAmount> {
|
||||
}
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
async fn current_upstream_peers<'a>(
|
||||
&self,
|
||||
ids: impl Iterator<Item = &'a str>,
|
||||
) -> HashMap<String, bool> {
|
||||
let Some(peer) = self.upstream_info.as_deref() else {
|
||||
return HashMap::new();
|
||||
};
|
||||
let ids = ids
|
||||
.map(str::to_owned)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
// 同一页重复账号只读取一次摘要,单个账号不存在或读取失败不阻断历史记录
|
||||
futures::stream::iter(ids)
|
||||
.map(|id: String| async move {
|
||||
let account_id = ProviderAccountId::new(id.clone()).ok()?;
|
||||
let upstream = peer.read_summary(&account_id).await.ok()??.upstream?;
|
||||
Some((id, upstream.discovered && upstream.observed_at.is_some()))
|
||||
})
|
||||
.buffer_unordered(8)
|
||||
.filter_map(|value| async move { value })
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn enrich_dashboard_quotas(&self, accounts: &mut [DashboardAccountUsage]) {
|
||||
if accounts.is_empty() {
|
||||
return;
|
||||
@@ -976,8 +1016,22 @@ impl DefaultObservabilityService {
|
||||
|
||||
/// 补充当前套餐展示名称,逐条尽力把可校验的总额升级为完整分解;单条脏数据(非法 Provider kind、
|
||||
/// 不支持的来源或费用规则失败)只保留该条已存的总额,不影响整页返回
|
||||
fn enrich_list_records(&self, records: &mut [crate::model::observability::UsageListRecord]) {
|
||||
async fn enrich_list_records(
|
||||
&self,
|
||||
records: &mut [crate::model::observability::UsageListRecord],
|
||||
) {
|
||||
let peers = self
|
||||
.current_upstream_peers(
|
||||
records
|
||||
.iter()
|
||||
.filter_map(|record| record.provider_account_ref.as_deref()),
|
||||
)
|
||||
.await;
|
||||
for record in records {
|
||||
record.provider_account_upstream_peer = record
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.and_then(|id| peers.get(id).copied());
|
||||
record.provider_account_plan_type_display =
|
||||
record.provider_kind.as_deref().and_then(|provider| {
|
||||
self.providers
|
||||
@@ -1071,7 +1125,7 @@ pub(super) fn health_timeline_at(
|
||||
.map(|index| {
|
||||
(
|
||||
start + Duration::minutes(HEALTH_TIMELINE_SLOT_MINUTES * index),
|
||||
HealthWindow::default(),
|
||||
RequestHealth::default(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -1097,7 +1151,7 @@ pub(super) fn health_timeline_at(
|
||||
let totals = buckets
|
||||
.iter()
|
||||
.filter(|(bucket_start, _)| *bucket_start <= current_slot)
|
||||
.fold(HealthWindow::default(), |mut totals, (_, bucket)| {
|
||||
.fold(RequestHealth::default(), |mut totals, (_, bucket)| {
|
||||
totals.add_window(*bucket);
|
||||
totals
|
||||
});
|
||||
@@ -1161,10 +1215,10 @@ fn trend(
|
||||
}
|
||||
}
|
||||
let average_latency_ms = average(metrics.latency_sum_ms, metrics.latency_count);
|
||||
let service_failure_count = service_failure_count(&metrics);
|
||||
let service_failure_count = RequestHealth::from_metrics(&metrics).failed_requests;
|
||||
let success_rate = ratio(
|
||||
metrics.success_count,
|
||||
HealthWindow::from_metrics(&metrics).eligible_requests(),
|
||||
RequestHealth::from_metrics(&metrics).eligible_requests(),
|
||||
);
|
||||
let cache_hit_request_rate = ratio(
|
||||
metrics.cache_hit_request_count,
|
||||
@@ -1213,7 +1267,7 @@ fn trend(
|
||||
}
|
||||
|
||||
fn trend_point(point: RequestMetricPoint) -> TrendPoint {
|
||||
let service_failure_count = service_failure_count(&point.metrics);
|
||||
let service_failure_count = RequestHealth::from_metrics(&point.metrics).failed_requests;
|
||||
TrendPoint {
|
||||
bucket_start: point.bucket_start,
|
||||
granularity: point.granularity,
|
||||
@@ -1233,7 +1287,7 @@ fn trend_point(point: RequestMetricPoint) -> TrendPoint {
|
||||
),
|
||||
success_rate: ratio(
|
||||
point.metrics.success_count,
|
||||
HealthWindow::from_metrics(&point.metrics).eligible_requests(),
|
||||
RequestHealth::from_metrics(&point.metrics).eligible_requests(),
|
||||
),
|
||||
metrics: point.metrics,
|
||||
cost_coverage: point.cost_coverage,
|
||||
@@ -1318,57 +1372,7 @@ fn rate_or_zero(numerator: u64, denominator: u64) -> f64 {
|
||||
ratio(numerator, denominator).unwrap_or(0.0)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
struct HealthWindow {
|
||||
success_requests: u64,
|
||||
failed_requests: u64,
|
||||
cancelled_requests: u64,
|
||||
incomplete_requests: u64,
|
||||
caller_error_requests: u64,
|
||||
}
|
||||
|
||||
impl HealthWindow {
|
||||
fn from_metrics(metrics: &RequestMetrics) -> Self {
|
||||
Self {
|
||||
success_requests: metrics.success_count,
|
||||
failed_requests: service_failure_count(metrics),
|
||||
cancelled_requests: metrics.cancelled_count,
|
||||
incomplete_requests: metrics.incomplete_count,
|
||||
caller_error_requests: metrics.caller_error_count,
|
||||
}
|
||||
}
|
||||
|
||||
// 未完成保留交付终态,但上游失败、超时和进程中断都不能视为服务成功
|
||||
fn non_success_requests(self) -> u64 {
|
||||
self.failed_requests
|
||||
.saturating_add(self.incomplete_requests)
|
||||
}
|
||||
|
||||
fn eligible_requests(self) -> u64 {
|
||||
self.success_requests
|
||||
.saturating_add(self.non_success_requests())
|
||||
}
|
||||
|
||||
fn add_metrics(&mut self, metrics: &RequestMetrics) {
|
||||
self.add_window(Self::from_metrics(metrics));
|
||||
}
|
||||
|
||||
fn add_window(&mut self, other: Self) {
|
||||
self.success_requests = self.success_requests.saturating_add(other.success_requests);
|
||||
self.failed_requests = self.failed_requests.saturating_add(other.failed_requests);
|
||||
self.cancelled_requests = self
|
||||
.cancelled_requests
|
||||
.saturating_add(other.cancelled_requests);
|
||||
self.incomplete_requests = self
|
||||
.incomplete_requests
|
||||
.saturating_add(other.incomplete_requests);
|
||||
self.caller_error_requests = self
|
||||
.caller_error_requests
|
||||
.saturating_add(other.caller_error_requests);
|
||||
}
|
||||
}
|
||||
|
||||
fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus {
|
||||
fn health_status(bucket: RequestHealth, is_future: bool) -> HealthStatus {
|
||||
let eligible_requests = bucket.eligible_requests();
|
||||
if is_future {
|
||||
HealthStatus::Future
|
||||
@@ -1389,14 +1393,6 @@ fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus {
|
||||
}
|
||||
}
|
||||
|
||||
fn health_reliability(bucket: HealthWindow) -> Option<f64> {
|
||||
let eligible_requests = bucket.eligible_requests();
|
||||
(eligible_requests > 0)
|
||||
.then(|| bucket.success_requests as f64 / eligible_requests as f64 * 100.0)
|
||||
}
|
||||
|
||||
fn service_failure_count(metrics: &RequestMetrics) -> u64 {
|
||||
metrics
|
||||
.failure_count
|
||||
.saturating_sub(metrics.caller_error_count)
|
||||
fn health_reliability(bucket: RequestHealth) -> Option<f64> {
|
||||
bucket.success_rate()
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::{
|
||||
model::{
|
||||
AdminError,
|
||||
peer_link::{PeerLinkExport, PeerLinkLimits},
|
||||
provider_credentials::ProviderQuotaReadRequest,
|
||||
provider_credentials::{ProviderQuotaReadRequest, explicit_plan_type},
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdminRegistry,
|
||||
@@ -117,6 +117,7 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.budget
|
||||
.limits
|
||||
.access(key.budget.daily_used_usd, key.budget.weekly_used_usd),
|
||||
key_limits: key.limits,
|
||||
budget: key.budget,
|
||||
budget_observed_at,
|
||||
pool: None,
|
||||
@@ -175,10 +176,16 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "peer account runtime"))?;
|
||||
let mut quotas = BTreeMap::new();
|
||||
if sharing.share_quota {
|
||||
if sharing.share_quota || sharing.share_plans {
|
||||
let mut window_count = 0usize;
|
||||
let mut by_provider = BTreeMap::new();
|
||||
for account in &accounts {
|
||||
// 仅公开套餐时只补读缺失套餐的本地额度,不回源刷新或扩大公开范围
|
||||
if !sharing.share_quota
|
||||
&& explicit_plan_type(account.plan_type.as_deref()).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
by_provider
|
||||
.entry(&account.provider_kind)
|
||||
.or_insert_with(Vec::new)
|
||||
@@ -200,9 +207,11 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.zip(provider.read_quotas(&requests).await)
|
||||
{
|
||||
if let Ok(quota) = quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
if sharing.share_quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
}
|
||||
}
|
||||
quotas.insert(account.id.clone(), quota);
|
||||
}
|
||||
@@ -215,6 +224,7 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
if sharing.share_pool {
|
||||
result.pool = Some(projection::pool_summary(
|
||||
&accounts,
|
||||
"as,
|
||||
&runtime,
|
||||
&self.providers,
|
||||
sharing.share_plans,
|
||||
|
||||
@@ -25,6 +25,7 @@ use crate::{
|
||||
|
||||
pub(super) fn pool_summary(
|
||||
accounts: &[AccountRecord],
|
||||
quotas: &BTreeMap<String, ProviderQuota>,
|
||||
runtime: &AccountRuntimeSnapshot,
|
||||
providers: &ProviderAdminRegistry,
|
||||
share_plans: bool,
|
||||
@@ -43,12 +44,13 @@ pub(super) fn pool_summary(
|
||||
AccountStatus::Error => summary.error += 1,
|
||||
}
|
||||
if let Some(plans) = &mut summary.plans
|
||||
&& let (Some(plan), Ok(provider)) = (
|
||||
&account.plan_type,
|
||||
providers.require(&account.provider_kind),
|
||||
&& let Some(plan) = providers.resolve_account_plan(
|
||||
account.provider_kind.as_str(),
|
||||
&mut account.plan_type.clone(),
|
||||
quotas.get(&account.id),
|
||||
)
|
||||
{
|
||||
*plans.entry(provider.plan_type_display(plan)).or_default() += 1;
|
||||
*plans.entry(plan).or_default() += 1;
|
||||
}
|
||||
}
|
||||
summary
|
||||
|
||||
@@ -5,7 +5,7 @@ use super::{
|
||||
};
|
||||
use crate::model::provider_credentials::ProviderQuota;
|
||||
use gateway_core::{
|
||||
account::{QuotaObservation, UpstreamConnection},
|
||||
account::{QuotaObservation, UpstreamConnection, UpstreamInfoFailure},
|
||||
error::StoreError,
|
||||
};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
@@ -56,6 +56,7 @@ impl PeerInfoService {
|
||||
checked_at: now,
|
||||
scheduling: None,
|
||||
summary: None,
|
||||
stored_failure: None,
|
||||
},
|
||||
));
|
||||
}
|
||||
@@ -85,22 +86,24 @@ impl PeerInfoService {
|
||||
connections
|
||||
.iter()
|
||||
.map(|connection| {
|
||||
let cached = observations
|
||||
.get(&connection.account_id)
|
||||
.filter(|cached| cached.connection == *connection);
|
||||
(
|
||||
connection.clone(),
|
||||
observations
|
||||
.get(&connection.account_id)
|
||||
.filter(|cached| cached.connection == *connection)
|
||||
.and_then(|cached| cached.summary.clone()),
|
||||
cached.and_then(|cached| cached.summary.clone()),
|
||||
cached.and_then(|cached| cached.stored_failure),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
Ok(summaries
|
||||
.into_iter()
|
||||
.map(|(connection, peer)| {
|
||||
.map(|(connection, peer, stored_failure)| {
|
||||
let quota = Self::project_peer(
|
||||
&connection,
|
||||
peer.as_deref(),
|
||||
stored_failure,
|
||||
self.last_error(&connection),
|
||||
now,
|
||||
);
|
||||
@@ -132,6 +135,8 @@ impl PeerInfoService {
|
||||
checked_at: now,
|
||||
scheduling,
|
||||
summary,
|
||||
stored_failure: observation
|
||||
.and_then(|value| snapshot::stored_failure(&value.quota, connection)),
|
||||
};
|
||||
let previous = self
|
||||
.observations
|
||||
@@ -140,4 +145,15 @@ impl PeerInfoService {
|
||||
.insert(connection.account_id.clone(), cached);
|
||||
drop(previous);
|
||||
}
|
||||
|
||||
pub(super) fn unsupported(&self, connection: &UpstreamConnection) -> bool {
|
||||
self.observations
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&connection.account_id)
|
||||
.is_some_and(|cached| {
|
||||
cached.connection == *connection
|
||||
&& cached.stored_failure == Some(UpstreamInfoFailure::Unsupported)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ use gateway_core::{
|
||||
error::StoreError,
|
||||
task::{WorkerContribution, WorkerDefinitionError},
|
||||
};
|
||||
pub(crate) use projection::account_capacity;
|
||||
use service::PeerInfoService;
|
||||
|
||||
/// 组合根独立注册公共互联 Worker,不属于任何具体 Provider 的任务集合
|
||||
@@ -62,6 +63,24 @@ pub fn initialize(
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl PeerUpstreamService for PeerInfoService {
|
||||
fn observed_concurrency_limit(
|
||||
&self,
|
||||
version: &gateway_core::account::QuotaObservationVersion,
|
||||
now: std::time::SystemTime,
|
||||
) -> Option<gateway_core::account::AccountConcurrencyLimit> {
|
||||
let observations = self
|
||||
.observations
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let cached = observations.get(&version.account_id)?;
|
||||
if cached.observed_at != Some(version.observed_at)
|
||||
|| cached.connection.credential_revision != version.credential_revision
|
||||
{
|
||||
return None;
|
||||
}
|
||||
cached.scheduling.as_ref()?.concurrency_limit(now)
|
||||
}
|
||||
|
||||
async fn account_changed(&self, id: &ProviderAccountId) -> Result<(), StoreError> {
|
||||
PeerInfoService::account_changed(self, id).await
|
||||
}
|
||||
|
||||
@@ -12,16 +12,35 @@ use crate::model::{
|
||||
},
|
||||
};
|
||||
use gateway_core::account::{QuotaAccessState, UpstreamConnection, UpstreamInfoFailure};
|
||||
use gateway_protocol::peer::{
|
||||
PeerComponentStatus, PeerLink, PeerPoolDisclosure, PeerQuotaDisclosure,
|
||||
};
|
||||
use gateway_protocol::peer::{PeerComponentStatus, PeerLink};
|
||||
|
||||
use super::{PeerInfoService, quota, snapshot};
|
||||
|
||||
pub(crate) fn account_capacity(
|
||||
account: &crate::model::accounts::AccountRecord,
|
||||
mut capacity: crate::model::accounts::AccountCapacity,
|
||||
quota: &ProviderQuota,
|
||||
) -> crate::model::accounts::AccountCapacity {
|
||||
if account.concurrency_limit.is_none()
|
||||
&& let Some(info) = "a.upstream
|
||||
&& !info.stale
|
||||
&& info
|
||||
.observed_at
|
||||
.is_some_and(|at| snapshot::fresh_at(at.into(), SystemTime::now()))
|
||||
&& let Some(limit) = info
|
||||
.max_concurrency
|
||||
.and_then(gateway_core::account::AccountConcurrencyLimit::from_key_limit)
|
||||
{
|
||||
capacity.total_slots = Some(u64::from(limit.get()));
|
||||
}
|
||||
capacity
|
||||
}
|
||||
|
||||
impl PeerInfoService {
|
||||
pub(super) fn project_peer(
|
||||
connection: &UpstreamConnection,
|
||||
peer: Option<&PeerLink>,
|
||||
stored_failure: Option<UpstreamInfoFailure>,
|
||||
last_error: Option<UpstreamInfoFailure>,
|
||||
now: SystemTime,
|
||||
) -> ProviderQuota {
|
||||
@@ -31,19 +50,24 @@ impl PeerInfoService {
|
||||
let stale = peer.as_ref().is_none_or(|peer| {
|
||||
!snapshot::component_fresh(Some(&peer.observed_at), Some(&peer.expires_at), now)
|
||||
});
|
||||
let limits = peer.and_then(|peer| peer.key_limits);
|
||||
let info = ProviderUpstreamInfo {
|
||||
discovered: peer.is_some(),
|
||||
discovered: peer.is_some() || stored_failure == Some(UpstreamInfoFailure::Unauthorized),
|
||||
observed_at,
|
||||
stale,
|
||||
connection_revision: connection.revision.clone(),
|
||||
last_error: last_error.map(|error| error.as_str().to_owned()),
|
||||
last_error: last_error
|
||||
.or(stored_failure)
|
||||
.map(|error| error.as_str().to_owned()),
|
||||
max_concurrency: limits.and_then(|limits| limits.max_concurrency),
|
||||
requests_per_minute: limits.and_then(|limits| limits.requests_per_minute),
|
||||
pool: peer
|
||||
.as_ref()
|
||||
.filter(|peer| peer.pool.disclosure == PeerPoolDisclosure::Summary)
|
||||
.and_then(|peer| {
|
||||
let states = peer.pool.states.as_ref()?;
|
||||
Some(UpstreamPool {
|
||||
account_count: peer.pool.account_count?,
|
||||
.and_then(|peer| peer.pool.as_ref())
|
||||
.map(|pool| {
|
||||
let states = &pool.states;
|
||||
UpstreamPool {
|
||||
account_count: pool.account_count,
|
||||
states: UpstreamPoolStates {
|
||||
ready: states.ready,
|
||||
cooldown: states.cooldown,
|
||||
@@ -52,8 +76,7 @@ impl PeerInfoService {
|
||||
error: states.error,
|
||||
unknown: states.unknown,
|
||||
},
|
||||
plans: peer
|
||||
.pool
|
||||
plans: pool
|
||||
.plans
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
@@ -63,19 +86,18 @@ impl PeerInfoService {
|
||||
count: plan.count,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}),
|
||||
};
|
||||
let mut windows = Vec::new();
|
||||
if let Some(peer) = &peer {
|
||||
if peer.usage.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &peer.usage.data
|
||||
if let Some(component) = &peer.usage
|
||||
&& component.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &component.data
|
||||
&& let Some(unit) = &usage.unit
|
||||
{
|
||||
for (key, label, seconds, window) in [
|
||||
("daily", "日额度", 86400, &usage.daily),
|
||||
("weekly", "周额度", 604800, &usage.weekly),
|
||||
] {
|
||||
// 不限额仍有日、周账本,只是不计算占用百分比
|
||||
for window in &usage.windows {
|
||||
// 不限额仍展示已提供的账本窗口,只是不计算占用百分比
|
||||
let total = window
|
||||
.total
|
||||
.as_deref()
|
||||
@@ -84,19 +106,17 @@ impl PeerInfoService {
|
||||
let used_percent = snapshot::decimal(&window.used)
|
||||
.zip(total)
|
||||
.map(|(used, total)| (used / total * 100.0).clamp(0.0, 100.0));
|
||||
let window_observed = peer
|
||||
.usage
|
||||
let window_observed = component
|
||||
.observed_at
|
||||
.as_deref()
|
||||
.and_then(snapshot::datetime);
|
||||
let window_stale = stale
|
||||
|| !peer
|
||||
.usage
|
||||
|| !component
|
||||
.observed_at
|
||||
.as_deref()
|
||||
.is_some_and(|value| snapshot::fresh(value, now))
|
||||
|| window
|
||||
.resets_at
|
||||
.reset_at
|
||||
.as_deref()
|
||||
.and_then(snapshot::timestamp)
|
||||
.is_some_and(|reset| reset <= now);
|
||||
@@ -106,27 +126,28 @@ impl PeerInfoService {
|
||||
observed_at: window_observed,
|
||||
stale: window_stale,
|
||||
amount: Some(UpstreamQuotaAmount {
|
||||
unit: usage.unit.clone(),
|
||||
unit: unit.clone(),
|
||||
used: window.used.clone(),
|
||||
total: window.total.clone(),
|
||||
remaining: window.remaining.clone(),
|
||||
}),
|
||||
}),
|
||||
key: format!("peer:budget:{key}"),
|
||||
key: format!("peer:budget:{}", window.key),
|
||||
group: "general".to_owned(),
|
||||
label: label.to_owned(),
|
||||
label: window.label.as_deref().unwrap_or(&window.key).to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: Some(if key == "weekly" {
|
||||
ProviderQuotaWindowRole::Secondary
|
||||
} else {
|
||||
ProviderQuotaWindowRole::Primary
|
||||
}),
|
||||
role: match window.key.as_str() {
|
||||
"daily" => Some(ProviderQuotaWindowRole::Primary),
|
||||
"weekly" => Some(ProviderQuotaWindowRole::Secondary),
|
||||
"monthly" => Some(ProviderQuotaWindowRole::Monthly),
|
||||
_ => None,
|
||||
},
|
||||
// Key 消费窗口覆盖此连接的全部请求,本机用量按同一时间边界聚合
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(seconds),
|
||||
window_seconds: window.window_seconds,
|
||||
used_percent,
|
||||
reset_at: window.resets_at.as_deref().and_then(snapshot::datetime),
|
||||
reset_at: window.reset_at.as_deref().and_then(snapshot::datetime),
|
||||
limit_reached: !window_stale
|
||||
&& quota::window_access(window, now) == QuotaAccessState::Exhausted,
|
||||
local_usage: None,
|
||||
@@ -135,8 +156,8 @@ impl PeerInfoService {
|
||||
}
|
||||
}
|
||||
// 池内窗口各自保留账号归属,不能将不同套餐相加为统一消费额度
|
||||
if peer.quota.disclosure == PeerQuotaDisclosure::Accounts {
|
||||
for (index, account) in peer.quota.accounts.iter().enumerate() {
|
||||
if let Some(quota) = &peer.quota {
|
||||
for (index, account) in quota.accounts.iter().enumerate() {
|
||||
let window_observed =
|
||||
account.observed_at.as_deref().and_then(snapshot::datetime);
|
||||
let window_stale = stale
|
||||
@@ -146,6 +167,7 @@ impl PeerInfoService {
|
||||
now,
|
||||
);
|
||||
for window in &account.windows {
|
||||
let group = window.group.as_deref().unwrap_or("general");
|
||||
let window_stale = window_stale
|
||||
|| window
|
||||
.reset_at
|
||||
@@ -156,7 +178,7 @@ impl PeerInfoService {
|
||||
"peer:{}:{}:{}",
|
||||
account.id.len(),
|
||||
account.id,
|
||||
window.limit_id.as_deref().unwrap_or(&window.group)
|
||||
window.limit_id.as_deref().unwrap_or(group)
|
||||
);
|
||||
windows.push(ProviderQuotaWindow {
|
||||
upstream: Some(UpstreamQuotaWindow {
|
||||
@@ -166,8 +188,8 @@ impl PeerInfoService {
|
||||
amount: None,
|
||||
}),
|
||||
key: format!("{scope}:{}", window.key),
|
||||
group: window.group.clone(),
|
||||
label: window.label.clone(),
|
||||
group: group.to_owned(),
|
||||
label: window.label.as_deref().unwrap_or(&window.key).to_owned(),
|
||||
limit_id: Some(scope),
|
||||
limit_name: Some(match window.limit_name.as_deref() {
|
||||
Some(name) => format!("账号 {} · {name}", index + 1),
|
||||
|
||||
@@ -13,24 +13,26 @@ pub(super) fn observe(
|
||||
started_at: SystemTime,
|
||||
now: SystemTime,
|
||||
) -> Option<QuotaState> {
|
||||
let usage = peer.usage.as_ref()?;
|
||||
if !fresh(&peer.observed_at, now)
|
||||
|| !component_fresh(Some(&peer.observed_at), Some(&peer.expires_at), now)
|
||||
|| peer.usage.status != PeerComponentStatus::Available
|
||||
|| !component_fresh(peer.usage.observed_at.as_deref(), None, now)
|
||||
|| usage.status != PeerComponentStatus::Available
|
||||
|| !component_fresh(usage.observed_at.as_deref(), None, now)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let observed_at = started_at
|
||||
.min(timestamp(&peer.observed_at)?)
|
||||
.min(timestamp(peer.usage.observed_at.as_deref()?)?);
|
||||
let usage = peer.usage.data.as_ref()?;
|
||||
.min(timestamp(usage.observed_at.as_deref()?)?);
|
||||
let usage = usage.data.as_ref()?;
|
||||
match access(usage, now) {
|
||||
QuotaAccessState::Allowed => Some(QuotaState::allowed(observed_at)),
|
||||
QuotaAccessState::Exhausted => {
|
||||
let reset_at = [&usage.daily, &usage.weekly]
|
||||
.into_iter()
|
||||
let reset_at = usage
|
||||
.windows
|
||||
.iter()
|
||||
.filter(|window| window_access(window, now) == QuotaAccessState::Exhausted)
|
||||
.filter_map(|window| window.resets_at.as_deref().and_then(timestamp))
|
||||
.filter_map(|window| window.reset_at.as_deref().and_then(timestamp))
|
||||
.min();
|
||||
Some(QuotaState::exhausted(
|
||||
if usage.allowed == Some(false) {
|
||||
@@ -51,8 +53,9 @@ pub(super) fn access(usage: &PeerKeyUsage, now: SystemTime) -> QuotaAccessState
|
||||
match usage.allowed {
|
||||
Some(true) => QuotaAccessState::Allowed,
|
||||
Some(false) => QuotaAccessState::Exhausted,
|
||||
None if [&usage.daily, &usage.weekly]
|
||||
.into_iter()
|
||||
None if usage
|
||||
.windows
|
||||
.iter()
|
||||
.any(|window| window_access(window, now) == QuotaAccessState::Exhausted) =>
|
||||
{
|
||||
QuotaAccessState::Exhausted
|
||||
@@ -64,7 +67,7 @@ pub(super) fn access(usage: &PeerKeyUsage, now: SystemTime) -> QuotaAccessState
|
||||
/// 过期窗口保持未知,不从重置时间或金额推断恢复
|
||||
pub(super) fn window_access(window: &PeerBudgetWindow, now: SystemTime) -> QuotaAccessState {
|
||||
if window
|
||||
.resets_at
|
||||
.reset_at
|
||||
.as_deref()
|
||||
.and_then(timestamp)
|
||||
.is_some_and(|reset| reset <= now)
|
||||
|
||||
@@ -43,6 +43,9 @@ impl PeerInfoService {
|
||||
}
|
||||
|
||||
pub(super) fn refresh_due(&self, connection: &UpstreamConnection) -> bool {
|
||||
if self.unsupported(connection) {
|
||||
return false;
|
||||
}
|
||||
let state = self.refresh_state(connection);
|
||||
state.try_lock().is_ok_and(|state| {
|
||||
state.connection != *connection || Instant::now() >= state.next_attempt
|
||||
@@ -91,18 +94,19 @@ impl PeerInfoService {
|
||||
.ok_or_else(|| ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))?;
|
||||
let slot = self.refresh_state(&connection);
|
||||
let mut state = slot.lock().await;
|
||||
if state.connection == connection
|
||||
&& Instant::now()
|
||||
< if manual {
|
||||
state.manual_after
|
||||
} else {
|
||||
state.next_attempt
|
||||
}
|
||||
if !manual && self.unsupported(&connection)
|
||||
|| state.connection == connection
|
||||
&& Instant::now()
|
||||
< if manual {
|
||||
state.manual_after
|
||||
} else {
|
||||
state.next_attempt
|
||||
}
|
||||
{
|
||||
let error = state.last_error;
|
||||
drop(state);
|
||||
if let Some(error) = error {
|
||||
return Err(refresh_error(error));
|
||||
if let Some(error) = error.and_then(refresh_error) {
|
||||
return Err(error);
|
||||
}
|
||||
return self
|
||||
.read_quota(id)
|
||||
@@ -126,25 +130,38 @@ impl PeerInfoService {
|
||||
let failure = result.as_ref().err().copied();
|
||||
let delay = match failure {
|
||||
None => Duration::from_secs(240 + stable_jitter(id.as_str())),
|
||||
Some(UpstreamInfoFailure::Unauthorized | UpstreamInfoFailure::Unsupported) => {
|
||||
Some(UpstreamInfoFailure::Unauthorized | UpstreamInfoFailure::SharingDisabled) => {
|
||||
Duration::from_secs(1800)
|
||||
}
|
||||
Some(_) => Duration::from_secs(30),
|
||||
}
|
||||
.max(retry_after.unwrap_or_default());
|
||||
state.next_attempt = Instant::now() + delay;
|
||||
state.manual_after = Instant::now()
|
||||
+ if failure.is_some() {
|
||||
delay
|
||||
} else {
|
||||
Duration::from_secs(15)
|
||||
};
|
||||
// 管理员修复授权后可主动复核,不继承自动重试的长退避,仍遵守上游限流
|
||||
state.manual_after =
|
||||
Instant::now() + Duration::from_secs(15).max(retry_after.unwrap_or_default());
|
||||
state.last_error = failure;
|
||||
let document = match result {
|
||||
Ok(document) => Some(document),
|
||||
// 空观测清除已撤销内容,并用共有观察时间阻止更早请求重新写回
|
||||
Err(UpstreamInfoFailure::Unauthorized) => {
|
||||
Some(OpaqueProviderData::new(Default::default()))
|
||||
Err(UpstreamInfoFailure::Unsupported) => Some(snapshot::unsupported(&connection)),
|
||||
Err(error @ UpstreamInfoFailure::SharingDisabled) => {
|
||||
Some(snapshot::unavailable(&connection, error))
|
||||
}
|
||||
Err(error @ UpstreamInfoFailure::Unauthorized) => {
|
||||
let discovered = self
|
||||
.read_summary(id)
|
||||
.await
|
||||
.map_err(store_error)?
|
||||
.and_then(|quota| quota.upstream)
|
||||
.is_some_and(|info| {
|
||||
info.connection_revision == connection.revision && info.discovered
|
||||
});
|
||||
// 首次探测失败不能证明对方是互联节点;已确认的连接只留下无共享内容的识别记录
|
||||
Some(if discovered {
|
||||
snapshot::unavailable(&connection, error)
|
||||
} else {
|
||||
OpaqueProviderData::new(Default::default())
|
||||
})
|
||||
}
|
||||
Err(_) => None,
|
||||
};
|
||||
@@ -185,8 +202,9 @@ impl PeerInfoService {
|
||||
);
|
||||
}
|
||||
drop(state);
|
||||
if let Some(error) = failure {
|
||||
return Err(refresh_error(error));
|
||||
// 不支持协议是正常发现结果,管理端保留分类,后台不重复报警
|
||||
if let Some(error) = failure.and_then(refresh_error) {
|
||||
return Err(error);
|
||||
}
|
||||
self.read_quota(id)
|
||||
.await
|
||||
@@ -201,12 +219,13 @@ fn stable_jitter(id: &str) -> u64 {
|
||||
}) % 60
|
||||
}
|
||||
|
||||
fn refresh_error(error: UpstreamInfoFailure) -> ProviderAdminError {
|
||||
ProviderAdminError::new(ProviderAdminErrorKind::BadGateway).with_public_message(match error {
|
||||
fn refresh_error(error: UpstreamInfoFailure) -> Option<ProviderAdminError> {
|
||||
let message = match error {
|
||||
UpstreamInfoFailure::Unauthorized => "上游未授权读取互联信息",
|
||||
UpstreamInfoFailure::Unsupported => "上游不支持互联信息查询",
|
||||
UpstreamInfoFailure::Unsupported | UpstreamInfoFailure::SharingDisabled => return None,
|
||||
UpstreamInfoFailure::RateLimited => "上游互联查询已限流",
|
||||
UpstreamInfoFailure::InvalidResponse => "上游互联响应无效",
|
||||
UpstreamInfoFailure::Unavailable => "上游互联服务暂不可用",
|
||||
})
|
||||
};
|
||||
Some(ProviderAdminError::new(ProviderAdminErrorKind::BadGateway).with_public_message(message))
|
||||
}
|
||||
|
||||
@@ -63,6 +63,7 @@ struct ModelObservation {
|
||||
|
||||
pub(super) struct SchedulingObservation {
|
||||
validity: Validity,
|
||||
concurrency_limit: Option<gateway_core::account::AccountConcurrencyLimit>,
|
||||
exhausted_budgets: Vec<Validity>,
|
||||
models: HashMap<String, Option<ModelObservation>>,
|
||||
}
|
||||
@@ -70,22 +71,23 @@ pub(super) struct SchedulingObservation {
|
||||
impl SchedulingObservation {
|
||||
pub(super) fn new(peer: &PeerLink, now: SystemTime) -> Option<Self> {
|
||||
let validity = Validity::component(Some(&peer.observed_at), Some(&peer.expires_at))?;
|
||||
let mut exhausted_budgets = Vec::with_capacity(2);
|
||||
if peer.usage.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &peer.usage.data
|
||||
let mut exhausted_budgets = Vec::new();
|
||||
if let Some(component) = &peer.usage
|
||||
&& component.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &component.data
|
||||
&& let Some(usage_validity) =
|
||||
Validity::component(peer.usage.observed_at.as_deref(), None)
|
||||
Validity::component(component.observed_at.as_deref(), None)
|
||||
&& quota::access(usage, now) == QuotaAccessState::Exhausted
|
||||
{
|
||||
if usage.allowed.is_some() {
|
||||
exhausted_budgets.push(usage_validity);
|
||||
} else {
|
||||
// 窗口提供的补充证据只在各自重置边界之前影响调度评分
|
||||
for window in [&usage.daily, &usage.weekly] {
|
||||
for window in &usage.windows {
|
||||
if quota::window_access(window, now) == QuotaAccessState::Exhausted {
|
||||
let mut validity = usage_validity;
|
||||
if let Some(reset) =
|
||||
window.resets_at.as_deref().and_then(snapshot::timestamp)
|
||||
window.reset_at.as_deref().and_then(snapshot::timestamp)
|
||||
{
|
||||
validity.expires_at = validity.expires_at.min(reset);
|
||||
}
|
||||
@@ -97,7 +99,11 @@ impl SchedulingObservation {
|
||||
|
||||
// 同一账号重复列出模型不增加候选数,多账号覆盖不计算池平均百分比
|
||||
let mut model_accounts = HashMap::<&str, Option<usize>>::new();
|
||||
for (index, account) in peer.quota.accounts.iter().enumerate() {
|
||||
let quota_accounts = peer
|
||||
.quota
|
||||
.as_ref()
|
||||
.map_or(&[][..], |quota| quota.accounts.as_slice());
|
||||
for (index, account) in quota_accounts.iter().enumerate() {
|
||||
for model in &account.models {
|
||||
model_accounts
|
||||
.entry(model)
|
||||
@@ -128,7 +134,7 @@ impl SchedulingObservation {
|
||||
accounts
|
||||
.entry(index)
|
||||
.or_insert_with(|| {
|
||||
let account = &peer.quota.accounts[index];
|
||||
let account = "a_accounts[index];
|
||||
let validity = Validity::component(
|
||||
account.observed_at.as_deref(),
|
||||
account.expires_at.as_deref(),
|
||||
@@ -161,43 +167,52 @@ impl SchedulingObservation {
|
||||
}
|
||||
Some(Self {
|
||||
validity,
|
||||
concurrency_limit: peer
|
||||
.key_limits
|
||||
.and_then(|limits| limits.max_concurrency)
|
||||
.and_then(gateway_core::account::AccountConcurrencyLimit::from_key_limit),
|
||||
exhausted_budgets,
|
||||
models,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn fresh(&self, now: SystemTime) -> bool {
|
||||
self.validity.fresh(now)
|
||||
&& now
|
||||
.duration_since(self.validity.observed_at)
|
||||
.unwrap_or_default()
|
||||
<= snapshot::FRESH_FOR
|
||||
self.validity.fresh(now) && snapshot::fresh_at(self.validity.observed_at, now)
|
||||
}
|
||||
|
||||
pub(super) fn signal(&self, model: &str, now: SystemTime) -> Option<UpstreamSchedulingSignal> {
|
||||
pub(super) fn concurrency_limit(
|
||||
&self,
|
||||
now: SystemTime,
|
||||
) -> Option<gateway_core::account::AccountConcurrencyLimit> {
|
||||
self.concurrency_limit.filter(|_| self.fresh(now))
|
||||
}
|
||||
|
||||
pub(super) fn signal(
|
||||
&self,
|
||||
model: Option<&str>,
|
||||
now: SystemTime,
|
||||
) -> Option<UpstreamSchedulingSignal> {
|
||||
if !self.fresh(now) {
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
let budget_exhausted = self
|
||||
.exhausted_budgets
|
||||
.iter()
|
||||
.any(|validity| validity.fresh(now))
|
||||
{
|
||||
return Some(UpstreamSchedulingSignal {
|
||||
exhausted: true,
|
||||
remaining_percent: None,
|
||||
});
|
||||
}
|
||||
let model = self.models.get(model)?.as_ref()?;
|
||||
if !model.validity.fresh(now) {
|
||||
.any(|validity| validity.fresh(now));
|
||||
let model = model
|
||||
.and_then(|id| self.models.get(id)?.as_ref())
|
||||
.filter(|model| model.validity.fresh(now));
|
||||
if !budget_exhausted && model.is_none() && self.concurrency_limit.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(UpstreamSchedulingSignal {
|
||||
exhausted: model.exhausted,
|
||||
remaining_percent: model
|
||||
.account
|
||||
.as_ref()
|
||||
.and_then(|account| account.remaining_percent(now)),
|
||||
concurrency_limit: self.concurrency_limit,
|
||||
exhausted: budget_exhausted || model.is_some_and(|model| model.exhausted),
|
||||
remaining_percent: if budget_exhausted {
|
||||
None
|
||||
} else {
|
||||
model.and_then(|model| model.account.as_ref()?.remaining_percent(now))
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ use futures::{StreamExt as _, stream};
|
||||
use gateway_core::{
|
||||
account::{
|
||||
LoadedCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
UpstreamConnection, UpstreamInfoSource, UpstreamSchedulingSignal,
|
||||
UpstreamConnection, UpstreamInfoFailure, UpstreamInfoSource, UpstreamSchedulingSignal,
|
||||
},
|
||||
error::{StoreError, StoreErrorKind},
|
||||
routing::ProviderKind,
|
||||
@@ -34,6 +34,7 @@ pub(super) struct CachedObservation {
|
||||
pub checked_at: SystemTime,
|
||||
pub scheduling: Option<Arc<SchedulingObservation>>,
|
||||
pub summary: Option<Arc<PeerLink>>,
|
||||
pub stored_failure: Option<UpstreamInfoFailure>,
|
||||
}
|
||||
|
||||
pub(crate) struct PeerInfoService {
|
||||
@@ -171,12 +172,16 @@ impl PeerInfoService {
|
||||
let peer = observation
|
||||
.as_ref()
|
||||
.and_then(|value| snapshot::decode(&value.quota));
|
||||
let stored_failure = observation
|
||||
.as_ref()
|
||||
.and_then(|value| snapshot::stored_failure(&value.quota, &connection));
|
||||
if self.current_connection(id).await?.as_ref() != Some(&connection) {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(Self::project_peer(
|
||||
&connection,
|
||||
peer.as_ref(),
|
||||
stored_failure,
|
||||
self.last_error(&connection),
|
||||
SystemTime::now(),
|
||||
)))
|
||||
@@ -280,7 +285,7 @@ impl UpstreamInfoSource for PeerInfoService {
|
||||
fn signal(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
model: &str,
|
||||
model: Option<&str>,
|
||||
now: SystemTime,
|
||||
) -> Option<UpstreamSchedulingSignal> {
|
||||
if !account.enabled() {
|
||||
|
||||
@@ -1,18 +1,71 @@
|
||||
//! 互联响应的有界校验、能力授权与时间有效性解释
|
||||
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::account::{OpaqueProviderData, UpstreamInfoFailure};
|
||||
use gateway_core::account::{OpaqueProviderData, UpstreamConnection, UpstreamInfoFailure};
|
||||
use gateway_protocol::peer::{
|
||||
MAX_ACCOUNTS, MAX_MODELS, MAX_QUOTA_WINDOWS, OBJECT, PeerLink, PeerPoolDisclosure,
|
||||
SCHEMA_VERSION,
|
||||
MAX_ACCOUNTS, MAX_MODELS, MAX_QUOTA_WINDOWS, OBJECT, PeerKeyUsage, PeerLink, PeerPool,
|
||||
PeerQuota, SCHEMA_VERSION,
|
||||
};
|
||||
use serde::Deserialize as _;
|
||||
|
||||
pub(super) const MAX_AGE: Duration = Duration::from_secs(30 * 60);
|
||||
pub(super) const FRESH_FOR: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
// 不支持协议是连接的识别结果,复用观测存储与版本保护,不伪造上游额度快照
|
||||
pub(super) fn unsupported(connection: &UpstreamConnection) -> OpaqueProviderData {
|
||||
OpaqueProviderData::new(serde_json::Map::from_iter([
|
||||
("object".into(), "cpr.peer.unsupported".into()),
|
||||
("schema_version".into(), 1.into()),
|
||||
(
|
||||
"connection_revision".into(),
|
||||
connection.revision.clone().into(),
|
||||
),
|
||||
]))
|
||||
}
|
||||
|
||||
// 授权撤销或共享关闭后只保存连接与失败原因,丢弃全部共享数据
|
||||
pub(super) fn unavailable(
|
||||
connection: &UpstreamConnection,
|
||||
reason: UpstreamInfoFailure,
|
||||
) -> OpaqueProviderData {
|
||||
OpaqueProviderData::new(serde_json::Map::from_iter([
|
||||
("object".into(), "cpr.peer.unavailable".into()),
|
||||
("schema_version".into(), 1.into()),
|
||||
(
|
||||
"connection_revision".into(),
|
||||
connection.revision.clone().into(),
|
||||
),
|
||||
("reason".into(), reason.as_str().into()),
|
||||
]))
|
||||
}
|
||||
|
||||
pub(super) fn stored_failure(
|
||||
data: &OpaqueProviderData,
|
||||
connection: &UpstreamConnection,
|
||||
) -> Option<UpstreamInfoFailure> {
|
||||
let fields = data.expose_to_provider();
|
||||
if fields.get("schema_version")?.as_u64()? != 1
|
||||
|| fields.get("connection_revision")?.as_str()? != connection.revision
|
||||
{
|
||||
return None;
|
||||
}
|
||||
match fields.get("object")?.as_str()? {
|
||||
"cpr.peer.unsupported" => Some(UpstreamInfoFailure::Unsupported),
|
||||
"cpr.peer.unavailable" => match fields.get("reason")?.as_str()? {
|
||||
"unauthorized" => Some(UpstreamInfoFailure::Unauthorized),
|
||||
"sharing_disabled" => Some(UpstreamInfoFailure::SharingDisabled),
|
||||
"unsupported" => Some(UpstreamInfoFailure::Unsupported),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn timestamp(value: &str) -> Option<SystemTime> {
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.ok()
|
||||
@@ -26,10 +79,12 @@ pub(super) fn datetime(value: &str) -> Option<DateTime<Utc>> {
|
||||
}
|
||||
|
||||
pub(super) fn fresh(observed: &str, now: SystemTime) -> bool {
|
||||
timestamp(observed).is_some_and(|observed| {
|
||||
observed <= now + Duration::from_secs(60)
|
||||
&& now.duration_since(observed).unwrap_or_default() <= FRESH_FOR
|
||||
})
|
||||
timestamp(observed).is_some_and(|observed| fresh_at(observed, now))
|
||||
}
|
||||
|
||||
pub(super) fn fresh_at(observed: SystemTime, now: SystemTime) -> bool {
|
||||
observed <= now + Duration::from_secs(60)
|
||||
&& now.duration_since(observed).unwrap_or_default() <= FRESH_FOR
|
||||
}
|
||||
|
||||
pub(super) fn component_fresh(
|
||||
@@ -51,14 +106,15 @@ pub(super) fn component_fresh(
|
||||
}
|
||||
|
||||
pub(super) fn decode(data: &OpaqueProviderData) -> Option<PeerLink> {
|
||||
PeerLink::deserialize(
|
||||
let peer = PeerLink::deserialize(
|
||||
serde::de::value::MapDeserializer::<_, serde_json::Error>::new(
|
||||
data.expose_to_provider()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.as_str(), value)),
|
||||
),
|
||||
)
|
||||
.ok()
|
||||
.ok()?;
|
||||
(peer.object == OBJECT && peer.schema_version == SCHEMA_VERSION).then_some(peer)
|
||||
}
|
||||
|
||||
pub(super) fn parse(
|
||||
@@ -67,11 +123,17 @@ pub(super) fn parse(
|
||||
) -> Result<OpaqueProviderData, UpstreamInfoFailure> {
|
||||
let mut peer: PeerLink =
|
||||
serde_json::from_slice(bytes).map_err(|_| UpstreamInfoFailure::InvalidResponse)?;
|
||||
let valid_text = |value: &str, max: usize| {
|
||||
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
|
||||
};
|
||||
let observed = timestamp(&peer.observed_at).ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let expires = timestamp(&peer.expires_at).ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let accounts = peer
|
||||
.quota
|
||||
.as_ref()
|
||||
.map_or(&[][..], |quota| quota.accounts.as_slice());
|
||||
let budget_windows = peer
|
||||
.usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.data.as_ref())
|
||||
.map_or(0, |usage| usage.windows.len());
|
||||
if peer.object != OBJECT
|
||||
|| peer.schema_version != SCHEMA_VERSION
|
||||
|| observed > now + Duration::from_secs(60)
|
||||
@@ -79,125 +141,42 @@ pub(super) fn parse(
|
||||
|| expires <= observed
|
||||
|| expires > observed + MAX_AGE
|
||||
|| peer.capabilities.len() > 32
|
||||
|| peer
|
||||
.quota
|
||||
.accounts
|
||||
|| accounts
|
||||
.iter()
|
||||
.map(|account| account.windows.len())
|
||||
.sum::<usize>()
|
||||
> MAX_QUOTA_WINDOWS
|
||||
|| peer.quota.accounts.len() > MAX_ACCOUNTS
|
||||
|| budget_windows > MAX_QUOTA_WINDOWS
|
||||
|| accounts.len() > MAX_ACCOUNTS
|
||||
|| peer.models.len() > MAX_MODELS
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
// 未授权能力不能借同名数据块越过逐层公开边界
|
||||
if !peer.capabilities.iter().any(|value| value == "key_usage") {
|
||||
peer.usage.data = None;
|
||||
let disclosed = |capability| peer.capabilities.iter().any(|value| value == capability);
|
||||
if !disclosed("key_limits") {
|
||||
peer.key_limits = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "pool_summary")
|
||||
{
|
||||
peer.pool = Default::default();
|
||||
if !disclosed("key_usage") {
|
||||
peer.usage = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "quota_windows")
|
||||
{
|
||||
peer.quota = Default::default();
|
||||
if !disclosed("pool_summary") {
|
||||
peer.pool = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "model_availability")
|
||||
{
|
||||
if !disclosed("quota_windows") {
|
||||
peer.quota = None;
|
||||
}
|
||||
if !disclosed("model_availability") {
|
||||
peer.models.clear();
|
||||
}
|
||||
if peer.pool.disclosure == PeerPoolDisclosure::Summary {
|
||||
let count = peer
|
||||
.pool
|
||||
.account_count
|
||||
.ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let states = peer
|
||||
.pool
|
||||
.states
|
||||
.as_ref()
|
||||
.ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let state_total = [
|
||||
states.ready,
|
||||
states.cooldown,
|
||||
states.exhausted,
|
||||
states.disabled,
|
||||
states.error,
|
||||
states.unknown,
|
||||
]
|
||||
.into_iter()
|
||||
.map(u64::from)
|
||||
.sum::<u64>();
|
||||
if u64::from(count) > MAX_ACCOUNTS as u64 || state_total != u64::from(count) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(plans) = &peer.pool.plans
|
||||
&& (plans.len() > MAX_ACCOUNTS
|
||||
|| plans
|
||||
.iter()
|
||||
.any(|plan| !valid_text(&plan.name, 256) || plan.count > count)
|
||||
|| plans.iter().map(|plan| u64::from(plan.count)).sum::<u64>() > u64::from(count))
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(pool) = &peer.pool {
|
||||
validate_pool(pool)?;
|
||||
}
|
||||
if let Some(usage) = &peer.usage.data {
|
||||
if !valid_text(&usage.unit, 16) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
for window in [&usage.daily, &usage.weekly] {
|
||||
if [
|
||||
Some(&window.used),
|
||||
window.total.as_ref(),
|
||||
window.remaining.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| decimal(value).is_none())
|
||||
|| window
|
||||
.resets_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
if let Some(usage) = peer.usage.as_ref().and_then(|usage| usage.data.as_ref()) {
|
||||
validate_usage(usage)?;
|
||||
}
|
||||
for account in &peer.quota.accounts {
|
||||
if !valid_text(&account.id, 256)
|
||||
|| account.models.len() > MAX_MODELS
|
||||
|| account.windows.iter().any(|window| {
|
||||
!valid_text(&window.key, 256)
|
||||
|| !valid_text(&window.label, 256)
|
||||
|| !valid_text(&window.group, 128)
|
||||
|| window
|
||||
.limit_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.limit_name
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.used_percent
|
||||
.is_some_and(|value| !value.is_finite() || !(0.0..=100.0).contains(&value))
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
})
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(quota) = &peer.quota {
|
||||
validate_quota(quota)?;
|
||||
}
|
||||
for model in &peer.models {
|
||||
if !valid_text(&model.model, 256)
|
||||
@@ -219,6 +198,113 @@ pub(super) fn parse(
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_text(value: &str, max: usize) -> bool {
|
||||
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
|
||||
}
|
||||
|
||||
fn validate_pool(pool: &PeerPool) -> Result<(), UpstreamInfoFailure> {
|
||||
let count = pool.account_count;
|
||||
let states = &pool.states;
|
||||
let state_total = [
|
||||
states.ready,
|
||||
states.cooldown,
|
||||
states.exhausted,
|
||||
states.disabled,
|
||||
states.error,
|
||||
states.unknown,
|
||||
]
|
||||
.into_iter()
|
||||
.map(u64::from)
|
||||
.sum::<u64>();
|
||||
if u64::from(count) > MAX_ACCOUNTS as u64 || state_total != u64::from(count) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(plans) = &pool.plans
|
||||
&& (plans.len() > MAX_ACCOUNTS
|
||||
|| plans
|
||||
.iter()
|
||||
.any(|plan| !valid_text(&plan.name, 256) || plan.count > count)
|
||||
|| plans.iter().map(|plan| u64::from(plan.count)).sum::<u64>() > u64::from(count))
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_usage(usage: &PeerKeyUsage) -> Result<(), UpstreamInfoFailure> {
|
||||
if usage
|
||||
.unit
|
||||
.as_deref()
|
||||
.is_some_and(|unit| !valid_text(unit, 16))
|
||||
|| !usage.windows.is_empty() && usage.unit.is_none()
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
let mut keys = HashSet::new();
|
||||
for window in &usage.windows {
|
||||
if !valid_text(&window.key, 256)
|
||||
|| !keys.insert(&window.key)
|
||||
|| window
|
||||
.label
|
||||
.as_deref()
|
||||
.is_some_and(|label| !valid_text(label, 256))
|
||||
|| window.window_seconds == Some(0)
|
||||
|| [
|
||||
Some(&window.used),
|
||||
window.total.as_ref(),
|
||||
window.remaining.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| decimal(value).is_none())
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_quota(quota: &PeerQuota) -> Result<(), UpstreamInfoFailure> {
|
||||
for account in "a.accounts {
|
||||
if !valid_text(&account.id, 256)
|
||||
|| account.models.len() > MAX_MODELS
|
||||
|| account.windows.iter().any(|window| {
|
||||
!valid_text(&window.key, 256)
|
||||
|| window
|
||||
.label
|
||||
.as_deref()
|
||||
.is_some_and(|label| !valid_text(label, 256))
|
||||
|| window
|
||||
.group
|
||||
.as_deref()
|
||||
.is_some_and(|group| !valid_text(group, 128))
|
||||
|| window
|
||||
.limit_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.limit_name
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.used_percent
|
||||
.is_some_and(|value| !value.is_finite() || !(0.0..=100.0).contains(&value))
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
})
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn decimal(value: &str) -> Option<f64> {
|
||||
if value.is_empty()
|
||||
|| value.len() > 64
|
||||
|
||||
@@ -217,7 +217,7 @@ impl PluginsService {
|
||||
artifact_sha256: metadata.sha256.clone(),
|
||||
enabled: !configuration_required && compatible,
|
||||
configuration: configuration.clone(),
|
||||
secrets: None,
|
||||
secrets: Default::default(),
|
||||
bindings: bindings.clone(),
|
||||
};
|
||||
match self.configure_instance(None, input, context).await {
|
||||
|
||||
@@ -28,17 +28,16 @@ pub(super) fn same_creation_request(
|
||||
existing: &PluginInstance,
|
||||
input: &ConfigurePluginInstance,
|
||||
) -> bool {
|
||||
let same_secrets = input
|
||||
let mut requested_secrets = input
|
||||
.secrets
|
||||
.as_ref()
|
||||
.map_or(existing.secrets.is_empty(), |secrets| {
|
||||
secrets.len() == existing.secrets.len()
|
||||
&& secrets.iter().all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
})
|
||||
.iter()
|
||||
.filter_map(|(key, value)| value.as_ref().map(|value| (key, value)));
|
||||
let same_secrets = requested_secrets.clone().count() == existing.secrets.len()
|
||||
&& requested_secrets.all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
});
|
||||
existing.name == input.name.trim()
|
||||
&& existing.artifact_sha256 == input.artifact_sha256
|
||||
|
||||
@@ -158,7 +158,6 @@ impl PluginsService {
|
||||
}
|
||||
let saved = if let Some(previous) = &existing
|
||||
&& previous.artifact_sha256 != input.artifact_sha256
|
||||
&& input.secrets.is_none()
|
||||
{
|
||||
self.store
|
||||
.load_version_configuration(&previous.id, &input.artifact_sha256)
|
||||
@@ -167,6 +166,21 @@ impl PluginsService {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut secrets = saved.map(|saved| saved.secrets).unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
});
|
||||
for (name, value) in input.secrets {
|
||||
match value {
|
||||
Some(value) => {
|
||||
secrets.insert(name, value);
|
||||
}
|
||||
None => {
|
||||
secrets.remove(&name);
|
||||
}
|
||||
}
|
||||
}
|
||||
let instance = PluginInstance {
|
||||
id: existing.as_ref().map_or_else(
|
||||
|| {
|
||||
@@ -182,14 +196,7 @@ impl PluginsService {
|
||||
enabled: input.enabled,
|
||||
trusted_process: true,
|
||||
configuration: input.configuration,
|
||||
secrets: input
|
||||
.secrets
|
||||
.or_else(|| saved.map(|saved| saved.secrets))
|
||||
.unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
}),
|
||||
secrets,
|
||||
bindings: input.bindings,
|
||||
revision: candidate_revision,
|
||||
};
|
||||
|
||||
@@ -366,6 +366,7 @@ fn target_environment_matches(target: &str) -> bool {
|
||||
match target {
|
||||
"x86_64-unknown-linux-gnu" | "aarch64-unknown-linux-gnu" => cfg!(target_env = "gnu"),
|
||||
"aarch64-apple-darwin" => cfg!(target_os = "macos"),
|
||||
"x86_64-pc-windows-msvc" | "aarch64-pc-windows-msvc" => cfg!(target_env = "msvc"),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -392,4 +393,18 @@ const SUPPORTED_PLATFORMS: &[Platform] = &[
|
||||
runtime_os: "macos",
|
||||
runtime_architecture: "aarch64",
|
||||
},
|
||||
Platform {
|
||||
release_os: "windows",
|
||||
release_architecture: "amd64",
|
||||
target: "x86_64-pc-windows-msvc",
|
||||
runtime_os: "windows",
|
||||
runtime_architecture: "x86_64",
|
||||
},
|
||||
Platform {
|
||||
release_os: "windows",
|
||||
release_architecture: "arm64",
|
||||
target: "aarch64-pc-windows-msvc",
|
||||
runtime_os: "windows",
|
||||
runtime_architecture: "aarch64",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -69,7 +69,7 @@ impl PluginsService {
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: existing.enabled,
|
||||
configuration: saved.configuration,
|
||||
secrets: Some(saved.secrets),
|
||||
secrets: Default::default(),
|
||||
bindings: saved.bindings,
|
||||
};
|
||||
// 沿用同一份快照的 CAS 和状态迁移流程,不能在检查目标后重新读取并覆盖并发配置
|
||||
@@ -202,7 +202,7 @@ impl PluginsService {
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: existing.enabled,
|
||||
configuration: configuration.configuration,
|
||||
secrets: Some(configuration.secrets),
|
||||
secrets: Default::default(),
|
||||
bindings: configuration.bindings,
|
||||
};
|
||||
// 同一次读取的 CAS 覆盖草稿计算、准备与提交,不能覆盖并发设置
|
||||
|
||||
@@ -361,9 +361,10 @@ impl ProxiesService for DefaultProxiesService {
|
||||
proxy: Option<&OutboundProxy>,
|
||||
detect_location: bool,
|
||||
) -> Result<ProxyTestResult, AdminError> {
|
||||
let _permit = self.test_slots.try_acquire().map_err(|_| {
|
||||
AdminError::new(AdminErrorKind::RateLimited, "代理测试繁忙,请稍后重试")
|
||||
})?;
|
||||
let _permit = self
|
||||
.test_slots
|
||||
.try_acquire()
|
||||
.map_err(|_| AdminError::new(AdminErrorKind::RateLimited, "代理测试已达并发上限"))?;
|
||||
Ok(self.probe.test(proxy, detect_location).await)
|
||||
}
|
||||
|
||||
@@ -374,9 +375,10 @@ impl ProxiesService for DefaultProxiesService {
|
||||
detect_location: bool,
|
||||
context: &MutationContext,
|
||||
) -> Result<ProxyRecord, AdminError> {
|
||||
let _permit = self.test_slots.try_acquire().map_err(|_| {
|
||||
AdminError::new(AdminErrorKind::RateLimited, "代理测试繁忙,请稍后重试")
|
||||
})?;
|
||||
let _permit = self
|
||||
.test_slots
|
||||
.try_acquire()
|
||||
.map_err(|_| AdminError::new(AdminErrorKind::RateLimited, "代理测试已达并发上限"))?;
|
||||
let record = self
|
||||
.store
|
||||
.get(id)
|
||||
|
||||
@@ -252,7 +252,7 @@ impl SettingsService for DefaultSettingsService {
|
||||
.is_some_and(|models| models.contains_key(model))
|
||||
})
|
||||
{
|
||||
return Err(AdminError::invalid("内置价目不能删除,仅支持人工覆盖"));
|
||||
return Err(AdminError::invalid("内置价目只能覆盖,不能删除"));
|
||||
}
|
||||
let defaults = gateway_core::metering::merge_pricing(catalog.defaults, &catalog.synced);
|
||||
let defaults = defaults.get(&command.provider);
|
||||
@@ -332,24 +332,35 @@ impl SettingsService for DefaultSettingsService {
|
||||
configuration: Option<&gateway_core::account::OpaqueProviderData>,
|
||||
) -> Result<gateway_core::account::OpaqueProviderData, AdminError> {
|
||||
let profile_provider = self.profile_provider(provider)?;
|
||||
let global;
|
||||
let (configuration, source) = if let Some(configuration) = configuration {
|
||||
(configuration, "override")
|
||||
let independent = configuration.filter(|key| {
|
||||
gateway_core::settings::request_profiles::inherited_fields(key).is_none()
|
||||
});
|
||||
let preview = if let Some(key) = independent {
|
||||
profile_provider.preview_key_client_profile(key, None)
|
||||
} else {
|
||||
global = self
|
||||
let global = self
|
||||
.load()
|
||||
.await?
|
||||
.client_profile(provider)
|
||||
.cloned()
|
||||
.or_else(|| profile_provider.default_client_profile())
|
||||
.ok_or_else(|| AdminError::internal("通用客户端身份尚未初始化"))?;
|
||||
(&global, "global")
|
||||
};
|
||||
let mut preview = profile_provider
|
||||
.preview_client_profile(configuration)
|
||||
.map_err(|error| super::map_provider_error(error, "client profile"))?
|
||||
.into_inner();
|
||||
preview.insert("source".to_owned(), serde_json::Value::from(source));
|
||||
if let Some(key) = configuration {
|
||||
profile_provider.preview_key_client_profile(key, Some(&global))
|
||||
} else {
|
||||
profile_provider.preview_client_profile(&global)
|
||||
}
|
||||
}
|
||||
.map_err(|error| super::map_provider_error(error, "client profile"))?;
|
||||
let mut preview = preview.into_inner();
|
||||
preview.insert(
|
||||
"source".to_owned(),
|
||||
serde_json::Value::from(if independent.is_some() {
|
||||
"override"
|
||||
} else {
|
||||
"global"
|
||||
}),
|
||||
);
|
||||
Ok(gateway_core::account::OpaqueProviderData::new(preview))
|
||||
}
|
||||
|
||||
@@ -419,7 +430,7 @@ impl SettingsService for DefaultSettingsService {
|
||||
crate::ports::store::AdminStoreErrorKind::Conflict
|
||||
| crate::ports::store::AdminStoreErrorKind::StaleRevision
|
||||
) {
|
||||
AdminError::conflict("运行设置已被其他调用修改,请重新读取并确认后提交")
|
||||
AdminError::conflict("运行设置已变更,请刷新核对后提交")
|
||||
} else {
|
||||
map_store_error(error, "runtime settings")
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::plugin_update::{ConfirmedPluginRestart, PluginSystemUpdatePreflight};
|
||||
use super::system_preflight::{ConfirmedPluginRestart, PluginSystemUpdatePreflight};
|
||||
use crate::model::{MutationContext, system::SystemRestartPlan};
|
||||
use async_trait::async_trait;
|
||||
|
||||
|
||||
@@ -15,6 +15,11 @@ use std::sync::Arc;
|
||||
|
||||
#[async_trait]
|
||||
pub trait UsageDataService: Send + Sync {
|
||||
async fn open_read(
|
||||
&self,
|
||||
query: UsageReadQuery,
|
||||
cancellation: gateway_core::lifecycle::CancellationToken,
|
||||
) -> UsageDataResult<Box<dyn UsageReadSession>>;
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
@@ -58,6 +63,21 @@ impl DefaultUsageDataService {
|
||||
|
||||
#[async_trait]
|
||||
impl UsageDataService for DefaultUsageDataService {
|
||||
async fn open_read(
|
||||
&self,
|
||||
query: UsageReadQuery,
|
||||
cancellation: gateway_core::lifecycle::CancellationToken,
|
||||
) -> UsageDataResult<Box<dyn UsageReadSession>> {
|
||||
let filter = query.filter();
|
||||
validation::query(&filter.records, None, None)?;
|
||||
if let Some(group) = &filter.routing_group_ref {
|
||||
validation::text(group, 128)?;
|
||||
}
|
||||
if !(1..=USAGE_READ_MAX_BATCH_ROWS).contains(&query.batch_size()) {
|
||||
return Err(UsageDataError::invalid("读取批次条数应为 1 到 1024"));
|
||||
}
|
||||
self.store.open_read(query, cancellation).await
|
||||
}
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
|
||||
@@ -7,7 +7,6 @@ use gateway_core::account::OpaqueProviderData;
|
||||
|
||||
use gateway_admin::model::{
|
||||
PageSize, Revision,
|
||||
accounts::AccountUsage,
|
||||
auth::LoginCommand,
|
||||
client_keys::ClientKeyPageSize,
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
@@ -146,57 +145,6 @@ fn representative_quota_should_prefer_account_wide_window_over_model_specific_wi
|
||||
assert_eq!(quota.representative_used_percent(), Some(5.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_window_should_not_use_daily_rolling_usage_for_weekly_statistics() {
|
||||
let usage = AccountUsage {
|
||||
account_id: "acct_xai".to_owned(),
|
||||
request_count: 5,
|
||||
success_count: 5,
|
||||
input_tokens: Some(100),
|
||||
output_tokens: Some(20),
|
||||
cached_input_tokens: Some(80),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
image_request_count: 0,
|
||||
image_request_failed_count: 0,
|
||||
total_tokens: Some(120),
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
last_used_at: Some(Utc::now()),
|
||||
request_buckets: Vec::new(),
|
||||
models: Vec::new(),
|
||||
};
|
||||
let quota = ProviderQuota {
|
||||
upstream: None,
|
||||
credits: None,
|
||||
plan_type: None,
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
upstream: None,
|
||||
key: "free-rolling-24h".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "日限额".to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(86_400),
|
||||
used_percent: None,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: Some(usage),
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
assert_eq!(quota.usage_window(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_quota_should_project_full_usage_to_only_the_representative_window() {
|
||||
let mut quota = ProviderQuota {
|
||||
|
||||
@@ -65,6 +65,36 @@ fn usage_window_should_not_fallback_to_short_daily_or_incomplete_windows() {
|
||||
assert_eq!(quota.usage_window(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_window_should_preserve_provider_rolling_daily_statistics() {
|
||||
let mut rolling = usage_window("rolling-day", "shortTerm", 86_400);
|
||||
rolling.reset_at = None;
|
||||
rolling.used_percent = None;
|
||||
let mut quota = usage_quota(vec![rolling.clone()]);
|
||||
assert_eq!(
|
||||
selected_usage_window("a),
|
||||
Some(("rolling-day", AccountUsagePeriod::Daily))
|
||||
);
|
||||
|
||||
quota.limit_reached = true;
|
||||
quota.apply_limit_reached_display();
|
||||
assert_eq!(
|
||||
selected_usage_window("a),
|
||||
Some(("rolling-day", AccountUsagePeriod::Daily))
|
||||
);
|
||||
|
||||
quota
|
||||
.windows
|
||||
.push(usage_window("week", "shortTerm", 7 * 86_400));
|
||||
assert_eq!(
|
||||
selected_usage_window("a),
|
||||
Some(("week", AccountUsagePeriod::Weekly))
|
||||
);
|
||||
|
||||
rolling.local_usage_attribution = QuotaLocalUsageAttribution::Unavailable;
|
||||
assert_eq!(usage_quota(vec![rolling]).usage_window(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_window_should_preserve_provider_order_for_equal_periods() {
|
||||
let quota = usage_quota(vec![
|
||||
|
||||
@@ -162,7 +162,10 @@ fn source_selection_keeps_provider_order_with_the_maximum_window_count() {
|
||||
fn short_or_model_specific_windows_are_not_account_capacity() {
|
||||
let mut model = window("model", 7);
|
||||
model.local_usage_attribution = QuotaLocalUsageAttribution::Unavailable;
|
||||
for item in forecast("a(vec![window("day", 1), model])) {
|
||||
let mut rolling = window("rolling-day", 1);
|
||||
rolling.reset_at = None;
|
||||
rolling.used_percent = None;
|
||||
for item in forecast("a(vec![window("day", 1), rolling, model])) {
|
||||
assert!(item.source.is_none());
|
||||
assert!(item.unavailable_reason.is_some());
|
||||
assert!(item.estimated_tokens.is_none());
|
||||
|
||||
@@ -373,6 +373,7 @@ fn group_record() -> AccountGroupRecord {
|
||||
|
||||
fn member(account_id: &str, total_slots: u64) -> AccountGroupMemberFact {
|
||||
AccountGroupMemberFact {
|
||||
inherited_version: None,
|
||||
group_id: group_id(),
|
||||
account_id: account_id.to_owned(),
|
||||
status: AccountStatusFacts {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
mod reset_credit_automation;
|
||||
|
||||
use gateway_core::engine::probe::AccountProbeTarget;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex},
|
||||
@@ -1166,6 +1167,7 @@ impl SettingsStore for StaticSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
@@ -1334,7 +1336,9 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
AccountProbeTarget::ConnectionTest(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
@@ -1371,7 +1375,9 @@ async fn connection_test_rate_limited_probe_returns_provider_failure() {
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
AccountProbeTarget::ConnectionTest(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
@@ -1404,7 +1410,9 @@ async fn connection_test_should_preserve_disabled_account_status() {
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
AccountProbeTarget::ConnectionTest(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
@@ -2685,12 +2693,15 @@ async fn peer_account_usage_should_follow_budget_windows_without_borrowing_pool_
|
||||
for longest in [86_400, 604_800] {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.concurrency_limit = None;
|
||||
account.authentication_kind = "api_key".to_owned();
|
||||
account.created_at = Utc::now() - TimeDelta::days(60);
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let reset_at = Utc::now() + TimeDelta::hours(1);
|
||||
let mut quota = empty_quota();
|
||||
quota.upstream = Some(ProviderUpstreamInfo {
|
||||
max_concurrency: Some(7),
|
||||
requests_per_minute: Some(60),
|
||||
discovered: true,
|
||||
observed_at: Some(Utc::now()),
|
||||
stale: false,
|
||||
@@ -2761,6 +2772,8 @@ async fn peer_account_usage_should_follow_budget_windows_without_borrowing_pool_
|
||||
page.items[0].usage.as_ref().unwrap().total_tokens,
|
||||
Some(1_234)
|
||||
);
|
||||
assert_eq!(page.items[0].capacity.total_slots, Some(7));
|
||||
assert_eq!(page.items[0].account.concurrency_limit, None);
|
||||
let queries = store.quota_window_queries();
|
||||
assert_eq!(queries.len(), window_usage.len());
|
||||
for (query, expected) in queries.iter().zip(&window_usage) {
|
||||
|
||||
@@ -143,7 +143,7 @@ impl UpstreamInfoSource for PendingPeer {
|
||||
fn signal(
|
||||
&self,
|
||||
_: &ProviderAccount,
|
||||
_: &str,
|
||||
_: Option<&str>,
|
||||
_: std::time::SystemTime,
|
||||
) -> Option<UpstreamSchedulingSignal> {
|
||||
None
|
||||
@@ -152,6 +152,14 @@ impl UpstreamInfoSource for PendingPeer {
|
||||
|
||||
#[async_trait]
|
||||
impl PeerUpstreamService for PendingPeer {
|
||||
fn observed_concurrency_limit(
|
||||
&self,
|
||||
_: &gateway_core::account::QuotaObservationVersion,
|
||||
_: std::time::SystemTime,
|
||||
) -> Option<gateway_core::account::AccountConcurrencyLimit> {
|
||||
None
|
||||
}
|
||||
|
||||
async fn account_changed(&self, id: &ProviderAccountId) -> Result<(), StoreError> {
|
||||
self.changed.lock().unwrap().push(id.as_str().into());
|
||||
Ok(())
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 账号保护恢复编排:到期探测、快照参数与失败顺延
|
||||
|
||||
use gateway_core::engine::probe::AccountProbeTarget;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -41,6 +42,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
@@ -241,6 +243,10 @@ impl AccountProbe for FailingProbe {
|
||||
request: AccountProbeRequest,
|
||||
_: Option<Arc<gateway_core::routing::RuntimeSnapshot>>,
|
||||
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
assert!(matches!(
|
||||
request.target,
|
||||
AccountProbeTarget::FreezeRecovery(_)
|
||||
));
|
||||
assert_eq!(request.upstream_model.as_str(), "snapshot-model");
|
||||
Box::pin(async {
|
||||
Err(AccountProbeError::new(
|
||||
|
||||
@@ -11,11 +11,11 @@ mod freeze_recovery;
|
||||
mod import_tasks;
|
||||
mod observability;
|
||||
mod peer_link;
|
||||
mod plugin_update;
|
||||
mod plugins;
|
||||
mod proxies;
|
||||
mod settings;
|
||||
mod system;
|
||||
mod system_preflight;
|
||||
mod usage_data;
|
||||
|
||||
use std::{
|
||||
|
||||
@@ -448,6 +448,7 @@ async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
store.replace_runtime_slots(Some(DashboardRuntimeSlots {
|
||||
inherited_versions: Vec::new(),
|
||||
inherited_accounts: 2,
|
||||
overridden_slots: 5,
|
||||
used_slots: Some(2),
|
||||
@@ -476,6 +477,7 @@ async fn dashboard_capacity_distinguishes_unlimited_inheritance_finite_overrides
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
store.replace_runtime_slots(Some(DashboardRuntimeSlots {
|
||||
inherited_versions: Vec::new(),
|
||||
inherited_accounts,
|
||||
overridden_slots,
|
||||
used_slots: Some(2),
|
||||
@@ -1212,7 +1214,7 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
range: query.range,
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
runtime_slots: *self.runtime_slots.lock().expect("runtime slots"),
|
||||
runtime_slots: self.runtime_slots.lock().expect("runtime slots").clone(),
|
||||
trend: self.trend.lock().expect("trend").clone(),
|
||||
account_usage: self.account_usage.lock().expect("account usage").clone(),
|
||||
recent_requests: Vec::new(),
|
||||
@@ -1384,6 +1386,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
@@ -1478,8 +1481,14 @@ fn total_record(
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
provider_account_upstream_peer: None,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
error_kind: None,
|
||||
websocket_close_code: None,
|
||||
continuation_unavailable_reason: None,
|
||||
upstream_connection_exit_reason: None,
|
||||
recovered: false,
|
||||
client_status_code: None,
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Succeeded,
|
||||
client_api_key_name: Some("Production".to_owned()),
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 公共互联按 Provider 声明发现连接,不将认证种类或调度观测绑在 OpenAI 上
|
||||
|
||||
mod refresh;
|
||||
|
||||
use super::super::UnavailableProvider;
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
@@ -239,7 +241,7 @@ async fn discovery_uses_provider_capabilities_and_publishes_neutral_signals() {
|
||||
let reader = UpstreamInfoReader::default();
|
||||
assert!(
|
||||
reader
|
||||
.signal(&initial_accounts[0], "model", SystemTime::now())
|
||||
.signal(&initial_accounts[0], Some("model"), SystemTime::now())
|
||||
.is_none()
|
||||
);
|
||||
let source = bundle.scheduling_source();
|
||||
@@ -271,7 +273,11 @@ async fn discovery_uses_provider_capabilities_and_publishes_neutral_signals() {
|
||||
);
|
||||
for account in &initial_accounts[..2] {
|
||||
if Some(account.provider().as_str()) == failed_provider {
|
||||
assert!(reader.signal(account, "model", SystemTime::now()).is_none());
|
||||
assert!(
|
||||
reader
|
||||
.signal(account, Some("model"), SystemTime::now())
|
||||
.is_none()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let observed = bundle
|
||||
@@ -286,21 +292,21 @@ async fn discovery_uses_provider_capabilities_and_publishes_neutral_signals() {
|
||||
);
|
||||
assert!(
|
||||
reader
|
||||
.signal(account, "model", SystemTime::now())
|
||||
.signal(account, Some("model"), SystemTime::now())
|
||||
.unwrap()
|
||||
.exhausted
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
reader
|
||||
.signal(&initial_accounts[2], "model", SystemTime::now())
|
||||
.signal(&initial_accounts[2], Some("model"), SystemTime::now())
|
||||
.is_none()
|
||||
);
|
||||
drop(worker);
|
||||
drop(bundle);
|
||||
assert!(
|
||||
reader
|
||||
.signal(&initial_accounts[0], "model", SystemTime::now())
|
||||
.signal(&initial_accounts[0], Some("model"), SystemTime::now())
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
@@ -330,7 +336,7 @@ async fn account_edits_preserve_observation_and_refresh_budget_until_connection_
|
||||
assert!(
|
||||
bundle
|
||||
.scheduling_source()
|
||||
.signal(&edited, "model", SystemTime::now())
|
||||
.signal(&edited, Some("model"), SystemTime::now())
|
||||
.is_some()
|
||||
);
|
||||
let after = service.refresh_quota(edited.id()).await.unwrap();
|
||||
@@ -345,7 +351,7 @@ async fn account_edits_preserve_observation_and_refresh_budget_until_connection_
|
||||
assert!(
|
||||
bundle
|
||||
.scheduling_source()
|
||||
.signal(&reconnected, "model", SystemTime::now())
|
||||
.signal(&reconnected, Some("model"), SystemTime::now())
|
||||
.is_none()
|
||||
);
|
||||
service.refresh_quota(reconnected.id()).await.unwrap();
|
||||
@@ -390,14 +396,13 @@ impl PeerTransport for TestTransport {
|
||||
.push(account.id().as_str().to_owned());
|
||||
let now = Utc::now();
|
||||
Ok(serde_json::to_vec(&serde_json::json!({
|
||||
"object":"cpr.peer", "schema_version":1,
|
||||
"object":"cpr.peer", "schema_version":3,
|
||||
"observed_at":now.to_rfc3339(),
|
||||
"expires_at":(now + chrono::Duration::minutes(5)).to_rfc3339(),
|
||||
"capabilities":["key_usage"],
|
||||
"usage":{"status":"available","observed_at":now.to_rfc3339(),"data":{"unit":"USD","allowed":false,
|
||||
"daily":{"total":"1","used":"1","remaining":"0","resetsAt":(now + chrono::Duration::days(1)).to_rfc3339()},
|
||||
"weekly":{"total":"0","used":"0","remaining":"0"}}},
|
||||
"pool":{"disclosure":"none"}, "quota":{"disclosure":"none"}, "models":[]
|
||||
"windows":[{"key":"daily","total":"1","used":"1","remaining":"0","reset_at":(now + chrono::Duration::days(1)).to_rfc3339()}]}},
|
||||
"models":[]
|
||||
})).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
//! 互联发现结果的持久恢复、连接失效、手动重查与临时失败退避
|
||||
|
||||
use super::*;
|
||||
use gateway_admin::PeerUpstreamBundle;
|
||||
use gateway_core::task::WorkerRegistration;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unsupported_detection_survives_restart_until_manual_rediscovery() {
|
||||
let (mut bundle, accounts, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
// 入库后的首查与手动刷新使用同一入口,不支持协议也是可保存的正常发现结果
|
||||
let initial = bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert!(!initial.upstream.unwrap().discovered);
|
||||
let first_worker = worker(&mut bundle);
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&first_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
drop(first_worker);
|
||||
drop(bundle);
|
||||
|
||||
let mut restarted = initialize(accounts, transport.clone());
|
||||
let info = restarted
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap();
|
||||
assert!(!info.discovered);
|
||||
assert_eq!(info.last_error.as_deref(), Some("unsupported"));
|
||||
let resumed_worker = worker(&mut restarted);
|
||||
cycle(&resumed_worker).await;
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&resumed_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
let summary = restarted
|
||||
.service()
|
||||
.read_summary(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
summary.upstream.unwrap().last_error.as_deref(),
|
||||
Some("unsupported")
|
||||
);
|
||||
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
let discovered = restarted.service().refresh_quota(&id).await.unwrap();
|
||||
assert!(discovered.upstream.unwrap().discovered);
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
tokio::time::advance(Duration::from_secs(360)).await;
|
||||
cycle(&resumed_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unsupported_detection_keeps_manual_refresh_throttling() {
|
||||
for retry_after in [None, Some(Duration::from_secs(90))] {
|
||||
let (bundle, _, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
*transport.failure.lock().unwrap() = Some(PeerFetchFailure {
|
||||
kind: UpstreamInfoFailure::Unsupported,
|
||||
retry_after,
|
||||
});
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
let delay = retry_after.unwrap_or(Duration::from_secs(15));
|
||||
tokio::time::advance(delay - Duration::from_secs(1)).await;
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
assert!(
|
||||
bundle
|
||||
.service()
|
||||
.refresh_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap()
|
||||
.discovered
|
||||
);
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn ordinary_edits_keep_detection_but_proxy_and_credential_changes_reprobe() {
|
||||
let (mut bundle, accounts, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
let worker = worker(&mut bundle);
|
||||
// 无识别记录的存量账号由周期任务补查
|
||||
cycle(&worker).await;
|
||||
let initial = accounts.accounts.lock().unwrap()[0].clone();
|
||||
let edited = initial.clone().with_notes(Some("修改备注".into()));
|
||||
*accounts.accounts.lock().unwrap() = vec![edited.clone()];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
|
||||
let reconnected = edited.with_outbound_proxy(Some(
|
||||
OutboundProxy::parse("http://proxy.invalid:8080").unwrap(),
|
||||
));
|
||||
*accounts.accounts.lock().unwrap() = vec![reconnected.clone()];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
|
||||
let replaced = ProviderAccount::new(
|
||||
id.clone(),
|
||||
initial.provider().clone(),
|
||||
"new credential".into(),
|
||||
None,
|
||||
"peer_test".into(),
|
||||
CredentialRevision::new(2).unwrap(),
|
||||
None,
|
||||
)
|
||||
.with_outbound_proxy(reconnected.outbound_proxy().cloned())
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
*accounts.accounts.lock().unwrap() = vec![replaced];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn uncertain_discovery_failures_retry_without_becoming_unsupported() {
|
||||
for failure in [
|
||||
UpstreamInfoFailure::Unauthorized,
|
||||
UpstreamInfoFailure::Unavailable,
|
||||
UpstreamInfoFailure::RateLimited,
|
||||
UpstreamInfoFailure::InvalidResponse,
|
||||
] {
|
||||
let (mut bundle, _, transport, id) = fixture(failure);
|
||||
let worker = worker(&mut bundle);
|
||||
cycle(&worker).await;
|
||||
let info = bundle
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap();
|
||||
assert_eq!(info.last_error.as_deref(), Some(failure.as_str()));
|
||||
assert!(!info.discovered);
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
tokio::time::advance(Duration::from_secs(
|
||||
if failure == UpstreamInfoFailure::Unauthorized {
|
||||
1800
|
||||
} else {
|
||||
30
|
||||
},
|
||||
))
|
||||
.await;
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
assert!(
|
||||
bundle
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap()
|
||||
.discovered
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn known_peer_identity_changes_only_on_definitive_detection() {
|
||||
for failure in [
|
||||
UpstreamInfoFailure::Unauthorized,
|
||||
UpstreamInfoFailure::Unsupported,
|
||||
UpstreamInfoFailure::SharingDisabled,
|
||||
UpstreamInfoFailure::Unavailable,
|
||||
UpstreamInfoFailure::RateLimited,
|
||||
UpstreamInfoFailure::InvalidResponse,
|
||||
] {
|
||||
let (bundle, accounts, transport, id) = fixture(failure);
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
let before = bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert!(!before.windows.is_empty());
|
||||
tokio::time::advance(Duration::from_secs(15)).await;
|
||||
*transport.failure.lock().unwrap() = Some(failure.into());
|
||||
let result = bundle.service().refresh_quota(&id).await;
|
||||
let clears_identity = matches!(
|
||||
failure,
|
||||
UpstreamInfoFailure::Unsupported | UpstreamInfoFailure::SharingDisabled
|
||||
);
|
||||
assert_eq!(result.is_ok(), clears_identity);
|
||||
let clears_data = clears_identity || failure == UpstreamInfoFailure::Unauthorized;
|
||||
let quota = bundle.service().read_quota(&id).await.unwrap().unwrap();
|
||||
let info = quota.upstream.as_ref().unwrap();
|
||||
assert_eq!(info.discovered, !clears_identity);
|
||||
assert_eq!(info.last_error.as_deref(), Some(failure.as_str()));
|
||||
assert_eq!(quota.windows.is_empty(), clears_data);
|
||||
assert_eq!(info.observed_at.is_none(), clears_data);
|
||||
assert_eq!(
|
||||
bundle
|
||||
.service()
|
||||
.read_summary(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream,
|
||||
quota.upstream
|
||||
);
|
||||
let account = accounts.accounts.lock().unwrap()[0].clone();
|
||||
assert_eq!(
|
||||
bundle
|
||||
.scheduling_source()
|
||||
.signal(&account, Some("model"), SystemTime::now())
|
||||
.is_none(),
|
||||
clears_data
|
||||
);
|
||||
|
||||
if failure == UpstreamInfoFailure::Unsupported {
|
||||
// 已运行 v3.40.0 的节点可能持有这种不支持记录,读取时应直接显示为普通 API Key
|
||||
accounts.records.lock().unwrap().get_mut(&id).unwrap().quota =
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"object": "cpr.peer.unavailable",
|
||||
"schema_version": 1,
|
||||
"connection_revision": info.connection_revision,
|
||||
"reason": "unsupported",
|
||||
}))
|
||||
.unwrap();
|
||||
}
|
||||
drop(bundle);
|
||||
let restarted = initialize(accounts, transport);
|
||||
let restored = restarted.service().read_quota(&id).await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
restored.upstream.as_ref().unwrap().discovered,
|
||||
!clears_identity
|
||||
);
|
||||
assert_eq!(restored.windows.is_empty(), clears_data);
|
||||
if clears_data {
|
||||
assert_eq!(
|
||||
restored.upstream.as_ref().unwrap().last_error.as_deref(),
|
||||
Some(failure.as_str())
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
restarted
|
||||
.service()
|
||||
.read_summary(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream,
|
||||
restored.upstream
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn peer_detection_after_revocation_is_scoped_to_the_connection() {
|
||||
for failure in [
|
||||
UpstreamInfoFailure::Unauthorized,
|
||||
UpstreamInfoFailure::Unsupported,
|
||||
] {
|
||||
let (bundle, accounts, transport, id) = fixture(failure);
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(15)).await;
|
||||
*transport.failure.lock().unwrap() = Some(failure.into());
|
||||
let _result = bundle.service().refresh_quota(&id).await;
|
||||
let initial = accounts.accounts.lock().unwrap()[0].clone();
|
||||
let edited = initial.clone().with_notes(Some("保留连接身份".into()));
|
||||
*accounts.accounts.lock().unwrap() = vec![edited.clone()];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
assert_eq!(
|
||||
bundle
|
||||
.service()
|
||||
.read_summary(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap()
|
||||
.discovered,
|
||||
failure == UpstreamInfoFailure::Unauthorized
|
||||
);
|
||||
|
||||
// 地址和密钥修改推进凭据版本,出口变化则改变同一版本的连接身份
|
||||
for replacement in [
|
||||
edited.with_outbound_proxy(Some(
|
||||
OutboundProxy::parse("http://other.invalid:8080").unwrap(),
|
||||
)),
|
||||
ProviderAccount::new(
|
||||
id.clone(),
|
||||
initial.provider().clone(),
|
||||
"替换连接".into(),
|
||||
None,
|
||||
"peer_test".into(),
|
||||
CredentialRevision::new(2).unwrap(),
|
||||
None,
|
||||
)
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
] {
|
||||
*accounts.accounts.lock().unwrap() = vec![replacement];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
let detail = bundle.service().read_quota(&id).await.unwrap().unwrap();
|
||||
assert!(!detail.upstream.as_ref().unwrap().discovered);
|
||||
assert!(detail.upstream.as_ref().unwrap().last_error.is_none());
|
||||
assert_eq!(
|
||||
bundle
|
||||
.service()
|
||||
.read_summary(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream,
|
||||
detail.upstream
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn manual_rediscovery_does_not_wait_for_authorization_backoff_but_respects_retry_after() {
|
||||
for retry_after in [None, Some(Duration::from_secs(90))] {
|
||||
let (mut bundle, _, transport, id) = fixture(UpstreamInfoFailure::Unauthorized);
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(15)).await;
|
||||
*transport.failure.lock().unwrap() = Some(PeerFetchFailure {
|
||||
kind: UpstreamInfoFailure::Unauthorized,
|
||||
retry_after,
|
||||
});
|
||||
assert!(bundle.service().refresh_quota(&id).await.is_err());
|
||||
let worker = worker(&mut bundle);
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
let delay = retry_after.unwrap_or(Duration::from_secs(15));
|
||||
tokio::time::advance(delay - Duration::from_secs(1)).await;
|
||||
assert!(bundle.service().refresh_quota(&id).await.is_err());
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
let recovered = bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 3);
|
||||
assert!(recovered.upstream.as_ref().unwrap().discovered);
|
||||
assert!(recovered.upstream.as_ref().unwrap().last_error.is_none());
|
||||
assert!(!recovered.windows.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture(
|
||||
failure: UpstreamInfoFailure,
|
||||
) -> (
|
||||
PeerUpstreamBundle,
|
||||
Arc<TestAccounts>,
|
||||
Arc<DetectionTransport>,
|
||||
ProviderAccountId,
|
||||
) {
|
||||
let record = account("acct_peer", "openai", "peer_test");
|
||||
let id = record.id().clone();
|
||||
let accounts = Arc::new(TestAccounts {
|
||||
accounts: Mutex::new(vec![record]),
|
||||
records: Mutex::default(),
|
||||
failed_provider: None,
|
||||
});
|
||||
let transport = Arc::new(DetectionTransport {
|
||||
requests: AtomicUsize::new(0),
|
||||
failure: Mutex::new(Some(failure.into())),
|
||||
success: TestTransport(Mutex::default()),
|
||||
});
|
||||
(
|
||||
initialize(accounts.clone(), transport.clone()),
|
||||
accounts,
|
||||
transport,
|
||||
id,
|
||||
)
|
||||
}
|
||||
|
||||
fn initialize(
|
||||
accounts: Arc<TestAccounts>,
|
||||
transport: Arc<DetectionTransport>,
|
||||
) -> PeerUpstreamBundle {
|
||||
let mut provider = UnavailableProvider::new("openai");
|
||||
provider.peer_enabled = true;
|
||||
let registry =
|
||||
ProviderAdminRegistry::new([Arc::new(provider) as Arc<dyn ProviderAdmin>]).unwrap();
|
||||
initialize_peer_upstream(accounts, ®istry, transport).unwrap()
|
||||
}
|
||||
|
||||
fn worker(bundle: &mut PeerUpstreamBundle) -> WorkerRegistration {
|
||||
let WorkerContribution::Registration(worker) = bundle.take_worker_contributions().remove(0)
|
||||
else {
|
||||
panic!("scheduled worker")
|
||||
};
|
||||
worker
|
||||
}
|
||||
|
||||
async fn cycle(worker: &WorkerRegistration) {
|
||||
let WorkerRunnable::Scheduled { task, .. } = &worker.runnable else {
|
||||
panic!("scheduled worker")
|
||||
};
|
||||
task.run_cycle(WorkerCycleContext::new(
|
||||
worker.id.clone(),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
struct DetectionTransport {
|
||||
requests: AtomicUsize,
|
||||
failure: Mutex<Option<PeerFetchFailure>>,
|
||||
success: TestTransport,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl PeerTransport for DetectionTransport {
|
||||
async fn fetch(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
connection: &PeerConnection,
|
||||
) -> Result<Vec<u8>, PeerFetchFailure> {
|
||||
self.requests.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(failure) = self.failure.lock().unwrap().as_ref() {
|
||||
return Err(PeerFetchFailure {
|
||||
kind: failure.kind,
|
||||
retry_after: failure.retry_after,
|
||||
});
|
||||
}
|
||||
self.success.fetch(account, connection).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
//! 实例敏感配置的逐字段更新、版本基准与提交隔离测试
|
||||
|
||||
use super::*;
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
fn secret_values(instance: &PluginInstance) -> serde_json::Value {
|
||||
serde_json::Value::Object(
|
||||
instance
|
||||
.secrets
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), json!(value.expose_secret())))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn secret_changes_apply_to_current_or_target_version_configuration() {
|
||||
for (target, saved_version) in [
|
||||
(OLD_ARTIFACT, false),
|
||||
(NEW_ARTIFACT, false),
|
||||
(NEW_ARTIFACT, true),
|
||||
] {
|
||||
let token = if saved_version {
|
||||
"saved-token"
|
||||
} else {
|
||||
"current-token"
|
||||
};
|
||||
let untouched = if saved_version {
|
||||
"saved-value"
|
||||
} else {
|
||||
"current-value"
|
||||
};
|
||||
for (changes, expected) in [
|
||||
(json!({}), json!({"token":token,"untouched":untouched})),
|
||||
(
|
||||
json!({"adminKey":"new-key"}),
|
||||
json!({"token":token,"untouched":untouched,"adminKey":"new-key"}),
|
||||
),
|
||||
(
|
||||
json!({"token":"new-token"}),
|
||||
json!({"token":"new-token","untouched":untouched}),
|
||||
),
|
||||
(json!({"token":null}), json!({"untouched":untouched})),
|
||||
(
|
||||
json!({"token":null,"adminKey":"new-key","absent":null}),
|
||||
json!({"untouched":untouched,"adminKey":"new-key"}),
|
||||
),
|
||||
] {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
{
|
||||
let mut data = fixture.data.lock().unwrap();
|
||||
data.snapshot.instances[0].secrets = BTreeMap::from([
|
||||
("token".into(), "current-token".into()),
|
||||
("untouched".into(), "current-value".into()),
|
||||
]);
|
||||
if saved_version {
|
||||
data.history.get_mut(NEW_ARTIFACT).unwrap().secrets = BTreeMap::from([
|
||||
("token".into(), "saved-token".into()),
|
||||
("untouched".into(), "saved-value".into()),
|
||||
]);
|
||||
} else {
|
||||
data.history.clear();
|
||||
}
|
||||
}
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.artifact_sha256 = target.into();
|
||||
request.expected_revision = Some(10);
|
||||
request.secrets = serde_json::from_value(changes).unwrap();
|
||||
let result = service(fixture.clone(), Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(secret_values(&result.instance), expected);
|
||||
assert_eq!(secret_values(&fixture.snapshot().instances[0]), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_secret_changes_leave_saved_values_unchanged() {
|
||||
for failure in ["stale", "preparation", "size"] {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
fixture.data.lock().unwrap().snapshot.instances[0].secrets =
|
||||
BTreeMap::from([("token".into(), "saved-token".into())]);
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.artifact_sha256 = OLD_ARTIFACT.into();
|
||||
request.expected_revision = Some(if failure == "stale" { 9 } else { 10 });
|
||||
request.secrets = BTreeMap::from([
|
||||
("token".into(), None),
|
||||
(
|
||||
"adminKey".into(),
|
||||
Some(
|
||||
if failure == "size" {
|
||||
"x".repeat(8193)
|
||||
} else {
|
||||
"new-key".into()
|
||||
}
|
||||
.into(),
|
||||
),
|
||||
),
|
||||
]);
|
||||
fixture.data.lock().unwrap().fail_preparation = failure == "preparation";
|
||||
let result = service(fixture.clone(), Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await;
|
||||
assert!(result.is_err(), "{failure} must reject the update");
|
||||
assert_eq!(
|
||||
secret_values(&fixture.snapshot().instances[0]),
|
||||
secret_values(&before.instances[0])
|
||||
);
|
||||
assert!(fixture.data.lock().unwrap().saves.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,7 @@ use serde_json::json;
|
||||
|
||||
use super::TestPluginPorts;
|
||||
|
||||
mod configuration;
|
||||
mod queries;
|
||||
|
||||
const OLD_ARTIFACT: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
@@ -759,7 +760,7 @@ fn input() -> ConfigurePluginInstance {
|
||||
artifact_sha256: NEW_ARTIFACT.into(),
|
||||
enabled: true,
|
||||
configuration: json!({}),
|
||||
secrets: None,
|
||||
secrets: Default::default(),
|
||||
bindings: vec![],
|
||||
}
|
||||
}
|
||||
@@ -942,6 +943,8 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
// 此用例验证创建重试,使用不需要私有状态迁移的固定版本
|
||||
value.artifact_sha256 = OLD_ARTIFACT.into();
|
||||
value.enabled = false;
|
||||
value.secrets =
|
||||
serde_json::from_value(json!({"token": "saved-token", "absent": null})).unwrap();
|
||||
value
|
||||
};
|
||||
let created = service
|
||||
@@ -955,6 +958,12 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
assert_eq!(created.instance.id, creation_id);
|
||||
assert_eq!(retried.instance.id, creation_id);
|
||||
assert_eq!(fixture.snapshot().instances.len(), 2);
|
||||
use secrecy::ExposeSecret as _;
|
||||
assert_eq!(
|
||||
retried.instance.secrets["token"].expose_secret(),
|
||||
"saved-token"
|
||||
);
|
||||
assert!(!retried.instance.secrets.contains_key("absent"));
|
||||
let before = fixture.snapshot().config_revision;
|
||||
let mut changed = request();
|
||||
changed.name = "changed draft".into();
|
||||
@@ -965,6 +974,17 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
.unwrap();
|
||||
assert_eq!(error.kind(), AdminErrorKind::Conflict);
|
||||
assert_eq!(fixture.snapshot().config_revision, before);
|
||||
let mut changed = request();
|
||||
changed
|
||||
.secrets
|
||||
.insert("token".into(), Some("different-token".into()));
|
||||
let error = service
|
||||
.configure_instance(None, changed, &context())
|
||||
.await
|
||||
.err()
|
||||
.unwrap();
|
||||
assert_eq!(error.kind(), AdminErrorKind::Conflict);
|
||||
assert_eq!(fixture.snapshot().config_revision, before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1435,23 +1455,6 @@ async fn version_switch_preparation_failure_preserves_current_and_saved_settings
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_edit_preserves_the_target_versions_secrets_when_omitted() {
|
||||
use secrecy::ExposeSecret as _;
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.secrets = None;
|
||||
let result = service(fixture, Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.instance.secrets["token"].expose_secret(),
|
||||
"old-version-secret"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rollback_requires_a_saved_configuration_instead_of_reusing_incompatible_current_settings()
|
||||
{
|
||||
|
||||
@@ -446,6 +446,8 @@ fn release_platform() -> (&'static str, &'static str, &'static str) {
|
||||
("linux", "x86_64") => ("linux", "amd64", "x86_64-unknown-linux-gnu"),
|
||||
("linux", "aarch64") => ("linux", "arm64", "aarch64-unknown-linux-gnu"),
|
||||
("macos", "aarch64") => ("darwin", "arm64", "aarch64-apple-darwin"),
|
||||
("windows", "x86_64") => ("windows", "amd64", "x86_64-pc-windows-msvc"),
|
||||
("windows", "aarch64") => ("windows", "arm64", "aarch64-pc-windows-msvc"),
|
||||
platform => panic!("unsupported test platform: {platform:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,14 @@ use super::TestPluginPorts;
|
||||
use gateway_admin::{model::usage_data::*, ports::usage_data::*};
|
||||
#[async_trait::async_trait]
|
||||
impl UsageDataStore for TestPluginPorts {
|
||||
async fn open_read(
|
||||
&self,
|
||||
_: UsageReadQuery,
|
||||
_: gateway_core::lifecycle::CancellationToken,
|
||||
) -> UsageDataResult<Box<dyn UsageReadSession>> {
|
||||
Err(UsageDataError::unavailable("test usage reader unavailable"))
|
||||
}
|
||||
|
||||
async fn get_record(&self, _: GetUsageRecord) -> UsageDataResult<UsageRecord> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
|
||||
@@ -334,6 +334,7 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -64,6 +64,7 @@ fn sdk_wire_types_match_host_contracts() {
|
||||
"value.rs",
|
||||
"record.rs",
|
||||
"query.rs",
|
||||
"read.rs",
|
||||
"aggregate.rs",
|
||||
"pricing.rs",
|
||||
"error.rs",
|
||||
|
||||
@@ -12,7 +12,7 @@ name = "main"
|
||||
path = "tests/main.rs"
|
||||
|
||||
[dependencies]
|
||||
axum = { version = "0.8.9", default-features = false, features = ["json", "multipart", "query", "ws"] }
|
||||
axum = { version = "0.8.9", default-features = false, features = ["json", "matched-path", "multipart", "query", "ws"] }
|
||||
base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
chrono = { version = "0.4.45", features = ["serde"] }
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 账号管理路由与 HTTP handler 编排
|
||||
|
||||
use gateway_core::engine::probe::AccountProbeTarget;
|
||||
|
||||
use crate::auth::SessionState;
|
||||
|
||||
use super::*;
|
||||
@@ -626,7 +628,10 @@ where
|
||||
let stream = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.test_connection(account_id, upstream_model)
|
||||
.test_connection(
|
||||
AccountProbeTarget::ConnectionTest(account_id),
|
||||
upstream_model,
|
||||
)
|
||||
.await
|
||||
.map_err(map_service_error)?
|
||||
.map(move |event| {
|
||||
|
||||
@@ -193,6 +193,7 @@ fn quota_forecast_view(
|
||||
period: match forecast.period {
|
||||
AccountUsagePeriod::Weekly => "weekly",
|
||||
AccountUsagePeriod::Monthly => "monthly",
|
||||
AccountUsagePeriod::Daily => "daily",
|
||||
},
|
||||
target_days: forecast.target_seconds as f64 / 86_400.0,
|
||||
extrapolated: forecast.extrapolated,
|
||||
@@ -389,6 +390,7 @@ pub(super) fn account_usage_view(
|
||||
};
|
||||
AccountUsageView {
|
||||
window_kind: match period {
|
||||
Some(AccountUsagePeriod::Daily) => AccountUsageWindowKind::Daily,
|
||||
Some(AccountUsagePeriod::Weekly) => AccountUsageWindowKind::Weekly,
|
||||
Some(AccountUsagePeriod::Monthly) => AccountUsageWindowKind::Monthly,
|
||||
None => AccountUsageWindowKind::Quota,
|
||||
@@ -453,20 +455,14 @@ pub(super) fn account_model_usage_view(
|
||||
.costs
|
||||
.iter()
|
||||
.find(|cost| cost.currency.eq_ignore_ascii_case("USD"));
|
||||
let success_rate = usage.health.success_rate();
|
||||
ModelUsageView {
|
||||
model: usage.model,
|
||||
request_count: usage.request_count,
|
||||
request_count_display: format_number(usage.request_count),
|
||||
success_rate: (usage.request_count > 0)
|
||||
.then(|| usage.success_count as f64 * 100.0 / usage.request_count as f64),
|
||||
success_rate_display: if usage.request_count == 0 {
|
||||
"—".to_owned()
|
||||
} else {
|
||||
format!(
|
||||
"{:.1}%",
|
||||
usage.success_count as f64 * 100.0 / usage.request_count as f64
|
||||
)
|
||||
},
|
||||
success_rate,
|
||||
success_rate_display: success_rate
|
||||
.map_or_else(|| "—".to_owned(), |rate| format!("{rate:.1}%")),
|
||||
input_tokens: usage.input_tokens,
|
||||
input_tokens_display: display_optional_tokens(usage.input_tokens),
|
||||
output_tokens: usage.output_tokens,
|
||||
@@ -493,9 +489,11 @@ pub(super) fn account_model_usage_view(
|
||||
partial_cost_count: u64::from(cost_estimate_status == "partial"),
|
||||
unknown_cost_count: usage.cost_coverage.unavailable_count,
|
||||
costs: usage.costs.iter().map(account_currency_cost_view).collect(),
|
||||
last_used_at: time.rfc3339(&usage.last_used_at),
|
||||
last_used_at_full_display: Some(time.datetime(&usage.last_used_at)),
|
||||
last_used_at_display: time.relative(usage.last_used_at, now),
|
||||
last_used_at: usage.last_used_at.map(|value| time.rfc3339(&value)),
|
||||
last_used_at_full_display: usage.last_used_at.map(|value| time.datetime(&value)),
|
||||
last_used_at_display: usage
|
||||
.last_used_at
|
||||
.map_or_else(|| "—".to_owned(), |value| time.relative(value, now)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ pub struct AccountUpstreamInfoView {
|
||||
pub stale: bool,
|
||||
pub connection_revision: String,
|
||||
pub last_error: Option<String>,
|
||||
pub max_concurrency: Option<u64>,
|
||||
pub requests_per_minute: Option<u64>,
|
||||
pub pool: Option<UpstreamPoolView>,
|
||||
}
|
||||
|
||||
@@ -78,6 +80,8 @@ pub(crate) fn view(
|
||||
stale: info.stale,
|
||||
connection_revision: info.connection_revision,
|
||||
last_error: info.last_error,
|
||||
max_concurrency: info.max_concurrency,
|
||||
requests_per_minute: info.requests_per_minute,
|
||||
pool: info.pool.map(|pool| UpstreamPoolView {
|
||||
account_count: pool.account_count,
|
||||
states: UpstreamPoolStatesView {
|
||||
|
||||
@@ -515,7 +515,7 @@ pub struct ModelUsageView {
|
||||
pub partial_cost_count: u64,
|
||||
pub unknown_cost_count: u64,
|
||||
pub costs: Vec<CurrencyCostView>,
|
||||
pub last_used_at: String,
|
||||
pub last_used_at: Option<String>,
|
||||
pub last_used_at_display: String,
|
||||
pub last_used_at_full_display: Option<String>,
|
||||
}
|
||||
|
||||
@@ -260,6 +260,11 @@ pub(crate) fn usage_list_record_view(
|
||||
UsageListRecordView {
|
||||
error_code: record.error_code,
|
||||
error_message: record.error_message,
|
||||
error_kind: record.error_kind,
|
||||
websocket_close_code: record.websocket_close_code,
|
||||
continuation_unavailable_reason: record.continuation_unavailable_reason,
|
||||
upstream_connection_exit_reason: record.upstream_connection_exit_reason,
|
||||
recovered: record.recovered,
|
||||
logical_outcome: record.outcome.as_str().to_owned(),
|
||||
client_api_key_name: record.client_api_key_name,
|
||||
id: record.id,
|
||||
@@ -271,6 +276,7 @@ pub(crate) fn usage_list_record_view(
|
||||
account_notes: record.provider_account_notes,
|
||||
account_plan_type: record.provider_account_plan_type,
|
||||
account_plan_type_display: record.provider_account_plan_type_display,
|
||||
account_upstream_peer: record.provider_account_upstream_peer,
|
||||
route: record.endpoint,
|
||||
model,
|
||||
requested_model: record.requested_model_id,
|
||||
@@ -356,6 +362,23 @@ pub(crate) fn usage_record_view(
|
||||
.or_else(|| record.requested_model_id.clone());
|
||||
let metadata = provider_metadata_fields(record.provider_metadata_json.as_deref());
|
||||
UsageRecordView {
|
||||
upstream_error: record
|
||||
.upstream_error
|
||||
.map(|error| super::wire::UsageUpstreamErrorView {
|
||||
code: error.code,
|
||||
error_type: error.error_type,
|
||||
message: error.message,
|
||||
param: error.param,
|
||||
close_reason: error.close_reason,
|
||||
last_event_type: error.last_event_type,
|
||||
}),
|
||||
retry_after_ms: record.retry_after_ms,
|
||||
error_code: record.provider_error_code,
|
||||
error_kind: record.error_kind,
|
||||
websocket_close_code: record.websocket_close_code,
|
||||
continuation_unavailable_reason: record.continuation_unavailable_reason,
|
||||
upstream_connection_exit_reason: record.upstream_connection_exit_reason,
|
||||
recovered: record.recovered,
|
||||
id: record.id.clone(),
|
||||
request_id: record.id,
|
||||
client_api_key_id: Some(record.client_api_key_ref),
|
||||
@@ -1351,6 +1374,7 @@ pub(crate) fn diagnostics_view(
|
||||
request_count: item.request_count,
|
||||
account_plan_type: item.account_plan_type,
|
||||
account_plan_type_display: item.account_plan_type_display,
|
||||
account_upstream_peer: item.account_upstream_peer,
|
||||
success_count: item.success_count,
|
||||
error_count: item.error_count,
|
||||
error_rate: item.error_rate,
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
pub use crate::time::parse_datetime;
|
||||
|
||||
/// 观测列表默认页大小
|
||||
pub const DEFAULT_PAGE_SIZE: u16 = 50;
|
||||
/// 观测列表允许的最大页大小
|
||||
@@ -256,16 +258,6 @@ impl DiagnosticDimension {
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析 RFC3339 时间;错误不回显原始值
|
||||
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
|
||||
let Some(value) = trimmed(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|value| Some(value.with_timezone(&Utc)))
|
||||
.map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
/// 解析 HTTP 状态码
|
||||
pub fn parse_status(value: Option<i64>) -> Result<Option<u16>, WireValidationError> {
|
||||
value
|
||||
|
||||
@@ -88,6 +88,11 @@ pub struct ImageBillingView {
|
||||
pub struct UsageListRecordView {
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub error_kind: Option<String>,
|
||||
pub websocket_close_code: Option<u16>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub recovered: bool,
|
||||
pub logical_outcome: String,
|
||||
pub client_api_key_name: Option<String>,
|
||||
pub id: String,
|
||||
@@ -99,6 +104,7 @@ pub struct UsageListRecordView {
|
||||
pub account_notes: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
@@ -121,10 +127,30 @@ pub struct UsageListRecordView {
|
||||
pub user_agent: Option<String>,
|
||||
}
|
||||
|
||||
/// 受控详情中的原始上游错误字段
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageUpstreamErrorView {
|
||||
pub code: Option<String>,
|
||||
pub error_type: Option<String>,
|
||||
pub message: Option<String>,
|
||||
pub param: Option<String>,
|
||||
pub close_reason: Option<String>,
|
||||
pub last_event_type: Option<String>,
|
||||
}
|
||||
|
||||
/// 单条逻辑请求详情展示
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageRecordView {
|
||||
pub upstream_error: Option<UsageUpstreamErrorView>,
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub error_code: Option<String>,
|
||||
pub error_kind: Option<String>,
|
||||
pub websocket_close_code: Option<u16>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub recovered: bool,
|
||||
pub id: String,
|
||||
pub request_id: String,
|
||||
pub client_api_key_id: Option<String>,
|
||||
@@ -849,6 +875,7 @@ pub struct DiagnosticItemView {
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub error_count: u64,
|
||||
|
||||
@@ -537,14 +537,10 @@ async fn probe<S>(
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let proxy = request
|
||||
.proxy_url
|
||||
.0
|
||||
.ok_or_else(|| AdminError::bad_request("代理 URL 不能为空"))?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.proxies()
|
||||
.probe(Some(&proxy), request.detect_location)
|
||||
.probe(request.proxy_url.0.as_ref(), request.detect_location)
|
||||
.await
|
||||
.map_err(map_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
|
||||
@@ -45,6 +45,7 @@ pub struct RuntimeSettingsFields {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
@@ -751,6 +752,7 @@ impl RuntimeSettingsFields {
|
||||
max_waiting_per_key: self.max_waiting_per_key,
|
||||
max_waiting_per_account: self.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: self.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: self.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: self.openai_account_affinity,
|
||||
max_account_rotations: self.max_account_rotations,
|
||||
@@ -790,6 +792,7 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: settings.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: settings.openai_account_affinity,
|
||||
max_account_rotations: settings.max_account_rotations,
|
||||
|
||||
@@ -9,25 +9,7 @@ use axum::{
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// 管理端 wire 字段校验错误;只携带稳定字段名,不回显输入值
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct WireValidationError {
|
||||
field: &'static str,
|
||||
}
|
||||
|
||||
impl WireValidationError {
|
||||
/// 构造字段校验错误
|
||||
#[must_use]
|
||||
pub const fn new(field: &'static str) -> Self {
|
||||
Self { field }
|
||||
}
|
||||
|
||||
/// 返回未通过校验的字段名
|
||||
#[must_use]
|
||||
pub const fn field(self) -> &'static str {
|
||||
self.field
|
||||
}
|
||||
}
|
||||
pub use crate::WireValidationError;
|
||||
|
||||
/// 成功响应的稳定业务码
|
||||
pub const ADMIN_OK_CODE: u32 = 200;
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
//! 数据面与公共 HTTP 中间件共用的客户端错误合同和重试投影
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
http::{HeaderValue, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// 认证边界提供的稳定拒绝原因,不携带认证输入
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct AccessRejection(pub &'static str);
|
||||
|
||||
/// 执行已交给 Core 观测,HTTP 入口只补充交付边界日志
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ExecutionObserved;
|
||||
|
||||
/// OpenAI 风格的协议错误 JSON
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolErrorBody {
|
||||
/// 错误对象
|
||||
pub error: ProtocolError,
|
||||
}
|
||||
|
||||
/// 不包含请求正文或 prompt 的错误对象
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolError {
|
||||
/// OpenAI 错误类别
|
||||
#[serde(rename = "type")]
|
||||
pub kind: &'static str,
|
||||
/// 稳定机器码
|
||||
pub code: &'static str,
|
||||
/// 安全的可读信息
|
||||
pub message: String,
|
||||
/// 出错字段;不包含字段值
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
}
|
||||
|
||||
impl ProtocolErrorBody {
|
||||
/// 序列化为 JSON value
|
||||
#[must_use]
|
||||
pub fn into_value(self) -> Value {
|
||||
serde_json::json!({ "error": self.error })
|
||||
}
|
||||
}
|
||||
|
||||
/// OpenAI 风格错误响应
|
||||
pub fn openai_error_response(
|
||||
status: StatusCode,
|
||||
message: &str,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
) -> (StatusCode, Json<Value>) {
|
||||
let body = json!({
|
||||
"error": {"message": message, "type": error_type, "code": code}
|
||||
});
|
||||
match capacity_error_for_client(&body) {
|
||||
Some(body) => (StatusCode::SERVICE_UNAVAILABLE, Json(body)),
|
||||
None => (status, Json(body)),
|
||||
}
|
||||
}
|
||||
|
||||
/// 无重试提示的容量拒绝及旧版 Codex 限流错误需要可重试投影
|
||||
/// 仅在交付边界转换,内部仍保留真实上游事实
|
||||
pub(crate) fn client_error_code(code: &str) -> &str {
|
||||
match code {
|
||||
"server_is_overloaded" | "slow_down" => "server_error",
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
/// 为 HTTP 错误或流内失败保留原有字段,仅投影客户端需要的重试信号
|
||||
pub(crate) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
const CODE_PATHS: [&str; 2] = ["/error/code", "/response/error/code"];
|
||||
let capacity_code = |value: &Value| {
|
||||
value
|
||||
.as_str()
|
||||
.is_some_and(|code| client_error_code(code) != code)
|
||||
};
|
||||
if !CODE_PATHS
|
||||
.iter()
|
||||
.any(|path| data.pointer(path).is_some_and(capacity_code))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut projected = data.clone();
|
||||
for path in CODE_PATHS {
|
||||
if let Some(code) = projected.pointer_mut(path)
|
||||
&& capacity_code(code)
|
||||
{
|
||||
*code = Value::String("server_error".to_owned());
|
||||
}
|
||||
}
|
||||
// WS 包装错误按 HTTP 状态分类;保留 400/429 会使客户端终止重试
|
||||
for field in ["status", "status_code"] {
|
||||
if let Some(status) = projected.get_mut(field)
|
||||
&& status.is_number()
|
||||
{
|
||||
*status = json!(503);
|
||||
}
|
||||
}
|
||||
Some(projected)
|
||||
}
|
||||
|
||||
/// Gateway 错误的 OpenAI HTTP 表达
|
||||
pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
let (status, default_type, default_code) = gateway_error_contract(error.kind());
|
||||
let mut response = openai_error_response(
|
||||
status,
|
||||
error.client_message(),
|
||||
error.client_error_type().unwrap_or(default_type),
|
||||
error.client_error_code().unwrap_or(default_code),
|
||||
)
|
||||
.into_response();
|
||||
response.extensions_mut().insert(error.clone());
|
||||
if let Some(delay) = error.retry_after()
|
||||
&& let Ok(value) = HeaderValue::from_str(
|
||||
&delay
|
||||
.as_secs()
|
||||
.saturating_add(u64::from(delay.subsec_nanos() > 0))
|
||||
.max(1)
|
||||
.to_string(),
|
||||
)
|
||||
{
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
kind: GatewayErrorKind,
|
||||
) -> (StatusCode, &'static str, &'static str) {
|
||||
match kind {
|
||||
GatewayErrorKind::InvalidRequest => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"invalid_request",
|
||||
),
|
||||
GatewayErrorKind::Unsupported => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"unsupported_capability",
|
||||
),
|
||||
GatewayErrorKind::Unauthorized => (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"invalid_request_error",
|
||||
"invalid_api_key",
|
||||
),
|
||||
GatewayErrorKind::PolicyDenied => (
|
||||
StatusCode::FORBIDDEN,
|
||||
"invalid_request_error",
|
||||
"policy_denied",
|
||||
),
|
||||
GatewayErrorKind::ModelNotFound => (
|
||||
StatusCode::NOT_FOUND,
|
||||
"invalid_request_error",
|
||||
"model_not_found",
|
||||
),
|
||||
GatewayErrorKind::NoAvailableProvider => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"no_available_provider",
|
||||
),
|
||||
GatewayErrorKind::AccountCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MiddlewareCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"middleware_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueFull => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_full",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueTimeout => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_timeout",
|
||||
),
|
||||
GatewayErrorKind::RateLimited => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"rate_limit_exceeded",
|
||||
),
|
||||
GatewayErrorKind::BudgetExhausted => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"usage_limit_reached",
|
||||
"usage_limit_reached",
|
||||
),
|
||||
GatewayErrorKind::UpstreamUnavailable => (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"upstream_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MessageTooBig => (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"invalid_request_error",
|
||||
"message_too_big",
|
||||
),
|
||||
GatewayErrorKind::Timeout => (
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"server_error",
|
||||
"request_timeout",
|
||||
),
|
||||
GatewayErrorKind::Cancelled => (
|
||||
StatusCode::REQUEST_TIMEOUT,
|
||||
"server_error",
|
||||
"request_cancelled",
|
||||
),
|
||||
GatewayErrorKind::Internal => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
_ => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
//! 数据面 HTTP 入口诊断;执行前失败进入运维事件,不伪造模型请求或上游状态
|
||||
|
||||
use std::{
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
extract::{MatchedPath, Request, State},
|
||||
http::StatusCode,
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure, TraceContext};
|
||||
use http_body::{Body as _, Frame, SizeHint};
|
||||
use serde_json::json;
|
||||
use tower_http::request_id::RequestId;
|
||||
|
||||
use crate::client_error::{AccessRejection, ProtocolError, ProtocolErrorBody};
|
||||
|
||||
pub(crate) async fn record_ingress(
|
||||
State(diagnostics): State<Arc<dyn OperationalDiagnostics>>,
|
||||
mut request: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
if request.uri().path() != "/v1" && !request.uri().path().starts_with("/v1/") {
|
||||
return next.run(request).await;
|
||||
}
|
||||
let correlation_id = request
|
||||
.extensions()
|
||||
.get::<RequestId>()
|
||||
.and_then(|id| id.header_value().to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let endpoint = request
|
||||
.extensions()
|
||||
.get::<MatchedPath>()
|
||||
.map(|path| path.as_str().to_owned());
|
||||
let method = request.method().clone();
|
||||
let dump = if tracing::enabled!(target: "request_dump", tracing::Level::INFO) {
|
||||
let trace = TraceContext::new(correlation_id.as_deref().unwrap_or("ingress"));
|
||||
trace.dump(
|
||||
"ingress.request",
|
||||
&serde_json::to_vec(&json!({
|
||||
"method":method.as_str(), "uri":request.uri().to_string(),
|
||||
}))
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
trace.headers(
|
||||
"ingress.request.headers",
|
||||
json!({}),
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
let body = std::mem::replace(request.body_mut(), Body::empty());
|
||||
*request.body_mut() = Body::new(DumpBody::new(body, trace.clone(), "ingress.request.body"));
|
||||
Some(trace)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut response = if endpoint.is_none() {
|
||||
// 数据面未知路径不能落入 SPA fallback,否则会把协议错误伪装成 HTML 成功
|
||||
crate::openai::error::protocol_error_response(
|
||||
StatusCode::NOT_FOUND,
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "invalid_request_error",
|
||||
code: "unknown_endpoint",
|
||||
message: "The requested API endpoint does not exist.".into(),
|
||||
param: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
} else {
|
||||
next.run(request).await
|
||||
};
|
||||
let protocol = response.extensions_mut().remove::<ProtocolErrorBody>();
|
||||
let gateway = response
|
||||
.extensions_mut()
|
||||
.remove::<gateway_core::error::GatewayError>();
|
||||
let access = response.extensions_mut().remove::<AccessRejection>();
|
||||
let execution_observed = response
|
||||
.extensions_mut()
|
||||
.remove::<crate::client_error::ExecutionObserved>()
|
||||
.is_some();
|
||||
let status = response.status();
|
||||
if status.is_client_error() || status.is_server_error() {
|
||||
let code = access
|
||||
.map(|error| error.0)
|
||||
.or_else(|| protocol.as_ref().map(|body| body.error.code))
|
||||
.or_else(|| gateway.as_ref().map(|error| error.kind().as_str()))
|
||||
.unwrap_or(match status {
|
||||
StatusCode::UNAUTHORIZED => "invalid_api_key",
|
||||
StatusCode::METHOD_NOT_ALLOWED => "method_not_allowed",
|
||||
StatusCode::NOT_FOUND => "not_found",
|
||||
_ => "http_rejection",
|
||||
});
|
||||
// 未知参数名可由客户端任意构造,只保留 decoder 生成的已知参数路径
|
||||
let parameter = protocol
|
||||
.as_ref()
|
||||
.filter(|body| body.error.code != "unknown_parameter")
|
||||
.and_then(|body| body.error.param.as_deref())
|
||||
.filter(|path| {
|
||||
path.len() <= 256
|
||||
&& path.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'[' | b']')
|
||||
})
|
||||
});
|
||||
let endpoint = endpoint.as_deref().unwrap_or("/v1/<unmatched>");
|
||||
let model_request_id = execution_observed
|
||||
.then(|| {
|
||||
response
|
||||
.headers()
|
||||
.get("x-gateway-request-id")
|
||||
.and_then(|id| id.to_str().ok())
|
||||
})
|
||||
.flatten();
|
||||
tracing::info!(target: "request_trace", stage="ingress.rejected", status=status.as_u16(),
|
||||
code, parameter, endpoint, model_request_id, method=method.as_str(), "Data-plane HTTP request rejected");
|
||||
if !execution_observed {
|
||||
let mut failure = OperationalFailure::new(
|
||||
"api",
|
||||
"http_ingress",
|
||||
code,
|
||||
json!({"method":method.as_str(), "endpoint":endpoint, "status":status.as_u16(),
|
||||
"code":code, "param":parameter})
|
||||
.to_string(),
|
||||
);
|
||||
failure.correlation_id = correlation_id;
|
||||
failure.details = gateway.as_ref().and_then(|error| error.error_details());
|
||||
if diagnostics.record_failure(failure).await.is_err() {
|
||||
tracing::warn!("HTTP ingress diagnostic could not be recorded");
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(trace) = dump {
|
||||
trace.dump(
|
||||
"ingress.response",
|
||||
&serde_json::to_vec(&json!({"status":status.as_u16()})).unwrap_or_default(),
|
||||
);
|
||||
trace.headers(
|
||||
"ingress.response.headers",
|
||||
json!({"status":status.as_u16()}),
|
||||
response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
let body = std::mem::replace(response.body_mut(), Body::empty());
|
||||
*response.body_mut() = Body::new(DumpBody::new(body, trace, "ingress.response.body"));
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
struct DumpBody {
|
||||
inner: Body,
|
||||
trace: TraceContext,
|
||||
stage: &'static str,
|
||||
complete: bool,
|
||||
}
|
||||
|
||||
impl DumpBody {
|
||||
fn new(inner: Body, trace: TraceContext, stage: &'static str) -> Self {
|
||||
let complete = inner.is_end_stream();
|
||||
Self {
|
||||
inner,
|
||||
trace,
|
||||
stage,
|
||||
complete,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl http_body::Body for DumpBody {
|
||||
type Data = Bytes;
|
||||
type Error = axum::Error;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<Frame<Bytes>, axum::Error>>> {
|
||||
let result = Pin::new(&mut self.inner).poll_frame(cx);
|
||||
if let Poll::Ready(Some(Ok(frame))) = &result
|
||||
&& let Some(bytes) = frame.data_ref()
|
||||
{
|
||||
self.trace.dump(self.stage, bytes);
|
||||
}
|
||||
if matches!(result, Poll::Ready(Some(Err(_)))) {
|
||||
self.complete = false;
|
||||
} else if matches!(result, Poll::Ready(None)) || self.inner.is_end_stream() {
|
||||
self.complete = true;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.is_end_stream()
|
||||
}
|
||||
fn size_hint(&self) -> SizeHint {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DumpBody {
|
||||
fn drop(&mut self) {
|
||||
// 不为诊断主动读取剩余正文;取消或未消费的请求必须明确标出转储不完整
|
||||
self.trace.dump(
|
||||
"ingress.body.finished",
|
||||
&serde_json::to_vec(&json!({
|
||||
"stage":self.stage, "complete":self.complete,
|
||||
}))
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//! `/healthz` 对 Core、Store 与 Host 关键 worker 健康事实的聚合
|
||||
//! 请求就绪检查与综合健康诊断
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
@@ -67,22 +67,7 @@ impl HealthStatus {
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
// 逐个探针共享截止时间,保留已完成结果并指出具体超时组件
|
||||
let deadline = tokio::time::Instant::now() + HEALTH_CHECK_TIMEOUT;
|
||||
let checks = self.probes.iter().map(|probe| async move {
|
||||
let (state, reason) = match tokio::time::timeout_at(deadline, probe.check()).await {
|
||||
Ok(HealthState::Healthy) => return None,
|
||||
Ok(HealthState::Degraded(reason)) => ("degraded", reason),
|
||||
Ok(HealthState::Unhealthy(reason)) => ("unhealthy", reason),
|
||||
Err(_) => ("timed_out", "health probe exceeded 2s deadline".to_owned()),
|
||||
};
|
||||
Some(HealthFailure {
|
||||
component: format!("probe:{}", probe.name()),
|
||||
state,
|
||||
reason,
|
||||
})
|
||||
});
|
||||
failures.extend(join_all(checks).await.into_iter().flatten());
|
||||
failures.extend(probe_failures(self.probes.iter()).await);
|
||||
let healthy = failures.is_empty();
|
||||
let changed = {
|
||||
let mut previous = self
|
||||
@@ -128,8 +113,42 @@ impl HealthStatus {
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_failures<'a>(
|
||||
probes: impl Iterator<Item = &'a Arc<dyn HealthProbe>>,
|
||||
) -> Vec<HealthFailure> {
|
||||
// 探针并行共用截止时间,保留已完成结果并指出具体超时组件
|
||||
let deadline = tokio::time::Instant::now() + HEALTH_CHECK_TIMEOUT;
|
||||
let checks = probes.map(|probe| async move {
|
||||
let (state, reason) = match tokio::time::timeout_at(deadline, probe.check()).await {
|
||||
Ok(HealthState::Healthy) => return None,
|
||||
Ok(HealthState::Degraded(reason)) => ("degraded", reason),
|
||||
Ok(HealthState::Unhealthy(reason)) => ("unhealthy", reason),
|
||||
Err(_) => ("timed_out", "health probe exceeded 2s deadline".to_owned()),
|
||||
};
|
||||
Some(HealthFailure {
|
||||
component: format!("probe:{}", probe.name()),
|
||||
state,
|
||||
reason,
|
||||
})
|
||||
});
|
||||
join_all(checks).await.into_iter().flatten().collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn readyz(State(state): State<ApiState>) -> StatusCode {
|
||||
let required = state
|
||||
.health()
|
||||
.probes
|
||||
.iter()
|
||||
.filter(|probe| probe.required_for_readiness());
|
||||
status_code(probe_failures(required).await.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) async fn healthz(State(state): State<ApiState>) -> StatusCode {
|
||||
if state.health().healthy().await {
|
||||
status_code(state.health().healthy().await)
|
||||
}
|
||||
|
||||
fn status_code(healthy: bool) -> StatusCode {
|
||||
if healthy {
|
||||
StatusCode::NO_CONTENT
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user