mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 08:05:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
36321a9af4 | ||
|
|
b485b8bd4a | ||
|
|
cae2c8656a | ||
|
|
acf574129f | ||
|
|
6426e0d3de | ||
|
|
eedc39b33a | ||
|
|
9672a2e000 | ||
|
|
0fffd2c550 | ||
|
|
5671a57dc8 | ||
|
|
166e8754a2 | ||
|
|
fbaf6c1bdd | ||
|
|
7cdc2dba6f | ||
|
|
a321ccf737 | ||
|
|
4533ae8850 | ||
|
|
7542ed365e | ||
|
|
bf4da7a3e7 | ||
|
|
0fe354b083 | ||
|
|
6d0c583599 | ||
|
|
2e861c03c4 | ||
|
|
d5fc778ea7 | ||
|
|
404cd41ff4 | ||
|
|
851f473b88 | ||
|
|
e97da4f551 | ||
|
|
5544b0fb29 | ||
|
|
428e744710 | ||
|
|
7b406765d6 | ||
|
|
92ae7df115 | ||
|
|
cc42e78334 | ||
|
|
0c90ed30fb | ||
|
|
5a81856510 | ||
|
|
3110b71512 | ||
|
|
eac1b2c39e | ||
|
|
337bb80230 | ||
|
|
82f2a0da06 | ||
|
|
f0d8dad71e | ||
|
|
f5c1d773d7 | ||
|
|
9148e6b11e | ||
|
|
a7d8c3d3df | ||
|
|
997eb86aa0 | ||
|
|
3866acc269 | ||
|
|
38477f4328 | ||
|
|
5168f6a924 | ||
|
|
3a407c635a | ||
|
|
6901e0ed45 | ||
|
|
9758c420fa | ||
|
|
6d2bf06ac7 | ||
|
|
b8faaf250f | ||
|
|
40137b1205 | ||
|
|
c67cde4e90 | ||
|
|
2e9213a784 | ||
|
|
87496b5656 | ||
|
|
c856e4306f | ||
|
|
92090f28bf | ||
|
|
12c2b612de | ||
|
|
e8af0b9c66 | ||
|
|
80776e8e03 | ||
|
|
b296c795ac | ||
|
|
9fc162a82f | ||
|
|
9532bd438b | ||
|
|
73319f1943 | ||
|
|
ee56d6a1d2 | ||
|
|
5d29ddfffc | ||
|
|
45903b684b | ||
|
|
a580874697 | ||
|
|
885369f19f | ||
|
|
24ff339c97 | ||
|
|
e582eb73d2 | ||
|
|
ec252b3b46 | ||
|
|
6503f36750 | ||
|
|
2366f66f33 | ||
|
|
cc0303c65e | ||
|
|
b11eeac29e | ||
|
|
ad5f3ba997 | ||
|
|
15455ca619 | ||
|
|
904315c209 | ||
|
|
95d720dd86 | ||
|
|
74f59cc4f8 | ||
|
|
e979454457 | ||
|
|
0af8610a48 | ||
|
|
8226089ade | ||
|
|
1bd40ffa67 | ||
|
|
34a6ccce43 | ||
|
|
c275119f12 | ||
|
|
9322b7aeb8 | ||
|
|
c6444b2bf1 | ||
|
|
55075677eb | ||
|
|
733aa5f88a | ||
|
|
d704910857 | ||
|
|
ad6c21392f | ||
|
|
55e87c3b16 | ||
|
|
0c4da6c838 | ||
|
|
225c1166c6 | ||
|
|
97b91d3409 | ||
|
|
c953f6f44d | ||
|
|
d6b9fedbc7 | ||
|
|
720eb0c227 | ||
|
|
495c19b5e4 | ||
|
|
237ba96233 | ||
|
|
ae46d89de6 | ||
|
|
69f73e8c5d | ||
|
|
245f87ef35 | ||
|
|
81e51db0b6 | ||
|
|
4634496638 | ||
|
|
4e2f0a9346 | ||
|
|
304b3d4772 | ||
|
|
74629f3ef2 | ||
|
|
c9d48188af | ||
|
|
7e7b88a59e | ||
|
|
5c66dc198d | ||
|
|
47efce7a90 | ||
|
|
ea4d3f0ba7 | ||
|
|
2d03e27dcf | ||
|
|
cf7ff80f71 | ||
|
|
93ab75e42b | ||
|
|
449dc7da79 | ||
|
|
3b129a483a | ||
|
|
3e0d60bb1f | ||
|
|
1d3645fc26 | ||
|
|
18bff86214 | ||
|
|
114084107c | ||
|
|
d60f7238da | ||
|
|
ea684dceeb | ||
|
|
dfd496b41a | ||
|
|
6125b10276 | ||
|
|
071271f003 | ||
|
|
13a9df2e07 | ||
|
|
37a5a6702b | ||
|
|
afb68e97d6 | ||
|
|
93b9741af2 | ||
|
|
013d592454 | ||
|
|
0f56e9b80c | ||
|
|
6ec5ecb9ac | ||
|
|
460d439351 | ||
|
|
f39406dd80 | ||
|
|
9de8e75238 | ||
|
|
fee34b6bd1 | ||
|
|
cbe8699322 | ||
|
|
066e5551f8 | ||
|
|
1d73b736a9 | ||
|
|
8ae6635b9c | ||
|
|
574f0d580e | ||
|
|
0879515ee8 | ||
|
|
be40783454 |
@@ -40,7 +40,7 @@ jobs:
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -61,9 +61,44 @@ jobs:
|
||||
--health-retries 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
# PR 场景需要 base 提交做冻结清单的 append-only 比对。
|
||||
fetch-depth: ${{ github.event_name == 'pull_request' && 0 || 1 }}
|
||||
|
||||
# 已应用迁移的字节被 sqlx 启动校验钉死:改动已发布迁移会让所有
|
||||
# 存量部署重启即失败。冻结清单保证旧文件不被改、新文件必须入册。
|
||||
- name: Migration freeze check
|
||||
working-directory: backend/migrations
|
||||
shell: bash
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sha256sum --check --strict .frozen-sha256
|
||||
for file in *.sql; do
|
||||
if ! awk '{print $2}' .frozen-sha256 | grep -Fxq "$file"; then
|
||||
echo "::error file=backend/migrations/$file::migration $file is not frozen; append its sha256 to backend/migrations/.frozen-sha256"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# 清单自证不足以防「改迁移 + 同 PR 刷新清单行」:校验对新哈希通过。
|
||||
# 对 PR,断言 .frozen-sha256 相对 base 只增不改——base 里的每一行
|
||||
# 必须原样保留(改哈希=删行,删迁移=删行,都会被挡)。
|
||||
if [ -n "${BASE_SHA:-}" ]; then
|
||||
base_manifest="$(git show "${BASE_SHA}:backend/migrations/.frozen-sha256" 2>/dev/null || true)"
|
||||
if [ -n "$base_manifest" ]; then
|
||||
while IFS= read -r line; do
|
||||
[ -z "$line" ] && continue
|
||||
if ! grep -Fxq "$line" .frozen-sha256; then
|
||||
file="$(printf '%s' "$line" | awk '{print $2}')"
|
||||
echo "::error file=backend/migrations/.frozen-sha256::frozen migration $file was modified or removed; released migrations are append-only — add a new numbered migration instead"
|
||||
exit 1
|
||||
fi
|
||||
done <<< "$base_manifest"
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
@@ -117,7 +152,7 @@ jobs:
|
||||
CI: 'true'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
@@ -156,7 +191,7 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
container: ${{ steps.filter.outputs.container }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -29,6 +29,7 @@ jobs:
|
||||
outputs:
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
docker_version: ${{ steps.version.outputs.docker_version }}
|
||||
build_time: ${{ steps.version.outputs.build_time }}
|
||||
git_sha: ${{ steps.version.outputs.git_sha }}
|
||||
platform_matrix: ${{ steps.version.outputs.platform_matrix }}
|
||||
@@ -40,7 +41,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
|
||||
fetch-depth: 0
|
||||
@@ -104,7 +105,7 @@ jobs:
|
||||
require_string(platform, key, "#{platforms_path}: platforms[#{index}]")
|
||||
end
|
||||
archive = platform["archive"]
|
||||
fail("#{platforms_path}: platforms[#{index}].archive must be tar.gz or zip") unless ["tar.gz", "zip"].include?(archive)
|
||||
fail("#{platforms_path}: platforms[#{index}].archive must be tar.gz") unless archive == "tar.gz"
|
||||
platform_key = "#{platform["os"]}-#{platform["arch"]}"
|
||||
fail("#{platforms_path}: duplicate platform #{platform_key}") if seen_platforms[platform_key]
|
||||
seen_platforms[platform_key] = true
|
||||
@@ -146,6 +147,8 @@ jobs:
|
||||
|
||||
puts "tag=#{release_tag}"
|
||||
puts "version=#{version}"
|
||||
# Docker tags forbid "+", so semver build metadata maps to "-" in image tags.
|
||||
puts "docker_version=#{version.tr("+", "-")}"
|
||||
puts "build_time=#{Time.now.utc.iso8601}"
|
||||
puts "git_sha=#{git_sha.strip}"
|
||||
puts "platform_matrix=#{matrix}"
|
||||
@@ -174,7 +177,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -234,7 +237,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -286,7 +289,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -321,7 +324,6 @@ jobs:
|
||||
import pathlib
|
||||
import shutil
|
||||
import tarfile
|
||||
import zipfile
|
||||
|
||||
APP_NAME = "codex-proxy-rs"
|
||||
|
||||
@@ -357,8 +359,7 @@ jobs:
|
||||
|
||||
binary_dest = package_root / platform["binary"]
|
||||
shutil.copy2(binary_src, binary_dest)
|
||||
if platform["os"] != "windows":
|
||||
binary_dest.chmod(0o755)
|
||||
binary_dest.chmod(0o755)
|
||||
shutil.copytree(frontend_dir, web_dist, dirs_exist_ok=True)
|
||||
|
||||
archive_name = f"{APP_NAME}_{version}_{platform['os']}_{platform['arch']}.{platform['archive']}"
|
||||
@@ -366,11 +367,6 @@ jobs:
|
||||
if platform["archive"] == "tar.gz":
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
archive.add(package_root, arcname=APP_NAME)
|
||||
elif platform["archive"] == "zip":
|
||||
with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
for path in sorted(package_root.rglob("*")):
|
||||
if path.is_file():
|
||||
archive.write(path, path.relative_to(package_dir))
|
||||
else:
|
||||
raise SystemExit(f"Unsupported archive format: {platform['archive']}")
|
||||
archives.append(archive_path)
|
||||
@@ -412,7 +408,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -458,7 +454,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -512,7 +508,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -529,7 +525,7 @@ jobs:
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -539,7 +535,7 @@ jobs:
|
||||
id: manifest
|
||||
shell: bash
|
||||
env:
|
||||
VERSION: ${{ needs.release-info.outputs.version }}
|
||||
DOCKER_VERSION: ${{ needs.release-info.outputs.docker_version }}
|
||||
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
|
||||
DOCKER_PLATFORMS: ${{ needs.release-info.outputs.docker_platforms }}
|
||||
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
|
||||
@@ -562,7 +558,7 @@ jobs:
|
||||
fi
|
||||
|
||||
target_args=(
|
||||
-t "${image}:${VERSION}"
|
||||
-t "${image}:${DOCKER_VERSION}"
|
||||
-t "${image}:sha-${GIT_SHA}"
|
||||
)
|
||||
if [ "$PRERELEASE" != true ]; then
|
||||
@@ -570,7 +566,7 @@ jobs:
|
||||
fi
|
||||
docker buildx imagetools create "${target_args[@]}" "${sources[@]}"
|
||||
|
||||
inspect_output="$(docker buildx imagetools inspect "${image}:${VERSION}")"
|
||||
inspect_output="$(docker buildx imagetools inspect "${image}:${DOCKER_VERSION}")"
|
||||
platform_list="$(awk '$1 == "Platform:" { print $2 }' <<< "${inspect_output}")"
|
||||
for platform in "${platforms[@]}"; do
|
||||
if ! grep -Eq "^${platform}(/|$)" <<< "${platform_list}"; then
|
||||
@@ -580,7 +576,7 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
digest="$(docker buildx imagetools inspect "${image}:${VERSION}" --format '{{ .Manifest.Digest }}')"
|
||||
digest="$(docker buildx imagetools inspect "${image}:${DOCKER_VERSION}" --format '{{ .Manifest.Digest }}')"
|
||||
if [ -z "${digest}" ]; then
|
||||
echo "missing image digest"
|
||||
exit 1
|
||||
@@ -598,7 +594,7 @@ jobs:
|
||||
shell: bash
|
||||
env:
|
||||
DIGEST: ${{ steps.manifest.outputs.digest }}
|
||||
VERSION: ${{ needs.release-info.outputs.version }}
|
||||
DOCKER_VERSION: ${{ needs.release-info.outputs.docker_version }}
|
||||
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
|
||||
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
|
||||
run: |
|
||||
@@ -609,7 +605,7 @@ jobs:
|
||||
fi
|
||||
|
||||
image="${REGISTRY}/${IMAGE_NAME}"
|
||||
tags=("${image}:${VERSION}" "${image}:sha-${GIT_SHA}")
|
||||
tags=("${image}:${DOCKER_VERSION}" "${image}:sha-${GIT_SHA}")
|
||||
if [ "$PRERELEASE" != true ]; then
|
||||
tags+=("${image}:latest")
|
||||
fi
|
||||
@@ -629,12 +625,15 @@ jobs:
|
||||
prerelease: ${{ needs.release-info.outputs.prerelease == 'true' }}
|
||||
make_latest: ${{ needs.release-info.outputs.prerelease == 'true' && 'false' || 'true' }}
|
||||
body: |
|
||||
> [!WARNING]
|
||||
> **Upgrading across major versions?** Database schema and configuration are only compatible within a major version, and the one-click self-update refuses cross-major targets (releases before 3.0 lack this gate — do not use their built-in update to reach a newer major). Deploy the new major fresh (new `.runtime/` data directory) and re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
|
||||
|
||||
## 📥 Installation
|
||||
|
||||
**Docker:**
|
||||
```bash
|
||||
# GitHub Container Registry
|
||||
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.version }}
|
||||
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.docker_version }}
|
||||
```
|
||||
|
||||
**Manual download:**
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
|
||||
@@ -21,15 +21,16 @@
|
||||
|
||||
## 能力
|
||||
|
||||
| 领域 | 实现 |
|
||||
| -------- | --------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI 保持 Responses 语义透明;xAI 的协议适配只驻留在 `providers/xai` |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因 |
|
||||
| 领域 | 实现 |
|
||||
| -------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI 请求保留未知字段和字段顺序,SSE 与 WebSocket 原样转发业务字节;xAI 在 Provider 内做协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 账号 | 导入、OAuth、credential 刷新、主动额度刷新,以及正常响应中的被动额度/状态观测 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因,并记录响应确认的实际 `service_tier` |
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -91,7 +92,12 @@ curl -i http://127.0.0.1:8080/healthz
|
||||
2. 在账号页选择对应 Provider:OpenAI 支持 OAuth、账号文件与 Agent Identity;xAI 支持 OAuth
|
||||
和兼容账号文件导入。
|
||||
3. 按需配置客户端模型到上游模型的全局映射;未命中映射的模型名原样透传。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置模型范围、速率与并发限制。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置速率与并发限制。
|
||||
|
||||
OpenAI 账号导入和首次 OAuth 创建在 credential 提交后会立即尝试一次额度观测;额度接口临时失败不会
|
||||
回滚已提交的账号。Free、K12 等套餐共用同一套额度与状态转换逻辑,套餐字段只作为账号事实并隔离
|
||||
对应的模型目录 cache。credential 刷新、额度刷新和正常请求中的被动额度观测是三条独立链路,详见
|
||||
[账号接口](docs/api.md#5-账号)。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> xAI 使用 OAuth session,不支持把 xAI API Key 作为上游 credential。
|
||||
@@ -145,7 +151,7 @@ curl http://127.0.0.1:8080/v1/responses \
|
||||
## 运维
|
||||
|
||||
```bash
|
||||
# 升级
|
||||
# 实例升级
|
||||
docker compose -f deploy/compose.yaml pull codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d --no-build
|
||||
|
||||
@@ -158,9 +164,15 @@ docker compose -f deploy/compose.yaml up -d
|
||||
```
|
||||
|
||||
> [!IMPORTANT]
|
||||
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点和自更新状态。删除该目录会永久清除
|
||||
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点密钥和自更新状态。删除该目录会永久清除
|
||||
> 运行状态。
|
||||
|
||||
> [!WARNING]
|
||||
> 跨大版本不是原地升级:数据库 schema 与配置只在同一大版本内兼容,内置一键自更新
|
||||
> 会拒绝跨大版本目标(3.0 之前的旧版没有这道门禁,不要用旧版界面的一键更新跨大版本)。
|
||||
> 升级到新的大版本需要全新部署(全新 `.runtime/` 数据目录),并重新导入或重新授权
|
||||
> Provider 账号与客户端 Key。
|
||||
|
||||
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
|
||||
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
|
||||
@@ -173,7 +185,8 @@ release/publish <version>
|
||||
```
|
||||
|
||||
脚本会更新 `release/version.yaml`、创建约定提交与带注释的 `v<version>` tag,并原子推送分支和 tag。
|
||||
不要手工修改版本后单独打 tag。
|
||||
不要手工修改版本后单独打 tag。该脚本只发布仓库版本和触发 Release 构建,不会登录服务器,也不会
|
||||
改变任何运行实例;源码提交、仓库发版和实例升级是三个独立状态。
|
||||
|
||||
## 文档
|
||||
|
||||
|
||||
Generated
+129
-62
@@ -142,7 +142,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -208,7 +208,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
|
||||
dependencies = [
|
||||
"axum-core",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
@@ -275,6 +275,12 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
@@ -444,7 +450,7 @@ dependencies = [
|
||||
"provider-openai",
|
||||
"provider-xai",
|
||||
"serde",
|
||||
"syn 2.0.118",
|
||||
"syn 3.0.3",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -669,7 +675,7 @@ dependencies = [
|
||||
"aead",
|
||||
"blake2",
|
||||
"crypto_secretbox",
|
||||
"curve25519-dalek",
|
||||
"curve25519-dalek 4.1.3",
|
||||
"salsa20",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -708,8 +714,23 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"curve25519-dalek-derive",
|
||||
"digest 0.10.7",
|
||||
"fiat-crypto",
|
||||
"fiat-crypto 0.2.9",
|
||||
"rustc_version",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
"curve25519-dalek-derive",
|
||||
"digest 0.11.3",
|
||||
"fiat-crypto 0.3.0",
|
||||
"rustc_version",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -723,7 +744,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -761,6 +782,16 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.8"
|
||||
@@ -799,7 +830,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -829,34 +860,35 @@ version = "0.16.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
|
||||
dependencies = [
|
||||
"der",
|
||||
"der 0.7.10",
|
||||
"digest 0.10.7",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
"signature",
|
||||
"spki",
|
||||
"signature 2.2.0",
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ed25519"
|
||||
version = "2.2.3"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
|
||||
checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a"
|
||||
dependencies = [
|
||||
"pkcs8",
|
||||
"signature",
|
||||
"pkcs8 0.11.0",
|
||||
"signature 3.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ed25519-dalek"
|
||||
version = "2.2.0"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9"
|
||||
checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de"
|
||||
dependencies = [
|
||||
"curve25519-dalek",
|
||||
"curve25519-dalek 5.0.0",
|
||||
"ed25519",
|
||||
"serde",
|
||||
"sha2 0.10.9",
|
||||
"sha2 0.11.0",
|
||||
"signature 3.0.0",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -883,7 +915,7 @@ dependencies = [
|
||||
"generic-array",
|
||||
"group",
|
||||
"pem-rfc7468",
|
||||
"pkcs8",
|
||||
"pkcs8 0.10.2",
|
||||
"rand_core 0.6.4",
|
||||
"sec1",
|
||||
"subtle",
|
||||
@@ -968,6 +1000,12 @@ version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
|
||||
|
||||
[[package]]
|
||||
name = "fiat-crypto"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24"
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.29"
|
||||
@@ -1100,7 +1138,7 @@ checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1144,7 +1182,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"chrono",
|
||||
"futures",
|
||||
"gateway-core",
|
||||
@@ -1152,9 +1190,11 @@ dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"subtle",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -1164,7 +1204,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
@@ -1175,6 +1215,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.29.0",
|
||||
"tower",
|
||||
"tower-http 0.7.0",
|
||||
"tracing",
|
||||
@@ -1188,12 +1229,12 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"futures-timer",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -1224,7 +1265,6 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
@@ -1372,7 +1412,7 @@ version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"headers-core",
|
||||
"http",
|
||||
@@ -1539,7 +1579,7 @@ version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
@@ -1737,19 +1777,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonwebtoken"
|
||||
version = "10.3.0"
|
||||
version = "11.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1"
|
||||
checksum = "881733cbc631fc9e472e24447ce32a64bedf2da498d6d8570b08edc87de71f65"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"getrandom 0.2.17",
|
||||
"js-sys",
|
||||
"pem",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"signature",
|
||||
"signature 2.2.0",
|
||||
"simple_asn1",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2016,7 +2057,7 @@ version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
@@ -2047,8 +2088,18 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||
dependencies = [
|
||||
"der",
|
||||
"spki",
|
||||
"der 0.7.10",
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.1",
|
||||
"spki 0.8.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2116,7 +2167,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"cookie",
|
||||
@@ -2126,7 +2177,7 @@ dependencies = [
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"gateway-protocol",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"hex",
|
||||
"jsonwebtoken",
|
||||
"reqwest",
|
||||
@@ -2150,7 +2201,6 @@ dependencies = [
|
||||
"url",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2159,13 +2209,14 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"gateway-protocol",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"jsonwebtoken",
|
||||
"p256",
|
||||
"reqwest",
|
||||
@@ -2410,7 +2461,7 @@ version = "0.12.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
@@ -2599,9 +2650,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"der",
|
||||
"der 0.7.10",
|
||||
"generic-array",
|
||||
"pkcs8",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -2797,6 +2848,12 @@ dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signature"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.9"
|
||||
@@ -2856,7 +2913,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der",
|
||||
"der 0.7.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2878,7 +2945,7 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"chrono",
|
||||
@@ -2919,7 +2986,7 @@ dependencies = [
|
||||
"quote",
|
||||
"sqlx-core",
|
||||
"sqlx-macros-core",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2942,7 +3009,7 @@ dependencies = [
|
||||
"sqlx-mysql",
|
||||
"sqlx-postgres",
|
||||
"sqlx-sqlite",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"url",
|
||||
@@ -2982,7 +3049,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e"
|
||||
dependencies = [
|
||||
"atoi",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"chrono",
|
||||
@@ -3067,9 +3134,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.118"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3104,7 +3171,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3125,7 +3192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -3239,7 +3306,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3415,7 +3482,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3692,7 +3759,7 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -3774,7 +3841,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3785,7 +3852,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3910,7 +3977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031"
|
||||
dependencies = [
|
||||
"assert-json-diff",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"deadpool",
|
||||
"futures",
|
||||
"http",
|
||||
@@ -3984,7 +4051,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4005,7 +4072,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4025,7 +4092,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4046,7 +4113,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4079,7 +4146,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -25,7 +25,7 @@ tokio = { version = "1.52.3", default-features = false, features = ["rt-multi-th
|
||||
|
||||
[dev-dependencies]
|
||||
config = { version = "0.15.23", default-features = false, features = ["yaml"] }
|
||||
syn = { version = "2.0.118", features = ["full"] }
|
||||
syn = { version = "3.0.3", features = ["full"] }
|
||||
tempfile = "3.27.0"
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
//! docs/architecture.md 依赖 DAG 与生产源码纪律的 workspace 级机器校验。
|
||||
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
/// workspace 成员冻结清单;新增 crate 必须同步扩展本文件的依赖规则。
|
||||
const WORKSPACE_MEMBERS: &[&str] = &[
|
||||
"apps/gateway",
|
||||
"crates/gateway-admin",
|
||||
"crates/gateway-api",
|
||||
"crates/gateway-core",
|
||||
"crates/gateway-host",
|
||||
"crates/gateway-protocol",
|
||||
"crates/gateway-store",
|
||||
"crates/providers/openai",
|
||||
"crates/providers/xai",
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn workspace_member_list_matches_the_frozen_dag_scope() {
|
||||
let manifest =
|
||||
fs::read_to_string(backend_root().join("Cargo.toml")).expect("read workspace manifest");
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
assert!(
|
||||
manifest.contains(&format!("\"{member}\"")),
|
||||
"workspace must include {member}"
|
||||
);
|
||||
}
|
||||
let member_count = manifest
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| line.starts_with("\"apps/") || line.starts_with("\"crates/"))
|
||||
.count();
|
||||
assert_eq!(
|
||||
member_count,
|
||||
WORKSPACE_MEMBERS.len(),
|
||||
"new workspace members must extend the dependency DAG rules"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_core_depends_on_no_http_db_redis_or_provider_crate() {
|
||||
assert_no_dependency(
|
||||
"crates/gateway-core",
|
||||
&[
|
||||
"axum",
|
||||
"hyper",
|
||||
"reqwest",
|
||||
"sqlx",
|
||||
"redis",
|
||||
"gateway-host",
|
||||
"gateway-store",
|
||||
"provider-openai",
|
||||
"provider-xai",
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_protocol_has_no_workspace_dependencies() {
|
||||
for name in dependency_names("crates/gateway-protocol") {
|
||||
assert!(
|
||||
!name.starts_with("gateway-") && !name.starts_with("provider-"),
|
||||
"gateway-protocol must not depend on workspace crate `{name}`"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_crates_do_not_depend_on_each_other() {
|
||||
assert_no_dependency("crates/providers/openai", &["provider-xai"]);
|
||||
assert_no_dependency("crates/providers/xai", &["provider-openai"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_admin_stays_free_of_infrastructure_dependencies() {
|
||||
assert_no_dependency(
|
||||
"crates/gateway-admin",
|
||||
&["axum", "sqlx", "redis", "reqwest"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_sources_do_not_host_tests() {
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
let src = backend_root().join(member).join("src");
|
||||
let sources = super::rust_files(&src);
|
||||
assert!(!sources.is_empty(), "{member} has no production sources");
|
||||
for relative in sources {
|
||||
let path = src.join(relative);
|
||||
let source = fs::read_to_string(&path).expect("read production source");
|
||||
assert!(
|
||||
!source.contains("#[cfg(test)]"),
|
||||
"{} hosts tests in production src",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn backend_root() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.canonicalize()
|
||||
.expect("backend workspace root")
|
||||
}
|
||||
|
||||
/// 提取成员 `[dependencies]` 段内声明的依赖名;段落以下一个 `[` 表头结束。
|
||||
fn dependency_names(member: &str) -> Vec<String> {
|
||||
let manifest = fs::read_to_string(backend_root().join(member).join("Cargo.toml"))
|
||||
.expect("read member manifest");
|
||||
let mut in_dependencies = false;
|
||||
let mut names = Vec::new();
|
||||
for line in manifest.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
in_dependencies = line == "[dependencies]";
|
||||
continue;
|
||||
}
|
||||
if !in_dependencies || line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if let Some((name, _)) = line.split_once('=') {
|
||||
names.push(name.trim().to_owned());
|
||||
}
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
fn assert_no_dependency(member: &str, forbidden: &[&str]) {
|
||||
for name in dependency_names(member) {
|
||||
assert!(
|
||||
!forbidden.contains(&name.as_str()),
|
||||
"{member} must not depend on `{name}`"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
mod architecture;
|
||||
mod bootstrap;
|
||||
|
||||
use std::{
|
||||
@@ -21,7 +22,11 @@ fn app_tree_matches_frozen_terminal_manifest() {
|
||||
);
|
||||
assert_eq!(
|
||||
rust_files(&root.join("tests")),
|
||||
BTreeSet::from([PathBuf::from("bootstrap.rs"), PathBuf::from("main.rs")]),
|
||||
BTreeSet::from([
|
||||
PathBuf::from("architecture.rs"),
|
||||
PathBuf::from("bootstrap.rs"),
|
||||
PathBuf::from("main.rs"),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ path = "tests/main.rs"
|
||||
[dependencies]
|
||||
argon2 = "0.5.3"
|
||||
async-trait = "0.1.89"
|
||||
base64 = "0.22.1"
|
||||
base64 = "0.23.0"
|
||||
chrono = "0.4.45"
|
||||
futures = "0.3.32"
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
@@ -24,9 +24,11 @@ serde = { version = "1.0.228", features = ["derive"] }
|
||||
secrecy = { version = "0.10.3", features = ["serde"] }
|
||||
subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -9,17 +9,8 @@ use gateway_core::routing::ProviderKind;
|
||||
|
||||
use super::{PageSize, Revision, observability::TimeRange};
|
||||
|
||||
/// Provider 账号当前可用性。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountAvailability {
|
||||
Unknown,
|
||||
Ready,
|
||||
Cooldown,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
Invalid,
|
||||
}
|
||||
/// Provider 账号当前可用性;权威定义与字符串编解码在 gateway-core,此处仅复用。
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -29,7 +20,6 @@ pub enum AccountStatus {
|
||||
QuotaExhausted,
|
||||
Disabled,
|
||||
Banned,
|
||||
Attention,
|
||||
}
|
||||
|
||||
/// 账号列表排序字段。
|
||||
@@ -179,18 +169,19 @@ pub struct AccountPage {
|
||||
}
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub attention: u64,
|
||||
pub unavailable: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SetAccountEnabled {
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_id: String,
|
||||
pub enabled: bool,
|
||||
}
|
||||
@@ -198,7 +189,6 @@ pub struct SetAccountEnabled {
|
||||
/// 账号批量删除命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DeleteAccounts {
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_ids: Vec<String>,
|
||||
}
|
||||
|
||||
@@ -222,6 +212,11 @@ pub enum AccountConnectionTestEvent {
|
||||
},
|
||||
Failed {
|
||||
message: String,
|
||||
provider_error_code: Option<String>,
|
||||
provider_error_type: Option<String>,
|
||||
upstream_status: Option<u16>,
|
||||
upstream_content_type: Option<String>,
|
||||
upstream_body: Option<String>,
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ impl AdminRequestContext {
|
||||
pub struct LoginCommand {
|
||||
pub username: Option<String>,
|
||||
pub password: String,
|
||||
pub source: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LoginCommand {
|
||||
@@ -48,7 +47,6 @@ impl std::fmt::Debug for LoginCommand {
|
||||
.debug_struct("LoginCommand")
|
||||
.field("username", &self.username)
|
||||
.field("password", &"[REDACTED]")
|
||||
.field("source", &self.source)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -65,8 +63,6 @@ pub struct LoginResult {
|
||||
pub enum LoginError {
|
||||
#[error("invalid administrator credentials")]
|
||||
InvalidCredentials,
|
||||
#[error("administrator login is throttled")]
|
||||
Throttled,
|
||||
#[error("administrator authentication is unavailable")]
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
@@ -141,7 +141,6 @@ impl fmt::Debug for ClientKeySecret {
|
||||
/// API 提交的 Client Key 创建命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreateClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub provider_kind: ProviderKind,
|
||||
@@ -151,7 +150,6 @@ pub struct CreateClientKey {
|
||||
/// 管理用例生成 ID 与明文后的持久化命令。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct NewClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
@@ -175,7 +173,6 @@ impl fmt::Debug for NewClientKey {
|
||||
/// 修改 Client Key 的公开策略字段。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
@@ -186,7 +183,6 @@ pub struct UpdateClientKey {
|
||||
/// 修改 Client Key 启用状态。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SetClientKeyEnabled {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
pub enabled: bool,
|
||||
}
|
||||
@@ -194,7 +190,6 @@ pub struct SetClientKeyEnabled {
|
||||
/// 删除 Client Key。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DeleteClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,20 @@
|
||||
|
||||
use std::{num::NonZeroU32, str::FromStr};
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use chrono::{DateTime, TimeDelta, Timelike as _, Utc};
|
||||
|
||||
use super::{AdminModelError, PageSize};
|
||||
|
||||
/// 观测日界所用的东八区(UTC+8)固定偏移秒数。
|
||||
const CHINA_OFFSET_SECONDS: i64 = 8 * 60 * 60;
|
||||
|
||||
/// 将 UTC 时刻截断到东八区(UTC+8)当日零点,返回值仍为 UTC。
|
||||
#[must_use]
|
||||
pub fn china_day_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = (value.timestamp() + CHINA_OFFSET_SECONDS).rem_euclid(24 * 60 * 60);
|
||||
value - TimeDelta::seconds(elapsed) - TimeDelta::nanoseconds(i64::from(value.nanosecond()))
|
||||
}
|
||||
|
||||
/// 外部观测查询的 UTC 时间范围。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TimeRange {
|
||||
@@ -320,6 +330,7 @@ pub struct LatencyPercentiles {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderBillingInput {
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
@@ -335,6 +346,7 @@ pub struct UsageCalculatedBillingFact {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub provider_kind: String,
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
@@ -448,6 +460,10 @@ pub struct RequestMetricPoint {
|
||||
}
|
||||
|
||||
/// 账号池容量统计。
|
||||
///
|
||||
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
|
||||
///
|
||||
/// `quota_exhausted` 仍单列,以便状态详情展示和诊断。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
@@ -564,6 +580,7 @@ pub struct UsageRecord {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;管理领域保持不透明。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
@@ -718,7 +735,8 @@ pub struct OpsError {
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
pub status_code: Option<u16>,
|
||||
pub client_status_code: Option<u16>,
|
||||
pub upstream_status_code: Option<u16>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub client_response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
|
||||
@@ -2,14 +2,15 @@
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId},
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId, ProviderAccountIdentity},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
};
|
||||
|
||||
use super::{
|
||||
MutationActor, MutationContext, PageSize, Revision,
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
};
|
||||
|
||||
@@ -101,7 +102,6 @@ pub struct CredentialDetails {
|
||||
/// Provider 正式文档批量导入命令。
|
||||
pub struct ImportCredentials {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub document: ProviderDocument,
|
||||
}
|
||||
|
||||
@@ -110,7 +110,6 @@ impl fmt::Debug for ImportCredentials {
|
||||
formatter
|
||||
.debug_struct("ImportCredentials")
|
||||
.field("context", &self.context)
|
||||
.field("expected_config_revision", &self.expected_config_revision)
|
||||
.field("document", &self.document)
|
||||
.finish()
|
||||
}
|
||||
@@ -169,17 +168,9 @@ pub struct PreparedCredentialImport {
|
||||
/// Admin 交给 Store 的导入事务命令。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CredentialImportCommit {
|
||||
pub expected_config_revision: Revision,
|
||||
pub prepared: PreparedCredentialImport,
|
||||
}
|
||||
|
||||
/// 可选的重新授权目标。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReauthorizationTarget {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub credential_revision: Revision,
|
||||
}
|
||||
|
||||
/// OAuth pending owner 的中立身份;不编码具体 Provider 的 Redis key 或 JSON。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub enum AuthorizationOwner {
|
||||
@@ -240,21 +231,18 @@ impl fmt::Debug for AuthorizationOwnerBinding {
|
||||
}
|
||||
|
||||
/// OAuth 完成时应创建新账号还是 CAS 更新既有 credential。
|
||||
///
|
||||
/// 重新授权只绑定稳定的账号身份:credential revision 由服务端在临近写入时读取,
|
||||
/// 长流程期间的后台刷新不得让恢复操作失效。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AuthorizationMutationTarget {
|
||||
Create {
|
||||
name: String,
|
||||
},
|
||||
Reauthorize {
|
||||
account_id: ProviderAccountId,
|
||||
expected_credential_revision: Revision,
|
||||
},
|
||||
Create { name: String },
|
||||
Reauthorize { account_id: ProviderAccountId },
|
||||
}
|
||||
|
||||
/// 必须完整进入 Provider opaque pending payload、并在 complete 后原样恢复的事务信封。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PendingAuthorizationMutation {
|
||||
expected_config_revision: Revision,
|
||||
provider_kind: ProviderKind,
|
||||
target: AuthorizationMutationTarget,
|
||||
owner_binding: AuthorizationOwnerBinding,
|
||||
@@ -263,24 +251,17 @@ pub struct PendingAuthorizationMutation {
|
||||
impl PendingAuthorizationMutation {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
expected_config_revision: Revision,
|
||||
provider_kind: ProviderKind,
|
||||
target: AuthorizationMutationTarget,
|
||||
owner_binding: AuthorizationOwnerBinding,
|
||||
) -> Self {
|
||||
Self {
|
||||
expected_config_revision,
|
||||
provider_kind,
|
||||
target,
|
||||
owner_binding,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn expected_config_revision(&self) -> Revision {
|
||||
self.expected_config_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
@@ -301,9 +282,8 @@ impl PendingAuthorizationMutation {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StartAuthorization {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub name: String,
|
||||
pub reauthorization: Option<ReauthorizationTarget>,
|
||||
pub reauthorization: Option<ProviderAccountId>,
|
||||
}
|
||||
|
||||
/// OAuth 流程启动结果。
|
||||
@@ -322,6 +302,20 @@ pub trait CredentialCommitGuard: Send + 'static {
|
||||
fn finish(self: Box<Self>);
|
||||
}
|
||||
|
||||
/// OAuth pending claim 在 Store 事务结果确定后执行的显式结算动作。
|
||||
///
|
||||
/// OAuth provider 必须在 credential 准备阶段持有 claim:Store 提交成功后消费 flow,提交或
|
||||
/// 校验失败后释放 flow。异步结算不能依赖 `Drop`,否则请求提前返回会让同一授权流无谓地
|
||||
/// 进入短暂的不可重试状态。
|
||||
#[async_trait]
|
||||
pub trait AuthorizationCommitGuard: Send + 'static {
|
||||
/// Store 已提交 OAuth credential 后消费对应的一次性 flow。
|
||||
async fn commit(self: Box<Self>) -> Result<(), AdminError>;
|
||||
|
||||
/// Store 未提交 OAuth credential 时释放 claim,允许同一 flow 重试。
|
||||
async fn abort(self: Box<Self>) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
/// OAuth complete 后由 Provider 返回的准备结果;Store 仍是唯一提交者。
|
||||
pub enum PreparedAuthorizationCredential {
|
||||
Create(PreparedCredentialCreate),
|
||||
@@ -332,6 +326,7 @@ pub enum PreparedAuthorizationCredential {
|
||||
pub struct PreparedAuthorizationCommit {
|
||||
pub pending: PendingAuthorizationMutation,
|
||||
pub credential: PreparedAuthorizationCredential,
|
||||
authorization_guard: Option<Box<dyn AuthorizationCommitGuard>>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for PreparedAuthorizationCommit {
|
||||
@@ -358,9 +353,37 @@ pub struct AuthorizationCommit {
|
||||
pub credential: AuthorizationCredentialCommit,
|
||||
}
|
||||
|
||||
/// OAuth 准备结果拆解后的 Store 命令与两个结算 guard。
|
||||
///
|
||||
/// Store 成功时先结束 credential guard,再消费 OAuth claim;失败时丢弃 credential guard 并
|
||||
/// 释放 OAuth claim,使同一授权流可以重试。
|
||||
pub(crate) struct AuthorizationCommitSettlement {
|
||||
pub(crate) command: AuthorizationCommit,
|
||||
pub(crate) credential_guard: Option<Box<dyn CredentialCommitGuard>>,
|
||||
pub(crate) authorization_guard: Option<Box<dyn AuthorizationCommitGuard>>,
|
||||
}
|
||||
|
||||
impl PreparedAuthorizationCommit {
|
||||
#[must_use]
|
||||
pub fn into_commit(self) -> (AuthorizationCommit, Option<Box<dyn CredentialCommitGuard>>) {
|
||||
pub fn new(
|
||||
pending: PendingAuthorizationMutation,
|
||||
credential: PreparedAuthorizationCredential,
|
||||
) -> Self {
|
||||
Self {
|
||||
pending,
|
||||
credential,
|
||||
authorization_guard: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让 OAuth claim 覆盖准备完成到 Store 事务结算之间的全部窗口。
|
||||
#[must_use]
|
||||
pub fn with_authorization_guard(mut self, guard: Box<dyn AuthorizationCommitGuard>) -> Self {
|
||||
self.authorization_guard = Some(guard);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn into_commit(self) -> AuthorizationCommitSettlement {
|
||||
let (credential, guard) = match self.credential {
|
||||
PreparedAuthorizationCredential::Create(credential) => {
|
||||
(AuthorizationCredentialCommit::Create(credential), None)
|
||||
@@ -373,13 +396,27 @@ impl PreparedAuthorizationCommit {
|
||||
)
|
||||
}
|
||||
};
|
||||
(
|
||||
AuthorizationCommit {
|
||||
AuthorizationCommitSettlement {
|
||||
command: AuthorizationCommit {
|
||||
pending: self.pending,
|
||||
credential,
|
||||
},
|
||||
guard,
|
||||
)
|
||||
credential_guard: guard,
|
||||
authorization_guard: self.authorization_guard,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn abort(self) -> Result<(), AdminError> {
|
||||
let AuthorizationCommitSettlement {
|
||||
credential_guard,
|
||||
authorization_guard,
|
||||
..
|
||||
} = self.into_commit();
|
||||
drop(credential_guard);
|
||||
if let Some(guard) = authorization_guard {
|
||||
guard.abort().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -406,7 +443,6 @@ impl fmt::Debug for CompleteAuthorization {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CredentialMutation {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_id: ProviderAccountId,
|
||||
}
|
||||
|
||||
@@ -422,7 +458,6 @@ pub struct CredentialMutationResult {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CredentialDeletion {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_ids: Vec<ProviderAccountId>,
|
||||
}
|
||||
|
||||
@@ -436,14 +471,12 @@ pub struct CredentialDeletionResult {
|
||||
/// Provider-owned token 轮换命令。
|
||||
pub struct RotateCredential {
|
||||
pub mutation: CredentialMutation,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub provider_material: ProviderDocument,
|
||||
}
|
||||
|
||||
/// Provider 校验手工轮换材料时所需的非事务输入。
|
||||
pub struct PrepareCredentialRotation {
|
||||
pub account: AccountRecord,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub provider_material: ProviderDocument,
|
||||
}
|
||||
|
||||
@@ -452,10 +485,6 @@ impl fmt::Debug for PrepareCredentialRotation {
|
||||
formatter
|
||||
.debug_struct("PrepareCredentialRotation")
|
||||
.field("account", &self.account)
|
||||
.field(
|
||||
"expected_credential_revision",
|
||||
&self.expected_credential_revision,
|
||||
)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.finish()
|
||||
}
|
||||
@@ -467,6 +496,7 @@ pub struct PreparedCredentialRotationFacts {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
@@ -520,7 +550,6 @@ impl fmt::Debug for PreparedCredentialRotation {
|
||||
/// Admin 交给 Store 的轮换或 refresh 事务命令。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CredentialRotationCommit {
|
||||
pub expected_config_revision: Revision,
|
||||
pub prepared: PreparedCredentialRotationFacts,
|
||||
}
|
||||
|
||||
@@ -529,10 +558,6 @@ impl fmt::Debug for RotateCredential {
|
||||
formatter
|
||||
.debug_struct("RotateCredential")
|
||||
.field("mutation", &self.mutation)
|
||||
.field(
|
||||
"expected_credential_revision",
|
||||
&self.expected_credential_revision,
|
||||
)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.finish()
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ pub struct RuntimeSettings {
|
||||
/// 原子替换运行设置的命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReplaceRuntimeSettings {
|
||||
pub expected_config_revision: Revision,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
|
||||
@@ -64,6 +64,12 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
/// 成功之后,不参与事务成败,也不得恢复或改写已经提交的账号状态。
|
||||
async fn account_unavailable(&self, account_id: &ProviderAccountId);
|
||||
|
||||
/// 账号资格事实已经由控制面提交,失效 Provider 持有的可重建派生状态。
|
||||
///
|
||||
/// 该通知发生在 Store 事务成功之后、下一份 RuntimeSnapshot 编译之前;通知
|
||||
/// 不参与已提交事务成败。没有账号派生状态的 Provider 可使用默认空实现。
|
||||
async fn account_facts_changed(&self, _account_ids: &[ProviderAccountId]) {}
|
||||
|
||||
/// 生成一次连接测试所需的 Provider-owned operation;Core 负责实际执行与落账。
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
|
||||
@@ -156,7 +156,7 @@ pub trait AccountStore: Send + Sync {
|
||||
) -> AdminStoreResult<()>;
|
||||
}
|
||||
|
||||
/// 管理员密码、会话、登录限流和安全审计。
|
||||
/// 管理员密码、会话和安全审计。
|
||||
#[async_trait]
|
||||
pub trait AuthStore: Send + Sync {
|
||||
async fn load_password_hash(&self, admin_user_id: &str) -> AdminStoreResult<Option<String>>;
|
||||
@@ -176,26 +176,10 @@ pub trait AuthStore: Send + Sync {
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AdminSession>>;
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
async fn clear_login_failures(&self, source: &str) -> AdminStoreResult<()>;
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()>;
|
||||
}
|
||||
|
||||
/// Client API Key 管理及其 revision CAS 写入。
|
||||
/// Client API Key 管理写入。
|
||||
#[async_trait]
|
||||
pub trait ClientKeyStore: Send + Sync {
|
||||
async fn list_client_keys(&self, query: ClientKeyListQuery) -> AdminStoreResult<ClientKeyPage>;
|
||||
@@ -281,7 +265,7 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage>;
|
||||
}
|
||||
|
||||
/// Runtime settings 与管理员 API Key 的 revision CAS 写入。
|
||||
/// Runtime settings 与管理员 API Key 写入。
|
||||
#[async_trait]
|
||||
pub trait SettingsStore: Send + Sync {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings>;
|
||||
@@ -296,14 +280,12 @@ pub trait SettingsStore: Send + Sync {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
key: AdminApiKey,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation>;
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation>;
|
||||
}
|
||||
|
||||
@@ -10,13 +10,12 @@ use gateway_core::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest},
|
||||
},
|
||||
error::GatewayErrorKind,
|
||||
routing::{ProviderKind, UpstreamModelId, snapshot::SnapshotControl},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, MutationContext, Revision,
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
@@ -55,7 +54,6 @@ pub trait AccountsService: Send + Sync {
|
||||
async fn refresh(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expected_config_revision: Revision,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError>;
|
||||
|
||||
@@ -355,7 +353,6 @@ impl AccountsService for DefaultAccountsService {
|
||||
async fn refresh(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expected_config_revision: Revision,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
@@ -368,12 +365,14 @@ impl AccountsService for DefaultAccountsService {
|
||||
validate_prepared_rotation(&account, &prepared, "provider credential refresh")?;
|
||||
let result = commit_credential_refresh(
|
||||
self.accounts.as_ref(),
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
context,
|
||||
"provider credential refresh",
|
||||
)
|
||||
.await?;
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
let account = self.load_directory_item(&result.account_id, false).await?;
|
||||
Ok(AccountRefreshResult {
|
||||
@@ -453,15 +452,33 @@ impl AccountsService for DefaultAccountsService {
|
||||
account_status: status,
|
||||
}))
|
||||
.collect(),
|
||||
Err(error) => vec![AccountConnectionTestEvent::Failed {
|
||||
message: match error.kind() {
|
||||
GatewayErrorKind::InvalidRequest
|
||||
| GatewayErrorKind::Unsupported
|
||||
| GatewayErrorKind::ModelNotFound => error.safe_message().to_owned(),
|
||||
_ => "Provider connection test failed".to_owned(),
|
||||
},
|
||||
account_status: status,
|
||||
}],
|
||||
Err(error) => {
|
||||
let upstream_status = error
|
||||
.upstream_response()
|
||||
.map(gateway_core::engine::probe::AccountProbeUpstreamResponse::status);
|
||||
let upstream_content_type = error
|
||||
.upstream_response()
|
||||
.and_then(|response| response.content_type())
|
||||
.and_then(|value| std::str::from_utf8(value).ok())
|
||||
.map(ToOwned::to_owned);
|
||||
let upstream_body = error
|
||||
.upstream_response()
|
||||
.map(|response| String::from_utf8_lossy(response.body()).into_owned());
|
||||
let message = upstream_body
|
||||
.as_deref()
|
||||
.filter(|body| !body.is_empty())
|
||||
.unwrap_or_else(|| error.client_message())
|
||||
.to_owned();
|
||||
vec![AccountConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status: status,
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
.flat_map(futures::stream::iter);
|
||||
@@ -472,22 +489,25 @@ impl AccountsService for DefaultAccountsService {
|
||||
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == crate::model::accounts::AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else {
|
||||
match account.availability {
|
||||
crate::model::accounts::AccountAvailability::Banned => AccountStatus::Banned,
|
||||
crate::model::accounts::AccountAvailability::QuotaExhausted => {
|
||||
AccountStatus::QuotaExhausted
|
||||
}
|
||||
crate::model::accounts::AccountAvailability::Expired => AccountStatus::Expired,
|
||||
_ if account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now) =>
|
||||
{
|
||||
AccountStatus::Expired
|
||||
}
|
||||
crate::model::accounts::AccountAvailability::Ready => AccountStatus::Active,
|
||||
_ => AccountStatus::Attention,
|
||||
}
|
||||
AccountStatus::Active
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,9 +22,6 @@ use crate::{
|
||||
|
||||
use super::map_store_error;
|
||||
|
||||
const LOGIN_FAILURE_LIMIT: u32 = 5;
|
||||
const LOGIN_FAILURE_WINDOW_SECONDS: u64 = 15 * 60;
|
||||
|
||||
/// API 鉴权与管理员登录消费的统一服务。
|
||||
#[async_trait]
|
||||
pub trait AuthService: Send + Sync {
|
||||
@@ -39,6 +36,10 @@ pub trait AuthService: Send + Sync {
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
/// 会话有效期上限(366 天);超出的配置按上限截断,保证 TTL 构造与
|
||||
/// `Utc::now() + session_ttl` 永不越界。
|
||||
const MAX_SESSION_TTL_MINUTES: i64 = 366 * 24 * 60;
|
||||
|
||||
/// 单管理员认证策略的最终实现。
|
||||
pub(crate) struct DefaultAuthService {
|
||||
default_admin_user_id: String,
|
||||
@@ -54,8 +55,8 @@ impl DefaultAuthService {
|
||||
store: Arc<dyn AuthStore>,
|
||||
) -> Self {
|
||||
let minutes = i64::try_from(session_ttl_minutes)
|
||||
.unwrap_or(i64::MAX)
|
||||
.max(1);
|
||||
.unwrap_or(MAX_SESSION_TTL_MINUTES)
|
||||
.clamp(1, MAX_SESSION_TTL_MINUTES);
|
||||
Self {
|
||||
default_admin_user_id: default_admin_user_id.into(),
|
||||
session_ttl: Duration::minutes(minutes),
|
||||
@@ -63,19 +64,6 @@ impl DefaultAuthService {
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_failed_login(&self, source: &str) -> Result<LoginError, LoginError> {
|
||||
let throttled = self
|
||||
.store
|
||||
.record_login_failure(source, LOGIN_FAILURE_LIMIT, LOGIN_FAILURE_WINDOW_SECONDS)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?;
|
||||
Ok(if throttled {
|
||||
LoginError::Throttled
|
||||
} else {
|
||||
LoginError::InvalidCredentials
|
||||
})
|
||||
}
|
||||
|
||||
fn auth_audit(&self, action: &str, occurred_at: chrono::DateTime<Utc>) -> AdminAuditEvent {
|
||||
AdminAuditEvent {
|
||||
id: format!("audit_{}", Uuid::now_v7().simple()),
|
||||
@@ -137,22 +125,13 @@ impl AuthService for DefaultAuthService {
|
||||
}
|
||||
|
||||
async fn login(&self, command: LoginCommand) -> Result<LoginResult, LoginError> {
|
||||
let source = normalized_login_source(&command.source);
|
||||
if self
|
||||
.store
|
||||
.login_source_is_throttled(source, LOGIN_FAILURE_LIMIT, LOGIN_FAILURE_WINDOW_SECONDS)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?
|
||||
{
|
||||
return Err(LoginError::Throttled);
|
||||
}
|
||||
if command
|
||||
.username
|
||||
.as_deref()
|
||||
.unwrap_or(&self.default_admin_user_id)
|
||||
!= self.default_admin_user_id
|
||||
{
|
||||
return Err(self.record_failed_login(source).await?);
|
||||
return Err(LoginError::InvalidCredentials);
|
||||
}
|
||||
let hash = self
|
||||
.store
|
||||
@@ -161,13 +140,9 @@ impl AuthService for DefaultAuthService {
|
||||
.map_err(|_| LoginError::Unavailable)?
|
||||
.ok_or(LoginError::InvalidCredentials)?;
|
||||
if !verify_admin_password(&command.password, &hash).map_err(|_| LoginError::Unavailable)? {
|
||||
return Err(self.record_failed_login(source).await?);
|
||||
return Err(LoginError::InvalidCredentials);
|
||||
}
|
||||
|
||||
self.store
|
||||
.clear_login_failures(source)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?;
|
||||
let session_id = random_session_token();
|
||||
let expires_at = Utc::now() + self.session_ttl;
|
||||
self.store
|
||||
@@ -246,8 +221,3 @@ fn valid_admin_api_key_shape(value: &str) -> bool {
|
||||
&& value.starts_with("admin-")
|
||||
&& value[6..].bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
fn normalized_login_source(source: &str) -> &str {
|
||||
let source = source.trim();
|
||||
if source.is_empty() { "unknown" } else { source }
|
||||
}
|
||||
|
||||
@@ -94,7 +94,6 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
.store
|
||||
.create_client_key(
|
||||
NewClientKey {
|
||||
expected_config_revision: command.expected_config_revision,
|
||||
id,
|
||||
name: command.name,
|
||||
label: command.label,
|
||||
|
||||
@@ -90,17 +90,10 @@ async fn pending_authorization(
|
||||
resource: &'static str,
|
||||
) -> Result<PendingAuthorizationMutation, AdminError> {
|
||||
let target = match &command.reauthorization {
|
||||
Some(target) => {
|
||||
let details =
|
||||
required_credential(accounts, provider_kind, &target.account_id, resource).await?;
|
||||
if details.credential.credential_revision != target.credential_revision {
|
||||
return Err(AdminError::conflict(format!(
|
||||
"{resource} reauthorization target is stale"
|
||||
)));
|
||||
}
|
||||
Some(account_id) => {
|
||||
required_credential(accounts, provider_kind, account_id, resource).await?;
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: target.account_id.clone(),
|
||||
expected_credential_revision: target.credential_revision,
|
||||
account_id: account_id.clone(),
|
||||
}
|
||||
}
|
||||
None => AuthorizationMutationTarget::Create {
|
||||
@@ -108,7 +101,6 @@ async fn pending_authorization(
|
||||
},
|
||||
};
|
||||
Ok(PendingAuthorizationMutation::new(
|
||||
command.expected_config_revision,
|
||||
provider_kind.clone(),
|
||||
target,
|
||||
crate::model::provider_credentials::AuthorizationOwnerBinding::from_context(
|
||||
@@ -143,7 +135,6 @@ fn validate_prepared_rotation(
|
||||
let facts = prepared.facts();
|
||||
if facts.account_id.as_str() != account.id.as_str()
|
||||
|| facts.provider_kind != account.provider_kind
|
||||
|| facts.expected_credential_revision != account.credential_revision
|
||||
{
|
||||
return Err(AdminError::conflict(format!(
|
||||
"{resource} prepared facts do not match the current credential"
|
||||
@@ -152,18 +143,27 @@ fn validate_prepared_rotation(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_authorization_commit(
|
||||
async fn validate_authorization_commit(
|
||||
provider_kind: &ProviderKind,
|
||||
context: &MutationContext,
|
||||
prepared: &PreparedAuthorizationCommit,
|
||||
prepared: PreparedAuthorizationCommit,
|
||||
resource: &'static str,
|
||||
) -> Result<(), AdminError> {
|
||||
) -> Result<PreparedAuthorizationCommit, AdminError> {
|
||||
if prepared.pending.provider_kind() != provider_kind
|
||||
|| !prepared.pending.owner_binding().matches_context(context)
|
||||
{
|
||||
return Err(AdminError::conflict(format!(
|
||||
let validation_error = AdminError::conflict(format!(
|
||||
"{resource} pending authorization binding is invalid"
|
||||
)));
|
||||
));
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
let matches_target = match (prepared.pending.target(), &prepared.credential) {
|
||||
(
|
||||
@@ -171,25 +171,29 @@ fn validate_authorization_commit(
|
||||
PreparedAuthorizationCredential::Create(credential),
|
||||
) => credential.provider_kind == *provider_kind,
|
||||
(
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
},
|
||||
AuthorizationMutationTarget::Reauthorize { account_id },
|
||||
PreparedAuthorizationCredential::Reauthorize(credential),
|
||||
) => {
|
||||
let facts = credential.facts();
|
||||
facts.provider_kind == *provider_kind
|
||||
&& facts.account_id == *account_id
|
||||
&& facts.expected_credential_revision == *expected_credential_revision
|
||||
facts.provider_kind == *provider_kind && facts.account_id == *account_id
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if !matches_target {
|
||||
return Err(AdminError::conflict(format!(
|
||||
let validation_error = AdminError::conflict(format!(
|
||||
"{resource} prepared credential does not match its pending target"
|
||||
)));
|
||||
));
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
Ok(())
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
async fn commit_authorization(
|
||||
@@ -198,16 +202,34 @@ async fn commit_authorization(
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (command, guard) = prepared.into_commit();
|
||||
let crate::model::provider_credentials::AuthorizationCommitSettlement {
|
||||
command,
|
||||
credential_guard,
|
||||
authorization_guard,
|
||||
} = prepared.into_commit();
|
||||
match accounts.commit_authorization(command, context).await {
|
||||
Ok(result) => {
|
||||
if let Some(guard) = guard {
|
||||
if let Some(guard) = credential_guard {
|
||||
guard.finish();
|
||||
}
|
||||
if let Some(guard) = authorization_guard {
|
||||
guard.commit().await?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(guard);
|
||||
drop(credential_guard);
|
||||
if let Some(guard) = authorization_guard
|
||||
&& let Err(settlement_error) = guard.abort().await
|
||||
{
|
||||
tracing::warn!(
|
||||
resource,
|
||||
store_error = %error,
|
||||
settlement_error = %settlement_error,
|
||||
"OAuth authorization claim release failed after Store commit failure"
|
||||
);
|
||||
return Err(settlement_error);
|
||||
}
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
@@ -215,20 +237,13 @@ async fn commit_authorization(
|
||||
|
||||
async fn commit_credential_rotation(
|
||||
accounts: &dyn AccountStore,
|
||||
expected_config_revision: crate::model::Revision,
|
||||
prepared: PreparedCredentialRotation,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_rotation(
|
||||
CredentialRotationCommit {
|
||||
expected_config_revision,
|
||||
prepared: facts,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.commit_credential_rotation(CredentialRotationCommit { prepared: facts }, context)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
@@ -244,20 +259,13 @@ async fn commit_credential_rotation(
|
||||
|
||||
async fn commit_credential_refresh(
|
||||
accounts: &dyn AccountStore,
|
||||
expected_config_revision: crate::model::Revision,
|
||||
prepared: PreparedCredentialRotation,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_refresh(
|
||||
CredentialRotationCommit {
|
||||
expected_config_revision,
|
||||
prepared: facts,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.commit_credential_refresh(CredentialRotationCommit { prepared: facts }, context)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
@@ -289,7 +297,6 @@ async fn set_credential_enabled(
|
||||
let revision = accounts
|
||||
.set_account_enabled(
|
||||
SetAccountEnabled {
|
||||
expected_config_revision: command.expected_config_revision,
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
enabled,
|
||||
},
|
||||
@@ -300,6 +307,9 @@ async fn set_credential_enabled(
|
||||
if !enabled {
|
||||
provider.account_unavailable(&account_id).await;
|
||||
}
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
Ok(CredentialMutationResult {
|
||||
config_revision: revision,
|
||||
account_id,
|
||||
@@ -320,7 +330,6 @@ async fn delete_credentials(
|
||||
let revision = accounts
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
expected_config_revision: command.expected_config_revision,
|
||||
account_ids: account_ids
|
||||
.iter()
|
||||
.map(|account_id| account_id.as_str().to_owned())
|
||||
@@ -333,6 +342,7 @@ async fn delete_credentials(
|
||||
for account_id in &account_ids {
|
||||
provider.account_unavailable(account_id).await;
|
||||
}
|
||||
provider.account_facts_changed(&account_ids).await;
|
||||
Ok(CredentialDeletionResult {
|
||||
config_revision: revision,
|
||||
account_ids,
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::{
|
||||
UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageInsights, UsageInsightsCost,
|
||||
UsageInsightsCostPoint, UsageInsightsHealth, UsageInsightsHealthPoint,
|
||||
UsageInsightsPerformance, UsageInsightsPerformancePoint, UsageOverview, UsagePage,
|
||||
UsageQuery, UsageSummary,
|
||||
UsageQuery, UsageSummary, china_day_start,
|
||||
},
|
||||
provider_credentials::ProviderQuotaRequest,
|
||||
},
|
||||
@@ -38,7 +38,6 @@ const HEALTH_TIMELINE_SLOTS: i64 = 24 * 4;
|
||||
const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10;
|
||||
const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3;
|
||||
const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 99.0;
|
||||
const CHINA_OFFSET_SECONDS: i64 = 8 * 60 * 60;
|
||||
|
||||
/// API 消费的观测控制面服务。
|
||||
#[async_trait]
|
||||
@@ -105,7 +104,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
self.store.dashboard_runtime_slots(range.end),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "dashboard"))?;
|
||||
self.enrich_billing(&mut observation.recent_requests)?;
|
||||
self.enrich_billing(&mut observation.recent_requests);
|
||||
self.enrich_dashboard_quotas(&mut observation.account_usage)
|
||||
.await;
|
||||
let today_start = china_day_start(observation.range.end);
|
||||
@@ -177,7 +176,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.list_usage_records(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage records"))?;
|
||||
self.enrich_billing(&mut page.items)?;
|
||||
self.enrich_billing(&mut page.items);
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
@@ -190,7 +189,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.usage_record_detail(request_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage record"))?;
|
||||
self.enrich_billing(std::slice::from_mut(&mut detail.request))?;
|
||||
self.enrich_billing(std::slice::from_mut(&mut detail.request));
|
||||
Ok(detail)
|
||||
}
|
||||
|
||||
@@ -429,6 +428,7 @@ fn recover_standard_costs(
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: fact.upstream_model_id,
|
||||
service_tier: fact.service_tier,
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
@@ -492,15 +492,14 @@ impl DefaultObservabilityService {
|
||||
}
|
||||
}
|
||||
|
||||
fn enrich_billing(
|
||||
&self,
|
||||
records: &mut [crate::model::observability::UsageRecord],
|
||||
) -> Result<(), AdminError> {
|
||||
/// 逐条尽力把可校验的总额升级为完整分解;单条脏数据(非法 Provider kind、
|
||||
/// 不支持的来源或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
fn enrich_billing(&self, records: &mut [crate::model::observability::UsageRecord]) {
|
||||
for record in records {
|
||||
let Some(UsageBilling::Total { source, total }) = record.billing.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if source != "calculated" {
|
||||
if !matches!(source.as_str(), "calculated" | "provider_reported") {
|
||||
continue;
|
||||
}
|
||||
let (Some(provider), Some(upstream_model_id)) = (
|
||||
@@ -509,25 +508,22 @@ impl DefaultObservabilityService {
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
let provider_kind = ProviderKind::new(provider.to_owned())
|
||||
.map_err(|_| AdminError::internal("Stored Provider kind is invalid"))?;
|
||||
let Ok(provider_kind) = ProviderKind::new(provider.to_owned()) else {
|
||||
continue;
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: upstream_model_id.clone(),
|
||||
service_tier: record.service_tier.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
total: total.clone(),
|
||||
};
|
||||
if let Some(breakdown) = self
|
||||
.providers
|
||||
.calculated_billing(&provider_kind, &input)
|
||||
.map_err(|error| map_provider_error(error, "usage billing"))?
|
||||
{
|
||||
if let Ok(Some(breakdown)) = self.providers.calculated_billing(&provider_kind, &input) {
|
||||
record.billing = Some(UsageBilling::Calculated(Box::new(breakdown)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -833,11 +829,6 @@ fn service_failure_count(metrics: &RequestMetrics) -> u64 {
|
||||
.saturating_sub(metrics.caller_error_count)
|
||||
}
|
||||
|
||||
fn china_day_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = (value.timestamp() + CHINA_OFFSET_SECONDS).rem_euclid(24 * 60 * 60);
|
||||
value - Duration::seconds(elapsed) - Duration::nanoseconds(i64::from(value.nanosecond()))
|
||||
}
|
||||
|
||||
fn quarter_hour_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = value
|
||||
.timestamp()
|
||||
|
||||
@@ -83,6 +83,27 @@ impl DefaultOpenAiService {
|
||||
snapshot,
|
||||
}
|
||||
}
|
||||
|
||||
async fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = self
|
||||
.provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -112,7 +133,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
command: ImportCredentials,
|
||||
) -> Result<CredentialImportResult, AdminError> {
|
||||
let context = command.context;
|
||||
let expected_config_revision = command.expected_config_revision;
|
||||
let prepared = self
|
||||
.provider
|
||||
.prepare_import(PrepareCredentialImport {
|
||||
@@ -127,27 +147,16 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
)?;
|
||||
let result = self
|
||||
.accounts
|
||||
.commit_credential_import(
|
||||
CredentialImportCommit {
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
},
|
||||
&context,
|
||||
)
|
||||
.commit_credential_import(CredentialImportCommit { prepared }, &context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "OpenAI credential import"))?;
|
||||
self.provider
|
||||
.account_facts_changed(&result.credential_ids)
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
// 导入已经提交,额度属于可重建观察事实;单个账号查询失败由周期任务重试。
|
||||
for account_id in &result.credential_ids {
|
||||
let _ = self
|
||||
.provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
// 导入完成后立即做一次观察;失败不回滚已经提交的 credential。
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id)
|
||||
.await;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -178,12 +187,13 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
.complete_authorization(command)
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "OpenAI authorization"))?;
|
||||
validate_authorization_commit(
|
||||
let prepared = validate_authorization_commit(
|
||||
self.provider.provider_kind(),
|
||||
&context,
|
||||
&prepared,
|
||||
prepared,
|
||||
"OpenAI authorization",
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
let result = commit_authorization(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
@@ -191,7 +201,15 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI authorization",
|
||||
)
|
||||
.await?;
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
)
|
||||
.await;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -200,7 +218,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
command: RotateCredential,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let context = command.mutation.context;
|
||||
let expected_config_revision = command.mutation.expected_config_revision;
|
||||
let account_id = command.mutation.account_id;
|
||||
let details = required_credential(
|
||||
self.accounts.as_ref(),
|
||||
@@ -209,17 +226,11 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
if details.credential.credential_revision != command.expected_credential_revision {
|
||||
return Err(AdminError::conflict(
|
||||
"OpenAI credential rotation revision is stale",
|
||||
));
|
||||
}
|
||||
let account = details.credential;
|
||||
let prepared = self
|
||||
.provider
|
||||
.prepare_rotation(PrepareCredentialRotation {
|
||||
account: account.clone(),
|
||||
expected_credential_revision: command.expected_credential_revision,
|
||||
provider_material: command.provider_material,
|
||||
})
|
||||
.await
|
||||
@@ -227,12 +238,14 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
validate_prepared_rotation(&account, &prepared, "OpenAI credential rotation")?;
|
||||
let result = commit_credential_rotation(
|
||||
self.accounts.as_ref(),
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
&context,
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -86,13 +86,12 @@ impl SettingsService for DefaultSettingsService {
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<RegeneratedAdminApiKey, AdminError> {
|
||||
let settings = self.load().await?;
|
||||
let mut bytes = [0_u8; 32];
|
||||
OsRng.fill_bytes(&mut bytes);
|
||||
let key = AdminApiKey::new(format!("admin-{}", hex::encode(bytes)));
|
||||
let mutation = self
|
||||
.store
|
||||
.replace_admin_api_key(settings.config_revision, key.clone(), context)
|
||||
.replace_admin_api_key(key.clone(), context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator API key"))?;
|
||||
publish_committed(self.snapshot.as_ref(), mutation.config_revision).await?;
|
||||
@@ -103,10 +102,9 @@ impl SettingsService for DefaultSettingsService {
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<AdminApiKeyMutation, AdminError> {
|
||||
let settings = self.load().await?;
|
||||
let mutation = self
|
||||
.store
|
||||
.delete_admin_api_key(settings.config_revision, context)
|
||||
.delete_admin_api_key(context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator API key"))?;
|
||||
publish_committed(self.snapshot.as_ref(), mutation.config_revision).await?;
|
||||
|
||||
@@ -89,7 +89,6 @@ impl XaiService for DefaultXaiService {
|
||||
command: ImportCredentials,
|
||||
) -> Result<CredentialImportResult, AdminError> {
|
||||
let context = command.context;
|
||||
let expected_config_revision = command.expected_config_revision;
|
||||
let prepared = self
|
||||
.provider
|
||||
.prepare_import(PrepareCredentialImport {
|
||||
@@ -104,13 +103,7 @@ impl XaiService for DefaultXaiService {
|
||||
)?;
|
||||
let result = self
|
||||
.accounts
|
||||
.commit_credential_import(
|
||||
CredentialImportCommit {
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
},
|
||||
&context,
|
||||
)
|
||||
.commit_credential_import(CredentialImportCommit { prepared }, &context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "xAI credential import"))?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
@@ -155,12 +148,13 @@ impl XaiService for DefaultXaiService {
|
||||
.complete_authorization(command)
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "xAI authorization"))?;
|
||||
validate_authorization_commit(
|
||||
let prepared = validate_authorization_commit(
|
||||
self.provider.provider_kind(),
|
||||
&context,
|
||||
&prepared,
|
||||
prepared,
|
||||
"xAI authorization",
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
let result = commit_authorization(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
|
||||
@@ -87,7 +87,6 @@ fn login_command_debug_should_redact_password() {
|
||||
let command = LoginCommand {
|
||||
username: None,
|
||||
password: "secret-password".to_owned(),
|
||||
source: "127.0.0.1".to_owned(),
|
||||
};
|
||||
assert!(!format!("{command:?}").contains("secret-password"));
|
||||
}
|
||||
|
||||
@@ -2,33 +2,34 @@ use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::{OpaqueProviderData, ProviderAccountId},
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::GatewayError,
|
||||
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
|
||||
operation::{GenerateRequest, Operation, ProtocolPayload},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
|
||||
use gateway_admin::{
|
||||
AdminServices,
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
AdminError, MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountAvailability, AccountListQuery, AccountPage, AccountRecord, AccountSummary,
|
||||
AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, DeleteAccounts,
|
||||
SetAccountEnabled,
|
||||
AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage,
|
||||
AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationCredentialCommit, AuthorizationMutationTarget,
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialCommitGuard, CredentialDetails,
|
||||
CredentialImportCommit, CredentialImportResult, CredentialListQuery,
|
||||
CredentialMutationResult, CredentialPage, CredentialRotationCommit,
|
||||
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
|
||||
PrepareCredentialRotation, PreparedAuthorizationCommit,
|
||||
AuthorizationCommit, AuthorizationCommitGuard, AuthorizationCredentialCommit,
|
||||
AuthorizationMutationTarget, AuthorizationStarted, CompleteAuthorization,
|
||||
CredentialCommitGuard, CredentialDetails, CredentialImportCommit,
|
||||
CredentialImportResult, CredentialListQuery, CredentialMutationResult, CredentialPage,
|
||||
CredentialRotationCommit, PendingAuthorizationMutation, PrepareCredentialImport,
|
||||
PrepareCredentialRefresh, PrepareCredentialRotation, PreparedAuthorizationCommit,
|
||||
PreparedAuthorizationCredential, PreparedCredentialCreate, PreparedCredentialImport,
|
||||
PreparedCredentialRotation, PreparedCredentialRotationFacts, ProviderDocument,
|
||||
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
|
||||
@@ -48,6 +49,7 @@ use gateway_admin::{
|
||||
},
|
||||
},
|
||||
};
|
||||
use serde_json::{Map, json};
|
||||
|
||||
pub(super) type EventLog = Arc<Mutex<Vec<&'static str>>>;
|
||||
|
||||
@@ -56,11 +58,13 @@ pub(super) struct FakeProviderAdmin {
|
||||
events: EventLog,
|
||||
failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
quota_failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
pending: Mutex<Option<PendingAuthorizationMutation>>,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
retry_authorization_after_abort: Mutex<bool>,
|
||||
export_inputs: Mutex<Vec<ProviderExportCredentialInput>>,
|
||||
import_account_ids: Mutex<Vec<String>>,
|
||||
quota_requests: Mutex<Vec<ProviderQuotaRequest>>,
|
||||
quota: Mutex<ProviderQuota>,
|
||||
current_credential_revision: Mutex<Revision>,
|
||||
}
|
||||
|
||||
impl FakeProviderAdmin {
|
||||
@@ -70,11 +74,13 @@ impl FakeProviderAdmin {
|
||||
events,
|
||||
failure: Mutex::new(None),
|
||||
quota_failure: Mutex::new(None),
|
||||
pending: Mutex::new(None),
|
||||
pending: Arc::new(Mutex::new(None)),
|
||||
retry_authorization_after_abort: Mutex::new(false),
|
||||
export_inputs: Mutex::new(Vec::new()),
|
||||
import_account_ids: Mutex::new(vec!["acct_prepared".to_owned()]),
|
||||
quota_requests: Mutex::new(Vec::new()),
|
||||
quota: Mutex::new(empty_quota()),
|
||||
current_credential_revision: Mutex::new(revision(1)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -111,6 +117,20 @@ impl FakeProviderAdmin {
|
||||
self.pending.lock().expect("pending authorization").clone()
|
||||
}
|
||||
|
||||
pub(super) fn retry_authorization_after_abort(&self) {
|
||||
*self
|
||||
.retry_authorization_after_abort
|
||||
.lock()
|
||||
.expect("authorization retry") = true;
|
||||
}
|
||||
|
||||
pub(super) fn set_current_credential_revision(&self, revision: Revision) {
|
||||
*self
|
||||
.current_credential_revision
|
||||
.lock()
|
||||
.expect("current credential revision") = revision;
|
||||
}
|
||||
|
||||
fn export_inputs(&self) -> Vec<ProviderExportCredentialInput> {
|
||||
self.export_inputs.lock().expect("export inputs").clone()
|
||||
}
|
||||
@@ -131,7 +151,11 @@ impl FakeProviderAdmin {
|
||||
PreparedCredentialRotationFacts {
|
||||
account_id: ProviderAccountId::new(account.id.clone()).expect("account ID"),
|
||||
provider_kind: account.provider_kind.clone(),
|
||||
expected_credential_revision: account.credential_revision,
|
||||
expected_credential_revision: *self
|
||||
.current_credential_revision
|
||||
.lock()
|
||||
.expect("current credential revision"),
|
||||
replacement_identity: None,
|
||||
name: account.name.clone(),
|
||||
email: account.email.clone(),
|
||||
plan_type: account.plan_type.clone(),
|
||||
@@ -157,12 +181,28 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
self.record("provider.account_unavailable");
|
||||
}
|
||||
|
||||
async fn account_facts_changed(&self, _: &[ProviderAccountId]) {
|
||||
self.record("provider.account_facts_changed");
|
||||
}
|
||||
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
_: &gateway_core::routing::UpstreamModelId,
|
||||
_: &str,
|
||||
model: &gateway_core::routing::UpstreamModelId,
|
||||
input: &str,
|
||||
) -> Result<gateway_core::operation::Operation, ProviderAdminError> {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!(model.as_str())),
|
||||
("input".to_owned(), json!(input)),
|
||||
("stream".to_owned(), json!(true)),
|
||||
("store".to_owned(), json!(false)),
|
||||
]),
|
||||
)
|
||||
.map_err(|_| ProviderAdminError::new(ProviderAdminErrorKind::Invalid))?;
|
||||
Ok(Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
payload,
|
||||
)))
|
||||
}
|
||||
|
||||
fn dashboard_wire_profile(
|
||||
@@ -232,23 +272,31 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
if !pending.owner_binding().matches_context(&command.context) {
|
||||
return Err(ProviderAdminError::new(ProviderAdminErrorKind::NotFound));
|
||||
}
|
||||
let retry_pending = (*self
|
||||
.retry_authorization_after_abort
|
||||
.lock()
|
||||
.expect("authorization retry"))
|
||||
.then(|| pending.clone());
|
||||
let credential = match pending.target() {
|
||||
AuthorizationMutationTarget::Create { name } => {
|
||||
PreparedAuthorizationCredential::Create(prepared_create(self.kind.clone(), name))
|
||||
}
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} => {
|
||||
AuthorizationMutationTarget::Reauthorize { account_id } => {
|
||||
let mut account = account_record(self.kind.as_str());
|
||||
account.id = account_id.as_str().to_owned();
|
||||
account.credential_revision = *expected_credential_revision;
|
||||
PreparedAuthorizationCredential::Reauthorize(self.prepared_rotation(&account))
|
||||
}
|
||||
};
|
||||
Ok(PreparedAuthorizationCommit {
|
||||
pending,
|
||||
credential,
|
||||
let prepared = PreparedAuthorizationCommit::new(pending, credential);
|
||||
Ok(match retry_pending {
|
||||
Some(pending) => {
|
||||
prepared.with_authorization_guard(Box::new(RetryableAuthorizationGuard::new(
|
||||
self.events.clone(),
|
||||
Arc::clone(&self.pending),
|
||||
pending,
|
||||
)))
|
||||
}
|
||||
None => prepared,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -326,20 +374,24 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
pub(super) struct FakeAccountStore {
|
||||
events: EventLog,
|
||||
account: AccountRecord,
|
||||
account_after_probe: Mutex<Option<AccountRecord>>,
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
authorization_revision: Mutex<Option<Revision>>,
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
}
|
||||
|
||||
impl FakeAccountStore {
|
||||
pub(super) fn new(kind: &str, events: EventLog) -> Arc<Self> {
|
||||
Self::with_account(account_record(kind), events)
|
||||
}
|
||||
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
account: account_record(kind),
|
||||
account,
|
||||
account_after_probe: Mutex::new(None),
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
authorization_revision: Mutex::new(None),
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
@@ -352,17 +404,17 @@ impl FakeAccountStore {
|
||||
self.audit_requests.lock().expect("audit requests").clone()
|
||||
}
|
||||
|
||||
pub(super) fn authorization_revision(&self) -> Option<Revision> {
|
||||
*self
|
||||
.authorization_revision
|
||||
.lock()
|
||||
.expect("authorization revision")
|
||||
}
|
||||
|
||||
pub(super) fn set_quota_window_usage(&self, usage: Vec<AccountUsageWindowResult>) {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
fn set_account_after_probe(&self, account: AccountRecord) {
|
||||
*self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe") = Some(account);
|
||||
}
|
||||
|
||||
fn record(&self, event: &'static str) {
|
||||
self.events.lock().expect("store events").push(event);
|
||||
}
|
||||
@@ -395,14 +447,20 @@ impl AccountStore for FakeAccountStore {
|
||||
total: 1,
|
||||
active: 1,
|
||||
quota_exhausted: 0,
|
||||
attention: 0,
|
||||
unavailable: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
self.record("store.load_account");
|
||||
Ok((account_id == self.account.id).then(|| self.account.clone()))
|
||||
let account = self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe")
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.account.clone());
|
||||
Ok((account_id == account.id.as_str()).then_some(account))
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -505,10 +563,6 @@ impl AccountStore for FakeAccountStore {
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
self.record("store.commit_authorization");
|
||||
self.record_context(context);
|
||||
*self
|
||||
.authorization_revision
|
||||
.lock()
|
||||
.expect("authorization revision") = Some(command.pending.expected_config_revision());
|
||||
self.require_commit()?;
|
||||
let (account_id, credential_revision) = match command.credential {
|
||||
AuthorizationCredentialCommit::Create(credential) => (credential.account_id, None),
|
||||
@@ -613,7 +667,6 @@ impl SettingsStore for StaticSettingsStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -622,7 +675,6 @@ impl SettingsStore for StaticSettingsStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
@@ -661,6 +713,51 @@ impl Drop for RecordingGuard {
|
||||
}
|
||||
}
|
||||
|
||||
struct RetryableAuthorizationGuard {
|
||||
events: EventLog,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
authorization: PendingAuthorizationMutation,
|
||||
}
|
||||
|
||||
impl RetryableAuthorizationGuard {
|
||||
fn new(
|
||||
events: EventLog,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
authorization: PendingAuthorizationMutation,
|
||||
) -> Self {
|
||||
Self {
|
||||
events,
|
||||
pending,
|
||||
authorization,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AuthorizationCommitGuard for RetryableAuthorizationGuard {
|
||||
async fn commit(self: Box<Self>) -> Result<(), AdminError> {
|
||||
self.events
|
||||
.lock()
|
||||
.expect("authorization guard events")
|
||||
.push("authorization_guard.commit");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn abort(self: Box<Self>) -> Result<(), AdminError> {
|
||||
let Self {
|
||||
events,
|
||||
pending,
|
||||
authorization,
|
||||
} = *self;
|
||||
*pending.lock().expect("pending authorization") = Some(authorization);
|
||||
events
|
||||
.lock()
|
||||
.expect("authorization guard events")
|
||||
.push("authorization_guard.abort");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_registry_should_resolve_custom_kind_without_central_match() {
|
||||
let provider = FakeProviderAdmin::new("custom-provider", events());
|
||||
@@ -686,6 +783,99 @@ fn provider_registry_should_reject_duplicate_kind() {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_probe_unavailable_account() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code: Some(code),
|
||||
provider_error_type: Some(error_type),
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
}) if message == "included usage exhausted"
|
||||
&& code == "usage_exhausted"
|
||||
&& error_type == "invalid_request_error"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_reloads_resettable_usage_limit_status_after_probe() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
let probe = Arc::new(FreeModelQuotaProbe {
|
||||
store: Arc::clone(&store),
|
||||
});
|
||||
let services = accounts_service_with_probe(provider, store, probe).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_preserve_disabled_account_status() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.enabled = false;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -724,7 +914,6 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
.accounts()
|
||||
.refresh(
|
||||
&context("refresh-request"),
|
||||
revision(1),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
@@ -743,6 +932,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
"provider.prepare_refresh",
|
||||
"store.commit_refresh",
|
||||
"guard.finish",
|
||||
"provider.account_facts_changed",
|
||||
"store.load_account",
|
||||
]
|
||||
);
|
||||
@@ -777,6 +967,36 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("usage-limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -837,7 +1057,6 @@ async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
|
||||
.accounts()
|
||||
.refresh(
|
||||
&context("refresh-provider-error"),
|
||||
revision(1),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
@@ -850,7 +1069,7 @@ async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_refresh_store_failure_should_drop_guard_after_cas_attempt() {
|
||||
async fn accounts_refresh_store_failure_should_drop_guard_after_commit_attempt() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
@@ -861,11 +1080,10 @@ async fn accounts_refresh_store_failure_should_drop_guard_after_cas_attempt() {
|
||||
.accounts()
|
||||
.refresh(
|
||||
&context("refresh-store-error"),
|
||||
revision(1),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
.expect_err("Store CAS must fail");
|
||||
.expect_err("Store commit must fail");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
@@ -994,7 +1212,7 @@ fn prepared_create_with_id(
|
||||
|
||||
fn rotation_result(command: CredentialRotationCommit) -> CredentialMutationResult {
|
||||
CredentialMutationResult {
|
||||
config_revision: revision(command.expected_config_revision.get() + 1),
|
||||
config_revision: revision(2),
|
||||
account_id: command.prepared.account_id,
|
||||
credential_revision: Some(revision(
|
||||
command.prepared.expected_credential_revision.get() + 1,
|
||||
@@ -1005,12 +1223,20 @@ fn rotation_result(command: CredentialRotationCommit) -> CredentialMutationResul
|
||||
async fn accounts_service(
|
||||
provider: Arc<FakeProviderAdmin>,
|
||||
store: Arc<FakeAccountStore>,
|
||||
) -> AdminServices {
|
||||
accounts_service_with_probe(provider, store, Arc::new(SuccessfulAccountProbe)).await
|
||||
}
|
||||
|
||||
async fn accounts_service_with_probe(
|
||||
provider: Arc<FakeProviderAdmin>,
|
||||
store: Arc<FakeAccountStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> AdminServices {
|
||||
super::AdminHarness::new()
|
||||
.accounts(store)
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.provider(provider)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.probe(probe)
|
||||
.build()
|
||||
.await
|
||||
}
|
||||
@@ -1021,7 +1247,7 @@ impl AccountProbe for SuccessfulAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Ok(AccountProbeResult {
|
||||
text: vec!["OK".to_owned()],
|
||||
@@ -1030,6 +1256,53 @@ impl AccountProbe for SuccessfulAccountProbe {
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingAccountProbe;
|
||||
|
||||
impl AccountProbe for FailingAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"included usage exhausted",
|
||||
Some("usage_exhausted".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
)
|
||||
.with_client_visible_upstream_error(detail)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FreeModelQuotaProbe {
|
||||
store: Arc<FakeAccountStore>,
|
||||
}
|
||||
|
||||
impl AccountProbe for FreeModelQuotaProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::hours(24));
|
||||
self.store.set_account_after_probe(account);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"xAI free model quota is exhausted",
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable() -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::{collections::BTreeMap, sync::Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{TimeDelta, Utc};
|
||||
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
@@ -15,7 +16,6 @@ struct MemoryAuthStore {
|
||||
password_hash: Mutex<Option<String>>,
|
||||
sessions: Mutex<BTreeMap<String, AdminSession>>,
|
||||
audits: Mutex<Vec<AdminAuditEvent>>,
|
||||
failures: Mutex<u32>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -66,31 +66,6 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(self.sessions.lock().expect("sessions").remove(session_id))
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
_: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(*self.failures.lock().expect("failures") >= failure_limit)
|
||||
}
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
_: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut failures = self.failures.lock().expect("failures");
|
||||
*failures += 1;
|
||||
Ok(*failures >= failure_limit)
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, _: &str) -> AdminStoreResult<()> {
|
||||
*self.failures.lock().expect("failures") = 0;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
self.audits.lock().expect("audits").push(event);
|
||||
Ok(())
|
||||
@@ -107,7 +82,6 @@ async fn successful_login_should_create_expiring_session_and_audit() {
|
||||
.login(LoginCommand {
|
||||
username: Some("admin".to_owned()),
|
||||
password: "strong-test-password".to_owned(),
|
||||
source: "127.0.0.1".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("login");
|
||||
@@ -142,9 +116,38 @@ async fn repeated_default_initialization_should_not_replace_password() {
|
||||
.login(LoginCommand {
|
||||
username: Some("admin".to_owned()),
|
||||
password: "first-strong-password".to_owned(),
|
||||
source: "127.0.0.1".to_owned(),
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn login_with_huge_session_ttl_should_clamp_expiry_instead_of_panicking() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new()
|
||||
.auth(store.clone())
|
||||
.session_ttl_minutes(i64::MAX as u64)
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let before = Utc::now();
|
||||
let result = services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
username: Some("admin".to_owned()),
|
||||
password: "strong-test-password".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("login with huge session TTL");
|
||||
|
||||
assert!(result.expires_at > before);
|
||||
assert!(result.expires_at <= before + TimeDelta::days(366) + TimeDelta::minutes(1));
|
||||
assert!(
|
||||
services
|
||||
.auth()
|
||||
.validate_session(Some(&result.session_id))
|
||||
.await
|
||||
.expect("validate")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ use gateway_admin::{
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
policy::ClientApiKeyId,
|
||||
@@ -69,6 +69,7 @@ use gateway_core::{
|
||||
|
||||
pub(super) struct AdminHarness {
|
||||
default_password: String,
|
||||
session_ttl_minutes: u64,
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
@@ -84,6 +85,7 @@ impl AdminHarness {
|
||||
let unavailable = Arc::new(UnavailableStore);
|
||||
Self {
|
||||
default_password: "strong-test-password".to_owned(),
|
||||
session_ttl_minutes: 60,
|
||||
accounts: unavailable.clone(),
|
||||
auth: Arc::new(BootstrapAuthStore::default()),
|
||||
client_keys: unavailable.clone(),
|
||||
@@ -103,6 +105,11 @@ impl AdminHarness {
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn session_ttl_minutes(mut self, minutes: u64) -> Self {
|
||||
self.session_ttl_minutes = minutes;
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn accounts(mut self, store: Arc<dyn AccountStore>) -> Self {
|
||||
self.accounts = store;
|
||||
self
|
||||
@@ -148,7 +155,7 @@ impl AdminHarness {
|
||||
pub(super) async fn build(self) -> AdminServices {
|
||||
gateway_admin::initialize(
|
||||
AdminConfig {
|
||||
session_ttl_minutes: 60,
|
||||
session_ttl_minutes: self.session_ttl_minutes,
|
||||
default_username: "admin".to_owned(),
|
||||
default_password: InitialAdminPassword::new(self.default_password),
|
||||
},
|
||||
@@ -210,18 +217,6 @@ impl AuthStore for BootstrapAuthStore {
|
||||
Err(unavailable("admin session"))
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(&self, _: &str, _: u32, _: u64) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn record_login_failure(&self, _: &str, _: u32, _: u64) -> AdminStoreResult<bool> {
|
||||
Err(unavailable("login failure"))
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, _: &str) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, _: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -454,7 +449,6 @@ impl SettingsStore for UnavailableStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -463,7 +457,6 @@ impl SettingsStore for UnavailableStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(unavailable("admin API key"))
|
||||
@@ -654,12 +647,13 @@ impl AccountProbe for UnavailableProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"test account probe is unavailable",
|
||||
))
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::{
|
||||
num::NonZeroU32,
|
||||
str::FromStr as _,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
@@ -9,13 +10,14 @@ use chrono::{DateTime, Duration, Timelike as _, Utc};
|
||||
use gateway_admin::{
|
||||
AdminServices,
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
MutationContext, PageSize, Revision,
|
||||
observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, CostCoverage, CurrencyCost, DashboardObservation,
|
||||
DashboardRuntimeSlots, DiagnosticDimension, DiagnosticObservation, Granularity,
|
||||
HealthStatus, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, RequestMetrics,
|
||||
TimeRange, TrendKind, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageOverview, UsagePage, UsageQuery,
|
||||
HealthStatus, OpsErrorPage, OpsErrorQuery, PageNumber, RequestMetricPoint,
|
||||
RequestMetrics, RequestOutcome, TimeRange, TrendKind, UsageBilling,
|
||||
UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageOverview, UsagePage,
|
||||
UsageQuery, UsageRecord, china_day_start,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -247,6 +249,7 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
bucket_start: quarter_hour_start(now),
|
||||
provider_kind: "openai".to_owned(),
|
||||
upstream_model_id: "gpt-5.5".to_owned(),
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
@@ -318,12 +321,80 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
assert_eq!(diagnostics.items[1].request_share, 0.25);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
let invalid_kind = "x".repeat(65);
|
||||
store.replace_usage_records(vec![
|
||||
total_record(
|
||||
"request_invalid_kind",
|
||||
Some(&invalid_kind),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record(
|
||||
"request_unregistered_kind",
|
||||
Some("anthropic"),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record("request_enriched", Some("openai"), "calculated", now),
|
||||
]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
let page = services
|
||||
.observability()
|
||||
.usage_records(usage_query(now))
|
||||
.await
|
||||
.expect("one bad record must not fail the whole usage list");
|
||||
|
||||
assert_eq!(page.items.len(), 3);
|
||||
assert!(
|
||||
matches!(page.items[0].billing, Some(UsageBilling::Total { .. })),
|
||||
"invalid Provider kind keeps the stored total"
|
||||
);
|
||||
assert!(
|
||||
matches!(page.items[1].billing, Some(UsageBilling::Total { .. })),
|
||||
"unregistered Provider kind keeps the stored total"
|
||||
);
|
||||
assert!(
|
||||
matches!(page.items[2].billing, Some(UsageBilling::Calculated(_))),
|
||||
"healthy record is still enriched"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_records_should_enrich_provider_reported_totals_when_pricing_matches() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
store.replace_usage_records(vec![total_record(
|
||||
"request_provider_reported",
|
||||
Some("openai"),
|
||||
"provider_reported",
|
||||
now,
|
||||
)]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
let page = services
|
||||
.observability()
|
||||
.usage_records(usage_query(now))
|
||||
.await
|
||||
.expect("usage records");
|
||||
|
||||
assert!(matches!(
|
||||
page.items[0].billing,
|
||||
Some(UsageBilling::Calculated(_))
|
||||
));
|
||||
}
|
||||
|
||||
struct FixtureObservabilityStore {
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
overview: Mutex<UsageOverview>,
|
||||
calculated_billing_facts: Mutex<Vec<UsageCalculatedBillingFact>>,
|
||||
diagnostics: Mutex<Vec<DiagnosticObservation>>,
|
||||
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
||||
usage_records: Mutex<Vec<UsageRecord>>,
|
||||
}
|
||||
|
||||
impl FixtureObservabilityStore {
|
||||
@@ -339,6 +410,7 @@ impl FixtureObservabilityStore {
|
||||
calculated_billing_facts: Mutex::new(Vec::new()),
|
||||
diagnostics: Mutex::new(Vec::new()),
|
||||
runtime_slots: Mutex::new(None),
|
||||
usage_records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +436,10 @@ impl FixtureObservabilityStore {
|
||||
fn replace_runtime_slots(&self, runtime_slots: Option<DashboardRuntimeSlots>) {
|
||||
*self.runtime_slots.lock().expect("runtime slots") = runtime_slots;
|
||||
}
|
||||
|
||||
fn replace_usage_records(&self, records: Vec<UsageRecord>) {
|
||||
*self.usage_records.lock().expect("usage records") = records;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -412,7 +488,13 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, _: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
Err(super::unavailable("usage records"))
|
||||
let items = self.usage_records.lock().expect("usage records").clone();
|
||||
let total = u64::try_from(items.len()).unwrap_or(u64::MAX);
|
||||
Ok(UsagePage {
|
||||
items,
|
||||
total,
|
||||
next_cursor: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
|
||||
@@ -471,7 +553,6 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -480,7 +561,6 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(super::unavailable("admin API key"))
|
||||
@@ -511,6 +591,98 @@ fn observation_range(end: DateTime<Utc>) -> TimeRange {
|
||||
TimeRange::new(end - Duration::hours(24), end).expect("observation range")
|
||||
}
|
||||
|
||||
fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
UsageQuery {
|
||||
range: observation_range(now),
|
||||
filter: UsageFilter::default(),
|
||||
cursor: None,
|
||||
page: PageNumber::new(NonZeroU32::new(1).expect("page number")),
|
||||
page_size: PageSize::new(50).expect("page size"),
|
||||
}
|
||||
}
|
||||
|
||||
fn total_record(
|
||||
id: &str,
|
||||
provider_kind: Option<&str>,
|
||||
source: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageRecord {
|
||||
UsageRecord {
|
||||
id: id.to_owned(),
|
||||
client_api_key_ref: "key_billing".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "gpt-5.5".to_owned(),
|
||||
provider_kind: provider_kind.map(str::to_owned),
|
||||
provider_account_ref: None,
|
||||
provider_account_name: None,
|
||||
provider_account_email: None,
|
||||
provider_account_authentication_kind: None,
|
||||
upstream_model_id: Some("gpt-5.5".to_owned()),
|
||||
upstream_transport: None,
|
||||
http_version: None,
|
||||
websocket_pool: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: None,
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1_000),
|
||||
cost_source: source.to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: Some(UsageBilling::Total {
|
||||
source: source.to_owned(),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
||||
.expect("billing total"),
|
||||
},
|
||||
}),
|
||||
transport_decision_wait_ms: None,
|
||||
connect_ms: None,
|
||||
headers_ms: None,
|
||||
first_event_ms: None,
|
||||
first_reasoning_ms: None,
|
||||
first_text_ms: None,
|
||||
first_token_ms: None,
|
||||
provider_processing_ms: None,
|
||||
latency_ms: Some(100),
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
image_generation_requested: false,
|
||||
image_generation_succeeded: None,
|
||||
started_at: now,
|
||||
deadline_at: now,
|
||||
completed_at: Some(now),
|
||||
}
|
||||
}
|
||||
|
||||
fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) -> RequestMetricPoint {
|
||||
RequestMetricPoint {
|
||||
bucket_start,
|
||||
@@ -535,11 +707,6 @@ fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn china_day_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = (value.timestamp() + 8 * 60 * 60).rem_euclid(24 * 60 * 60);
|
||||
value - Duration::seconds(elapsed) - Duration::nanoseconds(i64::from(value.nanosecond()))
|
||||
}
|
||||
|
||||
fn quarter_hour_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = value.timestamp().rem_euclid(15 * 60);
|
||||
value - Duration::seconds(elapsed) - Duration::nanoseconds(i64::from(value.nanosecond()))
|
||||
|
||||
@@ -6,7 +6,7 @@ use gateway_admin::{
|
||||
AdminServices,
|
||||
model::provider_credentials::{
|
||||
AuthorizationMutationTarget, CompleteAuthorization, CredentialDeletion, CredentialMutation,
|
||||
ImportCredentials, ProviderQuotaRequest, ReauthorizationTarget, StartAuthorization,
|
||||
ImportCredentials, ProviderQuotaRequest, StartAuthorization,
|
||||
},
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
@@ -30,7 +30,11 @@ async fn openai_enable_should_use_scoped_store_without_provider_mutation() {
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
["store.credential_details", "store.set_enabled"]
|
||||
[
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
}
|
||||
@@ -54,6 +58,7 @@ async fn openai_disable_should_commit_then_release_provider_resources() {
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
@@ -78,6 +83,7 @@ async fn openai_delete_should_commit_then_release_provider_resources() {
|
||||
"store.credential_details",
|
||||
"store.delete",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
@@ -114,7 +120,6 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -125,6 +130,7 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
[
|
||||
"provider.prepare_import",
|
||||
"store.commit_import",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
@@ -151,7 +157,6 @@ async fn openai_import_should_refresh_quota_for_every_imported_account() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai-batch"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -193,7 +198,6 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai-quota-failure"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -206,6 +210,135 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_authorization_create_should_observe_initial_quota() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = service(provider.clone(), store.clone()).await;
|
||||
|
||||
services
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai-create"),
|
||||
name: "new OpenAI credential".to_owned(),
|
||||
reauthorization: None,
|
||||
})
|
||||
.await
|
||||
.expect("start authorization");
|
||||
services
|
||||
.openai()
|
||||
.complete_authorization(CompleteAuthorization {
|
||||
context: context("oauth-start-openai-create"),
|
||||
flow_id: "flow-test".to_owned(),
|
||||
callback_url: "http://localhost/callback?code=test&state=test".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("complete authorization");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"provider.start_authorization",
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
provider.quota_requests(),
|
||||
[ProviderQuotaRequest {
|
||||
account_id: ProviderAccountId::new("acct_prepared").expect("account ID"),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
}]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["oauth-start-openai-create"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_authorization_create_should_remain_successful_when_initial_quota_fails() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
provider.fail_next_quota(ProviderAdminErrorKind::Unavailable);
|
||||
let store = FakeAccountStore::new("openai", events);
|
||||
let services = service(provider.clone(), store).await;
|
||||
|
||||
services
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai-quota-failure"),
|
||||
name: "new OpenAI credential".to_owned(),
|
||||
reauthorization: None,
|
||||
})
|
||||
.await
|
||||
.expect("start authorization");
|
||||
let result = services
|
||||
.openai()
|
||||
.complete_authorization(CompleteAuthorization {
|
||||
context: context("oauth-start-openai-quota-failure"),
|
||||
flow_id: "flow-test".to_owned(),
|
||||
callback_url: "http://localhost/callback?code=test&state=test".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("quota observation must not fail authorization");
|
||||
|
||||
assert_eq!(result.account_id.as_str(), "acct_prepared");
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_authorization_store_failure_should_release_claim_for_retry() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
provider.retry_authorization_after_abort();
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
store.fail_next_commit();
|
||||
let services = service(provider.clone(), store).await;
|
||||
let command = || CompleteAuthorization {
|
||||
context: context("oauth-store-retry"),
|
||||
flow_id: "flow-test".to_owned(),
|
||||
callback_url: "http://localhost/callback?code=test&state=test".to_owned(),
|
||||
};
|
||||
|
||||
services
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-store-retry"),
|
||||
name: "new OpenAI credential".to_owned(),
|
||||
reauthorization: None,
|
||||
})
|
||||
.await
|
||||
.expect("start authorization");
|
||||
services
|
||||
.openai()
|
||||
.complete_authorization(command())
|
||||
.await
|
||||
.expect_err("Store commit must fail");
|
||||
services
|
||||
.openai()
|
||||
.complete_authorization(command())
|
||||
.await
|
||||
.expect("released claim must permit retry");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"provider.start_authorization",
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"authorization_guard.abort",
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"authorization_guard.commit",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert!(provider.pending().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_import_provider_error_should_not_touch_store_transaction() {
|
||||
let events = events();
|
||||
@@ -218,7 +351,6 @@ async fn openai_import_provider_error_should_not_touch_store_transaction() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai-error"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -228,7 +360,7 @@ async fn openai_import_provider_error_should_not_touch_store_transaction() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_through_commit() {
|
||||
async fn openai_reauthorization_should_commit_after_credential_revision_advances() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
@@ -237,18 +369,13 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai"),
|
||||
expected_config_revision: revision(7),
|
||||
name: "reauthorize".to_owned(),
|
||||
reauthorization: Some(ReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
credential_revision: revision(1),
|
||||
}),
|
||||
reauthorization: Some(ProviderAccountId::new("acct_test").expect("account ID")),
|
||||
})
|
||||
.await
|
||||
.expect("start reauthorization");
|
||||
|
||||
let pending = provider.pending().expect("pending envelope");
|
||||
assert_eq!(pending.expected_config_revision(), revision(7));
|
||||
assert_eq!(pending.provider_kind().as_str(), "openai");
|
||||
assert_eq!(
|
||||
pending.owner_binding().started_request_id(),
|
||||
@@ -256,15 +383,14 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
);
|
||||
assert!(matches!(
|
||||
pending.target(),
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} if account_id.as_str() == "acct_test"
|
||||
&& *expected_credential_revision == revision(1)
|
||||
AuthorizationMutationTarget::Reauthorize { account_id }
|
||||
if account_id.as_str() == "acct_test"
|
||||
));
|
||||
assert!(!format!("{pending:?}").contains("admin-test"));
|
||||
|
||||
services
|
||||
provider.set_current_credential_revision(revision(2));
|
||||
|
||||
let result = services
|
||||
.openai()
|
||||
.complete_authorization(CompleteAuthorization {
|
||||
context: context("oauth-complete-openai"),
|
||||
@@ -274,6 +400,8 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.await
|
||||
.expect("complete reauthorization");
|
||||
|
||||
assert_eq!(result.credential_revision, Some(revision(3)));
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -282,9 +410,11 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"guard.finish",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.authorization_revision(), Some(revision(7)));
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
assert_eq!(store.audit_requests(), ["oauth-complete-openai"]);
|
||||
}
|
||||
|
||||
@@ -299,7 +429,6 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-openai"),
|
||||
expected_config_revision: revision(1),
|
||||
account_id: ProviderAccountId::new(account_id).expect("account ID"),
|
||||
}
|
||||
}
|
||||
@@ -307,7 +436,6 @@ fn mutation(account_id: &str) -> CredentialMutation {
|
||||
fn deletion(account_id: &str) -> CredentialDeletion {
|
||||
CredentialDeletion {
|
||||
context: context("request-openai"),
|
||||
expected_config_revision: revision(1),
|
||||
account_ids: vec![ProviderAccountId::new(account_id).expect("account ID")],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use async_trait::async_trait;
|
||||
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
AdminErrorKind, MutationContext, Revision,
|
||||
AdminErrorKind, MutationContext,
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
RuntimeSettings,
|
||||
@@ -33,7 +33,6 @@ impl SettingsStore for UnusedSettingsStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -42,7 +41,6 @@ impl SettingsStore for UnusedSettingsStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(unused())
|
||||
@@ -63,7 +61,6 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
request_id: "request-settings".to_owned(),
|
||||
},
|
||||
ReplaceRuntimeSettings {
|
||||
expected_config_revision: Revision::new(1).expect("revision"),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 0,
|
||||
refresh_concurrency: 1,
|
||||
|
||||
@@ -8,9 +8,7 @@ use gateway_admin::{
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
|
||||
use super::accounts::{
|
||||
FakeAccountStore, FakeProviderAdmin, context, document, events, recorded, revision,
|
||||
};
|
||||
use super::accounts::{FakeAccountStore, FakeProviderAdmin, context, document, events, recorded};
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_disable_should_commit_then_notify_the_stateless_provider() {
|
||||
@@ -31,6 +29,7 @@ async fn xai_disable_should_commit_then_notify_the_stateless_provider() {
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-xai"]);
|
||||
@@ -47,7 +46,6 @@ async fn xai_import_should_prepare_before_atomic_store_commit() {
|
||||
.xai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-xai"),
|
||||
expected_config_revision: revision(3),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -84,7 +82,6 @@ async fn xai_import_should_refresh_quota_for_every_imported_account() {
|
||||
.xai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-xai-batch"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -119,7 +116,6 @@ async fn xai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
.xai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-xai-quota-failure"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -140,7 +136,6 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-xai"),
|
||||
expected_config_revision: revision(1),
|
||||
account_id: ProviderAccountId::new(account_id).expect("account ID"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ path = "tests/main.rs"
|
||||
|
||||
[dependencies]
|
||||
axum = { version = "0.8.9", default-features = false, features = ["json", "query", "ws"] }
|
||||
base64 = "0.22.1"
|
||||
base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
chrono = { version = "0.4.45", features = ["serde"] }
|
||||
futures = "0.3.32"
|
||||
@@ -21,7 +21,7 @@ gateway-admin = { path = "../gateway-admin" }
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
gateway-protocol = { path = "../gateway-protocol" }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "time"] }
|
||||
tower-http = { version = "0.7.0", features = ["cors", "fs", "request-id", "timeout", "trace"] }
|
||||
@@ -31,7 +31,9 @@ uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
async-trait = "0.1.89"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "net", "rt-multi-thread"] }
|
||||
tokio-tungstenite = "0.29.0"
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -16,18 +16,18 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use gateway_admin::model::{
|
||||
AdminError as AdminServiceError, AdminErrorKind, PageSize, Revision,
|
||||
AdminError as AdminServiceError, AdminErrorKind, PageSize,
|
||||
accounts::{
|
||||
AccountAvailability, AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost,
|
||||
AccountListQuery, AccountModelUsage, AccountSort, AccountSortField,
|
||||
AccountStatus as DomainAccountStatus, AccountUsage, SortDirection,
|
||||
AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost, AccountListQuery,
|
||||
AccountModelUsage, AccountSort, AccountSortField, AccountStatus as DomainAccountStatus,
|
||||
AccountUsage, SortDirection,
|
||||
},
|
||||
provider_credentials::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialDeletion, CredentialDeletionResult,
|
||||
CredentialImportResult, CredentialMutation, CredentialMutationResult, ImportCredentials,
|
||||
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow,
|
||||
ReauthorizationTarget, RotateCredential, StartAuthorization,
|
||||
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow, RotateCredential,
|
||||
StartAuthorization,
|
||||
},
|
||||
};
|
||||
use gateway_core::{
|
||||
@@ -161,7 +161,6 @@ fn parse_sort_direction(value: &str) -> Option<SortDirection> {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountPageData {
|
||||
pub config_revision: u64,
|
||||
pub items: Vec<AccountView>,
|
||||
pub page: PageMeta,
|
||||
pub summary: AccountSummaryView,
|
||||
@@ -174,7 +173,7 @@ pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub attention: u64,
|
||||
pub unavailable: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -197,8 +196,6 @@ pub struct AccountView {
|
||||
pub token_refreshing: bool,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub credential_revision: u64,
|
||||
pub state_revision: Option<u64>,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
pub access_token_expires_at_display: Option<String>,
|
||||
pub refresh_token_expires_at: Option<String>,
|
||||
@@ -391,31 +388,21 @@ impl AccountActionRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 手工 OAuth 刷新会变更持久 credential,因此必须携带配置 CAS revision。
|
||||
/// 手工 OAuth 刷新会变更持久 credential。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountRefreshRequest {
|
||||
pub account_id: String,
|
||||
pub expected_config_revision: u64,
|
||||
}
|
||||
|
||||
impl AccountRefreshRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
if self.expected_config_revision == 0 {
|
||||
return Err(WireValidationError::new("expectedConfigRevision"));
|
||||
}
|
||||
Ok(())
|
||||
require_account_id(&self.account_id, "accountId")
|
||||
}
|
||||
|
||||
fn into_command(self) -> Result<(Revision, ProviderAccountId), WireValidationError> {
|
||||
fn into_command(self) -> Result<ProviderAccountId, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok((
|
||||
Revision::new(self.expected_config_revision)
|
||||
.map_err(|_| WireValidationError::new("expectedConfigRevision"))?,
|
||||
ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
))
|
||||
ProviderAccountId::new(self.account_id).map_err(|_| WireValidationError::new("accountId"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,7 +447,6 @@ pub struct AccountModelsData {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountRefreshData {
|
||||
pub config_revision: u64,
|
||||
pub account: AccountView,
|
||||
}
|
||||
|
||||
@@ -543,10 +529,20 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code,
|
||||
provider_error_type,
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status,
|
||||
} => serde_json::json!({
|
||||
"type": "error",
|
||||
"error": message,
|
||||
"providerErrorCode": provider_error_code,
|
||||
"providerErrorType": provider_error_type,
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
}),
|
||||
};
|
||||
@@ -575,14 +571,12 @@ impl AccountProvider {
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountImportRequest {
|
||||
pub provider: String,
|
||||
pub expected_config_revision: u64,
|
||||
pub data: Value,
|
||||
}
|
||||
|
||||
impl AccountImportRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")?;
|
||||
if !self.data.is_object()
|
||||
|| serde_json::to_vec(&self.data)
|
||||
.map_or(true, |encoded| encoded.len() > MAX_IMPORT_DATA_BYTES)
|
||||
@@ -602,10 +596,6 @@ impl AccountImportRequest {
|
||||
provider,
|
||||
ImportCredentials {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
document: provider_document(self.data, "data")?,
|
||||
},
|
||||
))
|
||||
@@ -616,28 +606,18 @@ impl AccountImportRequest {
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct StartAccountAuthorizationRequest {
|
||||
pub provider: String,
|
||||
pub expected_config_revision: u64,
|
||||
pub name: String,
|
||||
pub account_id: Option<String>,
|
||||
pub expected_credential_revision: Option<u64>,
|
||||
}
|
||||
|
||||
impl StartAccountAuthorizationRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")?;
|
||||
require_text(&self.name, MAX_NAME_BYTES, "name")?;
|
||||
match (
|
||||
self.account_id.as_deref(),
|
||||
self.expected_credential_revision,
|
||||
) {
|
||||
(None, None) => Ok(()),
|
||||
(Some(account_id), Some(credential_revision)) => {
|
||||
require_account_id(account_id, "accountId")?;
|
||||
require_positive(credential_revision, "expectedCredentialRevision")
|
||||
}
|
||||
_ => Err(WireValidationError::new("reauthorization")),
|
||||
if let Some(account_id) = self.account_id.as_deref() {
|
||||
require_account_id(account_id, "accountId")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -646,23 +626,15 @@ impl StartAccountAuthorizationRequest {
|
||||
) -> Result<(AccountProvider, StartAuthorization), WireValidationError> {
|
||||
self.validate()?;
|
||||
let provider = AccountProvider::parse(&self.provider)?;
|
||||
let reauthorization = match (self.account_id, self.expected_credential_revision) {
|
||||
(None, None) => None,
|
||||
(Some(account_id), Some(credential_revision)) => Some(ReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new(account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
credential_revision: revision(credential_revision, "expectedCredentialRevision")?,
|
||||
}),
|
||||
_ => return Err(WireValidationError::new("reauthorization")),
|
||||
};
|
||||
let reauthorization = self
|
||||
.account_id
|
||||
.map(ProviderAccountId::new)
|
||||
.transpose()
|
||||
.map_err(|_| WireValidationError::new("accountId"))?;
|
||||
Ok((
|
||||
provider,
|
||||
StartAuthorization {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
name: self.name,
|
||||
reauthorization,
|
||||
},
|
||||
@@ -717,14 +689,12 @@ impl CompleteAccountAuthorizationRequest {
|
||||
pub struct AccountMutationRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub expected_config_revision: u64,
|
||||
}
|
||||
|
||||
impl AccountMutationRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")
|
||||
require_account_id(&self.account_id, "accountId")
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -737,10 +707,6 @@ impl AccountMutationRequest {
|
||||
provider,
|
||||
CredentialMutation {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
},
|
||||
@@ -753,7 +719,6 @@ impl AccountMutationRequest {
|
||||
pub struct AccountDeletionRequest {
|
||||
pub provider: String,
|
||||
pub account_ids: Vec<String>,
|
||||
pub expected_config_revision: u64,
|
||||
}
|
||||
|
||||
impl AccountDeletionRequest {
|
||||
@@ -769,7 +734,7 @@ impl AccountDeletionRequest {
|
||||
return Err(WireValidationError::new("accountIds"));
|
||||
}
|
||||
}
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -782,10 +747,6 @@ impl AccountDeletionRequest {
|
||||
provider,
|
||||
CredentialDeletion {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
account_ids: self
|
||||
.account_ids
|
||||
.into_iter()
|
||||
@@ -802,8 +763,6 @@ impl AccountDeletionRequest {
|
||||
pub struct RotateAccountRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub expected_config_revision: u64,
|
||||
pub expected_credential_revision: u64,
|
||||
pub access_token: String,
|
||||
pub refresh_token: Option<String>,
|
||||
}
|
||||
@@ -814,11 +773,6 @@ impl RotateAccountRequest {
|
||||
return Err(WireValidationError::new("provider"));
|
||||
}
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")?;
|
||||
require_positive(
|
||||
self.expected_credential_revision,
|
||||
"expectedCredentialRevision",
|
||||
)?;
|
||||
validate_oauth_material(&self.access_token, self.refresh_token.as_deref())
|
||||
}
|
||||
|
||||
@@ -836,17 +790,9 @@ impl RotateAccountRequest {
|
||||
Ok(RotateCredential {
|
||||
mutation: CredentialMutation {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
},
|
||||
expected_credential_revision: revision(
|
||||
self.expected_credential_revision,
|
||||
"expectedCredentialRevision",
|
||||
)?,
|
||||
provider_material: ProviderDocument::new(OpaqueProviderData::new(material)),
|
||||
})
|
||||
}
|
||||
@@ -855,7 +801,6 @@ impl RotateAccountRequest {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountImportData {
|
||||
pub config_revision: u64,
|
||||
pub imported_count: usize,
|
||||
pub account_ids: Vec<String>,
|
||||
}
|
||||
@@ -868,7 +813,6 @@ impl From<CredentialImportResult> for AccountImportData {
|
||||
.map(|account_id| account_id.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
Self {
|
||||
config_revision: result.config_revision.get(),
|
||||
imported_count: account_ids.len(),
|
||||
account_ids,
|
||||
}
|
||||
@@ -896,18 +840,13 @@ impl From<AuthorizationStarted> for AccountAuthorizationData {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountMutationData {
|
||||
pub config_revision: u64,
|
||||
pub account_id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub credential_revision: Option<u64>,
|
||||
}
|
||||
|
||||
impl From<CredentialMutationResult> for AccountMutationData {
|
||||
fn from(result: CredentialMutationResult) -> Self {
|
||||
Self {
|
||||
config_revision: result.config_revision.get(),
|
||||
account_id: result.account_id.to_string(),
|
||||
credential_revision: result.credential_revision.map(Revision::get),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -915,7 +854,6 @@ impl From<CredentialMutationResult> for AccountMutationData {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountDeletionData {
|
||||
pub config_revision: u64,
|
||||
pub deleted_count: usize,
|
||||
pub account_ids: Vec<String>,
|
||||
}
|
||||
@@ -928,7 +866,6 @@ impl From<CredentialDeletionResult> for AccountDeletionData {
|
||||
.map(|account_id| account_id.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
Self {
|
||||
config_revision: result.config_revision.get(),
|
||||
deleted_count: account_ids.len(),
|
||||
account_ids,
|
||||
}
|
||||
@@ -1260,15 +1197,11 @@ async fn refresh_account<S>(
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (expected_config_revision, account_id) = request.into_command().map_err(map_wire_error)?;
|
||||
let account_id = request.into_command().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.refresh(
|
||||
&auth.context().mutation_context(),
|
||||
expected_config_revision,
|
||||
account_id,
|
||||
)
|
||||
.refresh(&auth.context().mutation_context(), account_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = account_refresh_data(result, Utc::now());
|
||||
@@ -1390,7 +1323,6 @@ fn account_page_data(
|
||||
u32::try_from(result.total.div_ceil(u64::from(page_size))).unwrap_or(u32::MAX)
|
||||
};
|
||||
AccountPageData {
|
||||
config_revision: result.config_revision.get(),
|
||||
items: result
|
||||
.items
|
||||
.into_iter()
|
||||
@@ -1401,14 +1333,13 @@ fn account_page_data(
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
attention: result.summary.attention,
|
||||
unavailable: result.summary.unavailable,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn account_refresh_data(result: AccountRefreshResult, now: DateTime<Utc>) -> AccountRefreshData {
|
||||
AccountRefreshData {
|
||||
config_revision: result.config_revision.get(),
|
||||
account: account_view(result.account, now),
|
||||
}
|
||||
}
|
||||
@@ -1456,10 +1387,8 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
token_refreshing: false,
|
||||
availability: account_availability_name(account.availability).to_owned(),
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
enabled: account.enabled,
|
||||
credential_revision: account.credential_revision.get(),
|
||||
state_revision: None,
|
||||
access_token_expires_at: expires_at,
|
||||
access_token_expires_at_display: account
|
||||
.access_token_expires_at
|
||||
@@ -1710,10 +1639,6 @@ fn empty_account_usage() -> AccountUsageView {
|
||||
}
|
||||
}
|
||||
|
||||
fn revision(value: u64, field: &'static str) -> Result<Revision, WireValidationError> {
|
||||
Revision::new(value).map_err(|_| WireValidationError::new(field))
|
||||
}
|
||||
|
||||
fn provider_document(
|
||||
value: Value,
|
||||
field: &'static str,
|
||||
@@ -1747,13 +1672,6 @@ fn require_text(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_positive(value: u64, field: &'static str) -> Result<(), WireValidationError> {
|
||||
if value == 0 || i64::try_from(value).is_err() {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_oauth_material(
|
||||
access_token: &str,
|
||||
refresh_token: Option<&str>,
|
||||
@@ -1798,19 +1716,6 @@ fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
DomainAccountStatus::Attention => "attention",
|
||||
}
|
||||
}
|
||||
|
||||
fn account_availability_name(availability: AccountAvailability) -> &'static str {
|
||||
match availability {
|
||||
AccountAvailability::Unknown => "unknown",
|
||||
AccountAvailability::Ready => "ready",
|
||||
AccountAvailability::Cooldown => "cooldown",
|
||||
AccountAvailability::QuotaExhausted => "quota_exhausted",
|
||||
AccountAvailability::Expired => "expired",
|
||||
AccountAvailability::Banned => "banned",
|
||||
AccountAvailability::Invalid => "invalid",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use std::fmt;
|
||||
|
||||
use axum::{
|
||||
Extension, Json, Router,
|
||||
Json, Router,
|
||||
extract::{FromRequestParts, State},
|
||||
http::{HeaderMap, HeaderValue, StatusCode, header::SET_COOKIE, request::Parts},
|
||||
response::{IntoResponse, Response},
|
||||
@@ -14,6 +14,7 @@ use gateway_admin::{
|
||||
model::auth::{AdminPrincipal, AdminRequestContext, LoginCommand, LoginError},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tower_http::request_id::RequestId;
|
||||
|
||||
use super::{AdminEnvelope, AdminError, AdminResponse};
|
||||
|
||||
@@ -51,22 +52,32 @@ where
|
||||
|
||||
async fn from_request_parts(parts: &mut Parts, state: &S) -> Result<Self, Self::Rejection> {
|
||||
let principal = require_admin_auth(state, &parts.headers).await?;
|
||||
let request_id = parts
|
||||
.headers
|
||||
.get(REQUEST_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
let request_id = admin_request_id(parts)
|
||||
.ok_or_else(|| AdminError::internal("Missing admin request context"))?;
|
||||
Ok(Self {
|
||||
context: AdminRequestContext {
|
||||
principal,
|
||||
request_id: request_id.to_owned(),
|
||||
request_id,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// request-id 层按配置的 header 名注入,同时写入与名字无关的扩展;
|
||||
/// 优先读扩展,使自定义 header 名不会让管理请求失去请求上下文。
|
||||
/// header 回退覆盖未装配该层的调用方。
|
||||
fn admin_request_id(parts: &Parts) -> Option<String> {
|
||||
parts
|
||||
.extensions
|
||||
.get::<RequestId>()
|
||||
.map(RequestId::header_value)
|
||||
.or_else(|| parts.headers.get(REQUEST_ID_HEADER))
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
pub async fn require_admin_session<S>(state: &S, headers: &HeaderMap) -> Result<String, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
@@ -185,22 +196,6 @@ impl Default for AdminLogoutData {
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求上下文解析出的客户端登录来源。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AdminLoginSource(String);
|
||||
|
||||
impl AdminLoginSource {
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造固定 GET/POST 管理员认证路由。
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
@@ -214,24 +209,16 @@ where
|
||||
|
||||
async fn login<S>(
|
||||
State(state): State<S>,
|
||||
source: Option<Extension<AdminLoginSource>>,
|
||||
Json(payload): Json<AdminLoginRequest>,
|
||||
) -> Result<Response, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let source = source
|
||||
.as_ref()
|
||||
.map_or("unknown", |Extension(source)| source.as_str());
|
||||
let (username, password) = payload.into_parts();
|
||||
let session = state
|
||||
.admin_services()
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
username,
|
||||
password,
|
||||
source: source.to_owned(),
|
||||
})
|
||||
.login(LoginCommand { username, password })
|
||||
.await
|
||||
.map_err(map_login_error)?;
|
||||
let mut response = AdminResponse::new(
|
||||
@@ -300,7 +287,6 @@ fn admin_session_cookie(headers: &HeaderMap) -> Option<String> {
|
||||
fn map_login_error(error: LoginError) -> AdminError {
|
||||
match error {
|
||||
LoginError::InvalidCredentials => AdminError::invalid_admin_credentials(),
|
||||
LoginError::Throttled => AdminError::too_many_login_attempts(),
|
||||
LoginError::Unavailable => AdminError::internal("Failed to create admin session"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,14 +4,11 @@ use std::fmt;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
client_keys::{
|
||||
ClientKeyCursor, ClientKeyCursorValue as DomainCursorValue, ClientKeyListQuery,
|
||||
ClientKeyMutation, ClientKeyPage, ClientKeyPageSize, ClientKeyRecord, ClientKeySecret,
|
||||
ClientKeySort as DomainSort, ClientKeySortField as DomainSortField, CreateClientKey,
|
||||
CreatedClientKey, DeleteClientKey, SetClientKeyEnabled, SortDirection, UpdateClientKey,
|
||||
},
|
||||
use gateway_admin::model::client_keys::{
|
||||
ClientKeyCursor, ClientKeyCursorValue as DomainCursorValue, ClientKeyListQuery,
|
||||
ClientKeyMutation, ClientKeyPage, ClientKeyPageSize, ClientKeyRecord, ClientKeySecret,
|
||||
ClientKeySort as DomainSort, ClientKeySortField as DomainSortField, CreateClientKey,
|
||||
CreatedClientKey, DeleteClientKey, SetClientKeyEnabled, SortDirection, UpdateClientKey,
|
||||
};
|
||||
use gateway_core::{
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
@@ -138,7 +135,6 @@ impl ClientKeySort {
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CreateClientKeyRequest {
|
||||
expected_config_revision: u64,
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
provider_kind: String,
|
||||
@@ -149,14 +145,12 @@ pub struct CreateClientKeyRequest {
|
||||
impl CreateClientKeyRequest {
|
||||
/// 校验 wire 边界并直接构造管理用例命令。
|
||||
pub fn into_command(self) -> Result<CreateClientKey, WireValidationError> {
|
||||
validate_revision(self.expected_config_revision)?;
|
||||
validate_required_text(&self.name, "name")?;
|
||||
validate_optional_text(self.label.as_deref(), "label")?;
|
||||
validate_provider_kind(&self.provider_kind)?;
|
||||
validate_limit(self.max_concurrency, "maxConcurrency")?;
|
||||
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
|
||||
Ok(CreateClientKey {
|
||||
expected_config_revision: revision(self.expected_config_revision)?,
|
||||
name: self.name,
|
||||
label: self.label,
|
||||
provider_kind: ProviderKind::new(self.provider_kind)
|
||||
@@ -174,7 +168,6 @@ impl CreateClientKeyRequest {
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateClientKeyRequest {
|
||||
id: String,
|
||||
expected_config_revision: u64,
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
provider_kind: String,
|
||||
@@ -186,14 +179,12 @@ impl UpdateClientKeyRequest {
|
||||
/// 校验 wire 边界并直接构造管理用例命令。
|
||||
pub fn into_command(self) -> Result<UpdateClientKey, WireValidationError> {
|
||||
validate_required_text(&self.id, "id")?;
|
||||
validate_revision(self.expected_config_revision)?;
|
||||
validate_required_text(&self.name, "name")?;
|
||||
validate_optional_text(self.label.as_deref(), "label")?;
|
||||
validate_provider_kind(&self.provider_kind)?;
|
||||
validate_limit(self.max_concurrency, "maxConcurrency")?;
|
||||
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
|
||||
Ok(UpdateClientKey {
|
||||
expected_config_revision: revision(self.expected_config_revision)?,
|
||||
id: client_key_id(self.id, "clientKeyMutationNotFound")?,
|
||||
name: self.name,
|
||||
label: self.label,
|
||||
@@ -207,12 +198,11 @@ impl UpdateClientKeyRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 只携带 ID 与配置 revision 的 Client Key mutation 请求。
|
||||
/// 只携带 ID 的 Client Key mutation 请求。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct ClientKeyRevisionRequest {
|
||||
pub struct ClientKeyMutationRequest {
|
||||
id: String,
|
||||
expected_config_revision: u64,
|
||||
}
|
||||
|
||||
/// 读取一次完整 Key 的 ID query。
|
||||
@@ -233,20 +223,15 @@ impl ClientKeyIdQuery {
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientKeyRevisionRequest {
|
||||
/// 校验请求并取出 ID 与 revision。
|
||||
pub fn into_parts(self) -> Result<(String, u64), WireValidationError> {
|
||||
impl ClientKeyMutationRequest {
|
||||
/// 校验请求并取出 ID。
|
||||
pub fn into_id(self) -> Result<String, WireValidationError> {
|
||||
validate_required_text(&self.id, "id")?;
|
||||
validate_revision(self.expected_config_revision)?;
|
||||
Ok((self.id, self.expected_config_revision))
|
||||
Ok(self.id)
|
||||
}
|
||||
|
||||
fn into_command_parts(self) -> Result<(ClientApiKeyId, Revision), WireValidationError> {
|
||||
let (id, expected_config_revision) = self.into_parts()?;
|
||||
Ok((
|
||||
client_key_id(id, "clientKeyMutationNotFound")?,
|
||||
revision(expected_config_revision)?,
|
||||
))
|
||||
fn into_domain_id(self) -> Result<ClientApiKeyId, WireValidationError> {
|
||||
client_key_id(self.into_id()?, "clientKeyMutationNotFound")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -289,7 +274,6 @@ impl From<ClientKeyRecord> for ClientKeyView {
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ClientKeyListData {
|
||||
config_revision: u64,
|
||||
items: Vec<ClientKeyView>,
|
||||
next_cursor: Option<String>,
|
||||
total: u64,
|
||||
@@ -298,14 +282,8 @@ pub struct ClientKeyListData {
|
||||
impl ClientKeyListData {
|
||||
/// 构造 Client Key 列表响应。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
config_revision: u64,
|
||||
items: Vec<ClientKeyView>,
|
||||
next_cursor: Option<String>,
|
||||
total: u64,
|
||||
) -> Self {
|
||||
pub fn new(items: Vec<ClientKeyView>, next_cursor: Option<String>, total: u64) -> Self {
|
||||
Self {
|
||||
config_revision,
|
||||
items,
|
||||
next_cursor,
|
||||
total,
|
||||
@@ -325,7 +303,6 @@ impl TryFrom<ClientKeyPage> for ClientKeyListData {
|
||||
.map(encode_client_key_cursor)
|
||||
.transpose()?;
|
||||
Ok(Self::new(
|
||||
page.config_revision.get(),
|
||||
page.items.into_iter().map(Into::into).collect(),
|
||||
next_cursor,
|
||||
page.total,
|
||||
@@ -337,7 +314,6 @@ impl TryFrom<ClientKeyPage> for ClientKeyListData {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CreatedClientKeyData {
|
||||
config_revision: u64,
|
||||
id: String,
|
||||
prefix: String,
|
||||
plaintext_key: String,
|
||||
@@ -346,9 +322,8 @@ pub struct CreatedClientKeyData {
|
||||
impl CreatedClientKeyData {
|
||||
/// 构造一次性创建响应。
|
||||
#[must_use]
|
||||
pub fn new(config_revision: u64, id: String, prefix: String, plaintext_key: String) -> Self {
|
||||
pub fn new(id: String, prefix: String, plaintext_key: String) -> Self {
|
||||
Self {
|
||||
config_revision,
|
||||
id,
|
||||
prefix,
|
||||
plaintext_key,
|
||||
@@ -359,7 +334,6 @@ impl CreatedClientKeyData {
|
||||
impl From<CreatedClientKey> for CreatedClientKeyData {
|
||||
fn from(created: CreatedClientKey) -> Self {
|
||||
Self::new(
|
||||
created.config_revision.get(),
|
||||
created.secret.record.id.to_string(),
|
||||
created.secret.record.prefix.clone(),
|
||||
created.secret.expose_for_response().to_owned(),
|
||||
@@ -371,7 +345,6 @@ impl fmt::Debug for CreatedClientKeyData {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("CreatedClientKeyData")
|
||||
.field("config_revision", &self.config_revision)
|
||||
.field("id", &self.id)
|
||||
.field("prefix", &self.prefix)
|
||||
.field("plaintext_key", &"[REDACTED]")
|
||||
@@ -417,24 +390,20 @@ impl fmt::Debug for RevealedClientKeyData {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MutatedClientKeyData {
|
||||
config_revision: u64,
|
||||
id: String,
|
||||
}
|
||||
|
||||
impl MutatedClientKeyData {
|
||||
/// 构造 mutation 响应。
|
||||
#[must_use]
|
||||
pub fn new(config_revision: u64, id: String) -> Self {
|
||||
Self {
|
||||
config_revision,
|
||||
id,
|
||||
}
|
||||
pub fn new(id: String) -> Self {
|
||||
Self { id }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ClientKeyMutation> for MutatedClientKeyData {
|
||||
fn from(mutation: ClientKeyMutation) -> Self {
|
||||
Self::new(mutation.config_revision.get(), mutation.id.to_string())
|
||||
Self::new(mutation.id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -582,18 +551,6 @@ fn wire_cursor(cursor: ClientKeyCursor) -> Result<ClientKeyCursorData, WireValid
|
||||
Ok(cursor)
|
||||
}
|
||||
|
||||
fn validate_revision(value: u64) -> Result<(), WireValidationError> {
|
||||
if value == 0 {
|
||||
return Err(WireValidationError::new("expectedConfigRevision"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn revision(value: u64) -> Result<Revision, WireValidationError> {
|
||||
validate_revision(value)?;
|
||||
Revision::new(value).map_err(|_| WireValidationError::new("expectedConfigRevision"))
|
||||
}
|
||||
|
||||
fn client_key_id(
|
||||
value: String,
|
||||
field: &'static str,
|
||||
@@ -768,23 +725,19 @@ where
|
||||
async fn disable_client_key<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(payload): Json<ClientKeyRevisionRequest>,
|
||||
Json(payload): Json<ClientKeyMutationRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (id, expected_config_revision) = payload.into_command_parts().map_err(map_wire_error)?;
|
||||
let id = payload.into_domain_id().map_err(map_wire_error)?;
|
||||
mutation_response(
|
||||
state
|
||||
.admin_services()
|
||||
.client_keys()
|
||||
.set_enabled(
|
||||
&auth.context().mutation_context(),
|
||||
SetClientKeyEnabled {
|
||||
expected_config_revision,
|
||||
id,
|
||||
enabled: false,
|
||||
},
|
||||
SetClientKeyEnabled { id, enabled: false },
|
||||
)
|
||||
.await,
|
||||
)
|
||||
@@ -793,23 +746,19 @@ where
|
||||
async fn enable_client_key<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(payload): Json<ClientKeyRevisionRequest>,
|
||||
Json(payload): Json<ClientKeyMutationRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (id, expected_config_revision) = payload.into_command_parts().map_err(map_wire_error)?;
|
||||
let id = payload.into_domain_id().map_err(map_wire_error)?;
|
||||
mutation_response(
|
||||
state
|
||||
.admin_services()
|
||||
.client_keys()
|
||||
.set_enabled(
|
||||
&auth.context().mutation_context(),
|
||||
SetClientKeyEnabled {
|
||||
expected_config_revision,
|
||||
id,
|
||||
enabled: true,
|
||||
},
|
||||
SetClientKeyEnabled { id, enabled: true },
|
||||
)
|
||||
.await,
|
||||
)
|
||||
@@ -818,23 +767,17 @@ where
|
||||
async fn delete_client_key<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(payload): Json<ClientKeyRevisionRequest>,
|
||||
Json(payload): Json<ClientKeyMutationRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (id, expected_config_revision) = payload.into_command_parts().map_err(map_wire_error)?;
|
||||
let id = payload.into_domain_id().map_err(map_wire_error)?;
|
||||
mutation_response(
|
||||
state
|
||||
.admin_services()
|
||||
.client_keys()
|
||||
.delete(
|
||||
&auth.context().mutation_context(),
|
||||
DeleteClientKey {
|
||||
expected_config_revision,
|
||||
id,
|
||||
},
|
||||
)
|
||||
.delete(&auth.context().mutation_context(), DeleteClientKey { id })
|
||||
.await,
|
||||
)
|
||||
}
|
||||
@@ -854,9 +797,6 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
"cursor" => AdminError::bad_request("Invalid client key cursor"),
|
||||
"clientKeyRevealNotFound" => AdminError::not_found("Client API key was not found"),
|
||||
"clientKeyMutationNotFound" => AdminError::not_found("client API key was not found"),
|
||||
"expectedConfigRevision" => {
|
||||
AdminError::bad_request("expectedConfigRevision must be positive")
|
||||
}
|
||||
_ => AdminError::bad_request("Invalid client API key request"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -801,8 +801,8 @@ pub struct DashboardCardsView {
|
||||
pub struct DashboardCredentialsCardView {
|
||||
pub total: String,
|
||||
pub total_value: u64,
|
||||
pub enabled: String,
|
||||
pub enabled_value: u64,
|
||||
pub available: String,
|
||||
pub available_value: u64,
|
||||
pub unavailable: String,
|
||||
pub unavailable_value: u64,
|
||||
}
|
||||
@@ -1235,7 +1235,6 @@ pub struct OpsErrorView {
|
||||
pub account_id: Option<String>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
@@ -1654,12 +1653,7 @@ fn format_token_price(cost: &domain::CurrencyCost) -> String {
|
||||
if cost.currency != "USD" {
|
||||
return format!("{} {} / 1M Token", cost.currency, cost.amount.as_str());
|
||||
}
|
||||
let value = cost
|
||||
.amount
|
||||
.as_str()
|
||||
.parse::<f64>()
|
||||
.map_or(0.0, |value| value);
|
||||
format!("${value:.4} / 1M Token")
|
||||
format!("${} / 1M Token", cost.amount.as_str())
|
||||
}
|
||||
|
||||
fn format_service_tier(service_tier: Option<&str>) -> String {
|
||||
@@ -1801,7 +1795,7 @@ fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView {
|
||||
model,
|
||||
requested_model: Some(record.requested_model_id),
|
||||
upstream_model: record.upstream_model_id,
|
||||
service_tier: None,
|
||||
service_tier: record.service_tier,
|
||||
status_code,
|
||||
transport,
|
||||
attempt_index: None,
|
||||
@@ -1954,7 +1948,6 @@ fn usage_page_view(
|
||||
}
|
||||
|
||||
fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
let status = error.status_code.map(i64::from);
|
||||
OpsErrorView {
|
||||
id: error.event_id,
|
||||
request_id: error.request_id,
|
||||
@@ -1964,9 +1957,8 @@ fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
account_id: error.provider_account_ref,
|
||||
route: error.operation,
|
||||
model: error.upstream_model_id,
|
||||
status_code: status,
|
||||
client_status_code: None,
|
||||
upstream_status_code: None,
|
||||
client_status_code: error.client_status_code.map(i64::from),
|
||||
upstream_status_code: error.upstream_status_code.map(i64::from),
|
||||
transport: error.upstream_transport,
|
||||
attempt_index: error.attempt_index,
|
||||
failure_class: error.failure_kind,
|
||||
@@ -2290,8 +2282,8 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
|
||||
credentials: DashboardCredentialsCardView {
|
||||
total: format_compact_number(provider_accounts.total),
|
||||
total_value: provider_accounts.total,
|
||||
enabled: format_compact_number(provider_accounts.enabled),
|
||||
enabled_value: provider_accounts.enabled,
|
||||
available: format_compact_number(provider_accounts.active),
|
||||
available_value: provider_accounts.active,
|
||||
unavailable: format_compact_number(provider_accounts.unavailable),
|
||||
unavailable_value: provider_accounts.unavailable,
|
||||
},
|
||||
|
||||
@@ -10,12 +10,8 @@ use axum::{
|
||||
routing::{get, post},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
settings::{
|
||||
ModelMappings as DomainModelMappings, ReplaceRuntimeSettings, RotationStrategy,
|
||||
RuntimeSettings,
|
||||
},
|
||||
use gateway_admin::model::settings::{
|
||||
ModelMappings as DomainModelMappings, ReplaceRuntimeSettings, RotationStrategy, RuntimeSettings,
|
||||
};
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -32,7 +28,6 @@ pub type ModelMappings = BTreeMap<String, String>;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RuntimeSettingsView {
|
||||
pub config_revision: u64,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -49,7 +44,6 @@ pub struct RuntimeSettingsView {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateRuntimeSettingsRequest {
|
||||
pub expected_config_revision: u64,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -64,9 +58,6 @@ pub struct UpdateRuntimeSettingsRequest {
|
||||
impl UpdateRuntimeSettingsRequest {
|
||||
/// 校验公共运行参数。
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
if self.expected_config_revision == 0 {
|
||||
return Err(WireValidationError::new("expectedConfigRevision"));
|
||||
}
|
||||
validate_model_mappings(&self.model_mappings)?;
|
||||
for (value, field) in [
|
||||
(self.refresh_margin_seconds, "refreshMarginSeconds"),
|
||||
@@ -93,8 +84,6 @@ impl UpdateRuntimeSettingsRequest {
|
||||
fn into_command(self) -> Result<ReplaceRuntimeSettings, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(ReplaceRuntimeSettings {
|
||||
expected_config_revision: Revision::new(self.expected_config_revision)
|
||||
.map_err(|_| WireValidationError::new("settingsRevisionOverflow"))?,
|
||||
model_mappings: domain_model_mappings(self.model_mappings)?,
|
||||
refresh_margin_seconds: self.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(self.refresh_concurrency)
|
||||
@@ -117,7 +106,6 @@ impl UpdateRuntimeSettingsRequest {
|
||||
impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
fn from(settings: RuntimeSettings) -> Self {
|
||||
Self {
|
||||
config_revision: settings.config_revision.get(),
|
||||
model_mappings: wire_model_mappings(settings.model_mappings),
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
@@ -358,7 +346,6 @@ fn valid_model_name(value: &str, max_len: usize) -> bool {
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
let message = match error.field() {
|
||||
"settingsRevisionOverflow" => "expectedConfigRevision must be positive".to_owned(),
|
||||
"settingsRefreshConcurrencyOverflow" => "Invalid refreshConcurrency".to_owned(),
|
||||
"settingsMaxConcurrencyOverflow" => "Invalid maxConcurrentPerAccount".to_owned(),
|
||||
"settingsUsageRetentionOverflow" => "Invalid usageRetentionDays".to_owned(),
|
||||
|
||||
@@ -201,11 +201,13 @@ struct RestartAcceptedView {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct SystemUpdateEventView {
|
||||
id: String,
|
||||
operation_id: Option<String>,
|
||||
level: &'static str,
|
||||
step: Option<String>,
|
||||
message: String,
|
||||
terminal: bool,
|
||||
progress_percent: Option<u8>,
|
||||
at: String,
|
||||
}
|
||||
|
||||
@@ -403,11 +405,13 @@ where
|
||||
impl From<SystemUpdateEvent> for SystemUpdateEventView {
|
||||
fn from(event: SystemUpdateEvent) -> Self {
|
||||
Self {
|
||||
id: event.id,
|
||||
operation_id: event.operation_id,
|
||||
level: update_event_level_name(event.level),
|
||||
step: event.step,
|
||||
message: event.message,
|
||||
terminal: event.terminal,
|
||||
progress_percent: event.progress_percent,
|
||||
at: event.occurred_at.to_rfc3339(),
|
||||
}
|
||||
}
|
||||
@@ -416,11 +420,13 @@ impl From<SystemUpdateEvent> for SystemUpdateEventView {
|
||||
impl SystemUpdateEventView {
|
||||
fn into_json(self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"id": self.id,
|
||||
"operationId": self.operation_id,
|
||||
"level": self.level,
|
||||
"step": self.step,
|
||||
"message": self.message,
|
||||
"terminal": self.terminal,
|
||||
"progressPercent": self.progress_percent,
|
||||
"at": self.at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use gateway_core::engine::execution::ExecutionService;
|
||||
use gateway_core::health::{HealthProbe, WorkerHealthSource};
|
||||
use gateway_core::lifecycle::ConnectionLifecycle;
|
||||
use serde::Deserialize;
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
|
||||
use tower_http::services::{ServeDir, ServeFile};
|
||||
use tower_http::timeout::TimeoutLayer;
|
||||
@@ -146,11 +146,17 @@ pub fn initialize(
|
||||
.map_err(|_| ApiError::Config(ApiConfigError::InvalidCorsOrigin))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
// credentials 模式禁止通配 allow_headers;显式列出鉴权与内容协商头。
|
||||
router = router.layer(
|
||||
CorsLayer::new()
|
||||
.allow_origin(origins)
|
||||
.allow_methods([Method::GET, Method::POST])
|
||||
.allow_headers(Any)
|
||||
.allow_headers([
|
||||
axum::http::header::AUTHORIZATION,
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
HeaderName::from_static("x-api-key"),
|
||||
request_id_header.clone(),
|
||||
])
|
||||
.allow_credentials(true),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
http::StatusCode,
|
||||
body::Body,
|
||||
http::{HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::{
|
||||
@@ -75,10 +76,14 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
if error.kind() == ProviderErrorKind::Unavailable
|
||||
&& error.send_state() == UpstreamSendState::NotSent =>
|
||||
{
|
||||
GatewayError::new(
|
||||
let gateway = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
)
|
||||
);
|
||||
match error.client_visible_upstream_error() {
|
||||
Some(upstream) => gateway.with_client_visible_upstream_error(upstream.clone()),
|
||||
None => gateway,
|
||||
}
|
||||
}
|
||||
EngineError::Provider(error) => GatewayError::from_provider(error),
|
||||
EngineError::Cancelled => {
|
||||
@@ -98,6 +103,7 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
| EngineError::ContinuationPinMismatch
|
||||
| EngineError::RequiredAccountMismatch
|
||||
| EngineError::DownstreamCommitRequired
|
||||
| EngineError::NoActiveAttempt
|
||||
| EngineError::InvalidDeliveryState => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
|
||||
}
|
||||
@@ -116,6 +122,33 @@ pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// 在下游尚未提交时优先交付 Provider 的原始 HTTP 失败响应。
|
||||
pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
if let EngineError::Provider(error) = error
|
||||
&& let Some(upstream) = error.client_visible_upstream_response()
|
||||
&& let Ok(status) = StatusCode::from_u16(upstream.status())
|
||||
{
|
||||
let mut response = Response::new(Body::from(upstream.body().clone()));
|
||||
*response.status_mut() = status;
|
||||
if let Some(content_type) = upstream.content_type()
|
||||
&& let Ok(content_type) = HeaderValue::from_bytes(content_type)
|
||||
{
|
||||
response.headers_mut().insert(CONTENT_TYPE, content_type);
|
||||
}
|
||||
for header in upstream.headers() {
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
gateway_error_response(&gateway_error_from_engine(error))
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用。
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Path, State},
|
||||
extract::{Path, Query, State},
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::routing::{PublicModelId, PublicModelProfile};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::ApiState;
|
||||
@@ -17,20 +18,50 @@ use super::{
|
||||
};
|
||||
|
||||
const MODEL_CREATED_TIMESTAMP: i64 = 1_700_000_000;
|
||||
const CODEX_BASE_INSTRUCTIONS: &str = "You are Codex, a coding agent. Follow the user's instructions and use the available tools to complete software engineering tasks. Inspect relevant files before editing, preserve unrelated changes, and verify the result.";
|
||||
|
||||
/// `GET /v1/models`。
|
||||
pub(crate) async fn models(State(state): State<ApiState>, headers: HeaderMap) -> Response {
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ModelsQuery {
|
||||
client_version: Option<String>,
|
||||
}
|
||||
|
||||
/// `GET /v1/models`。Codex 携带 `client_version` 时返回其专用目录合同。
|
||||
pub(crate) async fn models(
|
||||
State(state): State<ApiState>,
|
||||
Query(query): Query<ModelsQuery>,
|
||||
headers: HeaderMap,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return authentication_error_response(error),
|
||||
};
|
||||
|
||||
if query
|
||||
.client_version
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.trim().is_empty())
|
||||
{
|
||||
let models = service
|
||||
.public_model_profiles(&client)
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, profile)| codex_model_json(profile, index))
|
||||
.collect::<Vec<_>>();
|
||||
return (
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"models": models,
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
let data = service
|
||||
.public_models(&client)
|
||||
.into_iter()
|
||||
.map(|model| openai_model_json(&model))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
@@ -134,3 +165,92 @@ fn catalog_model_json(id: &str) -> Value {
|
||||
"source": "gateway",
|
||||
})
|
||||
}
|
||||
|
||||
fn codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
|
||||
let id = profile.model().as_str();
|
||||
let presentation = profile.presentation();
|
||||
let reasoning_levels = presentation
|
||||
.supported_reasoning_efforts()
|
||||
.iter()
|
||||
.map(|effort| {
|
||||
json!({
|
||||
"effort": effort,
|
||||
"description": reasoning_effort_description(effort),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let reasoning_supported = presentation
|
||||
.supported_reasoning_efforts()
|
||||
.iter()
|
||||
.any(|effort| effort != "none");
|
||||
let context_window = presentation.context_window_tokens();
|
||||
let input_modalities = if presentation.image_input() {
|
||||
vec!["text", "image"]
|
||||
} else {
|
||||
vec!["text"]
|
||||
};
|
||||
let additional_speed_tiers = presentation
|
||||
.service_tiers()
|
||||
.iter()
|
||||
.filter_map(|tier| tier.speed_tier())
|
||||
.collect::<Vec<_>>();
|
||||
let service_tiers = presentation
|
||||
.service_tiers()
|
||||
.iter()
|
||||
.map(|tier| {
|
||||
json!({
|
||||
"id": tier.id(),
|
||||
"name": tier.name(),
|
||||
"description": tier.description(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
json!({
|
||||
"slug": id,
|
||||
"display_name": presentation.display_name().unwrap_or(id),
|
||||
"description": presentation.description(),
|
||||
"default_reasoning_level": presentation.default_reasoning_effort(),
|
||||
"supported_reasoning_levels": reasoning_levels,
|
||||
"shell_type": "shell_command",
|
||||
"visibility": if presentation.hidden() { "hide" } else { "list" },
|
||||
"supported_in_api": true,
|
||||
"priority": index.saturating_add(1),
|
||||
"additional_speed_tiers": additional_speed_tiers,
|
||||
"service_tiers": service_tiers,
|
||||
"availability_nux": Value::Null,
|
||||
"upgrade": Value::Null,
|
||||
"base_instructions": CODEX_BASE_INSTRUCTIONS,
|
||||
"model_messages": Value::Null,
|
||||
"include_skills_usage_instructions": false,
|
||||
"supports_reasoning_summary_parameter": reasoning_supported,
|
||||
"default_reasoning_summary": "auto",
|
||||
"support_verbosity": presentation.verbosity(),
|
||||
"default_verbosity": Value::Null,
|
||||
"apply_patch_tool_type": presentation.agent_tools().then_some("freeform"),
|
||||
"web_search_tool_type": "text",
|
||||
"truncation_policy": { "mode": "tokens", "limit": 10_000 },
|
||||
"supports_parallel_tool_calls": presentation.parallel_tool_calls(),
|
||||
"supports_image_detail_original": presentation.image_detail_original(),
|
||||
"context_window": context_window,
|
||||
"max_context_window": context_window,
|
||||
"effective_context_window_percent": 95,
|
||||
"experimental_supported_tools": [],
|
||||
"input_modalities": input_modalities,
|
||||
"supports_search_tool": presentation.search_tool(),
|
||||
"use_responses_lite": false,
|
||||
})
|
||||
}
|
||||
|
||||
fn reasoning_effort_description(effort: &str) -> &'static str {
|
||||
match effort {
|
||||
"none" => "No reasoning",
|
||||
"minimal" => "Minimal reasoning",
|
||||
"low" => "Fast responses with lighter reasoning",
|
||||
"medium" => "Balances speed and reasoning depth for everyday tasks",
|
||||
"high" => "Greater reasoning depth for complex problems",
|
||||
"xhigh" => "Extra high reasoning depth for complex problems",
|
||||
"max" => "Maximum reasoning depth for the hardest problems",
|
||||
_ => "Provider-supported reasoning level",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,9 +152,6 @@ pub enum ResponseEncodeError {
|
||||
/// 协议原生流缺少可返回的终态 response object。
|
||||
#[error("responses wire stream has no terminal response")]
|
||||
MissingWireTerminal,
|
||||
/// 协议原生事件携带了不一致的响应身份。
|
||||
#[error("responses wire stream changed its response identity")]
|
||||
WireIdentityChanged,
|
||||
/// 终态 response object 无法序列化为完整 HTTP JSON 响应。
|
||||
#[error("responses wire terminal serialization failed")]
|
||||
Serialization,
|
||||
@@ -167,7 +164,7 @@ impl ResponseEncodeError {
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_canonical_response",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
|
||||
@@ -8,7 +8,7 @@ use axum::{
|
||||
body::{Body, Bytes},
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, StatusCode,
|
||||
HeaderMap, HeaderName, HeaderValue, StatusCode,
|
||||
header::{CACHE_CONTROL, CONTENT_TYPE, USER_AGENT},
|
||||
},
|
||||
response::{IntoResponse, Response},
|
||||
@@ -25,8 +25,8 @@ use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, authentication_error_response},
|
||||
error::{
|
||||
gateway_error_contract, gateway_error_from_engine, gateway_error_response,
|
||||
protocol_error_response, runtime_unavailable_response,
|
||||
engine_error_response, gateway_error_contract, gateway_error_from_engine,
|
||||
gateway_error_response, protocol_error_response, runtime_unavailable_response,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -180,7 +180,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -206,7 +206,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
@@ -237,9 +237,7 @@ fn encode_collected_events(
|
||||
) -> Result<Vec<u8>, BufferedResponseEncodeError> {
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
for event in events {
|
||||
encoder
|
||||
.push_sse(event)
|
||||
.map_err(BufferedResponseEncodeError::Canonical)?;
|
||||
encoder.push_sse(event);
|
||||
}
|
||||
let response = encoder
|
||||
.finish()
|
||||
@@ -253,23 +251,25 @@ fn json_body_response(encoded: Vec<u8>, response_headers: &[ProviderResponseHead
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_safe_response_headers(response, response_headers)
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_safe_response_headers(
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options = super::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
let Some(name) = super::safe_response_header_name(header.name()) else {
|
||||
if !super::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_str(header.value().as_str()) else {
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::HeaderName::from_static(name), value);
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -295,7 +295,7 @@ pub async fn stream_execution_response(
|
||||
return response;
|
||||
}
|
||||
Err(error) => {
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -310,18 +310,7 @@ pub async fn stream_execution_response(
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let mut frames = Vec::new();
|
||||
for event in &first_events {
|
||||
match encoder.push_sse(event) {
|
||||
Ok(encoded) => frames.extend(encoded),
|
||||
Err(error) => {
|
||||
let response = protocol_error_response(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
);
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
}
|
||||
frames.extend(encoder.push_sse(event));
|
||||
}
|
||||
if frames.is_empty() {
|
||||
let response = internal_gateway_response("gateway commit batch encoded no output");
|
||||
@@ -338,7 +327,7 @@ pub async fn stream_execution_response(
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
let Some(session) = execution.into_session() else {
|
||||
@@ -371,7 +360,7 @@ fn event_stream_response(body: Body, response_headers: &[ProviderResponseHeader]
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CACHE_CONTROL, HeaderValue::from_static("no-cache"));
|
||||
apply_safe_response_headers(response, response_headers)
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn internal_gateway_response(message: &'static str) -> Response {
|
||||
@@ -403,15 +392,10 @@ impl PendingExecution {
|
||||
}
|
||||
|
||||
async fn record_response_status(&mut self, response: Response) -> Response {
|
||||
let Some(session) = self.session.as_mut() else {
|
||||
return internal_gateway_response("gateway response session is unavailable");
|
||||
};
|
||||
if session
|
||||
.record_client_status(response.status().as_u16())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return internal_gateway_response("gateway client status could not be persisted");
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
let _ = session
|
||||
.record_client_status(response.status().as_u16())
|
||||
.await;
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -499,10 +483,16 @@ impl ResponsesStreamState {
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| session.is_finalized());
|
||||
self.finish_with_gateway_error(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without a terminal event",
|
||||
));
|
||||
if self.execution_terminal {
|
||||
self.pending
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
} else {
|
||||
self.finish_with_gateway_error(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without finalizing the execution",
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.execution_terminal = self
|
||||
@@ -521,18 +511,11 @@ impl ResponsesStreamState {
|
||||
}
|
||||
|
||||
async fn push_event(&mut self, event: ProviderEvent) {
|
||||
match self.encoder.push_sse(&event) {
|
||||
Ok(frames) => {
|
||||
if self.encoder.is_completed() {
|
||||
self.finish_completed(frames).await;
|
||||
} else {
|
||||
self.pending.extend(frames);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.cancel_execution().await;
|
||||
self.finish_with_encode_error(error);
|
||||
}
|
||||
let frames = self.encoder.push_sse(&event);
|
||||
if self.encoder.is_completed() {
|
||||
self.finish_completed(frames).await;
|
||||
} else {
|
||||
self.pending.extend(frames);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -607,20 +590,6 @@ impl ResponsesStreamState {
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
}
|
||||
|
||||
fn finish_with_encode_error(&mut self, error: ResponseEncodeError) {
|
||||
let protocol = error.protocol_body().error;
|
||||
self.pending
|
||||
.push_back(Bytes::from(response_failed_sse_event_with_id(
|
||||
self.encoder.response_id(),
|
||||
protocol.kind,
|
||||
protocol.code,
|
||||
&protocol.message,
|
||||
)));
|
||||
self.pending
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ResponsesStreamState {
|
||||
|
||||
@@ -17,13 +17,63 @@ pub use response::OpenAiResponsesEncoder;
|
||||
pub(crate) use websocket::responses_websocket;
|
||||
pub use websocket::{ResponseCreateFrameError, decode_response_create_with_context};
|
||||
|
||||
pub(super) fn safe_response_header_name(name: &str) -> Option<&'static str> {
|
||||
match name {
|
||||
"x-request-id" => Some("x-request-id"),
|
||||
"openai-model" => Some("openai-model"),
|
||||
"x-models-etag" => Some("x-models-etag"),
|
||||
"x-reasoning-included" => Some("x-reasoning-included"),
|
||||
"openai-processing-ms" => Some("openai-processing-ms"),
|
||||
_ => None,
|
||||
use gateway_core::event::ProviderResponseHeader;
|
||||
|
||||
pub(super) fn response_connection_options(headers: &[ProviderResponseHeader]) -> Vec<String> {
|
||||
headers
|
||||
.iter()
|
||||
.filter(|header| header.name().trim().eq_ignore_ascii_case("connection"))
|
||||
.filter_map(|header| std::str::from_utf8(header.value()).ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn response_header_is_forwardable(name: &str, connection_options: &[String]) -> bool {
|
||||
let name = name.trim().to_ascii_lowercase();
|
||||
if connection_options
|
||||
.iter()
|
||||
.any(|option| option.eq_ignore_ascii_case(&name))
|
||||
|| name.starts_with("sec-websocket-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name.as_str(),
|
||||
"connection"
|
||||
| "keep-alive"
|
||||
| "proxy-connection"
|
||||
| "te"
|
||||
| "trailer"
|
||||
| "transfer-encoding"
|
||||
| "upgrade"
|
||||
| "content-length"
|
||||
| "content-type"
|
||||
| "content-encoding"
|
||||
| "authorization"
|
||||
| "x-api-key"
|
||||
| "www-authenticate"
|
||||
| "authentication-info"
|
||||
| "proxy-authenticate"
|
||||
| "proxy-authorization"
|
||||
| "proxy-authentication-info"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "set-cookie"
|
||||
| "set-cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
| "x-codex-installation-id"
|
||||
| "x-codex-turn-state"
|
||||
| "x-codex-turn-metadata"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::{fmt, net::IpAddr};
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload, ProviderSessionState};
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
@@ -14,6 +15,7 @@ use super::error::RequestDecodeError;
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const REVIEW_SUBAGENT: &str = "review";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct OpenAiRequestHeaders {
|
||||
@@ -34,6 +36,7 @@ pub struct OpenAiRequestHeaders {
|
||||
responses_lite: Option<String>,
|
||||
memgen_request: Option<String>,
|
||||
subagent: Option<String>,
|
||||
passthrough_headers: Vec<Value>,
|
||||
}
|
||||
|
||||
impl OpenAiRequestHeaders {
|
||||
@@ -57,12 +60,8 @@ impl OpenAiRequestHeaders {
|
||||
turn_id: header_string(headers, "x-codex-turn-id"),
|
||||
responses_lite: header_string(headers, X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER),
|
||||
memgen_request: header_string(headers, X_OPENAI_MEMGEN_REQUEST_HEADER),
|
||||
subagent: header_string(headers, OPENAI_SUBAGENT_KEY).filter(|value| {
|
||||
matches!(
|
||||
value.as_str(),
|
||||
"review" | "compact" | "memory_consolidation" | "collab_spawn"
|
||||
)
|
||||
}),
|
||||
subagent: header_string(headers, OPENAI_SUBAGENT_KEY),
|
||||
passthrough_headers: passthrough_headers(headers),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +107,12 @@ impl OpenAiRequestHeaders {
|
||||
insert_protocol_context(&mut context, "turn_id", self.turn_id.as_ref());
|
||||
insert_protocol_context(&mut context, "responses_lite", self.responses_lite.as_ref());
|
||||
insert_protocol_context(&mut context, "memgen_request", self.memgen_request.as_ref());
|
||||
if !self.passthrough_headers.is_empty() {
|
||||
context.insert(
|
||||
PASSTHROUGH_HEADERS_CONTEXT_KEY.to_owned(),
|
||||
Value::Array(self.passthrough_headers.clone()),
|
||||
);
|
||||
}
|
||||
if let Some(use_websocket) = use_websocket {
|
||||
context.insert("use_websocket".to_owned(), Value::Bool(use_websocket));
|
||||
}
|
||||
@@ -310,16 +315,26 @@ pub(super) fn decode_request_inner(
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
let value =
|
||||
serde_json::from_slice::<Value>(body).map_err(|_| RequestDecodeError::MalformedJson)?;
|
||||
let Value::Object(mut object) = value else {
|
||||
let Value::Object(object) = value else {
|
||||
return Err(RequestDecodeError::ExpectedObject);
|
||||
};
|
||||
// `use_websocket` 只影响本地 transport。无论值是否可识别,都不得进入上游 body。
|
||||
let use_websocket = object
|
||||
.remove("use_websocket")
|
||||
.and_then(|value| value.as_bool());
|
||||
decode_request_object(object, review, request_headers)
|
||||
}
|
||||
|
||||
let model = required_non_empty_string(&object, "model", "model")?.trim();
|
||||
if model.is_empty() {
|
||||
/// 解码已解析的顶层 object;WebSocket 帧解码复用此内核,避免序列化往返。
|
||||
pub(super) fn decode_request_object(
|
||||
mut object: Map<String, Value>,
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
// 仅消费已识别的本地 transport 开关;未知同名值保留给未来上游协议。
|
||||
let use_websocket = object.get("use_websocket").and_then(Value::as_bool);
|
||||
if use_websocket.is_some() {
|
||||
object.remove("use_websocket");
|
||||
}
|
||||
|
||||
let model = required_non_empty_string(&object, "model", "model")?;
|
||||
if model.trim().is_empty() {
|
||||
return Err(RequestDecodeError::EmptyField {
|
||||
field: "model".to_owned(),
|
||||
});
|
||||
@@ -336,7 +351,6 @@ pub(super) fn decode_request_inner(
|
||||
let continuation = object
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|response_id| !response_id.is_empty())
|
||||
.map(|response_id| ContinuationIntent::PreviousResponseId(response_id.to_owned()))
|
||||
.unwrap_or(ContinuationIntent::None);
|
||||
|
||||
@@ -367,26 +381,98 @@ fn inject_subagent_metadata(body: &mut Map<String, Value>, subagent: &str) {
|
||||
let metadata = body
|
||||
.entry("client_metadata".to_owned())
|
||||
.or_insert_with(|| Value::Object(Map::new()));
|
||||
if !metadata.is_object() {
|
||||
*metadata = Value::Object(Map::new());
|
||||
}
|
||||
if let Some(metadata) = metadata.as_object_mut() {
|
||||
metadata.insert(
|
||||
OPENAI_SUBAGENT_KEY.to_owned(),
|
||||
Value::String(subagent.to_owned()),
|
||||
);
|
||||
}
|
||||
let Some(metadata) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
metadata.insert(
|
||||
OPENAI_SUBAGENT_KEY.to_owned(),
|
||||
Value::String(subagent.to_owned()),
|
||||
);
|
||||
}
|
||||
|
||||
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn passthrough_headers(headers: &HeaderMap) -> Vec<Value> {
|
||||
let connection_headers = headers
|
||||
.get_all("connection")
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| passthrough_header_name(name.as_str(), &connection_headers))
|
||||
.map(|(name, value)| {
|
||||
Value::Array(vec![
|
||||
Value::String(name.as_str().to_owned()),
|
||||
Value::String(STANDARD.encode(value.as_bytes())),
|
||||
])
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
if connection_headers
|
||||
.iter()
|
||||
.any(|connection_header| connection_header.eq_ignore_ascii_case(name))
|
||||
|| name.starts_with("sec-websocket-")
|
||||
|| name.starts_with("x-grok-")
|
||||
|| name.starts_with("x-xai-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name,
|
||||
// 逐跳头、上游 authority 和重序列化后的实体长度由 transport 重建。
|
||||
"connection"
|
||||
| "keep-alive"
|
||||
| "proxy-connection"
|
||||
| "proxy-authenticate"
|
||||
| "proxy-authorization"
|
||||
| "te"
|
||||
| "trailer"
|
||||
| "transfer-encoding"
|
||||
| "upgrade"
|
||||
| "host"
|
||||
| "content-length"
|
||||
// 下游连接诊断事实只用于本地观测,不能伪装成 OpenAI 请求事实。
|
||||
| "forwarded"
|
||||
| "x-forwarded-for"
|
||||
| "x-forwarded-host"
|
||||
| "x-forwarded-proto"
|
||||
| "x-forwarded-port"
|
||||
| "x-real-ip"
|
||||
| "true-client-ip"
|
||||
| "cf-connecting-ip"
|
||||
| "x-request-id"
|
||||
// 下游鉴权和账号 cookie 绝不能成为上游账号身份。
|
||||
| "authorization"
|
||||
| "x-api-key"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
)
|
||||
}
|
||||
|
||||
fn required_non_empty_string<'a>(
|
||||
object: &'a Map<String, Value>,
|
||||
key: &str,
|
||||
|
||||
@@ -11,15 +11,14 @@ const OPENAI_PROTOCOL: &str = "openai";
|
||||
|
||||
/// OpenAI Responses wire 的转发器。
|
||||
///
|
||||
/// OpenAI Provider 与 xAI adapter 都必须交付 OpenAI wire。这里仅保留同一
|
||||
/// 响应的身份和终态校验,不再从 canonical facts 重建客户端 JSON;因此
|
||||
/// OpenAI 上游的未知字段、字段顺序和事件语义不会被 API 层二次解释。
|
||||
/// OpenAI Provider 与 xAI adapter 都必须交付 OpenAI wire。wire 是客户端交付
|
||||
/// 与终态判断的事实来源;canonical facts 只在 wire 暂未携带 ID 时提供旁路观测,
|
||||
/// 不得反过来拒绝或改写上游事件。
|
||||
#[derive(Debug, Default)]
|
||||
pub struct OpenAiResponsesEncoder {
|
||||
response_id: Option<String>,
|
||||
wire_terminal: Option<Value>,
|
||||
wire_failure: bool,
|
||||
canonical_completed: bool,
|
||||
}
|
||||
|
||||
impl OpenAiResponsesEncoder {
|
||||
@@ -30,53 +29,41 @@ impl OpenAiResponsesEncoder {
|
||||
response_id: None,
|
||||
wire_terminal: None,
|
||||
wire_failure: false,
|
||||
canonical_completed: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 消费一个 Provider event,并返回 SSE frames。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 同一响应身份变化时返回错误。
|
||||
pub fn push_sse(&mut self, event: &ProviderEvent) -> Result<Vec<Bytes>, ResponseEncodeError> {
|
||||
self.observe_identity(event)?;
|
||||
pub fn push_sse(&mut self, event: &ProviderEvent) -> Vec<Bytes> {
|
||||
self.observe_canonical_identity(event);
|
||||
let Some(wire) = openai_wire(event) else {
|
||||
return Ok(Vec::new());
|
||||
return Vec::new();
|
||||
};
|
||||
self.observe_wire_terminal(wire);
|
||||
self.observe_wire(wire);
|
||||
if let Some(raw_sse_frame) = wire.raw_sse_frame() {
|
||||
return Ok(vec![raw_sse_frame.clone()]);
|
||||
return vec![raw_sse_frame.clone()];
|
||||
}
|
||||
Ok(vec![Bytes::from(encode_sse_event_with_metadata(
|
||||
vec![Bytes::from(encode_sse_event_with_metadata(
|
||||
wire.event_type().unwrap_or_default(),
|
||||
&wire.data().to_string(),
|
||||
wire.sse_id(),
|
||||
wire.sse_retry(),
|
||||
))])
|
||||
))]
|
||||
}
|
||||
|
||||
/// 消费一个 Provider event,并返回 WebSocket JSON messages。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 同一响应身份变化时返回错误。
|
||||
pub fn push_websocket(
|
||||
&mut self,
|
||||
event: &ProviderEvent,
|
||||
) -> Result<Vec<String>, ResponseEncodeError> {
|
||||
self.observe_identity(event)?;
|
||||
pub fn push_websocket(&mut self, event: &ProviderEvent) -> Vec<String> {
|
||||
self.observe_canonical_identity(event);
|
||||
let Some(wire) = openai_wire(event).filter(|wire| wire.has_json_data()) else {
|
||||
return Ok(Vec::new());
|
||||
return Vec::new();
|
||||
};
|
||||
self.observe_wire_terminal(wire);
|
||||
Ok(vec![wire.data().to_string()])
|
||||
self.observe_wire(wire);
|
||||
vec![wire.data().to_string()]
|
||||
}
|
||||
|
||||
/// 返回是否已经看到同一响应的 canonical 与 wire 终态。
|
||||
/// 返回是否已经看到客户端可见的 wire 终态。
|
||||
#[must_use]
|
||||
pub fn is_completed(&self) -> bool {
|
||||
self.wire_terminal.is_some() && self.canonical_completed
|
||||
self.wire_terminal.is_some()
|
||||
}
|
||||
|
||||
/// 返回是否已把 Provider 原生失败 event 交付给客户端。
|
||||
@@ -95,39 +82,31 @@ impl OpenAiResponsesEncoder {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 缺少 wire 或 canonical 终态时返回错误。
|
||||
/// 缺少可转换为非流式响应的 wire 终态时返回错误。
|
||||
pub fn finish(self) -> Result<Value, ResponseEncodeError> {
|
||||
if self.canonical_completed {
|
||||
self.wire_terminal
|
||||
.ok_or(ResponseEncodeError::MissingWireTerminal)
|
||||
} else {
|
||||
Err(ResponseEncodeError::MissingWireTerminal)
|
||||
}
|
||||
self.wire_terminal
|
||||
.ok_or(ResponseEncodeError::MissingWireTerminal)
|
||||
}
|
||||
|
||||
fn observe_identity(&mut self, event: &ProviderEvent) -> Result<(), ResponseEncodeError> {
|
||||
fn observe_canonical_identity(&mut self, event: &ProviderEvent) {
|
||||
for fact in event.canonical_facts() {
|
||||
let metadata = match fact {
|
||||
GatewayEvent::Started(metadata) => metadata,
|
||||
GatewayEvent::Completed(metadata) => {
|
||||
self.canonical_completed = true;
|
||||
metadata
|
||||
}
|
||||
GatewayEvent::Started(metadata) | GatewayEvent::Completed(metadata) => metadata,
|
||||
_ => continue,
|
||||
};
|
||||
if self
|
||||
.response_id
|
||||
.as_deref()
|
||||
.is_some_and(|current| current != metadata.response_id())
|
||||
{
|
||||
return Err(ResponseEncodeError::WireIdentityChanged);
|
||||
}
|
||||
self.response_id = Some(metadata.response_id().to_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_wire_terminal(&mut self, wire: &ProtocolWireEvent) {
|
||||
fn observe_wire(&mut self, wire: &ProtocolWireEvent) {
|
||||
if let Some(response_id) = wire
|
||||
.data()
|
||||
.pointer("/response/id")
|
||||
.or_else(|| wire.data().get("response_id"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
self.response_id = Some(response_id.to_owned());
|
||||
}
|
||||
let effective_type = wire
|
||||
.event_type()
|
||||
.or_else(|| wire.data().get("type").and_then(Value::as_str));
|
||||
|
||||
@@ -9,7 +9,7 @@ use axum::{
|
||||
connect_info::ConnectInfo,
|
||||
ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade, close_code},
|
||||
},
|
||||
http::{HeaderMap, StatusCode},
|
||||
http::{HeaderMap, HeaderName, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures::StreamExt;
|
||||
@@ -27,7 +27,6 @@ use crate::{
|
||||
openai::{
|
||||
auth::{authenticate_client, authentication_error_response},
|
||||
error::{gateway_error_contract, gateway_error_from_engine, runtime_unavailable_response},
|
||||
router::MAX_CLIENT_REQUEST_BODY_BYTES,
|
||||
service::OpenAiService,
|
||||
},
|
||||
};
|
||||
@@ -102,8 +101,10 @@ impl ResponsesWebSocketAdapter {
|
||||
_connection_guard: connection_guard,
|
||||
};
|
||||
websocket
|
||||
.max_message_size(MAX_CLIENT_REQUEST_BODY_BYTES)
|
||||
.max_frame_size(MAX_CLIENT_REQUEST_BODY_BYTES)
|
||||
// 覆盖 axum/tungstenite 的私有 64 MiB message 与 16 MiB frame 默认值。
|
||||
// Responses JSON 的协议可接受性由上游决定,代理不另设 wire 长度上限。
|
||||
.max_message_size(usize::MAX)
|
||||
.max_frame_size(usize::MAX)
|
||||
.on_upgrade(move |socket| async move {
|
||||
serve_responses_websocket(socket, session).await;
|
||||
})
|
||||
@@ -303,18 +304,7 @@ async fn forward_execution(
|
||||
if let Some(update) = event.take_session_update() {
|
||||
provider_state = Some(update);
|
||||
}
|
||||
match encoder.push_websocket(event) {
|
||||
Ok(messages) => first_messages.extend(messages),
|
||||
Err(error) => {
|
||||
return send_protocol_error(
|
||||
socket,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
&request_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
first_messages.extend(encoder.push_websocket(event));
|
||||
}
|
||||
if first_messages.is_empty() {
|
||||
let error = GatewayError::new(
|
||||
@@ -336,9 +326,7 @@ async fn forward_execution(
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
if let Err(error) = commit_connection_replay(replay, &encoder, provider_state.take()) {
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
if !send_text(
|
||||
socket,
|
||||
response_metadata_event(&request_id, &response_headers),
|
||||
@@ -378,18 +366,7 @@ async fn forward_execution(
|
||||
if let Some(update) = event.take_session_update() {
|
||||
provider_state = Some(update);
|
||||
}
|
||||
match encoder.push_websocket(event) {
|
||||
Ok(encoded) => messages.extend(encoded),
|
||||
Err(error) => {
|
||||
return send_protocol_error(
|
||||
socket,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
&request_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
messages.extend(encoder.push_websocket(event));
|
||||
}
|
||||
if encoder.is_completed() {
|
||||
if let Err(outcome) =
|
||||
@@ -397,11 +374,7 @@ async fn forward_execution(
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
if let Err(error) =
|
||||
commit_connection_replay(replay, &encoder, provider_state.take())
|
||||
{
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
if !send_messages(socket, messages).await {
|
||||
return ForwardOutcome::Disconnect;
|
||||
}
|
||||
@@ -413,9 +386,17 @@ async fn forward_execution(
|
||||
}
|
||||
}
|
||||
ActiveInput::Event(Ok(None)) => {
|
||||
if execution
|
||||
.session_mut()
|
||||
.is_some_and(|session| session.is_finalized())
|
||||
{
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
execution.disarm();
|
||||
return ForwardOutcome::Continue;
|
||||
}
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without a terminal event",
|
||||
"gateway response ended without finalizing the execution",
|
||||
);
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
@@ -441,15 +422,10 @@ fn commit_connection_replay(
|
||||
replay: &mut ConnectionReplaySnapshot,
|
||||
encoder: &OpenAiResponsesEncoder,
|
||||
provider_state: Option<ProviderSessionState>,
|
||||
) -> Result<(), GatewayError> {
|
||||
let response_id = encoder.response_id().ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway completed a response without an identity",
|
||||
)
|
||||
})?;
|
||||
replay.commit(response_id.to_owned(), provider_state);
|
||||
Ok(())
|
||||
) {
|
||||
if let Some(response_id) = encoder.response_id() {
|
||||
replay.commit(response_id.to_owned(), provider_state);
|
||||
}
|
||||
}
|
||||
|
||||
async fn confirm_completed_execution(
|
||||
@@ -549,9 +525,7 @@ fn decode_response_create_inner(
|
||||
if matches!(body.get("stream"), Some(value) if value.as_bool() != Some(true)) {
|
||||
return Err(ResponseCreateFrameError::StreamingRequired);
|
||||
}
|
||||
let encoded = serde_json::to_vec(&Value::Object(body))
|
||||
.map_err(|_| ResponseCreateFrameError::InvalidJson)?;
|
||||
super::request::decode_request_inner(&encoded, false, request_headers)
|
||||
super::request::decode_request_object(body, false, request_headers)
|
||||
.map_err(ResponseCreateFrameError::Request)
|
||||
}
|
||||
|
||||
@@ -597,21 +571,24 @@ fn response_metadata_event(
|
||||
request_id: &str,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> String {
|
||||
let mut headers = response_headers
|
||||
.iter()
|
||||
.filter_map(|header| {
|
||||
super::safe_response_header_name(header.name()).map(|name| {
|
||||
(
|
||||
name.to_owned(),
|
||||
Value::String(header.value().as_str().to_owned()),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Map<String, Value>>();
|
||||
headers.insert(
|
||||
"x-request-id".to_owned(),
|
||||
Value::String(request_id.to_owned()),
|
||||
);
|
||||
let connection_options = super::response_connection_options(response_headers);
|
||||
let mut headers = Map::new();
|
||||
for header in response_headers {
|
||||
if !super::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = std::str::from_utf8(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
// 官方 response.metadata 使用 string map,无法表达同名多值;后值保持既有覆盖语义。
|
||||
headers.insert(name.as_str().to_owned(), Value::String(value.to_owned()));
|
||||
}
|
||||
headers
|
||||
.entry("x-request-id".to_owned())
|
||||
.or_insert_with(|| Value::String(request_id.to_owned()));
|
||||
json!({
|
||||
"type": "response.metadata",
|
||||
"headers": headers,
|
||||
|
||||
@@ -13,9 +13,6 @@ use super::{
|
||||
|
||||
use crate::ApiState;
|
||||
|
||||
/// OpenAI 请求正文上限。
|
||||
pub const MAX_CLIENT_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// 构造 OpenAI 客户端协议路由。
|
||||
pub(crate) fn router() -> Router<ApiState> {
|
||||
Router::new()
|
||||
@@ -26,5 +23,7 @@ pub(crate) fn router() -> Router<ApiState> {
|
||||
.route("/v1/models/{model_id}/info", get(model_info))
|
||||
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
|
||||
.route("/v1/models/{model_id}", get(model_detail))
|
||||
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
|
||||
// Responses 正文属于 OpenAI/Codex 协议;代理不能用私有大小上限提前拒绝
|
||||
// 上游本可接受的未来 payload。
|
||||
.layer(DefaultBodyLimit::disable())
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ use gateway_core::engine::execution::{
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::lifecycle::{ConnectionDraining, ConnectionGuard, ConnectionLifecycle};
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::routing::{PublicModelId, PublicModelProfile};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::auth::ClientApiKeyAuthError;
|
||||
@@ -51,6 +51,13 @@ impl OpenAiService {
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn public_model_profiles(
|
||||
&self,
|
||||
client: &AuthenticatedClient,
|
||||
) -> Vec<PublicModelProfile> {
|
||||
self.execution.public_model_profiles(client)
|
||||
}
|
||||
|
||||
pub(crate) fn contains_public_model(
|
||||
&self,
|
||||
client: &AuthenticatedClient,
|
||||
@@ -74,16 +81,10 @@ impl OpenAiService {
|
||||
"requested model was not found",
|
||||
)
|
||||
})?;
|
||||
if !self.execution.contains_public_model(&client, &public_model) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
));
|
||||
}
|
||||
let previous_response_id = match metadata.continuation() {
|
||||
ContinuationIntent::None => None,
|
||||
ContinuationIntent::PreviousResponseId(value) => {
|
||||
PreviousResponseId::new(value.clone()).ok()
|
||||
Some(PreviousResponseId::new(value.clone()))
|
||||
}
|
||||
};
|
||||
self.execution
|
||||
|
||||
@@ -105,12 +105,15 @@ mod actions {
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus},
|
||||
provider_credentials::{CredentialDeletionResult, CredentialImportResult},
|
||||
provider_credentials::{
|
||||
CredentialDeletionResult, CredentialImportResult, CredentialMutationResult,
|
||||
},
|
||||
};
|
||||
use gateway_api::admin::accounts::{
|
||||
AccountActionRequest, AccountConnectionTestEvent, AccountDeletionData,
|
||||
AccountDeletionRequest, AccountExportData, AccountExportQuery, AccountIdQuery,
|
||||
AccountImportData, AccountImportRequest, AccountRefreshRequest, AccountTestQuery,
|
||||
AccountImportData, AccountImportRequest, AccountMutationData, AccountRefreshRequest,
|
||||
AccountTestQuery, RotateAccountRequest, StartAccountAuthorizationRequest,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use serde_json::json;
|
||||
@@ -147,7 +150,7 @@ mod actions {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_actions_should_require_frozen_account_ids_and_revision() {
|
||||
fn account_actions_should_require_frozen_account_ids_and_reject_unknown_revision() {
|
||||
let id: AccountIdQuery =
|
||||
serde_json::from_value(json!({ "accountId": "acct_1" })).expect("decode ID query");
|
||||
assert!(id.validate().is_ok());
|
||||
@@ -155,13 +158,68 @@ mod actions {
|
||||
serde_json::from_value(json!({ "accountId": "legacy-id" })).expect("decode action");
|
||||
assert_eq!(action.validate().unwrap_err().field(), "accountId");
|
||||
let refresh: AccountRefreshRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_1",
|
||||
"expectedConfigRevision": 0
|
||||
"accountId": "acct_1"
|
||||
}))
|
||||
.expect("decode refresh");
|
||||
assert!(refresh.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<AccountRefreshRequest>(json!({
|
||||
"accountId": "acct_1",
|
||||
"expectedConfigRevision": 0
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_recovery_requests_should_not_accept_client_revision_fences() {
|
||||
let authorization: StartAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"name": "reauthorize",
|
||||
"accountId": "acct_1"
|
||||
}))
|
||||
.expect("decode reauthorization");
|
||||
assert!(authorization.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<StartAccountAuthorizationRequest>(json!({
|
||||
"provider": "openai",
|
||||
"name": "reauthorize",
|
||||
"accountId": "acct_1",
|
||||
"expectedCredentialRevision": 1
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let rotation: RotateAccountRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_1",
|
||||
"accessToken": "header.payload.signature",
|
||||
"refreshToken": "refresh-token"
|
||||
}))
|
||||
.expect("decode rotation");
|
||||
assert!(rotation.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<RotateAccountRequest>(json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_1",
|
||||
"expectedCredentialRevision": 1,
|
||||
"accessToken": "header.payload.signature"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_mutation_response_should_not_expose_internal_revision() {
|
||||
let response = AccountMutationData::from(CredentialMutationResult {
|
||||
config_revision: Revision::new(8).expect("config revision"),
|
||||
account_id: ProviderAccountId::new("acct_1").expect("account ID"),
|
||||
credential_revision: Some(Revision::new(9).expect("credential revision")),
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
refresh.validate().unwrap_err().field(),
|
||||
"expectedConfigRevision"
|
||||
serde_json::to_value(response).expect("serialize credential mutation"),
|
||||
json!({ "accountId": "acct_1" })
|
||||
);
|
||||
}
|
||||
|
||||
@@ -169,7 +227,6 @@ mod actions {
|
||||
fn account_import_should_use_provider_and_opaque_data_fields() {
|
||||
let valid: AccountImportRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"data": {
|
||||
"providerOwnedUnknownField": {"nested": [1, 2, 3]},
|
||||
"accounts": [{"credentials": {"access_token": "provider-validates-this"}}]
|
||||
@@ -180,7 +237,6 @@ mod actions {
|
||||
|
||||
let invalid: AccountImportRequest = serde_json::from_value(json!({
|
||||
"provider": "xai",
|
||||
"expectedConfigRevision": 7,
|
||||
"data": []
|
||||
}))
|
||||
.expect("decode invalid account import");
|
||||
@@ -188,27 +244,32 @@ mod actions {
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"document": {}
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"data": {}
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_deletion_should_validate_one_provider_batch_and_emit_account_ids() {
|
||||
let request: AccountDeletionRequest = serde_json::from_value(json!({
|
||||
"provider": "xai",
|
||||
"accountIds": ["acct_1", "acct_2"],
|
||||
"expectedConfigRevision": 8
|
||||
"accountIds": ["acct_1", "acct_2"]
|
||||
}))
|
||||
.expect("decode account deletion");
|
||||
assert!(request.validate().is_ok());
|
||||
|
||||
let duplicate: AccountDeletionRequest = serde_json::from_value(json!({
|
||||
"provider": "xai",
|
||||
"accountIds": ["acct_1", "acct_1"],
|
||||
"expectedConfigRevision": 8
|
||||
"accountIds": ["acct_1", "acct_1"]
|
||||
}))
|
||||
.expect("decode duplicate account deletion");
|
||||
assert_eq!(duplicate.validate().unwrap_err().field(), "accountIds");
|
||||
@@ -223,7 +284,6 @@ mod actions {
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).expect("serialize account deletion"),
|
||||
json!({
|
||||
"configRevision": 9,
|
||||
"deletedCount": 2,
|
||||
"accountIds": ["acct_1", "acct_2"]
|
||||
})
|
||||
@@ -239,7 +299,6 @@ mod actions {
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).expect("serialize account import"),
|
||||
json!({
|
||||
"configRevision": 8,
|
||||
"importedCount": 1,
|
||||
"accountIds": ["acct_imported"]
|
||||
})
|
||||
@@ -276,7 +335,12 @@ mod actions {
|
||||
},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
account_status: AccountStatus::Active,
|
||||
provider_error_code: Some("usage_exhausted".to_owned()),
|
||||
provider_error_type: Some("invalid_request_error".to_owned()),
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::QuotaExhausted,
|
||||
},
|
||||
]
|
||||
.map(|event| AccountConnectionTestEvent::from(event).data);
|
||||
@@ -302,7 +366,16 @@ mod actions {
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "error", "error": "upstream unavailable", "accountStatus": "active" }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
"providerErrorCode": "usage_exhausted",
|
||||
"providerErrorType": "invalid_request_error",
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "quota_exhausted"
|
||||
}),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -51,7 +51,6 @@ async fn default_auth_service_should_initialize_login_validate_and_logout() {
|
||||
let service = fixture.services.auth();
|
||||
let session = service
|
||||
.login(LoginCommand {
|
||||
source: "127.0.0.1".to_owned(),
|
||||
username: Some("admin_1".to_owned()),
|
||||
password: "strong-admin-password".to_owned(),
|
||||
})
|
||||
@@ -84,42 +83,6 @@ async fn default_auth_service_should_initialize_login_validate_and_logout() {
|
||||
assert_eq!(fixture.auth.audit_count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_auth_service_should_throttle_repeated_invalid_identity() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
for attempt in 1..=5 {
|
||||
let error = fixture
|
||||
.services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
source: "shared-source".to_owned(),
|
||||
username: Some("wrong-user".to_owned()),
|
||||
password: "wrong-password".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect_err("invalid login");
|
||||
if attempt < 5 {
|
||||
assert_eq!(error, LoginError::InvalidCredentials);
|
||||
} else {
|
||||
assert_eq!(error, LoginError::Throttled);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
fixture
|
||||
.services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
source: "shared-source".to_owned(),
|
||||
username: None,
|
||||
password: "anything".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect_err("source remains throttled"),
|
||||
LoginError::Throttled
|
||||
);
|
||||
assert_eq!(fixture.auth.session_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_auth_service_should_verify_only_full_plaintext_admin_key() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
@@ -162,7 +125,6 @@ async fn audit_failure_should_revoke_new_session_before_returning_it() {
|
||||
.services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
source: "source".to_owned(),
|
||||
username: None,
|
||||
password: "strong-admin-password".to_owned(),
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use chrono::{DateTime, TimeZone as _, Utc};
|
||||
use gateway_api::admin::client_keys::{
|
||||
self, ClientKeyCursorData, ClientKeyCursorValue, ClientKeyListData, ClientKeyRevisionRequest,
|
||||
self, ClientKeyCursorData, ClientKeyCursorValue, ClientKeyListData, ClientKeyMutationRequest,
|
||||
ClientKeySort, ClientKeySortDirection, ClientKeySortField, ClientKeyView,
|
||||
CreateClientKeyRequest, CreatedClientKeyData, ListClientKeysQuery, MutatedClientKeyData,
|
||||
RevealedClientKeyData, UpdateClientKeyRequest, decode_client_key_cursor,
|
||||
@@ -101,9 +101,8 @@ fn client_key_queries_should_reject_unknown_zero_and_oversized_fields() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields() {
|
||||
fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
let valid = serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"name": "terminal key",
|
||||
"label": "production",
|
||||
"providerKind": "openai",
|
||||
@@ -113,22 +112,11 @@ fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields(
|
||||
.expect("deserialize create")
|
||||
.into_command()
|
||||
.expect("validate create");
|
||||
assert_eq!(valid.expected_config_revision.get(), 7);
|
||||
assert_eq!(valid.name, "terminal key");
|
||||
|
||||
for (payload, field) in [
|
||||
(
|
||||
json!({
|
||||
"expectedConfigRevision": 0,
|
||||
"name": "key",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": 0,
|
||||
"requestsPerMinute": 0
|
||||
}),
|
||||
"expectedConfigRevision",
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"expectedConfigRevision": 1,
|
||||
"name": " ",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": 0,
|
||||
@@ -138,7 +126,6 @@ fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields(
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"expectedConfigRevision": 1,
|
||||
"name": "key",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": u64::MAX,
|
||||
@@ -170,14 +157,23 @@ fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields(
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
let revision = serde_json::from_value::<ClientKeyRevisionRequest>(json!({
|
||||
"id": "key_1",
|
||||
"expectedConfigRevision": 3
|
||||
assert!(
|
||||
serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"name": "terminal key",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": 2,
|
||||
"requestsPerMinute": 60
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
let mutation_id = serde_json::from_value::<ClientKeyMutationRequest>(json!({
|
||||
"id": "key_1"
|
||||
}))
|
||||
.expect("deserialize revision mutation")
|
||||
.into_parts()
|
||||
.expect("validate revision mutation");
|
||||
assert_eq!(revision, ("key_1".to_owned(), 3));
|
||||
.expect("deserialize mutation")
|
||||
.into_id()
|
||||
.expect("validate mutation");
|
||||
assert_eq!(mutation_id, "key_1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -225,9 +221,9 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
updated_at: created_at,
|
||||
last_used_at: Some(created_at),
|
||||
});
|
||||
let list = serde_json::to_value(ClientKeyListData::new(9, vec![view], None, 1))
|
||||
.expect("serialize list");
|
||||
assert_eq!(list["configRevision"], 9);
|
||||
let list =
|
||||
serde_json::to_value(ClientKeyListData::new(vec![view], None, 1)).expect("serialize list");
|
||||
assert!(list.get("configRevision").is_none());
|
||||
assert_eq!(list["total"], 1);
|
||||
assert_eq!(list["items"][0]["id"], "key_visible");
|
||||
assert_eq!(list["items"][0]["providerKind"], "openai");
|
||||
@@ -248,7 +244,6 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
|
||||
let plaintext = format!("sk_{}", "a".repeat(43));
|
||||
let created = CreatedClientKeyData::new(
|
||||
10,
|
||||
"key_created".to_owned(),
|
||||
"sk_aaaaaaaaa".to_owned(),
|
||||
plaintext.clone(),
|
||||
@@ -263,9 +258,9 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
json!({ "id": "key_created", "plaintextKey": plaintext })
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(MutatedClientKeyData::new(11, "key_created".to_owned()))
|
||||
serde_json::to_value(MutatedClientKeyData::new("key_created".to_owned()))
|
||||
.expect("serialize mutation"),
|
||||
json!({ "configRevision": 11, "id": "key_created" })
|
||||
json!({ "id": "key_created" })
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -58,9 +58,8 @@ use gateway_api::admin::AdminSessionState;
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::GatewayError,
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
routing::{
|
||||
ConfigRevision, ProviderKind, PublicModelId, UpstreamModelId, snapshot::SnapshotControl,
|
||||
@@ -80,17 +79,24 @@ pub(super) struct AdminTestFixture {
|
||||
pub auth: Arc<MemoryAuthStore>,
|
||||
pub settings: Arc<MemorySettingsStore>,
|
||||
pub usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
}
|
||||
|
||||
impl AdminTestFixture {
|
||||
pub async fn new() -> Self {
|
||||
Self::with_system(Arc::new(UnusedSystem)).await
|
||||
}
|
||||
|
||||
pub async fn with_system(system: Arc<dyn SystemOperations>) -> Self {
|
||||
let api_key = Arc::new(Mutex::new(None));
|
||||
let auth = Arc::new(MemoryAuthStore::new(api_key.clone()));
|
||||
let settings = Arc::new(MemorySettingsStore::new(api_key));
|
||||
let client_keys = Arc::new(MemoryClientKeyStore);
|
||||
let usage_records = Arc::new(Mutex::new(Vec::new()));
|
||||
let dashboard_observation = Arc::new(Mutex::new(None));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
});
|
||||
let stores = AdminStorePorts::new(
|
||||
unused.clone(),
|
||||
@@ -113,7 +119,7 @@ impl AdminTestFixture {
|
||||
providers,
|
||||
Arc::new(NoopSnapshot),
|
||||
Arc::new(NoopProbe),
|
||||
Arc::new(UnusedSystem),
|
||||
system,
|
||||
)
|
||||
.await
|
||||
.expect("initialize test admin services");
|
||||
@@ -122,6 +128,7 @@ impl AdminTestFixture {
|
||||
auth,
|
||||
settings,
|
||||
usage_records,
|
||||
dashboard_observation,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,7 +149,6 @@ impl AdminSessionState for AdminTestState {
|
||||
pub(super) struct MemoryAuthStore {
|
||||
password_hash: Mutex<Option<String>>,
|
||||
sessions: Mutex<BTreeMap<String, AdminSession>>,
|
||||
failures: Mutex<BTreeMap<String, u32>>,
|
||||
audits: Mutex<Vec<AdminAuditEvent>>,
|
||||
api_key: Arc<Mutex<Option<AdminApiKey>>>,
|
||||
fail_audit: AtomicBool,
|
||||
@@ -153,7 +159,6 @@ impl MemoryAuthStore {
|
||||
Self {
|
||||
password_hash: Mutex::new(None),
|
||||
sessions: Mutex::new(BTreeMap::new()),
|
||||
failures: Mutex::new(BTreeMap::new()),
|
||||
audits: Mutex::new(Vec::new()),
|
||||
api_key,
|
||||
fail_audit: AtomicBool::new(false),
|
||||
@@ -235,37 +240,6 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(self.sessions.lock().expect("sessions").remove(session_id))
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(self
|
||||
.failures
|
||||
.lock()
|
||||
.expect("failures")
|
||||
.get(source)
|
||||
.is_some_and(|count| *count >= failure_limit))
|
||||
}
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut failures = self.failures.lock().expect("failures");
|
||||
let count = failures.entry(source.to_owned()).or_default();
|
||||
*count = count.saturating_add(1);
|
||||
Ok(*count >= failure_limit)
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, source: &str) -> AdminStoreResult<()> {
|
||||
self.failures.lock().expect("failures").remove(source);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
if self.fail_audit.load(Ordering::SeqCst) {
|
||||
return Err(unavailable("auth audit"));
|
||||
@@ -308,8 +282,9 @@ impl SettingsStore for MemorySettingsStore {
|
||||
command: ReplaceRuntimeSettings,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<RuntimeSettings> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
let updated = RuntimeSettings {
|
||||
config_revision: next_revision(command.expected_config_revision),
|
||||
config_revision: next_revision(settings.config_revision),
|
||||
model_mappings: command.model_mappings,
|
||||
refresh_margin_seconds: command.refresh_margin_seconds,
|
||||
refresh_concurrency: command.refresh_concurrency,
|
||||
@@ -321,31 +296,31 @@ impl SettingsStore for MemorySettingsStore {
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
updated_at: Utc::now(),
|
||||
};
|
||||
*self.settings.lock().expect("settings") = updated.clone();
|
||||
*settings = updated.clone();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
key: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
*self.api_key.lock().expect("API key") = Some(key);
|
||||
let revision = self.settings.lock().expect("settings").config_revision;
|
||||
Ok(AdminApiKeyMutation {
|
||||
config_revision: next_revision(expected_config_revision),
|
||||
config_revision: next_revision(revision),
|
||||
exists: true,
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
*self.api_key.lock().expect("API key") = None;
|
||||
let revision = self.settings.lock().expect("settings").config_revision;
|
||||
Ok(AdminApiKeyMutation {
|
||||
config_revision: next_revision(expected_config_revision),
|
||||
config_revision: next_revision(revision),
|
||||
exists: false,
|
||||
})
|
||||
}
|
||||
@@ -421,6 +396,7 @@ impl ClientKeyStore for MemoryClientKeyStore {
|
||||
|
||||
struct UnusedStore {
|
||||
usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -532,7 +508,11 @@ impl AccountStore for UnusedStore {
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
Err(unavailable("dashboard"))
|
||||
self.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation")
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("dashboard"))
|
||||
}
|
||||
|
||||
async fn dashboard_trend(&self, _: TimeRange) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
||||
@@ -703,7 +683,7 @@ impl AccountProbe for NoopProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async { panic!("unused account probe") })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -260,6 +260,80 @@ mod response {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headline_count() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
provider_accounts: AccountPoolMetrics {
|
||||
total: 807,
|
||||
enabled: 804,
|
||||
unavailable: 792,
|
||||
active: 15,
|
||||
expired: 2,
|
||||
quota_exhausted: 785,
|
||||
refreshing: None,
|
||||
disabled: 3,
|
||||
banned: 2,
|
||||
},
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_account_summary")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["credentials"]["totalValue"],
|
||||
&value["data"]["cards"]["credentials"]["availableValue"],
|
||||
&value["data"]["cards"]["credentials"]["unavailableValue"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!(807),
|
||||
&serde_json::json!(15),
|
||||
&serde_json::json!(792)
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
use axum::{
|
||||
@@ -291,12 +365,17 @@ async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
use std::str::FromStr as _;
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
|
||||
use gateway_admin::model::observability::{
|
||||
CalculatedBillingBreakdown, CurrencyCost, DecimalAmount, RequestOutcome, UsageBilling,
|
||||
UsageRecord,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
@@ -304,6 +383,10 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let usd = |amount: &str| CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: DecimalAmount::from_str(amount).expect("USD amount"),
|
||||
};
|
||||
fixture
|
||||
.usage_records
|
||||
.lock()
|
||||
@@ -326,6 +409,7 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: Some("reuse".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
provider_metadata_json: Some(
|
||||
serde_json::json!({
|
||||
"effectiveModel": "grok-4.5",
|
||||
@@ -358,7 +442,22 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: None,
|
||||
billing: Some(UsageBilling::Calculated(Box::new(
|
||||
CalculatedBillingBreakdown {
|
||||
input_amount: usd("0.03"),
|
||||
output_amount: usd("0.07"),
|
||||
cache_read_amount: usd("0.00"),
|
||||
cache_write_amount: usd("0.00"),
|
||||
standard_amount: usd("0.10"),
|
||||
total_amount: usd("0.10"),
|
||||
input_price_per_million: usd("10.0000"),
|
||||
output_price_per_million: usd("60.0000"),
|
||||
cache_read_price_per_million: usd("1.0000"),
|
||||
cache_write_price_per_million: usd("12.5000"),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
multiplier_percent: 100,
|
||||
},
|
||||
))),
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
headers_ms: Some(13),
|
||||
@@ -399,9 +498,23 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
.expect("usage response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
|
||||
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["inputPriceDisplay"],
|
||||
"$10 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["outputPriceDisplay"],
|
||||
"$60 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["cacheWritePriceDisplay"],
|
||||
"$12.5 / 1M Token"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"serviceTier": value["data"]["items"][0]["serviceTier"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
@@ -428,6 +541,7 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
}),
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"serviceTier": "priority",
|
||||
"authenticationKind": "oauth",
|
||||
"imageInputTokens": 31,
|
||||
"imageOutputTokens": 9,
|
||||
|
||||
@@ -37,7 +37,6 @@ async fn response_json(response: axum::response::Response) -> Value {
|
||||
|
||||
fn update_body() -> Value {
|
||||
json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
@@ -64,22 +63,11 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_require_positive_revision() {
|
||||
fn settings_request_should_reject_unknown_revision_field() {
|
||||
let mut body = update_body();
|
||||
body["expectedConfigRevision"] = json!(0);
|
||||
let request: UpdateRuntimeSettingsRequest =
|
||||
serde_json::from_value(body).expect("decode settings");
|
||||
assert_eq!(
|
||||
request.validate().unwrap_err().field(),
|
||||
"expectedConfigRevision"
|
||||
);
|
||||
body["expectedConfigRevision"] = json!(7);
|
||||
|
||||
let mut missing = update_body();
|
||||
missing
|
||||
.as_object_mut()
|
||||
.expect("settings object")
|
||||
.remove("expectedConfigRevision");
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(missing).is_err());
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -124,7 +112,7 @@ async fn settings_post_should_replace_global_model_mappings() {
|
||||
.expect("settings update response");
|
||||
let data = response_json(response).await["data"].clone();
|
||||
|
||||
assert_eq!(data["configRevision"], 8);
|
||||
assert!(data.get("configRevision").is_none());
|
||||
assert_eq!(data["modelMappings"]["gpt-5.4"], "gpt-5.5");
|
||||
assert_eq!(data["modelMappings"]["grok-latest"], "grok-4.5");
|
||||
}
|
||||
@@ -213,3 +201,26 @@ async fn settings_should_accept_admin_api_key_header() {
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_auth_should_accept_a_configured_request_id_header_name() {
|
||||
use axum::http::HeaderName;
|
||||
use tower_http::request_id::{MakeRequestUuid, SetRequestIdLayer};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
// 部署把 api.request_id_header 改名后,注入的 header 不再叫 x-request-id;
|
||||
// 管理请求仍须拿到请求上下文,而不是退化为 500。
|
||||
let custom = HeaderName::from_static("x-trace-id");
|
||||
let app = app(fixture.state()).layer(SetRequestIdLayer::new(custom, MakeRequestUuid));
|
||||
let unlabelled = Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri("/api/admin/settings")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.body(Body::empty())
|
||||
.expect("build settings request");
|
||||
|
||||
let response = app.oneshot(unlabelled).await.expect("settings response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,24 @@
|
||||
use gateway_api::admin::system::{UpdateDetailQuery, UpdateRequest};
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::stream;
|
||||
use gateway_admin::{
|
||||
model::system::{
|
||||
SystemOperationAccepted, SystemUpdateDetail, SystemUpdateEvent, SystemUpdateEventLevel,
|
||||
SystemUpdateStatus, SystemVersion,
|
||||
},
|
||||
ports::system::{SystemOperationError, SystemOperations, SystemUpdateEventStream},
|
||||
};
|
||||
use gateway_api::admin::system::{self, UpdateDetailQuery, UpdateRequest};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState, unavailable_system};
|
||||
|
||||
#[test]
|
||||
fn update_detail_query_should_default_to_no_refresh_and_reject_unknown_fields() {
|
||||
@@ -16,3 +36,84 @@ fn update_request_should_preserve_target_version_for_domain_validation() {
|
||||
serde_json::from_str::<UpdateRequest>(r#"{"targetVersion":"v0.2.0","extra":1}"#).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_event_stream_should_preserve_event_id_and_download_progress_percent() {
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem)).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = app(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/system/update/events")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_system_update_events")
|
||||
.body(Body::empty())
|
||||
.expect("update event request"),
|
||||
)
|
||||
.await
|
||||
.expect("update event response");
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("update event body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("UTF-8 event stream");
|
||||
|
||||
assert!(
|
||||
body.contains(r#""id":"update-progress-1""#),
|
||||
"unexpected update event stream: {body}"
|
||||
);
|
||||
assert!(
|
||||
body.contains(r#""progressPercent":40"#),
|
||||
"unexpected update event stream: {body}"
|
||||
);
|
||||
}
|
||||
|
||||
fn app(state: AdminTestState) -> Router {
|
||||
system::router::<AdminTestState>().with_state(state)
|
||||
}
|
||||
|
||||
struct ProgressSystem;
|
||||
|
||||
#[async_trait]
|
||||
impl SystemOperations for ProgressSystem {
|
||||
async fn version(&self) -> Result<SystemVersion, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn update_detail(&self, _: bool) -> Result<SystemUpdateDetail, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
fn update_events(&self) -> SystemUpdateEventStream {
|
||||
Box::pin(stream::once(async {
|
||||
SystemUpdateEvent {
|
||||
id: "update-progress-1".to_owned(),
|
||||
operation_id: Some("update-1".to_owned()),
|
||||
level: SystemUpdateEventLevel::Info,
|
||||
step: Some("download".to_owned()),
|
||||
message: "已下载 4.0 MiB / 10.0 MiB (40%)".to_owned(),
|
||||
terminal: true,
|
||||
progress_percent: Some(40),
|
||||
occurred_at: Utc::now(),
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async fn perform_update(
|
||||
&self,
|
||||
_: Option<String>,
|
||||
) -> Result<SystemOperationAccepted, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn update_status(&self) -> Result<SystemUpdateStatus, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn rollback(&self) -> Result<SystemOperationAccepted, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn restart(&self) -> Result<SystemOperationAccepted, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +135,35 @@ fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"no account is eligible for the requested model",
|
||||
Some("no_eligible_account".to_owned()),
|
||||
Some("account_unavailable_error".to_owned()),
|
||||
);
|
||||
let error = EngineError::Provider(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_error(detail),
|
||||
);
|
||||
let gateway = gateway_error_from_engine(&error);
|
||||
|
||||
assert_eq!(gateway.kind(), GatewayErrorKind::NoAvailableProvider);
|
||||
assert_eq!(
|
||||
gateway.client_message(),
|
||||
"no account is eligible for the requested model"
|
||||
);
|
||||
assert_eq!(gateway.client_error_code(), Some("no_eligible_account"));
|
||||
assert_eq!(
|
||||
gateway.client_error_type(),
|
||||
Some("account_unavailable_error")
|
||||
);
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"no upstream provider is currently available for this request"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
@@ -297,14 +326,11 @@ fn openai_error_response_should_preserve_only_safe_contract_fields() {
|
||||
async fn gateway_error_response_should_expose_only_structured_client_visible_upstream_fields() {
|
||||
let error = GatewayError::from_provider(
|
||||
&ProviderError::new(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
),
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)),
|
||||
);
|
||||
|
||||
let response = gateway_error_response(&error);
|
||||
|
||||
@@ -39,11 +39,18 @@ use gateway_core::routing::{
|
||||
};
|
||||
|
||||
pub(super) async fn api_router(execution: Arc<dyn ExecutionService>) -> axum::Router {
|
||||
api_router_with_origins(execution, Vec::new()).await
|
||||
}
|
||||
|
||||
pub(super) async fn api_router_with_origins(
|
||||
execution: Arc<dyn ExecutionService>,
|
||||
cors_allowed_origins: Vec<String>,
|
||||
) -> axum::Router {
|
||||
let admin = crate::admin::AdminTestFixture::new().await;
|
||||
gateway_api::initialize(
|
||||
gateway_api::ApiConfig {
|
||||
asset_directory: std::env::temp_dir(),
|
||||
cors_allowed_origins: Vec::new(),
|
||||
cors_allowed_origins,
|
||||
request_timeout_seconds: None,
|
||||
request_id_header: "x-request-id".to_owned(),
|
||||
},
|
||||
|
||||
@@ -10,19 +10,30 @@ use gateway_core::engine::execution::{
|
||||
StartedExecution,
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::routing::{
|
||||
ModelPresentation, ModelServiceTier, PublicModelId, PublicModelProfile,
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::{api_router, authenticated_client};
|
||||
|
||||
struct ModelsExecution {
|
||||
pub(super) struct ModelsExecution {
|
||||
client: AuthenticatedClient,
|
||||
profiles: bool,
|
||||
}
|
||||
|
||||
impl ModelsExecution {
|
||||
fn new() -> Arc<Self> {
|
||||
pub(super) fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_models_test"),
|
||||
profiles: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn with_profiles() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_models_test"),
|
||||
profiles: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -46,6 +57,36 @@ impl ExecutionService for ModelsExecution {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn public_model_profiles(&self, _: &AuthenticatedClient) -> Vec<PublicModelProfile> {
|
||||
if !self.profiles {
|
||||
return Vec::new();
|
||||
}
|
||||
vec![PublicModelProfile::new(
|
||||
PublicModelId::new("grok-4.5").expect("model"),
|
||||
ModelPresentation::new(
|
||||
Some("Grok 4.5".to_owned()),
|
||||
Some("xAI Grok 4.5 frontier model.".to_owned()),
|
||||
)
|
||||
.with_reasoning(
|
||||
Some("medium".to_owned()),
|
||||
["low", "medium", "high", "xhigh"]
|
||||
.map(str::to_owned)
|
||||
.to_vec(),
|
||||
)
|
||||
.with_context_window_tokens(Some(500_000))
|
||||
.with_image_input(true)
|
||||
.with_agent_tools(true, true)
|
||||
.with_service_tiers(vec![
|
||||
ModelServiceTier::new(
|
||||
"priority",
|
||||
"Fast",
|
||||
"Route the request through Codex fast mode.",
|
||||
)
|
||||
.with_speed_tier("fast"),
|
||||
]),
|
||||
)]
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
matches!(model.as_str(), "model-a" | "model-b")
|
||||
}
|
||||
@@ -91,6 +132,100 @@ async fn models_should_encode_the_service_visible_catalog() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_should_encode_provider_profiles_for_current_codex_clients() {
|
||||
let response = api_router(ModelsExecution::with_profiles())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models?client_version=0.145.0"))
|
||||
.await
|
||||
.expect("list Codex models response");
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read Codex models body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("models JSON");
|
||||
let model = &value["models"][0];
|
||||
|
||||
assert_eq!(
|
||||
value
|
||||
.as_object()
|
||||
.map(|catalog| catalog.keys().map(String::as_str).collect::<Vec<_>>()),
|
||||
Some(vec!["models"])
|
||||
);
|
||||
assert_eq!(model["slug"], "grok-4.5");
|
||||
assert_eq!(model["default_reasoning_level"], "medium");
|
||||
assert_eq!(model["context_window"], 500_000);
|
||||
assert_eq!(model["apply_patch_tool_type"], "freeform");
|
||||
assert_eq!(model["additional_speed_tiers"], serde_json::json!(["fast"]));
|
||||
assert_eq!(
|
||||
model["service_tiers"],
|
||||
serde_json::json!([{
|
||||
"id": "priority",
|
||||
"name": "Fast",
|
||||
"description": "Route the request through Codex fast mode."
|
||||
}])
|
||||
);
|
||||
assert_eq!(
|
||||
model["supported_reasoning_levels"],
|
||||
serde_json::json!([
|
||||
{
|
||||
"effort": "low",
|
||||
"description": "Fast responses with lighter reasoning"
|
||||
},
|
||||
{
|
||||
"effort": "medium",
|
||||
"description": "Balances speed and reasoning depth for everyday tasks"
|
||||
},
|
||||
{
|
||||
"effort": "high",
|
||||
"description": "Greater reasoning depth for complex problems"
|
||||
},
|
||||
{
|
||||
"effort": "xhigh",
|
||||
"description": "Extra high reasoning depth for complex problems"
|
||||
}
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_should_keep_the_codex_contract_when_profiles_are_empty() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models?client_version=0.145.0"))
|
||||
.await
|
||||
.expect("list empty Codex models response");
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read empty Codex models body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("Codex models JSON");
|
||||
|
||||
assert_eq!(value, serde_json::json!({ "models": [] }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_with_provider_profiles_should_keep_the_openai_list_contract() {
|
||||
let response = api_router(ModelsExecution::with_profiles())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models"))
|
||||
.await
|
||||
.expect("list compatible models response");
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read compatible models body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("models JSON");
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
serde_json::json!({
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id":"model-a","object":"model","created":1700000000_i64,"owned_by":"gateway"},
|
||||
{"id":"model-b","object":"model","created":1700000000_i64,"owned_by":"gateway"}
|
||||
]
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_detail_should_keep_the_official_path_id_contract() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
|
||||
@@ -8,7 +8,10 @@ use std::sync::{
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
extract::connect_info::ConnectInfo,
|
||||
http::{HeaderMap, Request, StatusCode, header::AUTHORIZATION},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, Request, StatusCode,
|
||||
header::{AUTHORIZATION, CONTENT_TYPE},
|
||||
},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::{BoxFuture, pending};
|
||||
@@ -18,8 +21,8 @@ use gateway_core::engine::execution::{
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
|
||||
SafeUpstreamValue,
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind,
|
||||
ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, GatewayEvent, ProtocolWireEvent, ProviderEvent,
|
||||
@@ -101,6 +104,8 @@ struct ContextCaptureExecution {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct CapturedClientContext {
|
||||
public_model: String,
|
||||
wire_model: Option<String>,
|
||||
client_ip: Option<IpAddr>,
|
||||
user_agent: Option<String>,
|
||||
endpoint: String,
|
||||
@@ -109,6 +114,7 @@ struct CapturedClientContext {
|
||||
client_metadata: Option<Value>,
|
||||
protocol_context: Option<Value>,
|
||||
prompt_cache_key: Option<String>,
|
||||
previous_response_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ExecutionService for ContextCaptureExecution {
|
||||
@@ -124,11 +130,14 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
Vec::new()
|
||||
["model-a", "model-b"]
|
||||
.into_iter()
|
||||
.map(|model| PublicModelId::new(model).expect("catalog model"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
|
||||
true
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
matches!(model.as_str(), "model-a" | "model-b")
|
||||
}
|
||||
|
||||
fn start(
|
||||
@@ -136,11 +145,16 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let public_model = request.public_model.as_str().to_owned();
|
||||
let operation_kind = request.operation.kind();
|
||||
let generation = match &request.operation {
|
||||
Operation::Generate(generation) => Some(generation),
|
||||
_ => None,
|
||||
};
|
||||
let wire_model = generation
|
||||
.and_then(|generation| generation.protocol_payload().body().get("model"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned);
|
||||
let (client_metadata, protocol_context, prompt_cache_key, input) =
|
||||
generation.map_or((None, None, None, None), |generation| {
|
||||
let payload = generation.protocol_payload();
|
||||
@@ -152,7 +166,14 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
payload.body().get("input").cloned(),
|
||||
)
|
||||
});
|
||||
let previous_response_id = request
|
||||
.metadata
|
||||
.previous_response_id
|
||||
.as_ref()
|
||||
.map(|value| value.as_str().to_owned());
|
||||
*self.observed.lock().expect("context capture lock") = Some(CapturedClientContext {
|
||||
public_model,
|
||||
wire_model,
|
||||
client_ip: request.metadata.client_ip,
|
||||
user_agent: request.metadata.user_agent,
|
||||
endpoint: request.metadata.endpoint,
|
||||
@@ -161,6 +182,7 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
client_metadata,
|
||||
protocol_context,
|
||||
prompt_cache_key,
|
||||
previous_response_id,
|
||||
});
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
@@ -503,14 +525,19 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
assert_eq!(
|
||||
captured_client_context(headers, peer).await,
|
||||
CapturedClientContext {
|
||||
public_model: "smart-code".to_owned(),
|
||||
wire_model: Some("smart-code".to_owned()),
|
||||
client_ip: Some("198.51.100.1".parse().expect("expected IP")),
|
||||
user_agent: Some("Codex-CLI/1.0".to_owned()),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
operation_kind: OperationKind::Generate,
|
||||
input: Some(json!("hello")),
|
||||
client_metadata: None,
|
||||
protocol_context: None,
|
||||
protocol_context: Some(json!({
|
||||
"opaque_request_headers": [["user-agent", "IENvZGV4LUNMSS8xLjAg"]]
|
||||
})),
|
||||
prompt_cache_key: None,
|
||||
previous_response_id: None,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -533,14 +560,58 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_forward_safe_codex_headers_without_projecting_xai_headers() {
|
||||
async fn stale_model_catalog_should_not_block_a_new_model_request() {
|
||||
let model = " gpt-future-codex ";
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({"model": model, "input": "hello"}),
|
||||
HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(captured.public_model, model);
|
||||
assert_eq!(captured.wire_model.as_deref(), Some(model));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_pass_opaque_previous_response_id_to_core() {
|
||||
for response_id in [
|
||||
format!("resp_{}", "x".repeat(257)),
|
||||
"resp_control\0opaque".to_owned(),
|
||||
String::new(),
|
||||
] {
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": response_id.clone()
|
||||
}),
|
||||
HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
captured.previous_response_id.as_deref(),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_forward_codex_headers_without_projecting_xai_headers() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-codex-turn-state", "turn-state".parse().expect("header"));
|
||||
headers.insert("conversation-id", "conversation-1".parse().expect("header"));
|
||||
headers.insert("session-id", "session-1".parse().expect("header"));
|
||||
headers.insert("x-grok-turn-idx", "7".parse().expect("header"));
|
||||
headers.insert("x-openai-subagent", "compact".parse().expect("header"));
|
||||
headers.insert(
|
||||
"x-openai-subagent",
|
||||
"future_codex_mode".parse().expect("header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-openai-internal-codex-responses-lite",
|
||||
"true".parse().expect("header"),
|
||||
@@ -566,10 +637,33 @@ async fn http_request_should_forward_safe_codex_headers_without_projecting_xai_h
|
||||
.client_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get("x-openai-subagent")),
|
||||
Some(&json!("compact"))
|
||||
Some(&json!("future_codex_mode"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subagent_header_should_not_replace_non_object_client_metadata() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-openai-subagent",
|
||||
"future_codex_mode".parse().expect("header"),
|
||||
);
|
||||
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "preserve metadata",
|
||||
"client_metadata": "opaque-client-value"
|
||||
}),
|
||||
headers,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(captured.client_metadata, Some(json!("opaque-client-value")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
@@ -632,6 +726,43 @@ async fn review_route_should_inject_review_subagent_and_record_its_endpoint() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_route_should_not_replace_non_object_client_metadata() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client("sk_context_test"),
|
||||
});
|
||||
let request = Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "review this",
|
||||
"client_metadata": ["opaque", "client", "value"]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("review request");
|
||||
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("review response");
|
||||
let captured = observed
|
||||
.lock()
|
||||
.expect("review capture lock")
|
||||
.clone()
|
||||
.expect("captured review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(
|
||||
captured.client_metadata,
|
||||
Some(json!(["opaque", "client", "value"]))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -653,19 +784,22 @@ async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_response_forwards_only_allowlisted_provider_headers() {
|
||||
async fn streaming_response_should_append_ordinary_headers_and_skip_unrepresentable_or_sensitive_ones()
|
||||
{
|
||||
let trace = Arc::new(Trace::default());
|
||||
let headers = vec![
|
||||
ProviderResponseHeader::new(
|
||||
"x-models-etag",
|
||||
SafeUpstreamValue::new("models-v2").expect("etag"),
|
||||
)
|
||||
.expect("safe header"),
|
||||
ProviderResponseHeader::new("x-models-etag", Bytes::from_static(b"models-v2")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
SafeUpstreamValue::new("should-not-cross-boundary").expect("header value"),
|
||||
)
|
||||
.expect("syntactically safe header"),
|
||||
Bytes::from_static(b"should-not-cross-boundary"),
|
||||
),
|
||||
ProviderResponseHeader::new("connection", Bytes::from_static(b"x-hop-secret")),
|
||||
ProviderResponseHeader::new("x-hop-secret", Bytes::from_static(b"hop-secret")),
|
||||
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
|
||||
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
|
||||
];
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
@@ -685,7 +819,29 @@ async fn streaming_response_forwards_only_allowlisted_provider_headers() {
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("models-v2")
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-future-multi")
|
||||
.iter()
|
||||
.map(HeaderValue::as_bytes)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![b"first".as_slice(), b"second".as_slice()]
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-future-bytes")
|
||||
.map(HeaderValue::as_bytes),
|
||||
Some(b"\xffopaque".as_slice())
|
||||
);
|
||||
assert!(response.headers().get("authorization").is_none());
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-hop-secret").is_none());
|
||||
assert_eq!(
|
||||
response.headers().get(CONTENT_TYPE),
|
||||
Some(&HeaderValue::from_static("text/event-stream"))
|
||||
);
|
||||
std::mem::forget(response);
|
||||
}
|
||||
|
||||
@@ -801,7 +957,107 @@ async fn streaming_rate_limit_before_first_frame_should_persist_the_returned_429
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
async fn streaming_upstream_http_failure_before_first_frame_should_preserve_raw_response() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_body = Bytes::from_static(
|
||||
b"{\"error\":{\"message\":\"rate limited\",\"future_field\":{\"kept\":true}},\"top_level\":17}\x00",
|
||||
);
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
429,
|
||||
Some(b"application/problem+json; charset=utf-8".to_vec()),
|
||||
raw_body.clone(),
|
||||
)
|
||||
.with_headers(vec![
|
||||
ProviderResponseHeader::new("retry-after", Bytes::from_static(b"17")),
|
||||
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req-failure")),
|
||||
ProviderResponseHeader::new("x-future-error", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-error", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
]),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![NextStep::Error(EngineError::Provider(error))],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"application/problem+json; charset=utf-8".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("retry-after")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"17".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-request-id")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"req-failure".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-future-error")
|
||||
.iter()
|
||||
.map(|value| value.as_bytes())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![b"first".as_slice(), b"second".as_slice()]
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-future-bytes")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"\xffopaque".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read raw error body"),
|
||||
raw_body
|
||||
);
|
||||
assert_eq!(trace.client_statuses(), vec![429]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_upstream_http_failure_should_preserve_raw_response() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_body = Bytes::from_static(b"not-json\xffstill-upstream");
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
503,
|
||||
Some(b"application/octet-stream".to_vec()),
|
||||
raw_body.clone(),
|
||||
));
|
||||
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
|
||||
.with_collect_error(EngineError::Provider(error));
|
||||
|
||||
let response = collect_execution_response(Box::new(session)).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read raw error body"),
|
||||
raw_body
|
||||
);
|
||||
assert_eq!(trace.client_statuses(), vec![503]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_canonical_identity_change_does_not_interrupt_wire_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
@@ -811,7 +1067,7 @@ async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
mismatched_terminal_event(),
|
||||
CommitRequirement::AlreadyCommitted,
|
||||
)),
|
||||
NextStep::FinalizeCancelled,
|
||||
NextStep::FinalizeSuccess,
|
||||
],
|
||||
);
|
||||
|
||||
@@ -820,12 +1076,16 @@ async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
|
||||
assert!(String::from_utf8_lossy(&body).contains("response.failed"));
|
||||
let body = String::from_utf8_lossy(&body);
|
||||
assert!(body.contains("response.completed"));
|
||||
assert!(body.contains("resp_other"));
|
||||
assert!(!body.contains("response.failed"));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert_eq!(trace.client_statuses(), vec![200]);
|
||||
assert!(trace.is_cancelled());
|
||||
assert!(!trace.is_cancelled());
|
||||
assert_eq!(
|
||||
trace.snapshot(),
|
||||
vec!["next_event", "commit", "next_event", "cancel_finalize"]
|
||||
vec!["next_event", "commit", "next_event", "next_end"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -874,6 +1134,38 @@ async fn streaming_success_should_emit_terminal_event_and_done_marker() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_finalized_unknown_wire_should_end_cleanly_without_synthetic_failure() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw = Bytes::from_static(
|
||||
b"event: response.future_terminal\ndata: {\"type\":\"response.future_terminal\",\"opaque\":true}\n\n",
|
||||
);
|
||||
let event = ProviderEvent::wire(
|
||||
ProtocolWireEvent::raw_sse("openai", raw.clone()).expect("raw future event"),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(delivery_provider(
|
||||
event,
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)),
|
||||
NextStep::FinalizeSuccess,
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read transparent SSE body");
|
||||
|
||||
assert!(body.as_ref().starts_with(raw.as_ref()));
|
||||
assert!(body.as_ref().ends_with(b"data: [DONE]\n\n"));
|
||||
assert!(!String::from_utf8_lossy(&body).contains("response.failed"));
|
||||
assert!(!trace.is_cancelled());
|
||||
assert_eq!(trace.snapshot(), vec!["next_event", "commit", "next_end"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_completed_event_without_finalized_execution_should_fail_closed() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -942,14 +1234,11 @@ async fn streaming_empty_terminal_should_emit_failure_done_and_cancel_execution(
|
||||
async fn streaming_error_should_emit_client_visible_upstream_details() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
);
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
));
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
@@ -1055,6 +1344,86 @@ async fn streaming_upstream_wire_failure_should_not_be_rewritten_as_a_gateway_er
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atomic_uncommitted_upstream_failure_batch_should_be_forwarded_once() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_created = Bytes::from_static(
|
||||
b"event: response.created\r\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_atomic_api\",\"model\":\"public-model\",\"status\":\"in_progress\"}}\r\n\r\n",
|
||||
);
|
||||
let raw_failure = Bytes::from_static(
|
||||
b"event: response.failed\r\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_atomic_api\",\"status\":\"failed\",\"error\":{\"code\":\"rate_limit_exceeded\",\"message\":\"atomic upstream marker\"}}}\r\n\r\n",
|
||||
);
|
||||
let started_wire = ProviderEvent::canonical_with_wire(
|
||||
vec![started()],
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_atomic_api",
|
||||
"model": "public-model",
|
||||
"status": "in_progress"
|
||||
}
|
||||
}),
|
||||
raw_created.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream started wire"),
|
||||
);
|
||||
let failed_wire = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": "resp_atomic_api",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"code": "rate_limit_exceeded",
|
||||
"message": "atomic upstream marker"
|
||||
}
|
||||
}
|
||||
}),
|
||||
raw_failure.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream failure wire"),
|
||||
);
|
||||
let batch = CoordinatedEvent::try_batch(
|
||||
vec![started_wire, failed_wire],
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)
|
||||
.expect("atomic failure delivery");
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(batch),
|
||||
NextStep::Error(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
))),
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("SSE is UTF-8");
|
||||
|
||||
assert_eq!(body.matches("atomic upstream marker").count(), 1);
|
||||
assert_eq!(body.matches("event: response.failed").count(), 1);
|
||||
assert!(body.contains(std::str::from_utf8(&raw_created).expect("created frame")));
|
||||
assert!(body.contains(std::str::from_utf8(&raw_failure).expect("failure frame")));
|
||||
assert!(!body.contains("upstream capacity is temporarily unavailable"));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert_eq!(trace.snapshot(), vec!["next_event", "commit", "next_error"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_second_commit_request_should_fail_and_finalize_once() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -1132,13 +1501,10 @@ async fn buffered_response_commits_only_after_complete_json_is_encoded() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_response_forwards_allowlisted_provider_headers() {
|
||||
async fn buffered_response_forwards_long_ordinary_provider_header_values() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let header = ProviderResponseHeader::new(
|
||||
"openai-model",
|
||||
SafeUpstreamValue::new("gpt-5.5-effective").expect("model header"),
|
||||
)
|
||||
.expect("safe header");
|
||||
let model = format!("gpt-{}", "x".repeat(512));
|
||||
let header = ProviderResponseHeader::new("openai-model", Bytes::from(model.clone()));
|
||||
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
|
||||
.with_response_headers(vec![header]);
|
||||
|
||||
@@ -1149,7 +1515,7 @@ async fn buffered_response_forwards_allowlisted_provider_headers() {
|
||||
.headers()
|
||||
.get("openai-model")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("gpt-5.5-effective")
|
||||
Some(model.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ mod http;
|
||||
mod websocket;
|
||||
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use gateway_api::openai::responses::{
|
||||
ContinuationIntent, DecodedResponsesRequest, OpenAiRequestHeaders, OpenAiResponsesEncoder,
|
||||
@@ -117,6 +118,28 @@ fn decoder_should_preserve_the_openai_body_and_only_derive_stable_routing_facts(
|
||||
assert!(decoded.metadata().store());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_opaque_client_model_values() {
|
||||
for model in [
|
||||
format!("future-{}", "x".repeat(512)),
|
||||
"future\0model".to_owned(),
|
||||
" future-model ".to_owned(),
|
||||
] {
|
||||
let decoded = generate_request(json!({
|
||||
"model": model.clone(),
|
||||
"input": "hello"
|
||||
}));
|
||||
|
||||
assert_eq!(decoded.metadata().public_model(), model);
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded)
|
||||
.get("model")
|
||||
.and_then(Value::as_str),
|
||||
Some(model.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_compaction_trigger_for_the_openai_provider() {
|
||||
let decoded = generate_request(json!({
|
||||
@@ -149,6 +172,21 @@ fn decoder_should_keep_transport_override_out_of_the_openai_wire_body() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unrecognized_use_websocket_values() {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"use_websocket": {"future": "transport-mode"}
|
||||
}));
|
||||
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded).get("use_websocket"),
|
||||
Some(&json!({"future": "transport-mode"}))
|
||||
);
|
||||
assert!(!openai_protocol_context(&decoded).contains_key("use_websocket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
let mut headers = HeaderMap::new();
|
||||
@@ -170,10 +208,142 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
("turn_state".to_owned(), json!("turn-state")),
|
||||
("turn_metadata".to_owned(), json!("{\"kind\":\"review\"}")),
|
||||
("conversation_id".to_owned(), json!("conversation")),
|
||||
(
|
||||
"opaque_request_headers".to_owned(),
|
||||
json!([
|
||||
["x-codex-turn-state", STANDARD.encode(b"turn-state")],
|
||||
[
|
||||
"x-codex-turn-metadata",
|
||||
STANDARD.encode(br#"{"kind":"review"}"#)
|
||||
],
|
||||
["conversation-id", STANDARD.encode(b"conversation")]
|
||||
]),
|
||||
),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_static("future-ascii"),
|
||||
);
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_bytes(b"\x80\xff").expect("opaque header bytes"),
|
||||
);
|
||||
headers.append("version", HeaderValue::from_static("future-v1"));
|
||||
headers.append(
|
||||
"version",
|
||||
HeaderValue::from_bytes(b"future-\x80").expect("opaque known header bytes"),
|
||||
);
|
||||
headers.insert(
|
||||
"accept",
|
||||
HeaderValue::from_static("application/vnd.openai.responses+json"),
|
||||
);
|
||||
headers.insert(
|
||||
"content-type",
|
||||
HeaderValue::from_static("application/vnd.openai.responses.request+json"),
|
||||
);
|
||||
headers.insert(
|
||||
"user-agent",
|
||||
HeaderValue::from_static("Codex future-client"),
|
||||
);
|
||||
headers.insert("originator", HeaderValue::from_static("future-codex"));
|
||||
headers.append(
|
||||
"openai-beta",
|
||||
HeaderValue::from_static("future_responses=v2"),
|
||||
);
|
||||
headers.append("openai-beta", HeaderValue::from_static("future_tools=v3"));
|
||||
headers.insert(
|
||||
"x-openai-internal-codex-residency",
|
||||
HeaderValue::from_static("future-region"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-codex-turn-state",
|
||||
HeaderValue::from_static("same-account-turn"),
|
||||
);
|
||||
|
||||
headers.insert(
|
||||
"connection",
|
||||
HeaderValue::from_static("keep-alive, x-private-hop"),
|
||||
);
|
||||
headers.insert("x-private-hop", HeaderValue::from_static("drop-me"));
|
||||
headers.insert("host", HeaderValue::from_static("downstream.invalid"));
|
||||
headers.insert("content-length", HeaderValue::from_static("999"));
|
||||
headers.insert("authorization", HeaderValue::from_static("Bearer client"));
|
||||
headers.insert("x-api-key", HeaderValue::from_static("client-key"));
|
||||
headers.insert("cookie", HeaderValue::from_static("client=cookie"));
|
||||
headers.insert(
|
||||
"chatgpt-account-id",
|
||||
HeaderValue::from_static("client-account"),
|
||||
);
|
||||
headers.insert(
|
||||
"chatgpt-project-id",
|
||||
HeaderValue::from_static("client-project"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-codex-installation-id",
|
||||
HeaderValue::from_static("client-installation"),
|
||||
);
|
||||
|
||||
let decoded =
|
||||
decode_request_with_headers(br#"{"model":"smart-code","input":"hello"}"#, &headers)
|
||||
.expect("opaque request headers should not affect body decoding");
|
||||
let entries = openai_protocol_context(&decoded)
|
||||
.get("opaque_request_headers")
|
||||
.and_then(Value::as_array)
|
||||
.expect("opaque request header context");
|
||||
let values = |target: &str| {
|
||||
entries
|
||||
.iter()
|
||||
.filter_map(Value::as_array)
|
||||
.filter(|entry| entry.first().and_then(Value::as_str) == Some(target))
|
||||
.filter_map(|entry| entry.get(1).and_then(Value::as_str))
|
||||
.map(|encoded| STANDARD.decode(encoded).expect("valid base64 header value"))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
values("x-openai-future-mode"),
|
||||
vec![b"future-ascii".to_vec(), b"\x80\xff".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("version"),
|
||||
vec![b"future-v1".to_vec(), b"future-\x80".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("openai-beta"),
|
||||
vec![b"future_responses=v2".to_vec(), b"future_tools=v3".to_vec()]
|
||||
);
|
||||
for preserved in [
|
||||
"accept",
|
||||
"content-type",
|
||||
"user-agent",
|
||||
"originator",
|
||||
"x-openai-internal-codex-residency",
|
||||
"x-codex-turn-state",
|
||||
] {
|
||||
assert_eq!(values(preserved).len(), 1, "missing {preserved}");
|
||||
}
|
||||
for excluded in [
|
||||
"connection",
|
||||
"x-private-hop",
|
||||
"host",
|
||||
"content-length",
|
||||
"authorization",
|
||||
"x-api-key",
|
||||
"cookie",
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-project-id",
|
||||
"x-codex-installation-id",
|
||||
] {
|
||||
assert!(values(excluded).is_empty(), "unexpected {excluded}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
let secret = "nested-private-value";
|
||||
@@ -191,25 +361,51 @@ fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_continuation_and_mark_the_routing_requirement() {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": "resp_private_continuation"
|
||||
}));
|
||||
fn decoder_should_preserve_unknown_arbitrary_precision_numbers() {
|
||||
let number = "12345678901234567890123456789012345678901234567890";
|
||||
let body = format!(r#"{{"model":"smart-code","input":"hello","future_number":{number}}}"#);
|
||||
|
||||
assert!(matches!(
|
||||
decoded.metadata().continuation(),
|
||||
ContinuationIntent::PreviousResponseId(value) if value == "resp_private_continuation"
|
||||
));
|
||||
assert!(
|
||||
decoded
|
||||
.operation()
|
||||
.capability_requirements()
|
||||
.features()
|
||||
.contains(&Feature::NativeContinuation)
|
||||
);
|
||||
assert!(!format!("{decoded:?}").contains("resp_private_continuation"));
|
||||
let decoded = decode_request_with_headers(body.as_bytes(), &HeaderMap::new())
|
||||
.expect("opaque numeric field should decode");
|
||||
let encoded = serde_json::to_string(openai_wire_body(&decoded)).expect("encode request body");
|
||||
|
||||
assert!(encoded.contains(&format!(r#""future_number":{number}"#)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_opaque_continuation_and_mark_the_routing_requirement() {
|
||||
for response_id in [
|
||||
"resp_private_continuation".to_owned(),
|
||||
format!("resp_{}\0opaque", "x".repeat(257)),
|
||||
String::new(),
|
||||
] {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": response_id.clone()
|
||||
}));
|
||||
|
||||
assert!(matches!(
|
||||
decoded.metadata().continuation(),
|
||||
ContinuationIntent::PreviousResponseId(value) if value == &response_id
|
||||
));
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded)
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
assert!(
|
||||
decoded
|
||||
.operation()
|
||||
.capability_requirements()
|
||||
.features()
|
||||
.contains(&Feature::NativeContinuation)
|
||||
);
|
||||
if !response_id.is_empty() {
|
||||
assert!(!format!("{decoded:?}").contains(&response_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -316,7 +512,7 @@ fn transparent_encoder_should_forward_raw_sse_frames_unknown_events_and_terminal
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let frames = events
|
||||
.iter()
|
||||
.flat_map(|event| encoder.push_sse(event).expect("wire event should encode"))
|
||||
.flat_map(|event| encoder.push_sse(event))
|
||||
.collect::<Vec<_>>();
|
||||
let response = encoder.finish().expect("wire response should finish");
|
||||
let body = frames
|
||||
@@ -370,15 +566,11 @@ fn transparent_encoder_should_use_identical_json_for_sse_and_websocket() {
|
||||
|
||||
let sse = events
|
||||
.iter()
|
||||
.flat_map(|event| sse_encoder.push_sse(event).expect("SSE wire event"))
|
||||
.flat_map(|event| sse_encoder.push_sse(event))
|
||||
.collect::<Vec<_>>();
|
||||
let websocket = events
|
||||
.iter()
|
||||
.flat_map(|event| {
|
||||
websocket_encoder
|
||||
.push_websocket(event)
|
||||
.expect("WebSocket wire event")
|
||||
})
|
||||
.flat_map(|event| websocket_encoder.push_websocket(event))
|
||||
.collect::<Vec<_>>();
|
||||
let sse_text = String::from_utf8(
|
||||
sse.iter()
|
||||
@@ -400,8 +592,40 @@ fn transparent_encoder_should_use_identical_json_for_sse_and_websocket() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transparent_encoder_should_require_matching_canonical_and_wire_terminals() {
|
||||
fn transparent_encoder_should_follow_wire_when_canonical_identity_changes() {
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let started = openai_wire_event(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_1", "gpt-test",
|
||||
))],
|
||||
"response.created",
|
||||
json!({"type": "response.created", "response": {"id": "wire_resp_1"}}),
|
||||
);
|
||||
encoder.push_sse(&started);
|
||||
let changed_identity = openai_wire_event(
|
||||
vec![GatewayEvent::Completed(ResponseMeta::new(
|
||||
"resp_2", "gpt-test",
|
||||
))],
|
||||
"response.completed",
|
||||
json!({"type": "response.completed", "response": {"id": "wire_resp_2"}}),
|
||||
);
|
||||
|
||||
let completed = encoder.push_sse(&changed_identity);
|
||||
|
||||
assert_eq!(completed.len(), 1);
|
||||
assert!(encoder.is_completed());
|
||||
assert_eq!(encoder.response_id(), Some("wire_resp_2"));
|
||||
assert_eq!(
|
||||
encoder
|
||||
.finish()
|
||||
.expect("wire terminal remains authoritative"),
|
||||
json!({"id": "wire_resp_2"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transparent_encoder_should_only_require_wire_terminal_for_buffered_conversion() {
|
||||
let mut missing_terminal = OpenAiResponsesEncoder::new();
|
||||
let started = openai_wire_event(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_1", "gpt-test",
|
||||
@@ -409,24 +633,8 @@ fn transparent_encoder_should_require_matching_canonical_and_wire_terminals() {
|
||||
"response.created",
|
||||
json!({"type": "response.created", "response": {"id": "resp_1"}}),
|
||||
);
|
||||
encoder.push_sse(&started).expect("started event");
|
||||
let changed_identity = openai_wire_event(
|
||||
vec![GatewayEvent::Completed(ResponseMeta::new(
|
||||
"resp_2", "gpt-test",
|
||||
))],
|
||||
"response.completed",
|
||||
json!({"type": "response.completed", "response": {"id": "resp_2"}}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
encoder
|
||||
.push_sse(&changed_identity)
|
||||
.expect_err("identity changes must fail"),
|
||||
ResponseEncodeError::WireIdentityChanged
|
||||
);
|
||||
|
||||
let mut missing_terminal = OpenAiResponsesEncoder::new();
|
||||
missing_terminal.push_sse(&started).expect("started event");
|
||||
missing_terminal.push_sse(&started);
|
||||
assert_eq!(
|
||||
missing_terminal
|
||||
.finish()
|
||||
|
||||
@@ -1,8 +1,35 @@
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::{SinkExt, StreamExt, future::BoxFuture};
|
||||
use gateway_api::openai::responses::ResponseCreateFrameError;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
|
||||
StartExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{GatewayError, ProviderError, ProviderErrorKind};
|
||||
use gateway_core::event::{
|
||||
GatewayEvent, ProtocolWireEvent, ProviderEvent, ProviderResponseHeader, ResponseMeta,
|
||||
};
|
||||
use gateway_core::operation::Operation;
|
||||
use serde_json::json;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use serde_json::{Value, json};
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::Message as ClientMessage;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::decode_response_create;
|
||||
use crate::openai::{api_router, authenticated_client, models::ModelsExecution};
|
||||
|
||||
#[test]
|
||||
fn response_create_should_default_to_the_websocket_streaming_contract() {
|
||||
@@ -141,3 +168,476 @@ fn response_create_should_reject_non_boolean_stream_without_disclosing_body_valu
|
||||
assert!(!rendered.contains(prompt));
|
||||
assert!(!rendered.contains(opaque_stream_value));
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct AtomicFailureTrace {
|
||||
next_calls: AtomicUsize,
|
||||
committed: AtomicBool,
|
||||
finalized: AtomicBool,
|
||||
}
|
||||
|
||||
struct AtomicFailureSession {
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
}
|
||||
|
||||
impl ExecutionSession for AtomicFailureSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
match self.trace.next_calls.fetch_add(1, Ordering::AcqRel) {
|
||||
0 => Ok(Some(atomic_failure_batch())),
|
||||
1 => {
|
||||
self.trace.finalized.store(true, Ordering::Release);
|
||||
Err(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)))
|
||||
}
|
||||
_ => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async { Err(EngineError::InvalidDeliveryState) })
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.response_headers
|
||||
}
|
||||
|
||||
fn commit_downstream(&mut self, _: Option<u16>) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
self.trace.committed.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.trace.finalized.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn cancel(&self) {}
|
||||
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move {
|
||||
self.trace.finalized.store(true, Ordering::Release);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct AtomicFailureExecution {
|
||||
client: AuthenticatedClient,
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
}
|
||||
|
||||
impl ExecutionService for AtomicFailureExecution {
|
||||
fn authenticate(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
|
||||
(plaintext == "sk_ws_atomic")
|
||||
.then(|| self.client.clone())
|
||||
.ok_or(ClientAuthenticationError::InvalidKey)
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
vec![PublicModelId::new("model-a").expect("public model")]
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
model.as_str() == "model-a"
|
||||
}
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
Ok(StartedExecution {
|
||||
request_id: gateway_core::engine::ModelRequestId::new("req_ws_atomic")
|
||||
.expect("request id"),
|
||||
created_at: SystemTime::now(),
|
||||
stream: request.metadata.stream,
|
||||
session: Box::new(AtomicFailureSession {
|
||||
trace: Arc::clone(&self.trace),
|
||||
response_headers: self.response_headers.clone(),
|
||||
}),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn atomic_failure_batch() -> CoordinatedEvent {
|
||||
let started = ProviderEvent::canonical_with_wire(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_ws_atomic",
|
||||
"model-a",
|
||||
))],
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_ws_atomic",
|
||||
"model": "model-a",
|
||||
"status": "in_progress"
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("created wire"),
|
||||
);
|
||||
let failed = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": "resp_ws_atomic",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"code": "rate_limit_exceeded",
|
||||
"message": "websocket atomic upstream marker"
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("failed wire"),
|
||||
);
|
||||
CoordinatedEvent::try_batch(
|
||||
vec![started, failed],
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)
|
||||
.expect("atomic WebSocket batch")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_atomic_upstream_failure_batch_should_be_forwarded_once() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace: Arc::clone(&trace),
|
||||
response_headers: Vec::new(),
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket
|
||||
.send(ClientMessage::Text(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "hello"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send response.create");
|
||||
|
||||
let mut messages = Vec::<Value>::new();
|
||||
loop {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
let failed = value.get("type").and_then(Value::as_str) == Some("response.failed");
|
||||
messages.push(value);
|
||||
if failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
messages
|
||||
.iter()
|
||||
.filter(|message| message.get("type").and_then(Value::as_str) == Some("response.failed"))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let failed = messages
|
||||
.iter()
|
||||
.find(|message| message.get("type").and_then(Value::as_str) == Some("response.failed"))
|
||||
.expect("upstream failed event");
|
||||
assert_eq!(
|
||||
failed
|
||||
.pointer("/response/error/message")
|
||||
.and_then(Value::as_str),
|
||||
Some("websocket atomic upstream marker")
|
||||
);
|
||||
assert!(trace.committed.load(Ordering::Acquire));
|
||||
assert_eq!(trace.next_calls.load(Ordering::Acquire), 2);
|
||||
assert!(trace.finalized.load(Ordering::Acquire));
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_metadata_should_preserve_ordinary_headers_without_blocking_events() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace,
|
||||
response_headers: vec![
|
||||
ProviderResponseHeader::new("x-future-header", Bytes::from_static(b"future-value")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
Bytes::from_static(b"should-not-cross-boundary"),
|
||||
),
|
||||
ProviderResponseHeader::new("connection", Bytes::from_static(b"x-private-hop")),
|
||||
ProviderResponseHeader::new("x-private-hop", Bytes::from_static(b"private-hop")),
|
||||
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
|
||||
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req_upstream")),
|
||||
],
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket
|
||||
.send(ClientMessage::Text(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "hello"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send response.create");
|
||||
|
||||
let mut metadata = None;
|
||||
let mut saw_failed = false;
|
||||
while metadata.is_none() || !saw_failed {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
match value.get("type").and_then(Value::as_str) {
|
||||
Some("response.metadata") => metadata = Some(value),
|
||||
Some("response.failed") => saw_failed = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let headers = metadata
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("headers"))
|
||||
.and_then(Value::as_object)
|
||||
.expect("response metadata headers");
|
||||
assert_eq!(headers.get("x-future-header"), Some(&json!("future-value")));
|
||||
// 官方 metadata 是 string map,同名多值只能保持既有的后值覆盖语义。
|
||||
assert_eq!(headers.get("x-future-multi"), Some(&json!("second")));
|
||||
assert_eq!(headers.get("x-request-id"), Some(&json!("req_upstream")));
|
||||
for omitted in [
|
||||
"x-future-bytes",
|
||||
"bad\0name",
|
||||
"authorization",
|
||||
"connection",
|
||||
"x-private-hop",
|
||||
"content-type",
|
||||
] {
|
||||
assert!(
|
||||
!headers.contains_key(omitted),
|
||||
"unexpected metadata header: {omitted:?}"
|
||||
);
|
||||
}
|
||||
assert!(saw_failed);
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace,
|
||||
response_headers: Vec::new(),
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
let payload = json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "x".repeat(16 * 1024 * 1024 + 1),
|
||||
})
|
||||
.to_string();
|
||||
assert!(payload.len() > 16 * 1024 * 1024);
|
||||
|
||||
socket
|
||||
.send(ClientMessage::Text(payload.into()))
|
||||
.await
|
||||
.expect("send response.create above the former frame limit");
|
||||
|
||||
let mut saw_upstream_event = false;
|
||||
for _ in 0..3 {
|
||||
let message = tokio::time::timeout(Duration::from_secs(10), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
if matches!(
|
||||
value.get("type").and_then(Value::as_str),
|
||||
Some("response.created" | "response.failed")
|
||||
) {
|
||||
saw_upstream_event = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(saw_upstream_event);
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
const TEST_WEBSOCKET_KEY: &str = "AAAAAAAAAAAAAAAAAAAAAA==";
|
||||
|
||||
fn upgrade_request(authorization: &str) -> Request<Body> {
|
||||
Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, authorization)
|
||||
.header("connection", "upgrade")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "13")
|
||||
.header("sec-websocket-key", TEST_WEBSOCKET_KEY)
|
||||
.body(Body::empty())
|
||||
.expect("build WebSocket upgrade request")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_responses_should_route_to_the_websocket_upgrade_boundary() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(upgrade_request("Bearer sk_models_test"))
|
||||
.await
|
||||
.expect("route WebSocket upgrade request");
|
||||
|
||||
// oneshot 请求不携带升级状态;426 证明 GET /v1/responses 进入的是
|
||||
// WebSocketUpgrade 边界而不是普通 HTTP handler。
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_upgradability_should_be_checked_before_authentication() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(upgrade_request("Bearer sk_invalid"))
|
||||
.await
|
||||
.expect("route unauthenticated upgrade request");
|
||||
|
||||
// 升级能力在 extractor 阶段先于 handler 内的 API Key 认证被校验,
|
||||
// 无效凭据得到的仍是升级失败而不是 401。
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_upgrade_should_reject_malformed_handshakes() {
|
||||
let missing_connection = Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "13")
|
||||
.header("sec-websocket-key", TEST_WEBSOCKET_KEY)
|
||||
.body(Body::empty())
|
||||
.expect("build request without connection header");
|
||||
let unsupported_version = Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.header("connection", "upgrade")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "12")
|
||||
.header("sec-websocket-key", TEST_WEBSOCKET_KEY)
|
||||
.body(Body::empty())
|
||||
.expect("build request with unsupported version");
|
||||
let missing_key = Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.header("connection", "upgrade")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "13")
|
||||
.body(Body::empty())
|
||||
.expect("build request without websocket key");
|
||||
|
||||
for request in [missing_connection, unsupported_version, missing_key] {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("route malformed WebSocket handshake");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +1,74 @@
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, Bytes},
|
||||
extract::DefaultBodyLimit,
|
||||
http::{Request, StatusCode},
|
||||
routing::post,
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode},
|
||||
};
|
||||
use gateway_api::openai::router::MAX_CLIENT_REQUEST_BODY_BYTES;
|
||||
use tower::ServiceExt;
|
||||
|
||||
fn body_limit_app() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
"/v1/responses",
|
||||
post(|body: Bytes| async move {
|
||||
if body.len() == MAX_CLIENT_REQUEST_BODY_BYTES {
|
||||
StatusCode::NO_CONTENT
|
||||
} else {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
}
|
||||
}),
|
||||
use super::api_router_with_origins;
|
||||
use super::models::ModelsExecution;
|
||||
|
||||
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
|
||||
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; REMOVED_BODY_LIMIT_BYTES + 1]))
|
||||
.expect("build request above the removed limit"),
|
||||
)
|
||||
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
|
||||
.await
|
||||
.expect("route request above the removed limit");
|
||||
|
||||
// The handler sees the body and rejects the missing credentials. A restored body limit
|
||||
// would return 413 before authentication runs.
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_limit_should_accept_exactly_sixteen_mibibytes() {
|
||||
let response = body_limit_app()
|
||||
async fn configured_cors_origin_assembles_router_and_answers_preflight() {
|
||||
let router = api_router_with_origins(
|
||||
ModelsExecution::new(),
|
||||
vec!["https://app.example.com".into()],
|
||||
)
|
||||
.await;
|
||||
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; MAX_CLIENT_REQUEST_BODY_BYTES]))
|
||||
.expect("build exact-limit request"),
|
||||
Request::builder()
|
||||
.method(Method::OPTIONS)
|
||||
.uri("/v1/models")
|
||||
.header("origin", "https://app.example.com")
|
||||
.header("access-control-request-method", "GET")
|
||||
.header("access-control-request-headers", "authorization")
|
||||
.body(Body::empty())
|
||||
.expect("build preflight request"),
|
||||
)
|
||||
.await
|
||||
.expect("route exact-limit request");
|
||||
.expect("route preflight request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_limit_should_reject_one_byte_over_sixteen_mibibytes() {
|
||||
let response = body_limit_app()
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; MAX_CLIENT_REQUEST_BODY_BYTES + 1]))
|
||||
.expect("build over-limit request"),
|
||||
)
|
||||
.await
|
||||
.expect("route over-limit request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("access-control-allow-origin")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("https://app.example.com")
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("access-control-allow-credentials")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("true")
|
||||
);
|
||||
let allow_headers = response
|
||||
.headers()
|
||||
.get("access-control-allow-headers")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("allow-headers present");
|
||||
assert!(allow_headers.contains("authorization"));
|
||||
assert!(allow_headers.contains("x-api-key"));
|
||||
assert!(allow_headers.contains("x-request-id"));
|
||||
}
|
||||
|
||||
@@ -14,12 +14,12 @@ path = "tests/main.rs"
|
||||
[dependencies]
|
||||
async-trait = "0.1.89"
|
||||
bytes = "1.11.1"
|
||||
chrono = "0.4.45"
|
||||
futures = "0.3.32"
|
||||
futures-timer = "3.0.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -8,24 +8,21 @@ use std::fmt;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::engine::credential::ProviderAccountId;
|
||||
use crate::error::{IdentifierError, SafeUpstreamValue, validate_text};
|
||||
use crate::operation::ProviderSessionState;
|
||||
use crate::routing::ProviderKind;
|
||||
|
||||
/// 客户端传入的 previous response ID。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
/// 客户端或 Provider 传递的 opaque response handle。
|
||||
///
|
||||
/// Codex 定义这个值的语义和格式。网关只将其作为亲和查找键或同账号上游
|
||||
/// continuation 的载体,不得按私有长度、字符集或前缀规则拒绝、归一化或改写。
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct PreviousResponseId(String);
|
||||
|
||||
impl PreviousResponseId {
|
||||
/// 校验并创建 previous response ID。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// ID 为空、过长或包含控制字符时返回错误。
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
validate_text(&value, 256, false, None)?;
|
||||
Ok(Self(value))
|
||||
/// 按原样创建 opaque response handle。
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -34,6 +31,12 @@ impl PreviousResponseId {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for PreviousResponseId {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("PreviousResponseId(<redacted>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 原生 response handle 的续接范围。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NativeContinuationScope {
|
||||
@@ -52,7 +55,7 @@ pub struct NativeContinuationPin {
|
||||
/// 客户端提交、用于查找可丢失会话亲和记录的 response ID。
|
||||
previous_response_id: PreviousResponseId,
|
||||
/// Provider 原生 response handle。
|
||||
upstream_response_id: SafeUpstreamValue,
|
||||
upstream_response_id: PreviousResponseId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
scope: NativeContinuationScope,
|
||||
@@ -63,7 +66,7 @@ impl NativeContinuationPin {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
previous_response_id: PreviousResponseId,
|
||||
upstream_response_id: SafeUpstreamValue,
|
||||
upstream_response_id: PreviousResponseId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
) -> Self {
|
||||
@@ -98,7 +101,7 @@ impl NativeContinuationPin {
|
||||
|
||||
/// 返回只允许发送给已冻结 Provider 目标的原生上游 handle。
|
||||
#[must_use]
|
||||
pub const fn upstream_response_id(&self) -> &SafeUpstreamValue {
|
||||
pub const fn upstream_response_id(&self) -> &PreviousResponseId {
|
||||
&self.upstream_response_id
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -44,6 +44,46 @@ impl fmt::Display for ProviderAccountId {
|
||||
}
|
||||
}
|
||||
|
||||
/// credential 轮换时可选替换的上游账号身份。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountIdentity {
|
||||
upstream_user_id: String,
|
||||
upstream_account_id: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderAccountIdentity {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderAccountIdentity")
|
||||
.field("upstream_user_id", &"<redacted>")
|
||||
.field(
|
||||
"upstream_account_id",
|
||||
&self.upstream_account_id.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderAccountIdentity {
|
||||
#[must_use]
|
||||
pub const fn new(upstream_user_id: String, upstream_account_id: Option<String>) -> Self {
|
||||
Self {
|
||||
upstream_user_id,
|
||||
upstream_account_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_user_id(&self) -> &str {
|
||||
&self.upstream_user_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_account_id(&self) -> Option<&str> {
|
||||
self.upstream_account_id.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
/// `provider_accounts.credential_revision` 的正数 CAS revision。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct CredentialRevision(NonZeroU64);
|
||||
@@ -602,6 +642,15 @@ pub trait ProviderAccountStore: Send + Sync {
|
||||
expected_revision: CredentialRevision,
|
||||
) -> Result<LoadedCredential, StoreError>;
|
||||
|
||||
/// 读取账号当前 credential 及其 revision,不做任何版本比对。
|
||||
///
|
||||
/// 管理写入必须在临近 CAS 时用它取 fence,而不是携带调用方持有的旧 revision:
|
||||
/// 后台刷新随时会推进 revision,用陈旧快照会把正常的恢复操作误判为冲突。
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, StoreError>;
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
update: CredentialCasUpdate,
|
||||
@@ -901,6 +950,63 @@ pub struct AccountSelectionContext {
|
||||
pub excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
pub preferred_account: Option<ProviderAccountId>,
|
||||
pub round_robin_cursor: u64,
|
||||
pub availability: AccountAvailabilityPolicy,
|
||||
}
|
||||
|
||||
/// 选择账号时是否信任本地可用性投影。
|
||||
///
|
||||
/// 管理端对指定账号执行诊断时,会直接向上游确认实际状态;该模式跳过
|
||||
/// `enabled`、可用性、冷却和 token 到期等本地投影,仍保留租约、并发和
|
||||
/// 请求间隔约束,且只能与固定账号约束组合使用。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AccountAvailabilityPolicy {
|
||||
#[default]
|
||||
Enforce,
|
||||
BypassForDiagnostic,
|
||||
}
|
||||
|
||||
impl AccountAvailabilityPolicy {
|
||||
#[must_use]
|
||||
pub const fn bypasses_local_availability(self) -> bool {
|
||||
matches!(self, Self::BypassForDiagnostic)
|
||||
}
|
||||
}
|
||||
|
||||
/// 候选账号未进入调度池的约束。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountSchedulingBlocker {
|
||||
LocalAvailability,
|
||||
Excluded,
|
||||
ConcurrencyLimit,
|
||||
RequestInterval,
|
||||
}
|
||||
|
||||
/// 优先账号在本次选择中的处理结果。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PreferredAccountSelection {
|
||||
NotRequested,
|
||||
Hit,
|
||||
Missing,
|
||||
Blocked(AccountSchedulingBlocker),
|
||||
}
|
||||
|
||||
/// 一次完整账号选择;同时保留优先账号决策,供 Provider 记录调度遥测。
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AccountSelection<'a> {
|
||||
candidate: &'a AccountCandidate,
|
||||
preferred: PreferredAccountSelection,
|
||||
}
|
||||
|
||||
impl<'a> AccountSelection<'a> {
|
||||
#[must_use]
|
||||
pub const fn candidate(self) -> &'a AccountCandidate {
|
||||
self.candidate
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn preferred(self) -> PreferredAccountSelection {
|
||||
self.preferred
|
||||
}
|
||||
}
|
||||
|
||||
/// 同一 target 内唯一的账号排序器。
|
||||
@@ -914,43 +1020,55 @@ impl AccountSelector {
|
||||
&self,
|
||||
candidates: &'a [AccountCandidate],
|
||||
context: &AccountSelectionContext,
|
||||
) -> Option<&'a AccountCandidate> {
|
||||
) -> Option<AccountSelection<'a>> {
|
||||
let preferred = if let Some(preferred) = context.preferred_account.as_ref() {
|
||||
match candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == preferred)
|
||||
{
|
||||
Some(candidate) => match self.scheduling_blocker(candidate, context) {
|
||||
None => {
|
||||
return Some(AccountSelection {
|
||||
candidate,
|
||||
preferred: PreferredAccountSelection::Hit,
|
||||
});
|
||||
}
|
||||
Some(blocker) => PreferredAccountSelection::Blocked(blocker),
|
||||
},
|
||||
None => PreferredAccountSelection::Missing,
|
||||
}
|
||||
} else {
|
||||
PreferredAccountSelection::NotRequested
|
||||
};
|
||||
|
||||
let mut eligible = candidates
|
||||
.iter()
|
||||
.filter(|candidate| eligible(candidate, context))
|
||||
.filter(|candidate| self.scheduling_blocker(candidate, context).is_none())
|
||||
.collect::<Vec<_>>();
|
||||
if eligible.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(preferred) = context.preferred_account.as_ref()
|
||||
&& let Some(candidate) = eligible
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == preferred)
|
||||
&& (context.policy.strategy() == RotationStrategy::Sticky
|
||||
|| (context.policy.strategy() == RotationStrategy::Smart
|
||||
&& !smart_preference_should_escape(&candidate.signals)))
|
||||
{
|
||||
return Some(candidate);
|
||||
}
|
||||
|
||||
match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => eligible.sort_by_key(|candidate| {
|
||||
(
|
||||
candidate.signals.quota_reset_at.is_none(),
|
||||
candidate.signals.quota_reset_at,
|
||||
candidate.signals.in_flight,
|
||||
candidate.signals.last_started_at,
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
}),
|
||||
let candidate = match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => {
|
||||
eligible.sort_by_key(|candidate| {
|
||||
(
|
||||
candidate.signals.quota_reset_at.is_none(),
|
||||
candidate.signals.quota_reset_at,
|
||||
candidate.signals.in_flight,
|
||||
candidate.signals.last_started_at,
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
RotationStrategy::RoundRobin => {
|
||||
eligible.sort_by_key(|candidate| candidate.account.id().clone());
|
||||
let index = context.round_robin_cursor as usize % eligible.len();
|
||||
return Some(eligible[index]);
|
||||
eligible.get(index).copied()?
|
||||
}
|
||||
RotationStrategy::Smart => {
|
||||
return select_smart_candidate(&eligible, context.round_robin_cursor);
|
||||
select_smart_candidate(&eligible, context.round_robin_cursor)?
|
||||
}
|
||||
RotationStrategy::Sticky => {
|
||||
eligible.sort_by_key(|candidate| {
|
||||
@@ -959,9 +1077,44 @@ impl AccountSelector {
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
};
|
||||
Some(AccountSelection {
|
||||
candidate,
|
||||
preferred,
|
||||
})
|
||||
}
|
||||
|
||||
fn scheduling_blocker(
|
||||
&self,
|
||||
candidate: &AccountCandidate,
|
||||
context: &AccountSelectionContext,
|
||||
) -> Option<AccountSchedulingBlocker> {
|
||||
if !context.availability.bypasses_local_availability()
|
||||
&& !candidate.account.is_schedulable(context.now)
|
||||
{
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
}
|
||||
eligible.into_iter().next()
|
||||
if context.excluded_accounts.contains(candidate.account.id()) {
|
||||
return Some(AccountSchedulingBlocker::Excluded);
|
||||
}
|
||||
if candidate.signals.in_flight >= context.policy.max_concurrent_per_account().get() {
|
||||
return Some(AccountSchedulingBlocker::ConcurrencyLimit);
|
||||
}
|
||||
if candidate
|
||||
.signals
|
||||
.last_started_at
|
||||
.is_some_and(|last_started| {
|
||||
!context
|
||||
.now
|
||||
.duration_since(last_started)
|
||||
.is_ok_and(|elapsed| elapsed >= context.policy.request_interval())
|
||||
})
|
||||
{
|
||||
return Some(AccountSchedulingBlocker::RequestInterval);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -969,8 +1122,6 @@ const SMART_LOAD_WEIGHT: f64 = 1.0;
|
||||
const SMART_QUOTA_WEIGHT: f64 = 0.8;
|
||||
const SMART_FAILURE_WEIGHT: f64 = 1.0;
|
||||
const SMART_LATENCY_WEIGHT: f64 = 0.5;
|
||||
const SMART_PREFERENCE_FAILURE_ESCAPE_BASIS_POINTS: u16 = 5_000;
|
||||
const SMART_PREFERENCE_LATENCY_ESCAPE_MS: u64 = 15_000;
|
||||
|
||||
struct SmartNormalization {
|
||||
max_in_flight: u32,
|
||||
@@ -1091,34 +1242,6 @@ fn higher_is_better(value: u64, minimum: u64, maximum: u64) -> f64 {
|
||||
(value.saturating_sub(minimum) as f64 / (maximum - minimum) as f64).clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
fn smart_preference_should_escape(signals: &AccountRuntimeSignals) -> bool {
|
||||
signals
|
||||
.failure_rate_basis_points
|
||||
.is_some_and(|rate| rate > SMART_PREFERENCE_FAILURE_ESCAPE_BASIS_POINTS)
|
||||
|| signals
|
||||
.first_output_latency_ms
|
||||
.is_some_and(|latency| latency > SMART_PREFERENCE_LATENCY_ESCAPE_MS)
|
||||
}
|
||||
|
||||
fn eligible(candidate: &AccountCandidate, context: &AccountSelectionContext) -> bool {
|
||||
if !candidate.account.is_schedulable(context.now)
|
||||
|| context.excluded_accounts.contains(candidate.account.id())
|
||||
|| candidate.signals.in_flight >= context.policy.max_concurrent_per_account().get()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
candidate
|
||||
.signals
|
||||
.last_started_at
|
||||
.is_none_or(|last_started| {
|
||||
context
|
||||
.now
|
||||
.duration_since(last_started)
|
||||
.is_ok_and(|elapsed| elapsed >= context.policy.request_interval())
|
||||
})
|
||||
}
|
||||
|
||||
/// Credential 值对象构造错误。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
pub enum CredentialError {
|
||||
|
||||
@@ -17,22 +17,29 @@ use crate::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, NativeContinuationPort, PreviousResponseId,
|
||||
};
|
||||
use crate::engine::coordinator::ResponseExecutionSession;
|
||||
use crate::engine::probe::{AccountProbe, AccountProbeRequest, AccountProbeResult};
|
||||
use crate::engine::probe::{
|
||||
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
|
||||
AccountProbeUpstreamResponse,
|
||||
};
|
||||
use crate::engine::provider::ProviderRegistry;
|
||||
use crate::engine::{
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CommitRequirement, CoordinatedEvent,
|
||||
EngineError, ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CoordinatedEvent, EngineError,
|
||||
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest, ProbeFailure, ProviderAccountId, ProviderAttemptOutcome, RecoveryReport,
|
||||
UpstreamSendState,
|
||||
};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderErrorKind, StoreError};
|
||||
use crate::event::{GatewayEvent, ProviderEvent, ProviderResponseHeader};
|
||||
use crate::operation::{Operation, ProviderSessionState};
|
||||
use crate::policy::{ClientApiKeyId, ClientPolicy};
|
||||
use crate::routing::snapshot::RuntimeSnapshotHandle;
|
||||
use crate::routing::{ProviderKind, PublicModelId, RoutingContext, RuntimeSnapshot};
|
||||
use crate::routing::{
|
||||
ProviderKind, PublicModelId, PublicModelProfile, RoutingContext, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
|
||||
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
|
||||
const CONTINUATION_AFFINITY_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const REDIS_COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClientTransport {
|
||||
@@ -139,6 +146,9 @@ pub trait ExecutionService: Send + Sync {
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError>;
|
||||
fn public_models(&self, client: &AuthenticatedClient) -> Vec<PublicModelId>;
|
||||
fn public_model_profiles(&self, _client: &AuthenticatedClient) -> Vec<PublicModelProfile> {
|
||||
Vec::new()
|
||||
}
|
||||
fn contains_public_model(&self, client: &AuthenticatedClient, model: &PublicModelId) -> bool;
|
||||
fn start(
|
||||
&self,
|
||||
@@ -182,6 +192,8 @@ pub struct DefaultExecutionService {
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
|
||||
probe_coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
|
||||
/// probe 自身走 transient store,探测失败仍写入持久 store 的 ops_events。
|
||||
observations: Arc<dyn ExecutionStore>,
|
||||
providers: ProviderRegistry,
|
||||
admissions: Arc<dyn ClientAdmissionPort>,
|
||||
circuits: Arc<dyn ProviderCircuitPort>,
|
||||
@@ -200,6 +212,7 @@ impl DefaultExecutionService {
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
|
||||
) -> Self {
|
||||
let observations = Arc::clone(&execution);
|
||||
let engine = GatewayEngine::<dyn ExecutionStore>::new(execution, providers.clone());
|
||||
let transient: Arc<dyn ExecutionStore> = Arc::new(TransientExecutionStore);
|
||||
let probe_engine = GatewayEngine::<dyn ExecutionStore>::new(transient, providers.clone());
|
||||
@@ -207,6 +220,7 @@ impl DefaultExecutionService {
|
||||
snapshots,
|
||||
coordinator: Arc::new(AttemptCoordinator::new(engine)),
|
||||
probe_coordinator: Arc::new(AttemptCoordinator::new(probe_engine)),
|
||||
observations,
|
||||
providers,
|
||||
admissions,
|
||||
circuits,
|
||||
@@ -222,15 +236,6 @@ impl DefaultExecutionService {
|
||||
request.client.policy.authorize().map_err(|_| {
|
||||
GatewayError::new(GatewayErrorKind::PolicyDenied, "client API key is disabled")
|
||||
})?;
|
||||
if !request.client.snapshot.contains_public_model_for_provider(
|
||||
&request.public_model,
|
||||
request.client.policy.provider_kind(),
|
||||
) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
));
|
||||
}
|
||||
let started_at = SystemTime::now();
|
||||
let deadline_at = started_at
|
||||
.checked_add(MODEL_REQUEST_DEADLINE)
|
||||
@@ -249,7 +254,7 @@ impl DefaultExecutionService {
|
||||
let continuation = match request.metadata.previous_response_id.as_ref() {
|
||||
Some(previous) => {
|
||||
let resolve = self.continuation.resolve(previous).fuse();
|
||||
let timeout = Delay::new(CONTINUATION_AFFINITY_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(resolve, timeout);
|
||||
let pin = select_biased! {
|
||||
result = resolve => result.ok().flatten(),
|
||||
@@ -387,14 +392,33 @@ impl DefaultExecutionService {
|
||||
&self,
|
||||
provider_kind: &ProviderKind,
|
||||
) -> Result<RoutingContext, GatewayError> {
|
||||
let blocked_providers = match self.circuits.decision(provider_kind).await.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"provider health state is temporarily unavailable",
|
||||
)
|
||||
})? {
|
||||
ProviderCircuitDecision::Allow => BTreeSet::new(),
|
||||
ProviderCircuitDecision::BlockedUntil(_) => BTreeSet::from([provider_kind.clone()]),
|
||||
let decision = self.circuits.decision(provider_kind).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(decision, timeout);
|
||||
let decision = select_biased! {
|
||||
result = decision => Some(result),
|
||||
_ = timeout => None,
|
||||
};
|
||||
let blocked_providers = match decision {
|
||||
Some(Ok(ProviderCircuitDecision::Allow)) => BTreeSet::new(),
|
||||
Some(Ok(ProviderCircuitDecision::BlockedUntil(_))) => {
|
||||
BTreeSet::from([provider_kind.clone()])
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
%error,
|
||||
"Provider circuit 读取失败,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
"Provider circuit 读取超时,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
}
|
||||
};
|
||||
Ok(RoutingContext {
|
||||
provider_kind: Some(provider_kind.clone()),
|
||||
@@ -405,13 +429,18 @@ impl DefaultExecutionService {
|
||||
async fn probe_inner(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> Result<AccountProbeResult, GatewayError> {
|
||||
) -> Result<AccountProbeResult, AccountProbeError> {
|
||||
let AccountProbeRequest {
|
||||
account_id,
|
||||
provider_kind,
|
||||
upstream_model,
|
||||
operation,
|
||||
} = request;
|
||||
let observed = ProbeObservation {
|
||||
provider_kind: provider_kind.clone(),
|
||||
account_id: account_id.clone(),
|
||||
upstream_model: upstream_model.clone(),
|
||||
};
|
||||
let snapshot = self.snapshots.acquire().map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
@@ -459,29 +488,44 @@ impl DefaultExecutionService {
|
||||
started_at,
|
||||
deadline_at,
|
||||
};
|
||||
let mut session = self
|
||||
let mut session = match self
|
||||
.probe_coordinator
|
||||
.start(
|
||||
.start_diagnostic(
|
||||
new_request,
|
||||
operation,
|
||||
plan,
|
||||
Some(account_id),
|
||||
account_id,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
{
|
||||
Ok(session) => session,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
let events = session.collect_uncommitted().await;
|
||||
publish_provider_attempt_outcomes(
|
||||
self.circuits.as_ref(),
|
||||
session.provider_attempt_outcomes(),
|
||||
)
|
||||
.await;
|
||||
let events = events.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
session
|
||||
.commit_downstream(Some(200))
|
||||
.await
|
||||
.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
let events = match events {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(200)).await {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
Ok(AccountProbeResult {
|
||||
text: events
|
||||
.into_iter()
|
||||
@@ -493,6 +537,63 @@ impl DefaultExecutionService {
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 探测失败先记录脱敏分类事实,再把请求局部的原始上游响应交给认证管理端。
|
||||
async fn observe_probe_failure(
|
||||
&self,
|
||||
observed: &ProbeObservation,
|
||||
started_at: SystemTime,
|
||||
error: &EngineError,
|
||||
) -> AccountProbeError {
|
||||
let upstream_response = match error {
|
||||
EngineError::Provider(provider_error) => provider_error
|
||||
.client_visible_upstream_response()
|
||||
.map(AccountProbeUpstreamResponse::from_client_response),
|
||||
_ => None,
|
||||
};
|
||||
if let EngineError::Provider(provider_error) = error {
|
||||
let latency = started_at.elapsed().unwrap_or_default();
|
||||
let latency_ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
|
||||
tracing::warn!(
|
||||
target: "gateway_probe",
|
||||
provider_kind = observed.provider_kind.as_str(),
|
||||
account_id = observed.account_id.as_str(),
|
||||
upstream_model = observed.upstream_model.as_str(),
|
||||
failure_kind = provider_error.kind().as_str(),
|
||||
send_state = ?provider_error.send_state(),
|
||||
upstream_status = ?provider_error.upstream_status(),
|
||||
provider_error_code = ?provider_error.upstream_code().map(|code| code.as_str()),
|
||||
latency_ms,
|
||||
"账号连接测试失败"
|
||||
);
|
||||
if let Err(store_error) = self
|
||||
.observations
|
||||
.record_probe_failure(ProbeFailure {
|
||||
provider_kind: observed.provider_kind.clone(),
|
||||
account_id: observed.account_id.clone(),
|
||||
upstream_model_id: observed.upstream_model.clone(),
|
||||
error: provider_error.clone(),
|
||||
latency,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
operation = "record_probe_failure",
|
||||
provider_kind = observed.provider_kind.as_str(),
|
||||
account_id = observed.account_id.as_str(),
|
||||
error_kind = ?store_error.kind(),
|
||||
"账号连接测试观测写入失败,测试结果不受影响"
|
||||
);
|
||||
}
|
||||
}
|
||||
AccountProbeError::new(gateway_error_from_engine(error), upstream_response)
|
||||
}
|
||||
}
|
||||
|
||||
struct ProbeObservation {
|
||||
provider_kind: ProviderKind,
|
||||
account_id: ProviderAccountId,
|
||||
upstream_model: UpstreamModelId,
|
||||
}
|
||||
|
||||
struct TransientExecutionStore;
|
||||
@@ -568,6 +669,12 @@ impl ExecutionService for DefaultExecutionService {
|
||||
.public_models_for_provider(client.policy.provider_kind())
|
||||
}
|
||||
|
||||
fn public_model_profiles(&self, client: &AuthenticatedClient) -> Vec<PublicModelProfile> {
|
||||
client
|
||||
.snapshot
|
||||
.public_model_profiles_for_provider(client.policy.provider_kind())
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, client: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
client
|
||||
.snapshot
|
||||
@@ -586,7 +693,7 @@ impl AccountProbe for DefaultExecutionService {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move { self.probe_inner(request).await })
|
||||
}
|
||||
}
|
||||
@@ -599,10 +706,13 @@ struct AdmissionLease {
|
||||
|
||||
impl AdmissionLease {
|
||||
async fn release(self) {
|
||||
let _ = self
|
||||
if let Err(error) = self
|
||||
.port
|
||||
.release(&self.client_api_key_id, &self.model_request_id)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(%error, "Client admission 释放失败,依赖租约 TTL 收敛");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -677,13 +787,7 @@ impl DefaultExecutionSession {
|
||||
return;
|
||||
};
|
||||
self.continuation_recorded = true;
|
||||
let record = self.continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(CONTINUATION_AFFINITY_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
_ = record => {},
|
||||
_ = timeout => {},
|
||||
}
|
||||
record_native_continuation(self.continuation.as_ref(), pin).await;
|
||||
}
|
||||
|
||||
async fn finalize_detached(&mut self) {
|
||||
@@ -691,8 +795,10 @@ impl DefaultExecutionSession {
|
||||
return;
|
||||
};
|
||||
core.cancel();
|
||||
if !core.is_finalized() {
|
||||
let _ = core.cancel_and_finalize().await;
|
||||
if !core.is_finalized()
|
||||
&& let Err(error) = core.cancel_and_finalize().await
|
||||
{
|
||||
tracing::warn!(%error, "Detached execution 终态收敛失败");
|
||||
}
|
||||
let pending = core
|
||||
.provider_attempt_outcomes()
|
||||
@@ -813,7 +919,32 @@ async fn publish_provider_attempt_outcomes(
|
||||
}
|
||||
Some(_) => continue,
|
||||
};
|
||||
let _ = result;
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(
|
||||
provider = outcome.provider_kind().as_str(),
|
||||
%error,
|
||||
"Provider circuit feedback 写入失败,数据面不受影响"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_native_continuation(
|
||||
continuation: &dyn NativeContinuationPort,
|
||||
pin: NativeContinuationPin,
|
||||
) {
|
||||
let record = continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
result = record => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "Continuation affinity 写入失败,后续请求将退化为外部续接");
|
||||
}
|
||||
},
|
||||
_ = timeout => {
|
||||
tracing::warn!("Continuation affinity 后台写入超时,已丢弃本次亲和记录");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -857,7 +988,7 @@ fn attach_continuation_session_state(operation: &mut Operation, pin: &NativeCont
|
||||
{
|
||||
return;
|
||||
}
|
||||
*operation = operation.clone().with_provider_session_state(state.clone());
|
||||
operation.set_provider_session_state(state.clone());
|
||||
}
|
||||
|
||||
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
@@ -878,8 +1009,3 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
_ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn commit_requirement(event: &CoordinatedEvent) -> CommitRequirement {
|
||||
event.commit_requirement()
|
||||
}
|
||||
|
||||
@@ -232,6 +232,7 @@ pub struct AccountAttemptContext {
|
||||
required_account: Option<ProviderAccountId>,
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
credential_recovery_attempted: bool,
|
||||
diagnostic_required_account: bool,
|
||||
}
|
||||
|
||||
impl AccountAttemptContext {
|
||||
@@ -246,6 +247,25 @@ impl AccountAttemptContext {
|
||||
required_account,
|
||||
state_owner,
|
||||
credential_recovery_attempted: false,
|
||||
diagnostic_required_account: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 为管理端对固定账号的上游诊断创建上下文。
|
||||
///
|
||||
/// 诊断只绕过本地的被动可用性投影,不能换号,并继续受账号租约保护。
|
||||
#[must_use]
|
||||
pub const fn diagnostic(
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
required_account: ProviderAccountId,
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
) -> Self {
|
||||
Self {
|
||||
excluded_accounts,
|
||||
required_account: Some(required_account),
|
||||
state_owner,
|
||||
credential_recovery_attempted: false,
|
||||
diagnostic_required_account: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,6 +295,12 @@ impl AccountAttemptContext {
|
||||
pub const fn credential_recovery_attempted(&self) -> bool {
|
||||
self.credential_recovery_attempted
|
||||
}
|
||||
|
||||
/// 此尝试是否要由真实上游而非本地投影确认固定账号的可用性。
|
||||
#[must_use]
|
||||
pub const fn is_diagnostic_required_account(&self) -> bool {
|
||||
self.diagnostic_required_account
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求中 Provider 账号绑定状态的唯一归属。
|
||||
@@ -474,6 +500,12 @@ impl AttemptContext {
|
||||
self.account.credential_recovery_attempted()
|
||||
}
|
||||
|
||||
/// 管理端固定账号诊断会跳过本地可用性投影,保留全部租约约束。
|
||||
#[must_use]
|
||||
pub const fn is_diagnostic_required_account(&self) -> bool {
|
||||
self.account.is_diagnostic_required_account()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn continuation(&self) -> Option<&ContinuationBinding> {
|
||||
self.continuation.as_ref()
|
||||
@@ -543,6 +575,16 @@ pub struct IntermediateFailure {
|
||||
pub latency: Duration,
|
||||
}
|
||||
|
||||
/// 不属于任何 `model_requests` 行的管理端账号探测失败。
|
||||
#[derive(Debug)]
|
||||
pub struct ProbeFailure {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub account_id: ProviderAccountId,
|
||||
pub upstream_model_id: UpstreamModelId,
|
||||
pub error: ProviderError,
|
||||
pub latency: Duration,
|
||||
}
|
||||
|
||||
/// `model_requests` 可用的毫秒级阶段耗时。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ModelRequestTimings {
|
||||
@@ -573,6 +615,7 @@ pub struct ModelRequestFinalization {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;Core 不解释字段。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub error: Option<GatewayError>,
|
||||
@@ -596,6 +639,19 @@ pub struct RecoveryReport {
|
||||
pub trait ExecutionStore: Send + Sync {
|
||||
async fn create_model_request(&self, request: NewModelRequest) -> Result<(), StoreError>;
|
||||
async fn record_attempt(&self, attempt: AttemptRecord) -> Result<(), StoreError>;
|
||||
/// 请求插入与首次 attempt 合并持久化;两者写同一行,支持合并写的
|
||||
/// store 可覆写为单次往返,缩短首 token 前的关键路径。
|
||||
///
|
||||
/// 覆写实现应保证原子性:默认的两步实现里第二步失败会留下一条
|
||||
/// `running` 请求行,由 deadline 回收器收敛。
|
||||
async fn create_model_request_with_attempt(
|
||||
&self,
|
||||
request: NewModelRequest,
|
||||
attempt: AttemptRecord,
|
||||
) -> Result<(), StoreError> {
|
||||
self.create_model_request(request).await?;
|
||||
self.record_attempt(attempt).await
|
||||
}
|
||||
async fn mark_send_state(
|
||||
&self,
|
||||
request_id: &ModelRequestId,
|
||||
@@ -616,6 +672,10 @@ pub trait ExecutionStore: Send + Sync {
|
||||
&self,
|
||||
failure: IntermediateFailure,
|
||||
) -> Result<(), StoreError>;
|
||||
/// 记录不挂在 `model_requests` 上的账号探测失败;默认丢弃。
|
||||
async fn record_probe_failure(&self, _failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn finalize_model_request(
|
||||
&self,
|
||||
finalization: ModelRequestFinalization,
|
||||
@@ -646,11 +706,7 @@ impl CoordinatedEvent {
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建一个必须在同一下游提交边界内按序编码的事件批次。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 空批次无法建立交付边界。
|
||||
/// 将一批 provider 事件合并为单个提交单元;空批次是无效投递状态。
|
||||
pub fn try_batch(
|
||||
events: Vec<ProviderEvent>,
|
||||
commit_requirement: CommitRequirement,
|
||||
@@ -701,6 +757,8 @@ pub enum EngineError {
|
||||
Deadline,
|
||||
#[error("routing plan has no candidate")]
|
||||
EmptyRoutingPlan,
|
||||
#[error("no provider attempt is active")]
|
||||
NoActiveAttempt,
|
||||
#[error("downstream delivery must be committed before execution can continue")]
|
||||
DownstreamCommitRequired,
|
||||
#[error("downstream delivery cannot be committed in the current state")]
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! 管理端账号连通性测试的真实执行链端口。
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::error::GatewayError;
|
||||
use crate::error::{ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind};
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
use crate::{engine::credential::ProviderAccountId, operation::Operation};
|
||||
|
||||
@@ -19,9 +22,111 @@ pub struct AccountProbeResult {
|
||||
pub text: Vec<String>,
|
||||
}
|
||||
|
||||
/// 仅供当前管理端连接测试展示的原始上游失败响应。
|
||||
///
|
||||
/// 正文不进入 `Debug`、日志或持久化;Core 只在探测终态从原始 Provider 错误显式复制。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct AccountProbeUpstreamResponse {
|
||||
status: u16,
|
||||
content_type: Option<Vec<u8>>,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
impl AccountProbeUpstreamResponse {
|
||||
pub(crate) fn from_client_response(response: &ClientVisibleUpstreamResponse) -> Self {
|
||||
Self {
|
||||
status: response.status(),
|
||||
content_type: response.content_type().map(<[u8]>::to_vec),
|
||||
body: response.body().clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn status(&self) -> u16 {
|
||||
self.status
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn content_type(&self) -> Option<&[u8]> {
|
||||
self.content_type.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for AccountProbeUpstreamResponse {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AccountProbeUpstreamResponse")
|
||||
.field("status", &self.status)
|
||||
.field(
|
||||
"content_type",
|
||||
&self.content_type.as_ref().map(|_| "<present>"),
|
||||
)
|
||||
.field("body", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理端连接测试的终态错误。
|
||||
///
|
||||
/// `gateway` 保留稳定分类,`upstream_response` 只面向本次认证管理请求展示源响应。
|
||||
#[derive(Debug)]
|
||||
pub struct AccountProbeError {
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
}
|
||||
|
||||
impl AccountProbeError {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
) -> Self {
|
||||
Self {
|
||||
gateway,
|
||||
upstream_response,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> GatewayErrorKind {
|
||||
self.gateway.kind()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_message(&self) -> &str {
|
||||
self.gateway.client_message()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_type(&self) -> Option<&str> {
|
||||
self.gateway.client_error_type()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_code(&self) -> Option<&str> {
|
||||
self.gateway.client_error_code()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn upstream_response(&self) -> Option<&AccountProbeUpstreamResponse> {
|
||||
self.upstream_response.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GatewayError> for AccountProbeError {
|
||||
fn from(gateway: GatewayError) -> Self {
|
||||
Self::new(gateway, None)
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AccountProbe: Send + Sync {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>>;
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>>;
|
||||
}
|
||||
|
||||
@@ -16,11 +16,13 @@ use crate::engine::credential::{
|
||||
};
|
||||
use crate::engine::{AttemptContext, UpstreamSendState};
|
||||
use crate::error::{
|
||||
IdentifierError, ProviderError, ProviderErrorKind, SafeUpstreamValue, validate_text,
|
||||
IdentifierError, OpaqueUpstreamValue, ProviderError, ProviderErrorKind, validate_text,
|
||||
};
|
||||
use crate::event::{EventSequenceValidator, ProviderEvent};
|
||||
use crate::operation::Operation;
|
||||
use crate::routing::{ModelCapabilities, ProviderCandidate, ProviderKind, UpstreamModelId};
|
||||
use crate::routing::{
|
||||
ModelCapabilities, ModelPresentation, ProviderCandidate, ProviderKind, UpstreamModelId,
|
||||
};
|
||||
|
||||
/// Box 只出现在 Provider Registry 的统一 event envelope 边界。
|
||||
pub type EventStream =
|
||||
@@ -123,7 +125,7 @@ pub struct ProviderCallMetadata {
|
||||
provider: ProviderKind,
|
||||
upstream_model: UpstreamModelId,
|
||||
resource: ProviderResource,
|
||||
upstream_request_id: Option<SafeUpstreamValue>,
|
||||
upstream_request_id: Option<OpaqueUpstreamValue>,
|
||||
transport: UpstreamTransport,
|
||||
}
|
||||
|
||||
@@ -147,7 +149,7 @@ impl ProviderCallMetadata {
|
||||
|
||||
/// 设置 adapter 已分类为非 bearer 的 request ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
pub fn with_upstream_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
@@ -184,7 +186,7 @@ impl ProviderCallMetadata {
|
||||
|
||||
/// 返回安全上游 request ID。
|
||||
#[must_use]
|
||||
pub const fn upstream_request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
pub const fn upstream_request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_request_id.as_ref()
|
||||
}
|
||||
|
||||
@@ -433,6 +435,7 @@ pub struct ProviderRequestObservation {
|
||||
pub struct ProviderModelCapabilities {
|
||||
upstream_model: UpstreamModelId,
|
||||
capabilities: ModelCapabilities,
|
||||
presentation: Option<ModelPresentation>,
|
||||
}
|
||||
|
||||
/// Provider 实时目录成功发布后的进程内单调代次。
|
||||
@@ -459,9 +462,16 @@ impl ProviderModelCapabilities {
|
||||
Self {
|
||||
upstream_model,
|
||||
capabilities,
|
||||
presentation: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_presentation(mut self, presentation: ModelPresentation) -> Self {
|
||||
self.presentation = Some(presentation);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn upstream_model(&self) -> &UpstreamModelId {
|
||||
&self.upstream_model
|
||||
@@ -471,6 +481,11 @@ impl ProviderModelCapabilities {
|
||||
pub const fn capabilities(&self) -> &ModelCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn presentation(&self) -> Option<&ModelPresentation> {
|
||||
self.presentation.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderRequest {
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::engine::UpstreamSendState;
|
||||
use crate::event::{ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
/// 应用层标识不满足约束。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -174,22 +176,18 @@ pub enum ContinuationFailure {
|
||||
HistoryUnavailable,
|
||||
}
|
||||
|
||||
/// Adapter 已明确分类为非 bearer、可用于诊断的上游值。
|
||||
/// Adapter 已明确分类为非 bearer 的不透明上游值。
|
||||
///
|
||||
/// Core 不解释或校验内容,只通过自定义 [`Debug`] 避免它意外进入日志。协议 adapter
|
||||
/// 可在原客户端响应中读取原值。
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub struct SafeUpstreamValue(String);
|
||||
pub struct OpaqueUpstreamValue(String);
|
||||
|
||||
impl SafeUpstreamValue {
|
||||
const MAX_BYTES: usize = 256;
|
||||
|
||||
/// 校验并创建诊断值。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 空值、超长值或控制字符会返回 [`IdentifierError`]。
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
validate_text(&value, Self::MAX_BYTES, false, None)?;
|
||||
Ok(Self(value))
|
||||
impl OpaqueUpstreamValue {
|
||||
/// 原样保存上游值。
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
/// 返回已经由 adapter 安全分类的原值。
|
||||
@@ -199,9 +197,9 @@ impl SafeUpstreamValue {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for SafeUpstreamValue {
|
||||
impl fmt::Debug for OpaqueUpstreamValue {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("SafeUpstreamValue(<redacted-from-Debug>)")
|
||||
formatter.write_str("OpaqueUpstreamValue(<redacted-from-Debug>)")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,31 +210,23 @@ impl fmt::Debug for SafeUpstreamValue {
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ClientVisibleUpstreamError {
|
||||
message: String,
|
||||
code: Option<SafeUpstreamValue>,
|
||||
error_type: Option<SafeUpstreamValue>,
|
||||
code: Option<OpaqueUpstreamValue>,
|
||||
error_type: Option<OpaqueUpstreamValue>,
|
||||
}
|
||||
|
||||
impl ClientVisibleUpstreamError {
|
||||
const MAX_MESSAGE_BYTES: usize = 8_192;
|
||||
|
||||
/// 从结构化上游 `message`、`code` 和 `type` 创建客户端可见错误。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// message 为空、过长或含控制字符时返回错误;可选 code/type 不满足安全
|
||||
/// 约束时会被省略,不会把整段上游 body 重新暴露。
|
||||
/// 原样保存结构化上游 `message`、`code` 和 `type`。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
message: impl Into<String>,
|
||||
code: Option<String>,
|
||||
error_type: Option<String>,
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let message = message.into();
|
||||
validate_text(&message, Self::MAX_MESSAGE_BYTES, false, None)?;
|
||||
Ok(Self {
|
||||
message,
|
||||
code: code.and_then(|value| SafeUpstreamValue::new(value).ok()),
|
||||
error_type: error_type.and_then(|value| SafeUpstreamValue::new(value).ok()),
|
||||
})
|
||||
) -> Self {
|
||||
Self {
|
||||
message: message.into(),
|
||||
code: code.map(OpaqueUpstreamValue::new),
|
||||
error_type: error_type.map(OpaqueUpstreamValue::new),
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回原上游的结构化 message。
|
||||
@@ -248,13 +238,13 @@ impl ClientVisibleUpstreamError {
|
||||
/// 返回原上游的结构化 code。
|
||||
#[must_use]
|
||||
pub fn code(&self) -> Option<&str> {
|
||||
self.code.as_ref().map(SafeUpstreamValue::as_str)
|
||||
self.code.as_ref().map(OpaqueUpstreamValue::as_str)
|
||||
}
|
||||
|
||||
/// 返回原上游的结构化 type。
|
||||
#[must_use]
|
||||
pub fn error_type(&self) -> Option<&str> {
|
||||
self.error_type.as_ref().map(SafeUpstreamValue::as_str)
|
||||
self.error_type.as_ref().map(OpaqueUpstreamValue::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,28 +254,110 @@ impl fmt::Debug for ClientVisibleUpstreamError {
|
||||
}
|
||||
}
|
||||
|
||||
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
|
||||
///
|
||||
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
|
||||
/// [`ProviderError`] 的普通 clone 会丢弃它,只有最终失败的原对象才能把响应交给
|
||||
/// 原客户端协议,或由认证管理端的账号连接测试显式复制到请求局部结果。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct ClientVisibleUpstreamResponse {
|
||||
status: u16,
|
||||
content_type: Option<Vec<u8>>,
|
||||
headers: Vec<ProviderResponseHeader>,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
impl ClientVisibleUpstreamResponse {
|
||||
/// 保存 transport 实际收到的状态码、Content-Type 原值和正文。
|
||||
#[must_use]
|
||||
pub fn new(status: u16, content_type: Option<Vec<u8>>, body: Bytes) -> Self {
|
||||
Self {
|
||||
status,
|
||||
content_type,
|
||||
headers: Vec::new(),
|
||||
body,
|
||||
}
|
||||
}
|
||||
|
||||
/// 附加 Provider 已剔除账号绑定与 framing 信息的普通响应头。
|
||||
#[must_use]
|
||||
pub fn with_headers(mut self, headers: Vec<ProviderResponseHeader>) -> Self {
|
||||
self.headers = headers;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn status(&self) -> u16 {
|
||||
self.status
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn content_type(&self) -> Option<&[u8]> {
|
||||
self.content_type.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.headers
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ClientVisibleUpstreamResponse {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ClientVisibleUpstreamResponse")
|
||||
.field("status", &self.status)
|
||||
.field(
|
||||
"content_type",
|
||||
&self.content_type.as_ref().map(|_| "<present>"),
|
||||
)
|
||||
.field("header_count", &self.headers.len())
|
||||
.field("body", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 单次 Provider 调用的稳定错误。
|
||||
///
|
||||
/// 该类型不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
|
||||
/// 稳定诊断字段不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
|
||||
/// code/request ID/response ID。Adapter 若捕获到可能含 secret 的上下文,只能调用
|
||||
/// [`ProviderError::redact_sensitive_context`] 丢弃正文并留下脱敏标记。
|
||||
#[derive(Clone)]
|
||||
///
|
||||
/// 唯一例外是 [`ProviderError::with_atomic_client_events`]:它只承载尚未交付客户端、
|
||||
/// 必须与本错误一起完成重试判断的协议事件。Core 会在任何持久化或日志记录前取走
|
||||
/// 这些事件,不能把它们当作诊断上下文使用。
|
||||
pub struct ProviderError {
|
||||
kind: ProviderErrorKind,
|
||||
send_state: UpstreamSendState,
|
||||
upstream_status: Option<u16>,
|
||||
upstream_code: Option<SafeUpstreamValue>,
|
||||
upstream_request_id: Option<SafeUpstreamValue>,
|
||||
upstream_response_id: Option<SafeUpstreamValue>,
|
||||
upstream_values: Option<Box<ProviderErrorUpstreamValues>>,
|
||||
retry_after: Option<Duration>,
|
||||
continuation_failure: Option<ContinuationFailure>,
|
||||
replay_safe: bool,
|
||||
pre_delivery_retry: bool,
|
||||
credential_recovery_required: bool,
|
||||
retry_same_account: bool,
|
||||
sensitive_context_redacted: bool,
|
||||
client_visible_upstream_error: Option<Box<ClientVisibleUpstreamError>>,
|
||||
client_visible_upstream_response: Option<Box<ClientVisibleUpstreamResponse>>,
|
||||
atomic_client_events: Option<Box<AtomicClientEvents>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProviderErrorUpstreamValues {
|
||||
code: Option<OpaqueUpstreamValue>,
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
response_id: Option<OpaqueUpstreamValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AtomicClientEvents(Vec<ProviderEvent>);
|
||||
|
||||
impl ProviderError {
|
||||
/// 创建 Provider 错误。
|
||||
#[must_use]
|
||||
@@ -294,16 +366,17 @@ impl ProviderError {
|
||||
kind,
|
||||
send_state,
|
||||
upstream_status: None,
|
||||
upstream_code: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
upstream_values: None,
|
||||
retry_after: None,
|
||||
continuation_failure: None,
|
||||
replay_safe: false,
|
||||
pre_delivery_retry: false,
|
||||
credential_recovery_required: false,
|
||||
retry_same_account: false,
|
||||
sensitive_context_redacted: false,
|
||||
client_visible_upstream_error: None,
|
||||
client_visible_upstream_response: None,
|
||||
atomic_client_events: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,25 +391,30 @@ impl ProviderError {
|
||||
|
||||
/// 附加 adapter 已分类为安全的上游错误 code。
|
||||
#[must_use]
|
||||
pub fn with_upstream_code(mut self, code: SafeUpstreamValue) -> Self {
|
||||
self.upstream_code = Some(code);
|
||||
pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().code = Some(code);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 adapter 已分类为非 bearer 的上游 request ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
self.upstream_request_id = Some(request_id);
|
||||
pub fn with_upstream_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 adapter 已分类为非 bearer 的上游 response ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_response_id(mut self, response_id: SafeUpstreamValue) -> Self {
|
||||
self.upstream_response_id = Some(response_id);
|
||||
pub fn with_upstream_response_id(mut self, response_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().response_id = Some(response_id);
|
||||
self
|
||||
}
|
||||
|
||||
fn upstream_values_mut(&mut self) -> &mut ProviderErrorUpstreamValues {
|
||||
self.upstream_values
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorUpstreamValues::default()))
|
||||
}
|
||||
|
||||
/// 附加 Provider 建议的冷却时间。
|
||||
#[must_use]
|
||||
pub const fn with_retry_after(mut self, retry_after: Duration) -> Self {
|
||||
@@ -358,6 +436,17 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 允许 Core 仅在客户端尚未收到任何事件时执行一次受预算约束的换号恢复。
|
||||
///
|
||||
/// 该标记不证明上游未执行,也不等价于 [`Self::with_replay_safe`];它只表达
|
||||
/// Provider 选择了“客户端无感恢复优先”的传输策略。Core 在 continuation、
|
||||
/// 指定账号或下游已经进入提交状态时必须忽略它。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_retry(mut self) -> Self {
|
||||
self.pre_delivery_retry = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次。
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry(mut self) -> Self {
|
||||
@@ -379,6 +468,27 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加仅可返回给当前请求方的原始上游 HTTP 失败响应。
|
||||
#[must_use]
|
||||
pub fn with_client_visible_upstream_response(
|
||||
mut self,
|
||||
response: ClientVisibleUpstreamResponse,
|
||||
) -> Self {
|
||||
self.client_visible_upstream_response = Some(Box::new(response));
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加必须先由 Core 判断重试、再决定丢弃或交付的客户端协议事件。
|
||||
///
|
||||
/// Provider 只能放入当前 attempt 尚未交付的事件;调用方应设置明确的有界缓冲,
|
||||
/// 并保证失败事件本身位于这批事件的末尾。
|
||||
#[must_use]
|
||||
pub fn with_atomic_client_events(mut self, events: Vec<ProviderEvent>) -> Self {
|
||||
self.atomic_client_events =
|
||||
(!events.is_empty()).then(|| Box::new(AtomicClientEvents(events)));
|
||||
self
|
||||
}
|
||||
|
||||
/// 丢弃敏感正文,只记录“发生过脱敏”这一事实。
|
||||
#[must_use]
|
||||
pub fn redact_sensitive_context(mut self, _sensitive_context: impl AsRef<str>) -> Self {
|
||||
@@ -406,20 +516,26 @@ impl ProviderError {
|
||||
|
||||
/// 返回安全分类的上游错误 code。
|
||||
#[must_use]
|
||||
pub fn upstream_code(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_code.as_ref()
|
||||
pub fn upstream_code(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.code.as_ref())
|
||||
}
|
||||
|
||||
/// 返回安全分类的上游 request ID。
|
||||
#[must_use]
|
||||
pub fn upstream_request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_request_id.as_ref()
|
||||
pub fn upstream_request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.request_id.as_ref())
|
||||
}
|
||||
|
||||
/// 返回安全分类的上游 response ID。
|
||||
#[must_use]
|
||||
pub fn upstream_response_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_response_id.as_ref()
|
||||
pub fn upstream_response_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.response_id.as_ref())
|
||||
}
|
||||
|
||||
/// 返回 Provider 建议的冷却时间。
|
||||
@@ -440,6 +556,12 @@ impl ProviderError {
|
||||
self.replay_safe
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否允许在首个客户端事件前换号恢复传输失败。
|
||||
#[must_use]
|
||||
pub const fn allows_pre_delivery_retry(&self) -> bool {
|
||||
self.pre_delivery_retry
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否已完成凭据恢复并要求原账号重放。
|
||||
#[must_use]
|
||||
pub const fn retries_same_account(&self) -> bool {
|
||||
@@ -462,6 +584,50 @@ impl ProviderError {
|
||||
pub fn client_visible_upstream_error(&self) -> Option<&ClientVisibleUpstreamError> {
|
||||
self.client_visible_upstream_error.as_deref()
|
||||
}
|
||||
|
||||
/// 返回只供当前协议响应使用的原始上游 HTTP 失败响应。
|
||||
#[must_use]
|
||||
pub fn client_visible_upstream_response(&self) -> Option<&ClientVisibleUpstreamResponse> {
|
||||
self.client_visible_upstream_response.as_deref()
|
||||
}
|
||||
|
||||
/// 取走只供本次重试/提交决策使用的原子客户端事件。
|
||||
///
|
||||
/// Core 必须在克隆错误、记录中间失败或构造终态前调用本方法,避免原始 wire
|
||||
/// 进入诊断或持久化对象。
|
||||
#[must_use]
|
||||
pub fn take_atomic_client_events(&mut self) -> Vec<ProviderEvent> {
|
||||
self.atomic_client_events
|
||||
.take()
|
||||
.map_or_else(Vec::new, |events| events.0)
|
||||
}
|
||||
|
||||
/// 返回是否携带尚未提交的原子客户端事件;不暴露事件内容。
|
||||
#[must_use]
|
||||
pub fn has_atomic_client_events(&self) -> bool {
|
||||
self.atomic_client_events.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for ProviderError {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
kind: self.kind,
|
||||
send_state: self.send_state,
|
||||
upstream_status: self.upstream_status,
|
||||
upstream_values: self.upstream_values.clone(),
|
||||
retry_after: self.retry_after,
|
||||
continuation_failure: self.continuation_failure,
|
||||
replay_safe: self.replay_safe,
|
||||
pre_delivery_retry: self.pre_delivery_retry,
|
||||
credential_recovery_required: self.credential_recovery_required,
|
||||
retry_same_account: self.retry_same_account,
|
||||
sensitive_context_redacted: self.sensitive_context_redacted,
|
||||
client_visible_upstream_error: self.client_visible_upstream_error.clone(),
|
||||
client_visible_upstream_response: None,
|
||||
atomic_client_events: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderError {
|
||||
@@ -473,25 +639,20 @@ impl fmt::Debug for ProviderError {
|
||||
.field("upstream_status", &self.upstream_status)
|
||||
.field(
|
||||
"upstream_code",
|
||||
&self.upstream_code.as_ref().map(|_| "<classified-safe>"),
|
||||
&self.upstream_code().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field(
|
||||
"upstream_request_id",
|
||||
&self
|
||||
.upstream_request_id
|
||||
.as_ref()
|
||||
.map(|_| "<classified-safe>"),
|
||||
&self.upstream_request_id().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field(
|
||||
"upstream_response_id",
|
||||
&self
|
||||
.upstream_response_id
|
||||
.as_ref()
|
||||
.map(|_| "<classified-safe>"),
|
||||
&self.upstream_response_id().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field("retry_after", &self.retry_after)
|
||||
.field("continuation_failure", &self.continuation_failure)
|
||||
.field("replay_safe", &self.replay_safe)
|
||||
.field("pre_delivery_retry", &self.pre_delivery_retry)
|
||||
.field(
|
||||
"credential_recovery_required",
|
||||
&self.credential_recovery_required,
|
||||
@@ -508,6 +669,20 @@ impl fmt::Debug for ProviderError {
|
||||
.as_ref()
|
||||
.map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"client_visible_upstream_response",
|
||||
&self
|
||||
.client_visible_upstream_response
|
||||
.as_ref()
|
||||
.map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"atomic_client_events",
|
||||
&self
|
||||
.atomic_client_events
|
||||
.as_ref()
|
||||
.map_or(0, |events| events.0.len()),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ use thiserror::Error;
|
||||
|
||||
use crate::accounting::{CalculatedCost, ProviderReportedCost, Usage};
|
||||
use crate::engine::provider::UpstreamTransport;
|
||||
use crate::error::{IdentifierError, SafeUpstreamValue, validate_text};
|
||||
use crate::error::{IdentifierError, OpaqueUpstreamValue, validate_text};
|
||||
use crate::operation::ProviderSessionState;
|
||||
|
||||
/// 一次响应的稳定元数据。
|
||||
@@ -199,7 +199,7 @@ impl ProtocolWireEvent {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名或显式事件名为空、过长或含控制字符时返回错误。
|
||||
/// 协议名不满足内部路由标识约束时返回错误。上游事件名保持不透明。
|
||||
pub fn json(
|
||||
protocol: impl Into<String>,
|
||||
event_type: Option<String>,
|
||||
@@ -212,7 +212,7 @@ impl ProtocolWireEvent {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名、显式事件名或 SSE ID 不满足 wire 安全约束时返回错误。
|
||||
/// 协议名不满足内部路由标识约束时返回错误。SSE 元数据保持不透明。
|
||||
pub fn json_with_sse_metadata(
|
||||
protocol: impl Into<String>,
|
||||
event_type: Option<String>,
|
||||
@@ -222,15 +222,6 @@ impl ProtocolWireEvent {
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let protocol = protocol.into();
|
||||
validate_text(&protocol, 64, true, None)?;
|
||||
if let Some(event_type) = event_type.as_deref() {
|
||||
validate_text(event_type, 256, true, None)?;
|
||||
}
|
||||
if sse_id
|
||||
.as_deref()
|
||||
.is_some_and(|id| id.contains(['\0', '\r', '\n']))
|
||||
{
|
||||
return Err(IdentifierError::ControlCharacter);
|
||||
}
|
||||
Ok(Self {
|
||||
protocol,
|
||||
event_type,
|
||||
@@ -440,24 +431,21 @@ impl fmt::Debug for ProviderResponseMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 已筛选、可由协议 adapter 再次按白名单表达的响应头。
|
||||
/// Provider 已筛选、可由协议 adapter 尝试表达的不透明响应头。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResponseHeader {
|
||||
name: String,
|
||||
value: SafeUpstreamValue,
|
||||
value: Bytes,
|
||||
}
|
||||
|
||||
impl ProviderResponseHeader {
|
||||
/// 创建规范化的小写响应头;非 ASCII token 名称直接拒绝。
|
||||
/// 保存 Provider 交付的原始名称和值;HTTP 可表达性由最终 adapter 判断。
|
||||
#[must_use]
|
||||
pub fn new(name: impl Into<String>, value: SafeUpstreamValue) -> Option<Self> {
|
||||
let name = name.into().trim().to_ascii_lowercase();
|
||||
let valid = !name.is_empty()
|
||||
&& name.len() <= 64
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-');
|
||||
valid.then_some(Self { name, value })
|
||||
pub fn new(name: impl Into<String>, value: Bytes) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
value,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -466,7 +454,7 @@ impl ProviderResponseHeader {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn value(&self) -> &SafeUpstreamValue {
|
||||
pub const fn value(&self) -> &Bytes {
|
||||
&self.value
|
||||
}
|
||||
}
|
||||
@@ -475,7 +463,7 @@ impl fmt::Debug for ProviderResponseHeader {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderResponseHeader")
|
||||
.field("name", &self.name)
|
||||
.field("name", &"<redacted>")
|
||||
.field("value", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
@@ -488,7 +476,8 @@ pub struct ProviderResponseObservation {
|
||||
http_version: Option<UpstreamHttpVersion>,
|
||||
websocket_pool: Option<WebSocketPoolKind>,
|
||||
status_code: Option<u16>,
|
||||
request_id: Option<SafeUpstreamValue>,
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
service_tier: Option<String>,
|
||||
timings: ProviderResponseTimings,
|
||||
client_headers: Vec<ProviderResponseHeader>,
|
||||
provider_metadata: Option<ProviderResponseMetadata>,
|
||||
@@ -503,6 +492,7 @@ impl ProviderResponseObservation {
|
||||
websocket_pool: None,
|
||||
status_code: None,
|
||||
request_id: None,
|
||||
service_tier: None,
|
||||
timings: ProviderResponseTimings::default(),
|
||||
client_headers: Vec::new(),
|
||||
provider_metadata: None,
|
||||
@@ -528,11 +518,36 @@ impl ProviderResponseObservation {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
pub fn with_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 Provider 在响应中确认的服务档位。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 档位为空、超过 64 字节或包含控制字符时返回错误。
|
||||
pub fn try_with_service_tier(
|
||||
mut self,
|
||||
service_tier: impl Into<String>,
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let service_tier = service_tier.into();
|
||||
validate_text(&service_tier, 64, false, None)?;
|
||||
self.service_tier = Some(service_tier);
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// 附加合法服务档位;不适合持久化的上游观测值会被忽略。
|
||||
#[must_use]
|
||||
pub fn with_service_tier_if_valid(mut self, service_tier: impl Into<String>) -> Self {
|
||||
let service_tier = service_tier.into();
|
||||
if validate_text(&service_tier, 64, false, None).is_ok() {
|
||||
self.service_tier = Some(service_tier);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_timings(mut self, timings: ProviderResponseTimings) -> Self {
|
||||
self.timings = timings;
|
||||
@@ -573,10 +588,15 @@ impl ProviderResponseObservation {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
pub const fn request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.request_id.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn service_tier(&self) -> Option<&str> {
|
||||
self.service_tier.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn timings(&self) -> ProviderResponseTimings {
|
||||
self.timings
|
||||
@@ -715,12 +735,6 @@ impl ProviderEvent {
|
||||
&self.canonical
|
||||
}
|
||||
|
||||
/// 返回 Core 可改写客户端响应身份的全部 facts。
|
||||
#[must_use]
|
||||
pub fn canonical_facts_mut(&mut self) -> &mut [GatewayEvent] {
|
||||
&mut self.canonical
|
||||
}
|
||||
|
||||
/// 返回协议原生表达。
|
||||
#[must_use]
|
||||
pub fn wire_event(&self) -> Option<&ProtocolWireEvent> {
|
||||
@@ -756,18 +770,6 @@ impl ProviderEvent {
|
||||
pub fn has_client_event(&self) -> bool {
|
||||
!self.canonical.is_empty() || self.wire.is_some()
|
||||
}
|
||||
|
||||
/// 返回是否包含足以冻结下游 commit barrier 的 canonical fact。
|
||||
///
|
||||
/// 单独的 wire event 不能冻结边界:协议层在看到 `Started`
|
||||
/// 携带的网关响应身份前无法安全编码它。Core 会保留这些
|
||||
/// 事件,直到首个 canonical 可见增量或终态再一并交付。
|
||||
#[must_use]
|
||||
pub fn is_commit_significant(&self) -> bool {
|
||||
self.canonical
|
||||
.iter()
|
||||
.any(GatewayEvent::is_commit_significant)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GatewayEvent> for ProviderEvent {
|
||||
@@ -776,23 +778,6 @@ impl From<GatewayEvent> for ProviderEvent {
|
||||
}
|
||||
}
|
||||
|
||||
impl GatewayEvent {
|
||||
/// 返回该 fact 是否足以冻结下游 commit barrier。
|
||||
///
|
||||
/// 生命周期、结构与结算 facts 可以在换号前安全丢弃;首个可见增量或
|
||||
/// Provider 正常终态才会使客户端输出不可撤回。
|
||||
#[must_use]
|
||||
pub const fn is_commit_significant(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::TextDelta(_)
|
||||
| Self::ReasoningDelta(_)
|
||||
| Self::ToolCallDelta(_)
|
||||
| Self::Completed(_)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical event 顺序错误。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum EventSequenceError {
|
||||
|
||||
@@ -269,10 +269,15 @@ impl GenerateRequest {
|
||||
/// 附着同一客户端连接上一轮由 Provider 返回的不透明状态。
|
||||
#[must_use]
|
||||
pub fn with_provider_session_state(mut self, state: ProviderSessionState) -> Self {
|
||||
Arc::make_mut(&mut self.payload).provider_session_state = Some(state);
|
||||
self.set_provider_session_state(state);
|
||||
self
|
||||
}
|
||||
|
||||
/// 原地附着会话状态;payload 独占时不复制正文。
|
||||
pub fn set_provider_session_state(&mut self, state: ProviderSessionState) {
|
||||
Arc::make_mut(&mut self.payload).provider_session_state = Some(state);
|
||||
}
|
||||
|
||||
/// 返回最大输出 token 数。
|
||||
#[must_use]
|
||||
pub fn max_output_tokens(&self) -> Option<u64> {
|
||||
@@ -297,7 +302,7 @@ impl GenerateRequest {
|
||||
self.body()
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|response_id| !response_id.is_empty())
|
||||
.is_some()
|
||||
}
|
||||
|
||||
/// 返回客户端是否请求了图片生成工具。
|
||||
@@ -597,6 +602,13 @@ impl Operation {
|
||||
}
|
||||
}
|
||||
|
||||
/// 原地附着 Provider 私有状态;payload 独占时不复制正文。
|
||||
pub fn set_provider_session_state(&mut self, state: ProviderSessionState) {
|
||||
if let Self::Generate(request) = self {
|
||||
request.set_provider_session_state(state);
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回稳定 operation 分类。
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> OperationKind {
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::engine::credential::{
|
||||
OpaqueProviderData, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use crate::error::{IdentifierError, validate_text};
|
||||
use crate::routing::ProviderKind;
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
|
||||
const MAX_PENDING_FLOW_TTL: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
@@ -474,9 +474,22 @@ pub trait ProviderCredentialStatePort: Send + Sync {
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 记录一次瞬态刷新失败并返回窗口内累计失败次数;每次失败刷新 TTL。
|
||||
fn record_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
|
||||
|
||||
/// 凭据完整成功轮换后清零失败计数,退避窗口重新从 base 起步。
|
||||
fn clear_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// 临时 cooldown 只保存调度截止时间;原因事实仍由 PostgreSQL 账号状态持有。
|
||||
/// 临时 cooldown 只保存可丢失的调度截止时间,不进入账号持久状态。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
@@ -514,6 +527,80 @@ impl ProviderCooldown {
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失 cooldown 的细粒度作用域。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum ProviderCooldownScope {
|
||||
/// 只阻止同一账号调用指定上游模型。
|
||||
UpstreamModel(UpstreamModelId),
|
||||
}
|
||||
|
||||
impl ProviderCooldownScope {
|
||||
#[must_use]
|
||||
pub const fn upstream_model(model: UpstreamModelId) -> Self {
|
||||
Self::UpstreamModel(model)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::UpstreamModel(_) => "model",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn value(&self) -> &str {
|
||||
match self {
|
||||
Self::UpstreamModel(model) => model.as_str(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 不进入账号持久状态的账号+作用域 cooldown。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderScopedCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
}
|
||||
|
||||
impl ProviderScopedCooldown {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
scope,
|
||||
until,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scope(&self) -> &ProviderCooldownScope {
|
||||
&self.scope
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCooldownPort: Send + Sync {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
@@ -530,6 +617,24 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>>;
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -562,36 +667,52 @@ impl ProviderRefreshPolicy {
|
||||
self.concurrency
|
||||
}
|
||||
|
||||
/// 按账号稳定扰动刷新提前量;相同账号与过期时间跨进程得到同一执行时刻。
|
||||
/// 按配置的精确提前量计算刷新时刻。
|
||||
pub fn next_attempt_at(
|
||||
self,
|
||||
account_id: &ProviderAccountId,
|
||||
_account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let remaining = access_token_expires_at
|
||||
.duration_since(observed_at)
|
||||
.map_err(|_| invalid_refresh_policy("schedule expired access token"))?;
|
||||
let base_margin_seconds = self.margin.as_secs();
|
||||
if base_margin_seconds == 0 {
|
||||
if self.margin >= remaining {
|
||||
return Ok(observed_at);
|
||||
}
|
||||
|
||||
let factor = stable_factor(account_id.as_str(), "normal", 850, 1_150);
|
||||
let jittered_seconds = base_margin_seconds
|
||||
.saturating_mul(u64::from(factor))
|
||||
.saturating_add(500)
|
||||
/ 1_000;
|
||||
if jittered_seconds >= remaining.as_secs() {
|
||||
return Ok(observed_at);
|
||||
}
|
||||
let lead = jittered_seconds.max(1);
|
||||
access_token_expires_at
|
||||
.checked_sub(Duration::from_secs(lead))
|
||||
.checked_sub(self.margin)
|
||||
.ok_or_else(|| invalid_refresh_policy("schedule access token refresh"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 指数退避基准延迟;attempt=1 即为该值。
|
||||
const REFRESH_BACKOFF_BASE_DELAY: Duration = Duration::from_secs(5);
|
||||
/// 每多一次连续失败,基准延迟乘以该因子。
|
||||
const REFRESH_BACKOFF_FACTOR: u32 = 3;
|
||||
/// 退避延迟上限,避免连续失败时无限增长。
|
||||
const REFRESH_BACKOFF_CAP: Duration = Duration::from_secs(300);
|
||||
|
||||
/// 基于连续失败计数的指数退避重试时刻;复用 `provider_refresh_retry_at` 的稳定扰动。
|
||||
///
|
||||
/// `attempt` 为窗口内累计失败次数(0 与 1 等价,均取基准延迟)。延迟按
|
||||
/// `base * factor^(attempt-1)` 增长并封顶到 `REFRESH_BACKOFF_CAP`。
|
||||
pub fn provider_refresh_backoff_at(
|
||||
account_id: &ProviderAccountId,
|
||||
observed_at: SystemTime,
|
||||
attempt: u32,
|
||||
reason: &'static str,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let exponent = attempt.saturating_sub(1);
|
||||
let multiplier = REFRESH_BACKOFF_FACTOR.saturating_pow(exponent);
|
||||
let scaled_seconds = REFRESH_BACKOFF_BASE_DELAY
|
||||
.as_secs()
|
||||
.saturating_mul(u64::from(multiplier))
|
||||
.min(REFRESH_BACKOFF_CAP.as_secs());
|
||||
let base_delay = Duration::from_secs(scaled_seconds);
|
||||
provider_refresh_retry_at(account_id, observed_at, base_delay, reason)
|
||||
}
|
||||
|
||||
/// 临时失败后的持久重试时刻;稳定扰动避免多实例同频重试。
|
||||
pub fn provider_refresh_retry_at(
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -738,35 +859,75 @@ pub enum OAuthPendingPutOutcome {
|
||||
AlreadyExists,
|
||||
}
|
||||
|
||||
/// OAuth 回调处理取得临时 flow 的独占权结果。
|
||||
///
|
||||
/// 与一次性消费不同,Provider 在上游换取令牌失败时可以释放 claim,让同一 flow
|
||||
/// 使用新的回调地址重试;只有完整校验成功后才会消费 flow。
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub enum OAuthPendingTakeOutcome {
|
||||
Taken(OpaqueProviderData),
|
||||
pub enum OAuthPendingClaimOutcome {
|
||||
Claimed(OpaqueProviderData),
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
InProgress,
|
||||
}
|
||||
|
||||
impl fmt::Debug for OAuthPendingTakeOutcome {
|
||||
impl fmt::Debug for OAuthPendingClaimOutcome {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Taken(_) => formatter.write_str("Taken([PROVIDER-OWNED])"),
|
||||
Self::Claimed(_) => formatter.write_str("Claimed([PROVIDER-OWNED])"),
|
||||
Self::NotFound => formatter.write_str("NotFound"),
|
||||
Self::OwnerMismatch => formatter.write_str("OwnerMismatch"),
|
||||
Self::InProgress => formatter.write_str("InProgress"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OAuthPendingReleaseOutcome {
|
||||
Released,
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
ClaimMismatch,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OAuthPendingConsumeOutcome {
|
||||
Consumed,
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
ClaimMismatch,
|
||||
}
|
||||
|
||||
pub trait OAuthPendingFlowPort: Send + Sync {
|
||||
fn put_if_absent(
|
||||
&self,
|
||||
flow: NewOAuthPendingFlow,
|
||||
) -> BoxFuture<'_, Result<OAuthPendingPutOutcome, ProviderStoreError>>;
|
||||
|
||||
fn take_if_owner<'a>(
|
||||
fn claim_if_owner<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingTakeOutcome, ProviderStoreError>>;
|
||||
claim: &'a OAuthPendingBinding,
|
||||
claim_ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingClaimOutcome, ProviderStoreError>>;
|
||||
|
||||
fn release_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingReleaseOutcome, ProviderStoreError>>;
|
||||
|
||||
fn consume_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingConsumeOutcome, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// Provider 只能按能力取用端口,无法取得 Redis client 或 repository 集合。
|
||||
|
||||
@@ -89,6 +89,18 @@ impl UpstreamModelId {
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
/// 从未命中显式映射的客户端模型名构造同名上游模型。
|
||||
///
|
||||
/// 目录和配置仍使用 [`Self::new`] 的内部标识约束;客户端 wire 只要求非空,
|
||||
/// 具体模型字符串是否可用由绑定 Provider 决定。
|
||||
pub fn from_client_wire(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
if value.is_empty() {
|
||||
return Err(IdentifierError::Empty);
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
@@ -136,6 +148,240 @@ pub struct ModelCapabilities {
|
||||
upstream_validates_features: bool,
|
||||
}
|
||||
|
||||
/// Provider 为客户端模型目录提供的展示与交互能力。
|
||||
///
|
||||
/// 该值不参与路由,也不把任一 Provider 的 wire 类型带入 Core。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ModelPresentation {
|
||||
display_name: Option<String>,
|
||||
description: Option<String>,
|
||||
default_reasoning_effort: Option<String>,
|
||||
supported_reasoning_efforts: Vec<String>,
|
||||
context_window_tokens: Option<u64>,
|
||||
image_input: bool,
|
||||
agent_tools: bool,
|
||||
parallel_tool_calls: bool,
|
||||
search_tool: bool,
|
||||
image_detail_original: bool,
|
||||
verbosity: bool,
|
||||
service_tiers: Vec<ModelServiceTier>,
|
||||
hidden: bool,
|
||||
}
|
||||
|
||||
/// Provider 声明给 Codex 客户端的服务档位。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ModelServiceTier {
|
||||
id: String,
|
||||
name: String,
|
||||
description: String,
|
||||
speed_tier: Option<String>,
|
||||
}
|
||||
|
||||
impl ModelServiceTier {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
id: impl Into<String>,
|
||||
name: impl Into<String>,
|
||||
description: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
name: name.into(),
|
||||
description: description.into(),
|
||||
speed_tier: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_speed_tier(mut self, speed_tier: impl Into<String>) -> Self {
|
||||
self.speed_tier = Some(speed_tier.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn id(&self) -> &str {
|
||||
&self.id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn description(&self) -> &str {
|
||||
&self.description
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn speed_tier(&self) -> Option<&str> {
|
||||
self.speed_tier.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelPresentation {
|
||||
#[must_use]
|
||||
pub fn new(display_name: Option<String>, description: Option<String>) -> Self {
|
||||
Self {
|
||||
display_name,
|
||||
description,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_reasoning(
|
||||
mut self,
|
||||
default_effort: Option<String>,
|
||||
supported_efforts: Vec<String>,
|
||||
) -> Self {
|
||||
self.default_reasoning_effort = default_effort;
|
||||
self.supported_reasoning_efforts = supported_efforts;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_context_window_tokens(mut self, context_window_tokens: Option<u64>) -> Self {
|
||||
self.context_window_tokens = context_window_tokens;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_image_input(mut self, image_input: bool) -> Self {
|
||||
self.image_input = image_input;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_agent_tools(mut self, agent_tools: bool, parallel_tool_calls: bool) -> Self {
|
||||
self.agent_tools = agent_tools;
|
||||
self.parallel_tool_calls = parallel_tool_calls;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_search_tool(mut self, search_tool: bool) -> Self {
|
||||
self.search_tool = search_tool;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_image_detail_original(mut self, image_detail_original: bool) -> Self {
|
||||
self.image_detail_original = image_detail_original;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_verbosity(mut self, verbosity: bool) -> Self {
|
||||
self.verbosity = verbosity;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_service_tiers(mut self, service_tiers: Vec<ModelServiceTier>) -> Self {
|
||||
self.service_tiers = service_tiers;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_hidden(mut self, hidden: bool) -> Self {
|
||||
self.hidden = hidden;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn display_name(&self) -> Option<&str> {
|
||||
self.display_name.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn description(&self) -> Option<&str> {
|
||||
self.description.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn default_reasoning_effort(&self) -> Option<&str> {
|
||||
self.default_reasoning_effort.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn supported_reasoning_efforts(&self) -> &[String] {
|
||||
&self.supported_reasoning_efforts
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn context_window_tokens(&self) -> Option<u64> {
|
||||
self.context_window_tokens
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn image_input(&self) -> bool {
|
||||
self.image_input
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn agent_tools(&self) -> bool {
|
||||
self.agent_tools
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn parallel_tool_calls(&self) -> bool {
|
||||
self.parallel_tool_calls
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn search_tool(&self) -> bool {
|
||||
self.search_tool
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn image_detail_original(&self) -> bool {
|
||||
self.image_detail_original
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn verbosity(&self) -> bool {
|
||||
self.verbosity
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn service_tiers(&self) -> &[ModelServiceTier] {
|
||||
&self.service_tiers
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn hidden(&self) -> bool {
|
||||
self.hidden
|
||||
}
|
||||
}
|
||||
|
||||
/// 一个公开模型及其 Provider 编译后的客户端画像。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PublicModelProfile {
|
||||
model: PublicModelId,
|
||||
presentation: ModelPresentation,
|
||||
}
|
||||
|
||||
impl PublicModelProfile {
|
||||
#[must_use]
|
||||
pub const fn new(model: PublicModelId, presentation: ModelPresentation) -> Self {
|
||||
Self {
|
||||
model,
|
||||
presentation,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn model(&self) -> &PublicModelId {
|
||||
&self.model
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn presentation(&self) -> &ModelPresentation {
|
||||
&self.presentation
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelCapabilities {
|
||||
#[must_use]
|
||||
pub fn new(operations: BTreeSet<OperationKind>, max_output_tokens: Option<u64>) -> Self {
|
||||
@@ -205,6 +451,7 @@ pub struct ProviderModel {
|
||||
provider: ProviderKind,
|
||||
upstream_model: UpstreamModelId,
|
||||
capabilities: ModelCapabilities,
|
||||
presentation: Option<ModelPresentation>,
|
||||
}
|
||||
|
||||
impl ProviderModel {
|
||||
@@ -218,9 +465,16 @@ impl ProviderModel {
|
||||
provider,
|
||||
upstream_model,
|
||||
capabilities,
|
||||
presentation: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_presentation(mut self, presentation: ModelPresentation) -> Self {
|
||||
self.presentation = Some(presentation);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider(&self) -> &ProviderKind {
|
||||
&self.provider
|
||||
@@ -230,6 +484,11 @@ impl ProviderModel {
|
||||
pub const fn upstream_model(&self) -> &UpstreamModelId {
|
||||
&self.upstream_model
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn presentation(&self) -> Option<&ModelPresentation> {
|
||||
self.presentation.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// 本次请求选择 Provider 时使用的动态过滤事实。
|
||||
|
||||
@@ -231,11 +231,15 @@ async fn compile_runtime_snapshot(
|
||||
continue;
|
||||
};
|
||||
provider_models.extend(models.into_iter().map(|model| {
|
||||
ProviderModel::new(
|
||||
let compiled = ProviderModel::new(
|
||||
provider.clone(),
|
||||
model.upstream_model().clone(),
|
||||
model.capabilities().clone(),
|
||||
)
|
||||
);
|
||||
match model.presentation().cloned() {
|
||||
Some(presentation) => compiled.with_presentation(presentation),
|
||||
None => compiled,
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -287,6 +291,8 @@ pub struct RuntimeSnapshot {
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
providers: Arc<BTreeSet<ProviderKind>>,
|
||||
provider_models: Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>>,
|
||||
provider_model_presentations:
|
||||
Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, super::ModelPresentation>>>,
|
||||
model_mappings: Arc<BTreeMap<String, String>>,
|
||||
provider_catalog_generations: Arc<BTreeMap<ProviderKind, ProviderCatalogGeneration>>,
|
||||
client_policies: Arc<BTreeMap<ClientApiKeyId, ClientPolicy>>,
|
||||
@@ -313,17 +319,24 @@ impl RuntimeSnapshot {
|
||||
|
||||
let mut model_map =
|
||||
BTreeMap::<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>::new();
|
||||
let mut presentation_map =
|
||||
BTreeMap::<ProviderKind, BTreeMap<UpstreamModelId, super::ModelPresentation>>::new();
|
||||
for model in provider_models {
|
||||
if !provider_set.contains(model.provider()) {
|
||||
let ProviderModel {
|
||||
provider,
|
||||
upstream_model,
|
||||
capabilities,
|
||||
presentation,
|
||||
} = model;
|
||||
if !provider_set.contains(&provider) {
|
||||
return Err(RoutingError::NotFound {
|
||||
entity: "provider",
|
||||
id: model.provider().to_string(),
|
||||
id: provider.to_string(),
|
||||
});
|
||||
}
|
||||
let models = model_map.entry(model.provider).or_default();
|
||||
let upstream_model = model.upstream_model;
|
||||
let models = model_map.entry(provider.clone()).or_default();
|
||||
if models
|
||||
.insert(upstream_model.clone(), model.capabilities)
|
||||
.insert(upstream_model.clone(), capabilities)
|
||||
.is_some()
|
||||
{
|
||||
return Err(RoutingError::DuplicateEntity {
|
||||
@@ -331,6 +344,12 @@ impl RuntimeSnapshot {
|
||||
id: upstream_model.to_string(),
|
||||
});
|
||||
}
|
||||
if let Some(presentation) = presentation {
|
||||
presentation_map
|
||||
.entry(provider)
|
||||
.or_default()
|
||||
.insert(upstream_model, presentation);
|
||||
}
|
||||
}
|
||||
|
||||
let mut client_policy_map = BTreeMap::new();
|
||||
@@ -350,6 +369,7 @@ impl RuntimeSnapshot {
|
||||
account_selection_policy,
|
||||
providers: Arc::new(provider_set),
|
||||
provider_models: Arc::new(model_map),
|
||||
provider_model_presentations: Arc::new(presentation_map),
|
||||
model_mappings: Arc::new(BTreeMap::new()),
|
||||
provider_catalog_generations: Arc::new(BTreeMap::new()),
|
||||
client_policies: Arc::new(client_policy_map),
|
||||
@@ -402,6 +422,38 @@ impl RuntimeSnapshot {
|
||||
models.into_iter().collect()
|
||||
}
|
||||
|
||||
/// 返回 Provider 已明确声明画像的公开模型;没有画像时不猜测 Provider 语义。
|
||||
#[must_use]
|
||||
pub fn public_model_profiles_for_provider(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
) -> Vec<super::PublicModelProfile> {
|
||||
let Some(presentations) = self.provider_model_presentations.get(provider) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut profiles = BTreeMap::new();
|
||||
for (model, presentation) in presentations {
|
||||
if let Ok(public_model) = PublicModelId::new(model.as_str().to_owned()) {
|
||||
profiles.insert(public_model, presentation.clone());
|
||||
}
|
||||
}
|
||||
for alias in self.model_mappings.keys() {
|
||||
let target = self.mapped_model(alias);
|
||||
let Some(presentation) = presentations.iter().find_map(|(model, presentation)| {
|
||||
(model.as_str() == target).then_some(presentation)
|
||||
}) else {
|
||||
continue;
|
||||
};
|
||||
if let Ok(public_model) = PublicModelId::new(alias.clone()) {
|
||||
profiles.insert(public_model, presentation.clone());
|
||||
}
|
||||
}
|
||||
profiles
|
||||
.into_iter()
|
||||
.map(|(model, presentation)| super::PublicModelProfile::new(model, presentation))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn public_models(&self) -> Vec<PublicModelId> {
|
||||
self.providers
|
||||
@@ -436,15 +488,11 @@ impl RuntimeSnapshot {
|
||||
|
||||
#[must_use]
|
||||
pub fn mapped_model(&self, requested: &str) -> String {
|
||||
let original = requested.trim();
|
||||
let original = requested;
|
||||
let mut current = original.to_owned();
|
||||
let mut seen = BTreeSet::new();
|
||||
for _ in 0..20 {
|
||||
let Some(target) = self
|
||||
.model_mappings
|
||||
.get(¤t)
|
||||
.map(|target| target.trim())
|
||||
else {
|
||||
let Some(target) = self.model_mappings.get(¤t).map(String::as_str) else {
|
||||
return current;
|
||||
};
|
||||
if !seen.insert(current.clone()) || seen.contains(target) {
|
||||
@@ -477,8 +525,14 @@ impl RuntimeSnapshot {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let upstream_model = UpstreamModelId::new(self.mapped_model(public_model.as_str()))
|
||||
.map_err(|_| RoutingError::InvalidIdentifier)?;
|
||||
let requested_model = public_model.as_str();
|
||||
let mapped_model = self.mapped_model(requested_model);
|
||||
let upstream_model = if self.model_mappings.contains_key(requested_model) {
|
||||
UpstreamModelId::new(mapped_model)
|
||||
} else {
|
||||
UpstreamModelId::from_client_wire(mapped_model)
|
||||
}
|
||||
.map_err(|_| RoutingError::InvalidIdentifier)?;
|
||||
let emulated_features = match self
|
||||
.provider_models
|
||||
.get(provider)
|
||||
|
||||
@@ -2,13 +2,12 @@ use gateway_core::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::error::SafeUpstreamValue;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
fn pin() -> NativeContinuationPin {
|
||||
NativeContinuationPin::new(
|
||||
PreviousResponseId::new("response-private").expect("valid response"),
|
||||
SafeUpstreamValue::new("upstream-private").expect("valid upstream response"),
|
||||
PreviousResponseId::new("response-private"),
|
||||
PreviousResponseId::new("upstream-private"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
ProviderAccountId::new("acct_codex").expect("valid account"),
|
||||
)
|
||||
@@ -23,13 +22,20 @@ fn native_pin_debug_should_redact_previous_response_id() {
|
||||
|
||||
#[test]
|
||||
fn external_binding_debug_should_redact_previous_response_id() {
|
||||
let binding = ContinuationBinding::External(
|
||||
PreviousResponseId::new("external-private").expect("valid response"),
|
||||
);
|
||||
let binding = ContinuationBinding::External(PreviousResponseId::new("external-private"));
|
||||
|
||||
assert!(!format!("{binding:?}").contains("external-private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_response_id_preserves_any_codex_string_without_debug_disclosure() {
|
||||
let value = format!("resp_{}\0opaque", "x".repeat(257));
|
||||
let response_id = PreviousResponseId::new(value.clone());
|
||||
|
||||
assert_eq!(response_id.as_str(), value);
|
||||
assert!(!format!("{response_id:?}").contains(&value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_pin_should_reject_different_account() {
|
||||
assert!(!pin().matches(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,10 +5,11 @@ use std::time::{Duration, SystemTime};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountCandidate, AccountFeedbackStats,
|
||||
AccountQuotaSignals, AccountRuntimeSignals, AccountSelectionContext, AccountSelectionPolicy,
|
||||
AccountSelector, CredentialCasUpdate, CredentialRevision, OpaqueProviderData,
|
||||
PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountUpdate,
|
||||
AccountAttemptFeedback, AccountAvailability, AccountAvailabilityPolicy, AccountCandidate,
|
||||
AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker,
|
||||
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate,
|
||||
RotationStrategy,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -51,6 +52,7 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
preferred_account: None,
|
||||
round_robin_cursor: 0,
|
||||
availability: AccountAvailabilityPolicy::Enforce,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +74,18 @@ fn opaque_provider_data_should_not_expose_quota_values_in_debug() {
|
||||
assert!(!format!("{quota:?}").contains("private-window"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_account_identity_debug_should_redact_upstream_ids() {
|
||||
let identity = ProviderAccountIdentity::new(
|
||||
"private-user-id".to_owned(),
|
||||
Some("private-account-id".to_owned()),
|
||||
);
|
||||
let debug = format!("{identity:?}");
|
||||
|
||||
assert!(!debug.contains("private-user-id"));
|
||||
assert!(!debug.contains("private-account-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn availability_should_round_trip_all_database_values() {
|
||||
assert_eq!(
|
||||
@@ -80,6 +94,55 @@ fn availability_should_round_trip_all_database_values() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
let exhausted = AccountCandidate {
|
||||
account: account("acct_exhausted").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let disabled = AccountCandidate {
|
||||
account: account("acct_disabled").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let mut context = context(RotationStrategy::Sticky);
|
||||
context.availability = AccountAvailabilityPolicy::BypassForDiagnostic;
|
||||
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(std::slice::from_ref(&exhausted), &context)
|
||||
.map(|selection| selection.candidate().account.id()),
|
||||
Some(exhausted.account.id())
|
||||
);
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(std::slice::from_ref(&disabled), &context)
|
||||
.map(|selection| selection.candidate().account.id()),
|
||||
Some(disabled.account.id())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account =
|
||||
@@ -119,7 +182,7 @@ fn smart_selector_should_prefer_lower_inflight_count() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_idle");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_idle");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -136,15 +199,13 @@ fn smart_selector_should_balance_signals_instead_of_using_lexicographic_load() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_healthy");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_keep_healthy_preferred_account_at_escape_boundary() {
|
||||
fn smart_selector_should_report_a_schedulable_preferred_account_hit() {
|
||||
let fallback = candidate("acct_fallback", 0, Some(100));
|
||||
let mut preferred = candidate("acct_preferred", 0, Some(1));
|
||||
preferred.signals.failure_rate_basis_points = Some(5_000);
|
||||
preferred.signals.first_output_latency_ms = Some(15_000);
|
||||
let preferred = candidate("acct_preferred", 0, Some(1));
|
||||
let preferred_id = preferred.account.id().clone();
|
||||
let mut selection = context(RotationStrategy::Smart);
|
||||
selection.preferred_account = Some(preferred_id);
|
||||
@@ -154,16 +215,22 @@ fn smart_selector_should_keep_healthy_preferred_account_at_escape_boundary() {
|
||||
.select(&candidates, &selection)
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_preferred");
|
||||
assert_eq!(
|
||||
(
|
||||
selected.candidate().account.id().as_str(),
|
||||
selected.preferred(),
|
||||
),
|
||||
("acct_preferred", PreferredAccountSelection::Hit)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_escape_preferred_account_after_failure_threshold() {
|
||||
fn smart_selector_should_keep_preferred_account_despite_failure_signals() {
|
||||
let mut fallback = candidate("acct_fallback", 0, Some(100));
|
||||
fallback.signals.failure_rate_basis_points = Some(0);
|
||||
fallback.signals.first_output_latency_ms = Some(100);
|
||||
let mut preferred = candidate("acct_preferred", 0, Some(100));
|
||||
preferred.signals.failure_rate_basis_points = Some(5_001);
|
||||
preferred.signals.failure_rate_basis_points = Some(10_000);
|
||||
preferred.signals.first_output_latency_ms = Some(100);
|
||||
let preferred_id = preferred.account.id().clone();
|
||||
let mut selection = context(RotationStrategy::Smart);
|
||||
@@ -172,19 +239,19 @@ fn smart_selector_should_escape_preferred_account_after_failure_threshold() {
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("fallback candidate available");
|
||||
.expect("preferred candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_preferred");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_escape_preferred_account_after_latency_threshold() {
|
||||
fn smart_selector_should_keep_preferred_account_despite_latency_signals() {
|
||||
let mut fallback = candidate("acct_fallback", 0, Some(100));
|
||||
fallback.signals.failure_rate_basis_points = Some(0);
|
||||
fallback.signals.first_output_latency_ms = Some(100);
|
||||
let mut preferred = candidate("acct_preferred", 0, Some(100));
|
||||
preferred.signals.failure_rate_basis_points = Some(0);
|
||||
preferred.signals.first_output_latency_ms = Some(15_001);
|
||||
preferred.signals.first_output_latency_ms = Some(120_000);
|
||||
let preferred_id = preferred.account.id().clone();
|
||||
let mut selection = context(RotationStrategy::Smart);
|
||||
selection.preferred_account = Some(preferred_id);
|
||||
@@ -192,9 +259,9 @@ fn smart_selector_should_escape_preferred_account_after_latency_threshold() {
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("fallback candidate available");
|
||||
.expect("preferred candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_preferred");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -210,7 +277,16 @@ fn smart_selector_should_escape_preferred_account_at_concurrency_limit() {
|
||||
.select(&candidates, &selection)
|
||||
.expect("fallback candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(
|
||||
(
|
||||
selected.candidate().account.id().as_str(),
|
||||
selected.preferred(),
|
||||
),
|
||||
(
|
||||
"acct_fallback",
|
||||
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::ConcurrencyLimit),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -296,7 +372,10 @@ fn smart_selector_should_use_provider_quota_rank_after_load_is_equal() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_high_quota");
|
||||
assert_eq!(
|
||||
selected.candidate().account.id().as_str(),
|
||||
"acct_high_quota"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -312,7 +391,7 @@ fn quota_reset_selector_should_prefer_known_earliest_window() {
|
||||
.select(&candidates, &context(RotationStrategy::QuotaResetPriority))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_earlier");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_earlier");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -325,7 +404,7 @@ fn smart_selector_should_prefer_known_quota_over_unknown_after_load_is_equal() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_known");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_known");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -338,7 +417,7 @@ fn selector_should_reject_account_at_concurrency_limit() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_available");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_available");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -360,7 +439,7 @@ fn selector_should_enforce_request_interval_until_boundary() {
|
||||
.select(&candidates, &context)
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_ready");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_ready");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -377,7 +456,7 @@ fn sticky_selector_should_fall_back_when_requested_account_is_excluded() {
|
||||
.select(&candidates, &context)
|
||||
.expect("fallback candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_fallback");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -397,11 +476,38 @@ fn round_robin_selector_should_be_stable_for_unsorted_candidates() {
|
||||
.expect("second candidate available");
|
||||
|
||||
assert_eq!(
|
||||
(first.account.id().as_str(), second.account.id().as_str()),
|
||||
(
|
||||
first.candidate().account.id().as_str(),
|
||||
second.candidate().account.id().as_str(),
|
||||
),
|
||||
("acct_first", "acct_second")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_robin_selector_should_honor_a_schedulable_preferred_account_before_the_cursor() {
|
||||
let candidates = vec![
|
||||
candidate("acct_first", 0, None),
|
||||
candidate("acct_second", 0, None),
|
||||
];
|
||||
let mut context = context(RotationStrategy::RoundRobin);
|
||||
context.round_robin_cursor = 0;
|
||||
context.preferred_account =
|
||||
Some(ProviderAccountId::new("acct_second").expect("preferred account"));
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.expect("preferred account available");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
selected.candidate().account.id().as_str(),
|
||||
selected.preferred(),
|
||||
),
|
||||
("acct_second", PreferredAccountSelection::Hit)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_should_honor_request_local_exclusion() {
|
||||
let candidates = vec![
|
||||
@@ -416,7 +522,7 @@ fn selector_should_honor_request_local_exclusion() {
|
||||
.select(&candidates, &context)
|
||||
.expect("second account available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_second");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -432,7 +538,7 @@ fn sticky_selector_should_prefer_requested_account() {
|
||||
.select(&candidates, &context)
|
||||
.expect("sticky account available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_second");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -447,7 +553,7 @@ fn sticky_selector_without_request_binding_should_reuse_latest_account() {
|
||||
.select(&candidates, &context(RotationStrategy::Sticky))
|
||||
.expect("sticky candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_latest");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_latest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -462,7 +568,7 @@ fn round_robin_selector_should_use_frozen_cursor() {
|
||||
.select(&candidates, &context)
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_second");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4,10 +4,11 @@ use std::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
time::{Duration, Instant, SystemTime},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{executor::block_on, future::BoxFuture};
|
||||
use gateway_core::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionError, ClientAdmissionPort, ClientAdmissionRecovery,
|
||||
@@ -20,21 +21,28 @@ use gateway_core::engine::credential::{
|
||||
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
ClientApiKeyUsageSink, DefaultExecutionService, ExecutionService, ProviderCircuitDecision,
|
||||
ProviderCircuitError, ProviderCircuitPort, provider_failure_affects_circuit,
|
||||
ClientApiKeyUsageSink, ClientTransport, DefaultExecutionService, ExecutionRequestMetadata,
|
||||
ExecutionService, ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
StartExecution, provider_failure_affects_circuit,
|
||||
};
|
||||
use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest};
|
||||
use gateway_core::engine::provider::ProviderRegistry;
|
||||
use gateway_core::engine::provider::{
|
||||
Provider, ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRegistry,
|
||||
ProviderRequest, ProviderStream,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest, RecoveryReport, UpstreamSendState,
|
||||
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamResponse, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError,
|
||||
StoreErrorKind,
|
||||
};
|
||||
use gateway_core::error::{GatewayErrorKind, ProviderErrorKind, StoreError};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, RuntimeSnapshot,
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
use serde_json::json;
|
||||
@@ -78,6 +86,112 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() {
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_failures_should_be_observable_without_a_model_request_row() {
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let providers = ProviderRegistry::new([Arc::new(FailingProvider) as Arc<dyn Provider>])
|
||||
.expect("provider registry");
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(probe_snapshot()),
|
||||
store.clone(),
|
||||
providers,
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
|
||||
let error = block_on(service.probe(AccountProbeRequest {
|
||||
account_id: ProviderAccountId::new("acct_probe").expect("account ID"),
|
||||
provider_kind: ProviderKind::new("openai").expect("provider kind"),
|
||||
upstream_model: UpstreamModelId::new("gpt-probe").expect("model ID"),
|
||||
operation: probe_operation(),
|
||||
}))
|
||||
.expect_err("the provider rejects every probe");
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
let upstream = error
|
||||
.upstream_response()
|
||||
.expect("probe must preserve its request-local upstream response");
|
||||
assert_eq!(upstream.status(), 502);
|
||||
assert_eq!(
|
||||
upstream.content_type(),
|
||||
Some(b"application/json".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
upstream.body(),
|
||||
&Bytes::from_static(br#"{"error":{"message":"source upstream failure"}}"#),
|
||||
);
|
||||
assert!(!format!("{error:?}").contains("source upstream failure"));
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
assert_eq!(store.probe_failures(), vec!["transport".to_owned()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_observation_store_failure_preserves_the_provider_error() {
|
||||
let store = Arc::new(TrackingExecutionStore {
|
||||
fail_probe_observation: AtomicBool::new(true),
|
||||
..TrackingExecutionStore::default()
|
||||
});
|
||||
let providers = ProviderRegistry::new([Arc::new(FailingProvider) as Arc<dyn Provider>])
|
||||
.expect("provider registry");
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(probe_snapshot()),
|
||||
store.clone(),
|
||||
providers,
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
|
||||
let error = block_on(service.probe(AccountProbeRequest {
|
||||
account_id: ProviderAccountId::new("acct_probe").expect("account ID"),
|
||||
provider_kind: ProviderKind::new("openai").expect("provider kind"),
|
||||
upstream_model: UpstreamModelId::new("gpt-probe").expect("model ID"),
|
||||
operation: probe_operation(),
|
||||
}))
|
||||
.expect_err("the provider error must survive observation failure");
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
assert!(store.probe_failures().is_empty());
|
||||
}
|
||||
|
||||
struct FailingProvider;
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for FailingProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
"openai"
|
||||
}
|
||||
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
_request: ProviderRequest,
|
||||
_context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
502,
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from_static(br#"{"error":{"message":"source upstream failure"}}"#),
|
||||
)),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_authentication_should_record_client_key_usage() {
|
||||
let usage = Arc::new(RecordingClientApiKeyUsage::default());
|
||||
@@ -98,6 +212,111 @@ fn successful_authentication_should_record_client_key_usage() {
|
||||
assert_eq!(usage.recorded(), vec!["key_usage_test".to_owned()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_store_failure_should_fail_open_during_request_start() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(FailingDecisionCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
|
||||
let started = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::new("gpt-start").expect("public model"),
|
||||
operation: start_operation(),
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("recoverable circuit state must not reject the request");
|
||||
|
||||
assert!(!started.session.is_finalized());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slow_circuit_store_should_time_out_and_fail_open_during_request_start() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(PendingDecisionCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
let started_at = Instant::now();
|
||||
|
||||
let started = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::new("gpt-start").expect("public model"),
|
||||
operation: start_operation(),
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("slow recoverable circuit state must not reject the request");
|
||||
|
||||
assert!(started_at.elapsed() < Duration::from_secs(2));
|
||||
assert!(!started.session.is_finalized());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
let model = "gpt-future-not-in-catalog";
|
||||
|
||||
let started = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::from_client_wire(model).expect("client model"),
|
||||
operation: start_operation_for_model(model),
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("stale model catalog must not reject the request");
|
||||
|
||||
assert!(!started.session.is_finalized());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingClientApiKeyUsage {
|
||||
ids: Mutex<Vec<String>>,
|
||||
@@ -121,12 +340,21 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage {
|
||||
#[derive(Default)]
|
||||
struct TrackingExecutionStore {
|
||||
touched: AtomicBool,
|
||||
probe_failures: Mutex<Vec<String>>,
|
||||
fail_probe_observation: AtomicBool,
|
||||
}
|
||||
|
||||
impl TrackingExecutionStore {
|
||||
fn touch(&self) {
|
||||
self.touched.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn probe_failures(&self) -> Vec<String> {
|
||||
self.probe_failures
|
||||
.lock()
|
||||
.expect("probe failures lock")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -170,6 +398,17 @@ impl ExecutionStore for TrackingExecutionStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
if self.fail_probe_observation.load(Ordering::SeqCst) {
|
||||
return Err(StoreError::new(StoreErrorKind::Unavailable));
|
||||
}
|
||||
self.probe_failures
|
||||
.lock()
|
||||
.expect("probe failures lock")
|
||||
.push(failure.error.kind().as_str().to_owned());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finalize_model_request(&self, _: ModelRequestFinalization) -> Result<(), StoreError> {
|
||||
self.touch();
|
||||
Ok(())
|
||||
@@ -232,6 +471,56 @@ impl ProviderCircuitPort for UnusedCircuits {
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingDecisionCircuits;
|
||||
|
||||
impl ProviderCircuitPort for FailingDecisionCircuits {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<ProviderCircuitDecision, ProviderCircuitError>> {
|
||||
Box::pin(async { Err(ProviderCircuitError) })
|
||||
}
|
||||
|
||||
fn observe_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn observe_success<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
struct PendingDecisionCircuits;
|
||||
|
||||
impl ProviderCircuitPort for PendingDecisionCircuits {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<ProviderCircuitDecision, ProviderCircuitError>> {
|
||||
Box::pin(futures::future::pending())
|
||||
}
|
||||
|
||||
fn observe_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn observe_success<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
struct UnusedContinuation;
|
||||
|
||||
impl NativeContinuationPort for UnusedContinuation {
|
||||
@@ -294,6 +583,34 @@ fn client_snapshot() -> RuntimeSnapshot {
|
||||
.expect("client snapshot")
|
||||
}
|
||||
|
||||
fn start_snapshot() -> RuntimeSnapshot {
|
||||
let provider = ProviderKind::new("openai").expect("provider kind");
|
||||
let capabilities =
|
||||
ModelCapabilities::new(BTreeSet::from([OperationKind::Generate]), Some(16_000));
|
||||
RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
std::num::NonZeroU32::new(1).expect("concurrency"),
|
||||
Duration::from_millis(1),
|
||||
),
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-start").expect("model ID"),
|
||||
capabilities,
|
||||
)],
|
||||
vec![ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_start_test").expect("client API key ID"),
|
||||
PlaintextClientApiKey::new("sk_start_test").expect("plaintext client API key"),
|
||||
provider,
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
)
|
||||
.expect("start snapshot")
|
||||
}
|
||||
|
||||
fn probe_operation() -> Operation {
|
||||
let body = json!({
|
||||
"model": "gpt-probe",
|
||||
@@ -304,3 +621,18 @@ fn probe_operation() -> Operation {
|
||||
.expect("OpenAI payload"),
|
||||
))
|
||||
}
|
||||
|
||||
fn start_operation() -> Operation {
|
||||
start_operation_for_model("gpt-start")
|
||||
}
|
||||
|
||||
fn start_operation_for_model(model: &str) -> Operation {
|
||||
let body = json!({
|
||||
"model": model,
|
||||
"input": [{"type": "message", "role": "user", "content": "ping"}],
|
||||
});
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object("openai", body.as_object().expect("request object").clone())
|
||||
.expect("OpenAI payload"),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
use bytes::Bytes;
|
||||
use gateway_core::engine::UpstreamSendState;
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, GatewayError, ProviderError, ProviderErrorKind, SafeUpstreamValue,
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, GatewayError, OpaqueUpstreamValue,
|
||||
ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn provider_error_debug_should_not_expose_sensitive_context() {
|
||||
@@ -15,13 +19,22 @@ fn provider_error_debug_should_not_expose_sensitive_context() {
|
||||
#[test]
|
||||
fn provider_error_debug_should_not_print_classified_upstream_values() {
|
||||
let diagnostic = "request-visible-only-through-explicit-accessor";
|
||||
let value = SafeUpstreamValue::new(diagnostic).expect("test diagnostic is valid");
|
||||
let value = OpaqueUpstreamValue::new(diagnostic);
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_upstream_request_id(value);
|
||||
|
||||
assert!(!format!("{error:?}").contains(diagnostic));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_upstream_value_should_preserve_arbitrary_text_without_logging_it() {
|
||||
let original = format!("\0{}\n", "x".repeat(9_000));
|
||||
let value = OpaqueUpstreamValue::new(original.clone());
|
||||
|
||||
assert_eq!(value.as_str(), original);
|
||||
assert!(!format!("{value:?}").contains(&original));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_replay_proof_should_default_to_false() {
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent);
|
||||
@@ -37,18 +50,82 @@ fn provider_error_replay_proof_should_be_explicit() {
|
||||
assert!(error.replay_is_safe());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_atomic_client_events_should_be_take_only_and_debug_redacted() {
|
||||
let marker = "atomic-wire-must-not-enter-debug";
|
||||
let event = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({"type": "response.failed", "message": marker}),
|
||||
)
|
||||
.expect("atomic wire"),
|
||||
);
|
||||
let mut error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_atomic_client_events(vec![event]);
|
||||
|
||||
assert!(error.has_atomic_client_events());
|
||||
assert!(!format!("{error:?}").contains(marker));
|
||||
assert_eq!(error.take_atomic_client_events().len(), 1);
|
||||
assert!(!error.has_atomic_client_events());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_clone_should_drop_request_local_upstream_response() {
|
||||
let marker = Bytes::from_static(b"raw-client-response-only");
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
429,
|
||||
Some(b"application/problem+json".to_vec()),
|
||||
marker.clone(),
|
||||
)
|
||||
.with_headers(vec![ProviderResponseHeader::new(
|
||||
"x-future-error",
|
||||
Bytes::from_static(b"opaque-header-value"),
|
||||
)]),
|
||||
);
|
||||
|
||||
let cloned = error.clone();
|
||||
|
||||
assert_eq!(
|
||||
error
|
||||
.client_visible_upstream_response()
|
||||
.map(ClientVisibleUpstreamResponse::body),
|
||||
Some(&marker)
|
||||
);
|
||||
assert!(cloned.client_visible_upstream_response().is_none());
|
||||
assert!(!format!("{error:?}").contains("raw-client-response-only"));
|
||||
assert!(!format!("{error:?}").contains("opaque-header-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_clone_should_drop_atomic_client_events() {
|
||||
let event = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({"type": "response.failed"}),
|
||||
)
|
||||
.expect("atomic wire"),
|
||||
);
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_atomic_client_events(vec![event]);
|
||||
let cloned = error.clone();
|
||||
|
||||
assert!(error.has_atomic_client_events());
|
||||
assert!(!cloned.has_atomic_client_events());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnostics() {
|
||||
let message = "Your Codex quota is exhausted";
|
||||
let error = ProviderError::new(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
message,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
);
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
message,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
));
|
||||
let gateway = GatewayError::from_provider(&error);
|
||||
|
||||
assert_eq!(
|
||||
@@ -60,3 +137,20 @@ fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnost
|
||||
assert_eq!(gateway.client_error_type(), Some("rate_limit_error"));
|
||||
assert!(!format!("{gateway:?}").contains(message));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
|
||||
let message = format!("\0{}\n", "m".repeat(9_000));
|
||||
let code = format!("\0{}", "c".repeat(300));
|
||||
let error_type = String::new();
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
message.clone(),
|
||||
Some(code.clone()),
|
||||
Some(error_type.clone()),
|
||||
);
|
||||
|
||||
assert_eq!(detail.message(), message);
|
||||
assert_eq!(detail.code(), Some(code.as_str()));
|
||||
assert_eq!(detail.error_type(), Some(error_type.as_str()));
|
||||
assert!(!format!("{detail:?}").contains(&message));
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
use bytes::Bytes;
|
||||
use gateway_core::engine::provider::UpstreamTransport;
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, EventSequenceError, EventSequenceValidator, GatewayEvent,
|
||||
ProtocolWireEvent, ProviderEvent, ProviderResponseObservation, ReasoningDelta, ResponseMeta,
|
||||
TextDelta, ToolCallDelta,
|
||||
ProtocolWireEvent, ProviderEvent, ProviderResponseHeader, ProviderResponseObservation,
|
||||
ReasoningDelta, ResponseMeta, TextDelta, ToolCallDelta,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -21,6 +22,28 @@ fn provider_event_keeps_large_wire_and_observation_payloads_behind_one_indirecti
|
||||
assert!(!observation.has_client_event());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_response_header_should_preserve_opaque_name_and_bytes_without_debug_disclosure() {
|
||||
let name = format!("x-future-{}\0", "n".repeat(512));
|
||||
let value = Bytes::from_static(b"\xffopaque-response-header-secret");
|
||||
let header = ProviderResponseHeader::new(name.clone(), value.clone());
|
||||
|
||||
assert_eq!(header.name(), name);
|
||||
assert_eq!(header.value(), &value);
|
||||
let rendered = format!("{header:?}");
|
||||
assert!(!rendered.contains(&name));
|
||||
assert!(!rendered.contains("opaque-response-header-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_observed_service_tier_should_be_ignored() {
|
||||
let observation =
|
||||
ProviderResponseObservation::new(UpstreamTransport::new("http_sse").expect("transport"))
|
||||
.with_service_tier_if_valid(format!("priority\0{}", "x".repeat(128)));
|
||||
|
||||
assert_eq!(observation.service_tier(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_wire_event_should_preserve_sse_metadata_without_exposing_id_in_debug() {
|
||||
let wire = ProtocolWireEvent::json_with_sse_metadata(
|
||||
@@ -37,6 +60,23 @@ fn protocol_wire_event_should_preserve_sse_metadata_without_exposing_id_in_debug
|
||||
assert!(!format!("{wire:?}").contains("upstream-event-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_wire_event_should_preserve_opaque_sse_metadata() {
|
||||
let event_type = format!("\0{}", "event".repeat(80));
|
||||
let sse_id = "id\0with\r\ncontrols".to_owned();
|
||||
let wire = ProtocolWireEvent::json_with_sse_metadata(
|
||||
"openai",
|
||||
Some(event_type.clone()),
|
||||
json!({"type": "future.event"}),
|
||||
Some(sse_id.clone()),
|
||||
None,
|
||||
)
|
||||
.expect("internal protocol name is valid");
|
||||
|
||||
assert_eq!(wire.event_type(), Some(event_type.as_str()));
|
||||
assert_eq!(wire.sse_id(), Some(sse_id.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validator_should_accept_started_content_delta_completed_sequence() {
|
||||
let mut validator = EventSequenceValidator::new();
|
||||
|
||||
@@ -57,7 +57,7 @@ fn refresh_policy_requires_a_positive_margin() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_schedule_is_stable_and_never_uses_query_time_margin() {
|
||||
fn refresh_schedule_should_use_the_exact_configured_margin() {
|
||||
let account = ProviderAccountId::new("acct_stable").expect("valid account");
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
@@ -70,13 +70,7 @@ fn refresh_schedule_is_stable_and_never_uses_query_time_margin() {
|
||||
let first = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh");
|
||||
let second = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh again");
|
||||
|
||||
assert_eq!(first, second);
|
||||
assert!(first > observed_at);
|
||||
assert!(first < expires_at);
|
||||
assert_eq!(first, expires_at - Duration::from_secs(3_600));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -190,21 +190,36 @@ fn global_mapping_should_apply_to_every_provider() {
|
||||
#[test]
|
||||
fn unmapped_model_should_pass_through_unchanged() {
|
||||
let snapshot = snapshot();
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("future-openai-model").expect("model"),
|
||||
&operation(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("unknown model remains transparent");
|
||||
for requested in [
|
||||
"future-openai-model".to_owned(),
|
||||
format!("future-{}", "x".repeat(512)),
|
||||
"future\0model".to_owned(),
|
||||
" future-openai-model ".to_owned(),
|
||||
] {
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::from_client_wire(requested.clone()).expect("client model"),
|
||||
&operation(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("unknown model remains transparent");
|
||||
|
||||
assert_eq!(
|
||||
plan.candidates()[0].upstream_model().as_str(),
|
||||
"future-openai-model"
|
||||
);
|
||||
assert_eq!(plan.candidates()[0].upstream_model().as_str(), requested);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_mapping_should_use_exact_client_keys() {
|
||||
let snapshot = snapshot().with_model_mappings(BTreeMap::from([(
|
||||
" exact-alias ".to_owned(),
|
||||
"gpt-5.5".to_owned(),
|
||||
)]));
|
||||
|
||||
assert_eq!(snapshot.mapped_model(" exact-alias "), "gpt-5.5");
|
||||
assert_eq!(snapshot.mapped_model("exact-alias"), "exact-alias");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -24,8 +24,8 @@ use gateway_core::routing::snapshot::{
|
||||
SnapshotSubscriptionError, SnapshotSubscriptionPort, runtime_revision_needs_refresh,
|
||||
};
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, PublicModelId, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
ConfigRevision, ModelCapabilities, ModelPresentation, ProviderKind, PublicModelId,
|
||||
RuntimeSnapshot, UpstreamModelId,
|
||||
};
|
||||
use gateway_core::task::WorkerKind;
|
||||
|
||||
@@ -135,13 +135,19 @@ impl Provider for PublishingCatalogProvider {
|
||||
if self.queries.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
self.generation.store(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(vec![ProviderModelCapabilities::new(
|
||||
UpstreamModelId::new("upstream-model").expect("model"),
|
||||
ModelCapabilities::new(
|
||||
std::collections::BTreeSet::from([OperationKind::Generate]),
|
||||
Ok(vec![
|
||||
ProviderModelCapabilities::new(
|
||||
UpstreamModelId::new("upstream-model").expect("model"),
|
||||
ModelCapabilities::new(
|
||||
std::collections::BTreeSet::from([OperationKind::Generate]),
|
||||
None,
|
||||
),
|
||||
)
|
||||
.with_presentation(ModelPresentation::new(
|
||||
Some("Upstream Model".to_owned()),
|
||||
None,
|
||||
),
|
||||
)])
|
||||
)),
|
||||
])
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
@@ -215,6 +221,15 @@ fn compiler_retries_when_provider_publishes_catalog_during_compilation() {
|
||||
.map(|generation| generation.get()),
|
||||
Some(1),
|
||||
);
|
||||
let profiles =
|
||||
snapshot.public_model_profiles_for_provider(&ProviderKind::new("alpha").expect("provider"));
|
||||
assert_eq!(
|
||||
profiles
|
||||
.iter()
|
||||
.map(|profile| profile.model().as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["public-model", "upstream-model"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -28,7 +28,6 @@ tokio = { version = "1.52.3", default-features = false, features = ["macros", "r
|
||||
tracing = "0.1.44"
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
filetime = "0.2.27"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
pub mod config;
|
||||
mod logging;
|
||||
mod serve;
|
||||
pub mod serve;
|
||||
pub mod system_update;
|
||||
pub mod workers;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! HTTP 监听、OS signal、原子连接注册与优雅 drain。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Router;
|
||||
@@ -12,7 +12,12 @@ use tokio::sync::Notify;
|
||||
const DRAINING_BIT: usize = 1usize << (usize::BITS - 1);
|
||||
const ACTIVE_MASK: usize = !DRAINING_BIT;
|
||||
|
||||
pub(crate) struct ConnectionTracker {
|
||||
/// 自重启交接没有显式握手:替换进程按 CPR_RESTART_DELAY_MS 估算等待,
|
||||
/// 可能在旧进程释放监听端口前尝试绑定,因此对 AddrInUse 保留有限重试窗口。
|
||||
const BIND_RETRY_WINDOW: Duration = Duration::from_secs(10);
|
||||
const BIND_RETRY_MAX_DELAY: Duration = Duration::from_secs(1);
|
||||
|
||||
pub struct ConnectionTracker {
|
||||
state: Arc<ConnectionState>,
|
||||
cancellation: CancellationToken,
|
||||
}
|
||||
@@ -23,7 +28,8 @@ struct ConnectionState {
|
||||
}
|
||||
|
||||
impl ConnectionTracker {
|
||||
pub(crate) fn new(cancellation: CancellationToken) -> Self {
|
||||
#[must_use]
|
||||
pub fn new(cancellation: CancellationToken) -> Self {
|
||||
Self {
|
||||
state: Arc::new(ConnectionState {
|
||||
value: AtomicUsize::new(0),
|
||||
@@ -41,10 +47,15 @@ impl ConnectionTracker {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_idle(&self, timeout: Duration) {
|
||||
pub async fn wait_until_idle(&self, timeout: Duration) {
|
||||
let wait = async {
|
||||
loop {
|
||||
let notified = self.state.idle.notified();
|
||||
tokio::pin!(notified);
|
||||
// notify_waiters 不保存许可:必须先 enable 注册等待者,再读
|
||||
// 活跃计数;否则最后一个 guard 在读数与首次 poll 之间 drop
|
||||
// 时唤醒丢失,只能等满整个超时。
|
||||
notified.as_mut().enable();
|
||||
if self.state.value.load(Ordering::Acquire) & ACTIVE_MASK == 0 {
|
||||
return;
|
||||
}
|
||||
@@ -102,6 +113,27 @@ impl Drop for ActiveConnection {
|
||||
}
|
||||
}
|
||||
|
||||
/// 绑定失败即进程退出、服务彻底离线,因此 AddrInUse(典型为自重启交接
|
||||
/// 时旧进程尚未关闭 listener)在窗口内指数退避重试;其他错误立即上抛。
|
||||
pub async fn bind_listener(address: &str) -> std::io::Result<tokio::net::TcpListener> {
|
||||
let deadline = tokio::time::Instant::now() + BIND_RETRY_WINDOW;
|
||||
let mut delay = Duration::from_millis(100);
|
||||
loop {
|
||||
match tokio::net::TcpListener::bind(address).await {
|
||||
Ok(listener) => return Ok(listener),
|
||||
Err(error)
|
||||
if error.kind() == std::io::ErrorKind::AddrInUse
|
||||
&& tokio::time::Instant::now() < deadline =>
|
||||
{
|
||||
tracing::warn!(target: "gateway_startup", address, "监听地址仍被占用,退避后重试绑定");
|
||||
tokio::time::sleep(delay).await;
|
||||
delay = (delay * 2).min(BIND_RETRY_MAX_DELAY);
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn serve_router(
|
||||
router: Router,
|
||||
host: &str,
|
||||
@@ -110,7 +142,7 @@ pub(crate) async fn serve_router(
|
||||
connections: Arc<ConnectionTracker>,
|
||||
drain_timeout: Duration,
|
||||
) -> Result<(), ServeError> {
|
||||
let listener = tokio::net::TcpListener::bind(format!("{host}:{port}"))
|
||||
let listener = bind_listener(&format!("{host}:{port}"))
|
||||
.await
|
||||
.map_err(ServeError::Bind)?;
|
||||
tracing::info!(target: "gateway_startup", host, port, "网关开始监听");
|
||||
@@ -121,16 +153,44 @@ pub(crate) async fn serve_router(
|
||||
wait_for_shutdown(&shutdown_cancellation).await;
|
||||
shutdown_connections.begin_draining();
|
||||
};
|
||||
axum::serve(
|
||||
// axum 的优雅关闭会无限等待存量连接结束,而 SSE 等长响应体不观察
|
||||
// 取消信号:慢客户端可以把关闭拖住直到被 SIGKILL。整个 drain(axum
|
||||
// 优雅关闭 + 游离 guard 等待)共享同一个从取消信号起算的绝对截止点,
|
||||
// 逾期即放弃等待,存量连接随进程退出终止。
|
||||
let drain_deadline_at = Arc::new(OnceLock::new());
|
||||
let drain_deadline = {
|
||||
let cancellation = cancellation.clone();
|
||||
let deadline_at = Arc::clone(&drain_deadline_at);
|
||||
async move {
|
||||
cancellation.cancelled().await;
|
||||
let deadline = tokio::time::Instant::now() + drain_timeout;
|
||||
let _ = deadline_at.set(deadline);
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
}
|
||||
};
|
||||
let serve = axum::serve(
|
||||
listener,
|
||||
router.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
)
|
||||
.with_graceful_shutdown(shutdown)
|
||||
.await
|
||||
.map_err(ServeError::Serve)?;
|
||||
|
||||
connections.begin_draining();
|
||||
connections.wait_until_idle(drain_timeout).await;
|
||||
.with_graceful_shutdown(shutdown);
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = serve => {
|
||||
result.map_err(ServeError::Serve)?;
|
||||
connections.begin_draining();
|
||||
let remaining = drain_deadline_at.get().map_or(drain_timeout, |deadline| {
|
||||
deadline.saturating_duration_since(tokio::time::Instant::now())
|
||||
});
|
||||
connections.wait_until_idle(remaining).await;
|
||||
}
|
||||
() = drain_deadline => {
|
||||
tracing::warn!(
|
||||
target: "gateway_shutdown",
|
||||
timeout_secs = drain_timeout.as_secs(),
|
||||
"优雅 drain 超时,放弃等待存量连接"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ pub(crate) async fn download_file(
|
||||
.map_err(|error| super::internal(format!("failed to create download file: {error}")))?;
|
||||
let mut stream = response.bytes_stream();
|
||||
let mut downloaded = 0_u64;
|
||||
let mut last_percent = 0_u8;
|
||||
let mut next_progress = 10_u8;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|error| upstream(format!("download stream failed: {error}")))?;
|
||||
downloaded = downloaded.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
|
||||
@@ -59,9 +59,17 @@ pub(crate) async fn download_file(
|
||||
file.write_all(&chunk)
|
||||
.map_err(|error| super::internal(format!("failed to write download: {error}")))?;
|
||||
let percent = download_percent(downloaded, expected_size);
|
||||
if percent > last_percent {
|
||||
last_percent = percent;
|
||||
events.emit_progress(operation_id, "downloading release archive", percent);
|
||||
if percent >= next_progress || downloaded == expected_size {
|
||||
events.emit_progress(
|
||||
operation_id,
|
||||
&format!(
|
||||
"已下载 {} / {} ({percent}%)",
|
||||
format_bytes(downloaded),
|
||||
format_bytes(expected_size)
|
||||
),
|
||||
percent,
|
||||
);
|
||||
next_progress = percent.saturating_add(10);
|
||||
}
|
||||
}
|
||||
file.sync_all()
|
||||
@@ -155,3 +163,15 @@ fn download_percent(downloaded: u64, total: u64) -> u8 {
|
||||
)
|
||||
.unwrap_or(100)
|
||||
}
|
||||
|
||||
pub(crate) fn format_bytes(bytes: u64) -> String {
|
||||
const KIB: u64 = 1024;
|
||||
const MIB: u64 = KIB * 1024;
|
||||
if bytes >= MIB {
|
||||
return format!("{:.1} MiB", bytes as f64 / MIB as f64);
|
||||
}
|
||||
if bytes >= KIB {
|
||||
return format!("{:.1} KiB", bytes as f64 / KIB as f64);
|
||||
}
|
||||
format!("{bytes} B")
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ use self::download::{MAX_CHECKSUM_SIZE, MAX_DOWNLOAD_SIZE, download_file, verify
|
||||
use self::process::{environment_value, spawn_replacement};
|
||||
use self::release::{
|
||||
ReleaseCache, confirmed_target, detail_from_release, fetch_latest, select_archive,
|
||||
versions_cross_major,
|
||||
};
|
||||
use self::state::{
|
||||
OperationFileLock, UpdateTempDir, finish, operation_id, read_status, set_running,
|
||||
@@ -226,14 +227,32 @@ impl ProcessSystemOperations {
|
||||
return Err(conflict(reason));
|
||||
}
|
||||
let target = confirmed_target(target_version)?;
|
||||
if versions_cross_major(&self.config.version, &target).unwrap_or(false) {
|
||||
let reason = "跨大版本升级不提供自动更新,请按发布说明手动迁移";
|
||||
self.events.error_terminal(None, Some("preflight"), reason);
|
||||
return Err(conflict(reason));
|
||||
}
|
||||
let repository = self
|
||||
.config
|
||||
.update_repository
|
||||
.as_deref()
|
||||
.ok_or_else(|| conflict("update repository is not configured"))?;
|
||||
self.events
|
||||
.info(None, Some("release"), "fetching latest release");
|
||||
let release = fetch_latest(&self.config.github_api_base, repository).await?;
|
||||
.info(None, Some("release"), "正在获取最新 Release 信息");
|
||||
let release = match fetch_latest(
|
||||
&self.config.github_api_base,
|
||||
repository,
|
||||
&self.config.update_channel,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(release) => release,
|
||||
Err(error) => {
|
||||
self.events
|
||||
.error_terminal(None, Some("release"), error.to_string());
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let detail = detail_from_release(&self.config, &release);
|
||||
if detail.latest_version != target {
|
||||
self.events.warning_terminal(
|
||||
@@ -258,12 +277,17 @@ impl ProcessSystemOperations {
|
||||
Some(&target),
|
||||
&self.config.version,
|
||||
)?;
|
||||
self.events.info(
|
||||
Some(&operation_id),
|
||||
Some("prepare"),
|
||||
format!("准备更新到 v{target}"),
|
||||
);
|
||||
let result = self.install_release(&release, &target, &operation_id).await;
|
||||
match &result {
|
||||
Ok(()) => self.events.success_terminal(
|
||||
Some(&operation_id),
|
||||
Some("done"),
|
||||
"release files replaced",
|
||||
"更新文件已替换,等待服务重启生效",
|
||||
),
|
||||
Err(error) => {
|
||||
self.events
|
||||
@@ -294,7 +318,23 @@ impl ProcessSystemOperations {
|
||||
version: &str,
|
||||
operation_id: &str,
|
||||
) -> Result<(), OperationError> {
|
||||
self.events.info(
|
||||
Some(operation_id),
|
||||
Some("asset"),
|
||||
"正在选择匹配当前平台的更新包",
|
||||
);
|
||||
let archive = select_archive(release, version)?;
|
||||
self.events.info(
|
||||
Some(operation_id),
|
||||
Some("asset"),
|
||||
format!(
|
||||
"已选择更新包 {} ({})",
|
||||
archive.name,
|
||||
download::format_bytes(archive.size)
|
||||
),
|
||||
);
|
||||
self.events
|
||||
.info(Some(operation_id), Some("verify"), "正在校验更新资源");
|
||||
if archive.size == 0 || archive.size > MAX_DOWNLOAD_SIZE {
|
||||
return Err(invalid("release archive size is invalid"));
|
||||
}
|
||||
@@ -306,12 +346,16 @@ impl ProcessSystemOperations {
|
||||
if checksum.size == 0 || checksum.size > MAX_CHECKSUM_SIZE {
|
||||
return Err(invalid("release checksum size is invalid"));
|
||||
}
|
||||
self.events
|
||||
.info(Some(operation_id), Some("prepare"), "正在创建临时更新目录");
|
||||
fs::create_dir_all(&self.config.update_temp_dir)
|
||||
.map_err(|error| internal(format!("failed to prepare update temp dir: {error}")))?;
|
||||
let temp_root = fs::canonicalize(&self.config.update_temp_dir)
|
||||
.map_err(|error| internal(format!("failed to resolve update temp dir: {error}")))?;
|
||||
let temp = UpdateTempDir::create(&temp_root)?;
|
||||
let archive_path = temp.path().join(&archive.name);
|
||||
self.events
|
||||
.info(Some(operation_id), Some("download"), "开始下载更新包");
|
||||
download_file(
|
||||
&archive.browser_download_url,
|
||||
&archive_path,
|
||||
@@ -321,6 +365,10 @@ impl ProcessSystemOperations {
|
||||
&self.events,
|
||||
)
|
||||
.await?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("download"), "更新包下载完成");
|
||||
self.events
|
||||
.info(Some(operation_id), Some("checksum"), "正在校验 checksum");
|
||||
verify_checksum(
|
||||
&archive_path,
|
||||
&archive.name,
|
||||
@@ -329,12 +377,23 @@ impl ProcessSystemOperations {
|
||||
&self.config.github_api_base,
|
||||
)
|
||||
.await?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("checksum"), "checksum 校验通过");
|
||||
self.events
|
||||
.info(Some(operation_id), Some("extract"), "正在解压更新包");
|
||||
let extracted = extract_release(&archive_path, temp.path())?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("extract"), "更新包解压完成");
|
||||
self.events
|
||||
.info(Some(operation_id), Some("replace"), "正在替换应用文件");
|
||||
replace_release_files(
|
||||
&self.config.executable_path()?,
|
||||
&self.config.web_dist_dir,
|
||||
extracted,
|
||||
)
|
||||
)?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("replace"), "应用文件替换完成");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,6 +501,12 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
}
|
||||
|
||||
async fn restart(&self) -> Result<SystemOperationAccepted, OperationError> {
|
||||
// 与 update/rollback 互斥:更新替换文件期间触发自重启会让新进程
|
||||
// 载入半成品产物。
|
||||
let _operation = self
|
||||
.operation_lock
|
||||
.try_lock()
|
||||
.map_err(|_| conflict("system operation is already running"))?;
|
||||
if !self.config.self_restart_enabled {
|
||||
return Err(conflict("self restart is disabled"));
|
||||
}
|
||||
@@ -515,6 +580,17 @@ impl UpdateEvents {
|
||||
);
|
||||
}
|
||||
|
||||
fn success(&self, operation_id: Option<&str>, step: Option<&str>, message: impl Into<String>) {
|
||||
self.emit(
|
||||
SystemUpdateEventLevel::Success,
|
||||
operation_id,
|
||||
step,
|
||||
message,
|
||||
None,
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
fn warning_terminal(
|
||||
&self,
|
||||
operation_id: Option<&str>,
|
||||
|
||||
@@ -20,6 +20,8 @@ pub(crate) struct GitHubRelease {
|
||||
pub(crate) html_url: Option<String>,
|
||||
pub(crate) prerelease: bool,
|
||||
#[serde(default)]
|
||||
pub(crate) draft: bool,
|
||||
#[serde(default)]
|
||||
pub(crate) assets: Vec<GitHubAsset>,
|
||||
}
|
||||
|
||||
@@ -57,7 +59,7 @@ impl ReleaseCache {
|
||||
if !refresh && let Some(detail) = self.cached(&key).await {
|
||||
return Ok(detail);
|
||||
}
|
||||
match fetch_latest(&config.github_api_base, repository).await {
|
||||
match fetch_latest(&config.github_api_base, repository, &config.update_channel).await {
|
||||
Ok(release) => {
|
||||
let detail = detail_from_release(config, &release);
|
||||
*self.entry.lock().await = Some(CachedRelease {
|
||||
@@ -85,13 +87,27 @@ impl ReleaseCache {
|
||||
pub(crate) async fn fetch_latest(
|
||||
api_base: &str,
|
||||
repository: &str,
|
||||
update_channel: &str,
|
||||
) -> Result<GitHubRelease, OperationError> {
|
||||
validate_api_base(api_base).map_err(conflict)?;
|
||||
validate_repository(repository)?;
|
||||
let url = format!(
|
||||
"{}/{repository}/releases/latest",
|
||||
api_base.trim_end_matches('/')
|
||||
);
|
||||
let base = api_base.trim_end_matches('/');
|
||||
// GitHub 的 /releases/latest 永远只返回正式版;preview 渠道需要覆盖
|
||||
// prerelease,因此改查列表端点并取最新一条非草稿 Release。
|
||||
if update_channel == "preview" {
|
||||
let releases: Vec<GitHubRelease> =
|
||||
fetch_release_json(&format!("{base}/{repository}/releases?per_page=10")).await?;
|
||||
return releases
|
||||
.into_iter()
|
||||
.find(|release| !release.draft)
|
||||
.ok_or_else(|| upstream("no published release available"));
|
||||
}
|
||||
fetch_release_json(&format!("{base}/{repository}/releases/latest")).await
|
||||
}
|
||||
|
||||
async fn fetch_release_json<T: serde::de::DeserializeOwned>(
|
||||
url: &str,
|
||||
) -> Result<T, OperationError> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
@@ -111,7 +127,7 @@ pub(crate) async fn fetch_latest(
|
||||
)));
|
||||
}
|
||||
response
|
||||
.json::<GitHubRelease>()
|
||||
.json::<T>()
|
||||
.await
|
||||
.map_err(|error| upstream(format!("invalid GitHub release response: {error}")))
|
||||
}
|
||||
@@ -121,12 +137,13 @@ pub(crate) fn detail_from_release(
|
||||
release: &GitHubRelease,
|
||||
) -> SystemUpdateDetail {
|
||||
let latest = normalize_version(&release.tag_name);
|
||||
let available = (config.update_channel != "stable" || !release.prerelease)
|
||||
let newer = (config.update_channel != "stable" || !release.prerelease)
|
||||
&& version_is_newer(&config.version, &latest).unwrap_or(false);
|
||||
let cross_major = versions_cross_major(&config.version, &latest).unwrap_or(false);
|
||||
SystemUpdateDetail {
|
||||
current_version: config.version.clone(),
|
||||
latest_version: latest,
|
||||
has_update: available,
|
||||
latest_version: latest.clone(),
|
||||
has_update: newer && !cross_major,
|
||||
deployment_mode: config.deployment_mode.clone(),
|
||||
build_type: config.build_type.clone(),
|
||||
release_url: release.html_url.clone(),
|
||||
@@ -134,7 +151,9 @@ pub(crate) fn detail_from_release(
|
||||
cached: false,
|
||||
update_supported: config.update_support_error().is_none(),
|
||||
unsupported_reason: config.update_support_error(),
|
||||
warning: None,
|
||||
warning: (newer && cross_major).then(|| {
|
||||
format!("检测到新的大版本 v{latest}:跨大版本升级不提供自动更新,请按发布说明手动迁移")
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,6 +284,13 @@ fn version_is_newer(current: &str, latest: &str) -> Option<bool> {
|
||||
Some(latest > current)
|
||||
}
|
||||
|
||||
/// 自动更新只在同 major 内提供;跨大版本的配置与数据迁移契约不保证兼容。
|
||||
pub(crate) fn versions_cross_major(current: &str, latest: &str) -> Option<bool> {
|
||||
let current = semver::Version::parse(&normalize_version(current)).ok()?;
|
||||
let latest = semver::Version::parse(&normalize_version(latest)).ok()?;
|
||||
Some(current.major != latest.major)
|
||||
}
|
||||
|
||||
fn repository_character(character: char) -> bool {
|
||||
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.')
|
||||
}
|
||||
@@ -272,7 +298,6 @@ fn repository_character(character: char) -> bool {
|
||||
fn platform_os_aliases() -> &'static [&'static str] {
|
||||
match env::consts::OS {
|
||||
"macos" => &["macos", "darwin"],
|
||||
"windows" => &["windows", "win32"],
|
||||
"linux" => &["linux"],
|
||||
_ => &[env::consts::OS],
|
||||
}
|
||||
|
||||
@@ -108,22 +108,19 @@ impl WorkerSupervisor {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 先协作取消;超时后 abort 不合作任务,lease 依赖 TTL 兜底。
|
||||
/// 先协作取消;所有任务共享同一绝对截止点,逾期任务被 abort,
|
||||
/// lease 依赖 TTL 兜底。每个句柄恰好 join 一次:已在截止点前退出的
|
||||
/// 不会被二次 poll,逾期的在 abort 后等待其真正终止。
|
||||
pub async fn shutdown(&self, timeout: Duration) {
|
||||
self.cancellation.cancel();
|
||||
let mut handles = std::mem::take(&mut *lock_unpoisoned(&self.handles));
|
||||
let joined = tokio::time::timeout(timeout, async {
|
||||
for (_, handle) in &mut handles {
|
||||
let _ = handle.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok();
|
||||
if !joined {
|
||||
for (_, handle) in &handles {
|
||||
let handles = std::mem::take(&mut *lock_unpoisoned(&self.handles));
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
for (_, mut handle) in handles {
|
||||
if tokio::time::timeout_at(deadline, &mut handle)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
handle.abort();
|
||||
}
|
||||
for (_, handle) in handles {
|
||||
let _ = handle.await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::engine::CancellationToken;
|
||||
use gateway_core::lifecycle::ConnectionLifecycle as _;
|
||||
use gateway_host::HostBundle;
|
||||
use gateway_host::serve::{ConnectionTracker, bind_listener};
|
||||
|
||||
#[test]
|
||||
fn host_bundle_serve_is_a_consuming_process_entrypoint() {
|
||||
@@ -6,3 +12,83 @@ fn host_bundle_serve_is_a_consuming_process_entrypoint() {
|
||||
|
||||
assert_eq!(std::mem::size_of_val(&_serve), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_until_idle_should_return_immediately_without_active_connections() {
|
||||
let tracker = ConnectionTracker::new(CancellationToken::new());
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
tracker.wait_until_idle(Duration::from_secs(30)).await;
|
||||
|
||||
assert_eq!(started.elapsed(), Duration::ZERO);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_until_idle_should_give_up_at_timeout_while_connections_remain() {
|
||||
let tracker = ConnectionTracker::new(CancellationToken::new());
|
||||
let _guard = tracker.try_register().expect("register connection");
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
tracker.wait_until_idle(Duration::from_secs(30)).await;
|
||||
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(30));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_until_idle_should_wake_when_last_guard_drops_after_first_poll() {
|
||||
let tracker = ConnectionTracker::new(CancellationToken::new());
|
||||
let guard = tracker.try_register().expect("register connection");
|
||||
let mut wait = Box::pin(tracker.wait_until_idle(Duration::from_secs(30)));
|
||||
assert!(futures::poll!(wait.as_mut()).is_pending());
|
||||
|
||||
drop(guard);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), wait)
|
||||
.await
|
||||
.expect("woken by last guard drop instead of waiting out the timeout");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn wait_until_idle_should_observe_guard_drop_racing_the_idle_check() {
|
||||
// 回归约束:notified 必须在读取活跃计数前完成注册(enable),否则最后
|
||||
// 一个 guard 在计数检查与首次 poll 之间 drop 时唤醒丢失,只能等满整个
|
||||
// 超时。多线程反复交错,任何一次丢唤醒都会撞上 2s 超时并触发断言。
|
||||
for _ in 0..256 {
|
||||
let tracker = Arc::new(ConnectionTracker::new(CancellationToken::new()));
|
||||
let guard = tracker.try_register().expect("register connection");
|
||||
let waiter = tokio::spawn({
|
||||
let tracker = Arc::clone(&tracker);
|
||||
async move { tracker.wait_until_idle(Duration::from_secs(2)).await }
|
||||
});
|
||||
let dropper = std::thread::spawn(move || drop(guard));
|
||||
let started = std::time::Instant::now();
|
||||
waiter.await.expect("waiter completes");
|
||||
dropper.join().expect("dropper completes");
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_millis(1_900),
|
||||
"idle wakeup was lost and the waiter slept until the drain timeout"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bind_listener_should_retry_until_previous_listener_releases_port() {
|
||||
let holder = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("holder listener");
|
||||
let address = holder.local_addr().expect("holder address").to_string();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
drop(holder);
|
||||
});
|
||||
|
||||
let listener = tokio::time::timeout(Duration::from_secs(8), bind_listener(&address))
|
||||
.await
|
||||
.expect("bound within the retry window")
|
||||
.expect("bound after the previous listener released the port");
|
||||
|
||||
assert_eq!(
|
||||
listener.local_addr().expect("bound address").to_string(),
|
||||
address
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use filetime::FileTime;
|
||||
@@ -18,7 +19,8 @@ use tar::{Builder, EntryType, Header};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
const TARGET_VERSION: &str = "9.9.9";
|
||||
const TARGET_VERSION: &str = "1.9.9";
|
||||
const CROSS_MAJOR_VERSION: &str = "2.0.0";
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_should_not_shutdown_when_replacement_spawn_fails() {
|
||||
@@ -78,6 +80,49 @@ async fn restart_should_spawn_replacement_before_shutdown_outside_docker() {
|
||||
.expect("replacement executed before fixture cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_should_conflict_while_another_system_operation_is_running() {
|
||||
let fixture = Fixture::new();
|
||||
// 裸 TCP 监听不回包:update 持有操作锁后停在 release 拉取阶段,accept
|
||||
// 信号保证断言时锁一定已被占用。
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("release listener");
|
||||
let api_base = format!("http://{}/repos", listener.local_addr().expect("addr"));
|
||||
let (connected_tx, connected_rx) = tokio::sync::oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
let connection = listener.accept().await;
|
||||
let _ = connected_tx.send(());
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
drop(connection);
|
||||
});
|
||||
let mut config = fixture.config(&api_base);
|
||||
config.self_restart_enabled = true;
|
||||
config.deployment_mode = "docker".to_owned();
|
||||
let shutdown = CancellationToken::new();
|
||||
let service = Arc::new(ProcessSystemOperations::new(shutdown.clone(), config));
|
||||
let update = tokio::spawn({
|
||||
let service = Arc::clone(&service);
|
||||
async move {
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), connected_rx)
|
||||
.await
|
||||
.expect("update reaches release fetch while holding the operation lock")
|
||||
.expect("connected signal");
|
||||
|
||||
let error = service.restart().await.expect_err("restart during update");
|
||||
assert_eq!(error.kind(), SystemOperationErrorKind::Conflict);
|
||||
assert!(!shutdown.is_cancelled());
|
||||
|
||||
update.abort();
|
||||
let _ = update.await;
|
||||
service.restart().await.expect("restart after lock release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rollback_should_restore_binary_web_and_version_state() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -134,6 +179,40 @@ async fn update_detail_should_fallback_to_cache_when_refresh_fetch_fails() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_offer_latest_non_draft_prerelease_on_preview_channel() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/repos/owner/repository/releases"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
|
||||
{
|
||||
"tag_name": "v1.2.0-beta.2",
|
||||
"name": "Draft 1.2.0-beta.2",
|
||||
"prerelease": true,
|
||||
"draft": true,
|
||||
"assets": [],
|
||||
},
|
||||
{
|
||||
"tag_name": "v1.1.0-beta.1",
|
||||
"name": "Release 1.1.0-beta.1",
|
||||
"body": "notes",
|
||||
"html_url": "https://github.com/owner/repository/releases",
|
||||
"prerelease": true,
|
||||
"assets": [],
|
||||
},
|
||||
])))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let mut config = fixture.config(&format!("{}/repos", server.uri()));
|
||||
config.update_channel = "preview".to_owned();
|
||||
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
|
||||
|
||||
let detail = service.update_detail(true).await.expect("detail");
|
||||
assert_eq!(detail.latest_version, "1.1.0-beta.1");
|
||||
assert!(detail.has_update);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_reject_untrusted_github_api_base() {
|
||||
let fixture = Fixture::new();
|
||||
@@ -174,6 +253,43 @@ async fn update_detail_should_use_cached_release_when_not_refreshed() {
|
||||
assert!(service.update_detail(false).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_withhold_cross_major_release() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release_once(&server, CROSS_MAJOR_VERSION)
|
||||
.await;
|
||||
|
||||
let detail = fixture
|
||||
.service(&server)
|
||||
.update_detail(true)
|
||||
.await
|
||||
.expect("detail");
|
||||
assert_eq!(detail.latest_version, CROSS_MAJOR_VERSION);
|
||||
assert!(!detail.has_update);
|
||||
assert!(detail.warning.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_reject_cross_major_target_before_fetching_release() {
|
||||
let fixture = Fixture::new();
|
||||
let service = ProcessSystemOperations::new(
|
||||
CancellationToken::new(),
|
||||
fixture.config("http://127.0.0.1:1/repos"),
|
||||
);
|
||||
|
||||
let error = service
|
||||
.perform_update(Some(CROSS_MAJOR_VERSION.to_owned()))
|
||||
.await
|
||||
.expect_err("cross-major target must be rejected");
|
||||
assert_eq!(error.kind(), SystemOperationErrorKind::Conflict);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("binary"),
|
||||
b"old-binary"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_events_should_close_after_terminal_update_log() {
|
||||
let fixture = Fixture::new();
|
||||
@@ -204,6 +320,68 @@ async fn update_events_should_open_authenticated_sse_stream() {
|
||||
assert!(stream.next().await.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_events_should_preserve_complete_release_stage_sequence() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::Safe,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
let events = service.update_events();
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
.expect("update");
|
||||
let steps = events
|
||||
.map(|event| event.step.expect("release update step"))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
steps,
|
||||
[
|
||||
"release", "prepare", "asset", "asset", "verify", "prepare", "download", "download",
|
||||
"download", "checksum", "checksum", "extract", "extract", "replace", "replace", "done",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_events_should_report_download_bytes_and_percent() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::Safe,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
let events = service.update_events();
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
.expect("update");
|
||||
let progress = events
|
||||
.filter_map(
|
||||
|event| async move { event.progress_percent.map(|value| (value, event.message)) },
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(progress.iter().any(|(percent, message)| {
|
||||
*percent == 100 && message.starts_with("已下载 ") && message.ends_with(" (100%)")
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_fail_when_release_checksum_is_missing() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -335,7 +513,7 @@ async fn update_should_reject_when_confirmed_target_differs_from_remote_latest()
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(&server, "9.9.8", ArchiveKind::Safe, ChecksumKind::Valid)
|
||||
.mount_release(&server, "1.9.8", ArchiveKind::Safe, ChecksumKind::Valid)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -626,6 +626,55 @@ async fn shutdown_timeout_aborts_uncooperative_task_and_drops_guard() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_timeout_must_not_rejoin_workers_that_already_exited() {
|
||||
// 挂死任务放在 join 顺序最末的 kind(Retention),保证超时触发时
|
||||
// 排在它前面的合作任务句柄都已被 join 过一次。
|
||||
let hung = HungTask {
|
||||
entered: Arc::new(AtomicBool::new(false)),
|
||||
notification: Arc::new(Notify::new()),
|
||||
};
|
||||
let cooperative = CancellationTask {
|
||||
entered: Arc::new(AtomicUsize::new(0)),
|
||||
observed: Arc::new(AtomicUsize::new(0)),
|
||||
};
|
||||
let mut plan = ACTIVE_KINDS
|
||||
.into_iter()
|
||||
.map(|kind| {
|
||||
if kind == WorkerKind::Retention {
|
||||
registration(kind, kind.as_str(), hung.clone(), test_schedule(), true)
|
||||
} else {
|
||||
registration(
|
||||
kind,
|
||||
kind.as_str(),
|
||||
cooperative.clone(),
|
||||
test_schedule(),
|
||||
true,
|
||||
)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
plan.push(WorkerContribution::Disabled {
|
||||
kind: WorkerKind::OpsFlush,
|
||||
reason: WorkerDisabledReason::NoBufferedOpsEvents,
|
||||
});
|
||||
let leases = Arc::new(FakeLeasePort::default());
|
||||
let supervisor = supervisor();
|
||||
supervisor
|
||||
.start(plan, leases.clone())
|
||||
.expect("complete plan");
|
||||
hung.notification.notified().await;
|
||||
wait_until(|| cooperative.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len() - 1).await;
|
||||
|
||||
supervisor.shutdown(Duration::from_millis(100)).await;
|
||||
|
||||
assert_eq!(
|
||||
cooperative.observed.load(Ordering::SeqCst),
|
||||
ACTIVE_KINDS.len() - 1
|
||||
);
|
||||
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn running_worker_becomes_unhealthy_when_last_success_is_stale() {
|
||||
let task = SucceedsThenHangsTask {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user